.devcontainer/Dockerfile

@@ -0,0 +1,26 @@
+ARG VARIANT="3.11-bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
+
+# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+ARG NODE_VERSION="none"
+RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+
+# Poetry
+ARG POETRY_VERSION="none"
+RUN if [ "${POETRY_VERSION}" != "none" ]; then su vscode -c "umask 0002 && pip3 install poetry==${POETRY_VERSION}"; fi
+
+# Nox
+ARG NOX_VERSION="none"
+RUN if [ "${NOX_VERSION}" != "none" ]; then su vscode -c "umask 0002 && pip3 install nox-poetry nox==${NOX_VERSION}"; fi
+
+#[Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
+#COPY requirements.txt /tmp/pip-tmp/
+#RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+#  && rm -rf /tmp/pip-tmp
+
+#[Optional] Uncomment this section to install additional OS packages.
+#RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#  && apt-get -y install --no-install-recommends <your-package-list-here>
+
+#[Optional] Uncomment this line to install global node packages.
+#RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1

.devcontainer/devcontainer.json

@@ -0,0 +1,57 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.231.6/containers/python-3
+{
+  "name": "Python 3",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "context": "..",
+    "args": {
+      // Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
+      // Append -bullseye or -buster to pin to an OS version.
+      // Use -bullseye variants on local on arm64/Apple Silicon.
+      "VARIANT": "3.10-bullseye",
+      // Options
+      "NODE_VERSION": "lts/*"
+    }
+  },
+  "customizations": {
+    "vscode": {
+      // Set *default* container specific settings.json values on container create.
+      "settings": {
+        "python.defaultInterpreterPath": "/usr/local/bin/python",
+        "python.linting.enabled": true,
+        "python.linting.pylintEnabled": true,
+        "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+        "python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+        "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+        "python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+        "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+        "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+        "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+        "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+        "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
+      },
+      // Add the IDs of extensions you want installed when the container is created.
+      "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance"
+      ]
+    }
+  },
+  // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+  "remoteUser": "vscode",
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  "forwardPorts": [
+    8081
+  ],
+  // Install project dependencies
+  // "postCreateCommand": "poetry install",
+  "postCreateCommand": "bash ./.devcontainer/post-install.sh",
+  "features": {
+    "github-cli": "latest"
+  },
+  "mounts": [
+    // Re-use local Git configuration
+    "source=${localEnv:HOME}/.gitconfig,target=/home/vscode/.gitconfig,type=bind,consistency=cached"
+  ]
+}

.devcontainer/post-install.sh

@@ -0,0 +1 @@
+#!/usr/bin/env bash

.dockerignore

@@ -1,13 +1,4 @@
-**/.idea
+.git
-**/.git
+.venv
-**/.venv
+ui/.angular
-.github/
-.vscode/
-.devcontainer/
-ui/.nuxt
-ui/.output
 ui/node_modules
-ui/dist
-**/.env
-./var/*
-!./var/.gitignore

.editorconfig

@@ -15,4 +15,3 @@ trim_trailing_whitespace = false
 
 [*.py]
 indent_size = 4
-max_line_length = 120

.github/scripts/generate_changelog.py

@@ -1,107 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import json
-import sys
-from datetime import UTC, datetime
-
-try:
-    import git  # type: ignore
-except ImportError:
-    print("Please install GitPython: pip install GitPython", file=sys.stderr)  # noqa: T201
-    sys.exit(1)
-
-
-def get_sorted_tags(repo):
-    """Returns all tags sorted by their commit date (newest first)."""
-    return sorted(repo.tags, key=lambda t: t.commit.committed_datetime, reverse=True)
-
-
-def get_commits_between(repo, start, end):
-    """Return commits between two refs (no merges)."""
-    return list(repo.iter_commits(f"{start}..{end}", no_merges=True))
-
-
-def generate_changelog(repo_path, changelog_path):
-    repo = git.Repo(repo_path)
-    tags = get_sorted_tags(repo)
-    changelog = []
-
-    if not tags:
-        start = repo.commit(repo.git.rev_list("--max-parents=0", "HEAD"))
-        commits = get_commits_between(repo, start.hexsha, "HEAD")
-        changelog.append(
-            {
-                "tag": "Initial Release",
-                "date": datetime.now(UTC).isoformat(),
-                "commits": [
-                    {
-                        "sha": c.hexsha[:8],
-                        "full_sha": c.hexsha,
-                        "message": c.message.strip(),
-                        "author": c.author.name,
-                        "date": c.committed_datetime.astimezone(UTC).isoformat(),
-                    }
-                    for c in commits
-                ],
-            }
-        )
-    else:
-        for i in range(len(tags) - 1):
-            newer, older = tags[i], tags[i + 1]
-            commits = get_commits_between(repo, older.commit.hexsha, newer.commit.hexsha)
-            if not commits:
-                continue
-            changelog.append(
-                {
-                    "tag": newer.name,
-                    "full_sha": newer.commit.hexsha,
-                    "date": newer.commit.committed_datetime.astimezone(UTC).isoformat(),
-                    "commits": [
-                        {
-                            "sha": c.hexsha[:8],
-                            "full_sha": c.hexsha,
-                            "message": c.message.strip(),
-                            "author": c.author.name,
-                            "date": c.committed_datetime.astimezone(UTC).isoformat(),
-                        }
-                        for c in commits
-                    ],
-                }
-            )
-
-        # Add HEAD -> latest tag if ahead
-        head = repo.head.commit
-        if head.hexsha != tags[0].commit.hexsha:
-            commits = get_commits_between(repo, tags[0].commit.hexsha, head.hexsha)
-            if commits:
-                changelog.insert(
-                    0,
-                    {
-                        "tag": f"Unreleased ({head.hexsha[:8]})",
-                        "full_sha": head.hexsha,
-                        "date": head.committed_datetime.astimezone(UTC).isoformat(),
-                        "commits": [
-                            {
-                                "sha": c.hexsha[:8],
-                                "full_sha": c.hexsha,
-                                "message": c.message.strip(),
-                                "author": c.author.name,
-                                "date": c.committed_datetime.astimezone(UTC).isoformat(),
-                            }
-                            for c in commits
-                        ],
-                    },
-                )
-
-    with open(changelog_path, "w", encoding="utf-8") as f:
-        json.dump(changelog, f, indent=2)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Generate CHANGELOG.json from Git tags.")
-    parser.add_argument("--repo_path", "-p", default=".", help="Path to git repo")
-    parser.add_argument("--changelog_path", "-f", default="CHANGELOG.json", help="Output file path")
-    args = parser.parse_args()
-
-    generate_changelog(args.repo_path, args.changelog_path)

.github/workflows/build-pr.yml

@@ -1,108 +0,0 @@
-name: build-pr
-
-permissions:
-  contents: read
-
-on:
-  pull_request:
-    branches:
-      - master
-      - dev
-      - testing
-    paths-ignore:
-      - "**.md"
-      - ".github/ISSUE_TEMPLATE/**"
-
-env:
-  BUN_VERSION: latest
-  NODE_VERSION: 22
-  PYTHON_VERSION: "3.13"
-
-jobs:
-  test:
-    name: Run Tests & Build Validation
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Install uv
-        run: pip install uv
-
-      - name: Install Python dependencies
-        run: uv sync
-
-      - name: Run Python linting
-        run: uv run ruff check app/
-
-      - name: Run Python tests
-        run: uv run pytest app/tests/ -v --tb=short
-
-      - name: Cache bun installation
-        id: cache-bun
-        uses: actions/cache@v4
-        with:
-          path: ~/.bun/install/cache
-          key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-bun-
-
-      - name: Install Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: ${{ env.NODE_VERSION }}
-
-      - name: Install bun
-        uses: oven-sh/setup-bun@v2
-        with:
-          bun-version: ${{ env.BUN_VERSION }}
-
-      - name: Install frontend dependencies
-        working-directory: ui
-        env:
-          NODE_ENV: development
-        run: bun install --frozen-lockfile
-
-      - name: Prepare frontend (nuxt prepare)
-        working-directory: ui
-        env:
-          NODE_ENV: development
-        run: bun nuxt prepare
-
-      - name: Run frontend linting
-        working-directory: ui
-        run: bun run lint
-
-      - name: Run frontend type checking
-        working-directory: ui
-        run: bun run typecheck
-
-      - name: Run frontend tsc
-        working-directory: ui
-        run: bun run lint:tsc
-
-      - name: Run frontend tests
-        working-directory: ui
-        run: bun run test:ci
-
-      - name: Build frontend
-        working-directory: ui
-        run: bun run generate
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Validate Docker build
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          platforms: linux/amd64
-          push: false
-          cache-from: type=gha,scope=${{ github.workflow }}
-          cache-to: type=gha,mode=max,scope=${{ github.workflow }}
-          provenance: false

.github/workflows/delete-builds.yml

@@ -1,4 +1,4 @@
-name: delete-builds
+name: delete builds
 
 on:
   workflow_dispatch:

.github/workflows/main.yml

@@ -1,228 +1,74 @@
-name: build-docker-containers
+name: build
 
 on:
   workflow_dispatch:
     inputs:
-      build:
+      logLevel:
-        description: "Build and publish native multi-arch image"
+        description: "Log level"
-        type: boolean
+        required: true
-        default: true
+        default: "warning"
-      update_readme:
+        type: choice
-        description: "Also sync DockerHub README"
+        options:
-        type: boolean
+          - info
-        default: false
+          - warning
+          - debug
   push:
     branches:
-      - dev
+      - "*"
-      - master
+    paths-ignore:
-      - testing
+      - "**.md"
-    tags:
+      - ".github/**"
-      - "v*"
+  pull_request:
+    branches:
+      - "master"
     paths-ignore:
       - "**.md"
       - ".github/ISSUE_TEMPLATE/**"
 
 env:
-  REGISTRY: ${{ vars.REGISTRY != '' && vars.REGISTRY || '' }}
-  BUILD_AMD64: ${{ vars.BUILD_AMD64 != 'false' && 'true' || 'false' }}
-  BUILD_ARM64: ${{ vars.BUILD_ARM64 != 'false' && 'true' || 'false' }}
   DOCKERHUB_SLUG: arabcoders/ytptube
   GHCR_SLUG: ghcr.io/arabcoders/ytptube
-  BUN_VERSION: latest
+  PLATFORMS: linux/amd64
-  NODE_VERSION: 22
-  PYTHON_VERSION: "3.13"
 
 jobs:
-  validate-build-config:
+  push-build:
-    name: Validate Build Configuration
     runs-on: ubuntu-latest
-    permissions: {}
-    steps:
-      - name: Check if at least one architecture is enabled
-        run: |
-          if [ "${{ vars.BUILD_AMD64 }}" = "false" ] && [ "${{ vars.BUILD_ARM64 }}" = "false" ]; then
-            echo "::error::Both BUILD_AMD64 and BUILD_ARM64 are disabled. At least one architecture must be enabled."
-            exit 1
-          fi
-          echo "Build configuration is valid"
-          echo "BUILD_AMD64: ${{ vars.BUILD_AMD64 != 'false' && 'true' || 'false' }}"
-          echo "BUILD_ARM64: ${{ vars.BUILD_ARM64 != 'false' && 'true' || 'false' }}"
-
-  test:
-    name: Run Tests & Prepare Frontend
-    needs: [validate-build-config]
-    permissions:
-      contents: read
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Install uv
-        run: pip install uv
-
-      - name: Install Python dependencies
-        run: uv sync
-
-      - name: Run Python linting
-        run: uv run ruff check app/
-
-      - name: Run Python tests
-        run: uv run pytest app/tests/ -v --tb=short
-
-      - name: Cache bun installation
-        id: cache-bun
-        uses: actions/cache@v4
-        with:
-          path: ~/.bun/install/cache
-          key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-bun-
-
-      - name: Install Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: ${{ env.NODE_VERSION }}
-
-      - name: Install bun
-        uses: oven-sh/setup-bun@v2
-        with:
-          bun-version: ${{ env.BUN_VERSION }}
-
-      - name: Install frontend dependencies
-        working-directory: ui
-        env:
-          NODE_ENV: development
-        run: bun install --frozen-lockfile
-
-      - name: Prepare frontend (nuxt prepare)
-        working-directory: ui
-        env:
-          NODE_ENV: development
-        run: bun nuxt prepare
-
-      - name: Run frontend linting
-        working-directory: ui
-        run: bun run lint
-
-      - name: Run frontend type checking
-        working-directory: ui
-        run: bun run typecheck
-
-      - name: Run frontend tsc
-        working-directory: ui
-        run: bun run lint:tsc
-
-      - name: Run frontend tests
-        working-directory: ui
-        run: bun run test:ci
-
-      - name: Remove dev dependencies
-        working-directory: ui
-        env:
-          NODE_ENV: production
-        run: bun install --frozen-lockfile --production --ignore-scripts
-
-      - name: Build frontend
-        working-directory: ui
-        env:
-          NODE_ENV: production
-        run: bun run generate
-
-      - name: Install GitPython
-        run: pip install gitpython
-
-      - name: Generate CHANGELOG.json
-        run: |
-          python3 .github/scripts/generate_changelog.py -p . -f ./ui/exported/CHANGELOG.json
-
-      - name: Upload frontend build
-        uses: actions/upload-artifact@v4
-        if: env.REGISTRY == ''
-        with:
-          name: frontend-build
-          path: ui/exported/
-          retention-days: 1
-
-  docker-build-amd64:
-    name: Build Container (amd64)
-    runs-on: ubuntu-latest
-    needs: [test, validate-build-config]
-    if: vars.BUILD_AMD64 != 'false'
     permissions:
       packages: write
       contents: write
-    env:
-      ARCH: amd64
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+
+      - name: Update Version File
+        uses: ArabCoders/write-version-to-file@master
         with:
-          fetch-depth: 0
+          filename: "/app/version.py"
+          placeholder: "dev-master"
 
-      - name: Download frontend build
+      - name: Set up QEMU
-        uses: actions/download-artifact@v4
+        uses: docker/setup-qemu-action@v3
-        if: env.REGISTRY == ''
-        with:
-          name: frontend-build
-          path: ui/exported/
 
-      - name: Update app/library/version.py
+      - name: Set up Docker Buildx
-        run: |
+        uses: docker/setup-buildx-action@v3
-          VERSION="${GITHUB_REF##*/}"
-          SHA=$(git rev-parse HEAD)
-          DATE=$(date -u +"%Y%m%d")
-          BRANCH=$(echo "${GITHUB_REF#refs/heads/}" | sed 's/\//-/g')
-
-          echo "APP_VERSION=${VERSION}" >> "$GITHUB_ENV"
-          echo "APP_SHA=${SHA}" >> "$GITHUB_ENV"
-          echo "APP_DATE=${DATE}" >> "$GITHUB_ENV"
-          echo "APP_BRANCH=${BRANCH}" >> "$GITHUB_ENV"
-
-          sed -i \
-            -e "s/^APP_VERSION = \".*\"/APP_VERSION = \"${VERSION}\"/" \
-            -e "s/^APP_COMMIT_SHA = \".*\"/APP_COMMIT_SHA = \"${SHA}\"/" \
-            -e "s/^APP_BUILD_DATE = \".*\"/APP_BUILD_DATE = \"${DATE}\"/" \
-            -e "s/^APP_BRANCH = \".*\"/APP_BRANCH = \"${BRANCH}\"/" \
-            app/library/version.py
-
-          echo "Updated version info:"
-          cat app/library/version.py
 
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
         with:
           images: |
-            name=${{ env.DOCKERHUB_SLUG }},enable=${{ env.REGISTRY == '' }}
+            ${{ env.DOCKERHUB_SLUG }}
-            name=${{ env.GHCR_SLUG }},enable=${{ env.REGISTRY == '' }}
+            ${{ env.GHCR_SLUG }}
-            name=${{ env.REGISTRY }}/${{ github.repository }},enable=${{ env.REGISTRY != '' }}
-          flavor: |
-            latest=false
-            suffix=-${{ env.ARCH }}
           tags: |
+            type=raw,value=latest,enable=${{ endsWith(github.ref, github.event.repository.default_branch) }}
             type=ref,event=branch
             type=ref,event=tag
-            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
+            type=raw,value={{branch}}{{base_ref}}-{{date 'YYYYMMDD'}}-{{sha}}
-
+          flavor: |
-      - name: Login to Private Registry
+            latest=false
-        uses: docker/login-action@v3
-        if: env.REGISTRY != ''
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GIT_TOKEN && secrets.GIT_TOKEN || secrets.GITHUB_TOKEN }}
 
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
-        if: env.REGISTRY == ''
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -230,267 +76,37 @@ jobs:
 
       - name: Login to DockerHub
         uses: docker/login-action@v3
-        if: env.REGISTRY == ''
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
       - name: Build and push
         uses: docker/build-push-action@v5
         with:
           context: .
-          platforms: linux/${{ env.ARCH }}
+          platforms: ${{ env.PLATFORMS }}
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-to: type=gha,mode=max,scope=${{ github.workflow }}
+          cache-from: type=gha, scope=${{ github.workflow }}
-          cache-from: type=gha,scope=${{ github.workflow }}
+          cache-to: type=gha, scope=${{ github.workflow }}
-          provenance: true
 
-  docker-build-arm64:
+      - name: Version tag
-    name: Build Container (arm64)
+        if: github.event_name != 'pull_request'
-    runs-on: ubuntu-latest
+        uses: arabcoders/action-python-autotagger@master
-    needs: [test, validate-build-config]
-    if: vars.BUILD_ARM64 != 'false'
-    permissions:
-      packages: write
-      contents: write
-    env:
-      ARCH: arm64
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
         with:
-          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
-
+          repo_name: arabcoders/ytptube
-      - name: Download frontend build
+          path: app/version.py
-        uses: actions/download-artifact@v4
+          variable: APP_VERSION
-        if: env.REGISTRY == ''
-        with:
-          name: frontend-build
-          path: ui/exported/
-
-      - name: Update app/library/version.py
-        run: |
-          VERSION="${GITHUB_REF##*/}"
-          SHA=$(git rev-parse HEAD)
-          DATE=$(date -u +"%Y%m%d")
-          BRANCH=$(echo "${GITHUB_REF#refs/heads/}" | sed 's/\//-/g')
-
-          echo "APP_VERSION=${VERSION}" >> "$GITHUB_ENV"
-          echo "APP_SHA=${SHA}" >> "$GITHUB_ENV"
-          echo "APP_DATE=${DATE}" >> "$GITHUB_ENV"
-          echo "APP_BRANCH=${BRANCH}" >> "$GITHUB_ENV"
-
-          sed -i \
-            -e "s/^APP_VERSION = \".*\"/APP_VERSION = \"${VERSION}\"/" \
-            -e "s/^APP_COMMIT_SHA = \".*\"/APP_COMMIT_SHA = \"${SHA}\"/" \
-            -e "s/^APP_BUILD_DATE = \".*\"/APP_BUILD_DATE = \"${DATE}\"/" \
-            -e "s/^APP_BRANCH = \".*\"/APP_BRANCH = \"${BRANCH}\"/" \
-            app/library/version.py
-
-          echo "Updated version info:"
-          cat app/library/version.py
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            name=${{ env.DOCKERHUB_SLUG }},enable=${{ env.REGISTRY == '' }}
-            name=${{ env.GHCR_SLUG }},enable=${{ env.REGISTRY == '' }}
-            name=${{ env.REGISTRY }}/${{ github.repository }},enable=${{ env.REGISTRY != '' }}
-          flavor: |
-            latest=false
-            suffix=-${{ env.ARCH }}
-          tags: |
-            type=ref,event=branch
-            type=ref,event=tag
-            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        if: env.REGISTRY != ''
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GIT_TOKEN && secrets.GIT_TOKEN || secrets.GITHUB_TOKEN }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        if: env.REGISTRY == ''
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        if: env.REGISTRY == ''
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          platforms: linux/${{ env.ARCH }}
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-to: type=gha,mode=max,scope=${{ github.workflow }}
-          cache-from: type=gha,scope=${{ github.workflow }}
-          provenance: true
-
-  docker-publish-manifest:
-    name: Publish multi-arch manifest
-    runs-on: ubuntu-latest
-    needs: [docker-build-amd64, docker-build-arm64]
-    if: |
-      !cancelled() &&
-      (needs.docker-build-amd64.result == 'success' || needs.docker-build-arm64.result == 'success')
-    permissions:
-      packages: write
-      contents: write
-    steps:
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            name=${{ env.DOCKERHUB_SLUG }},enable=${{ env.REGISTRY == '' }}
-            name=${{ env.GHCR_SLUG }},enable=${{ env.REGISTRY == '' }}
-            name=${{ env.REGISTRY }}/${{ github.repository }},enable=${{ env.REGISTRY != '' }}
-          flavor: |
-            latest=false
-          tags: |
-            type=ref,event=branch
-            type=ref,event=tag
-            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
-
-      - name: Login to Private Registry
-        uses: docker/login-action@v3
-        if: env.REGISTRY != ''
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GIT_TOKEN && secrets.GIT_TOKEN || secrets.GITHUB_TOKEN }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        if: env.REGISTRY == ''
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        if: env.REGISTRY == ''
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Create and push manifest list
-        shell: bash
-        run: |
-          IFS=$'\n'
-          BUILD_AMD64="${{ env.BUILD_AMD64 }}"
-          BUILD_ARM64="${{ env.BUILD_ARM64 }}"
-
-          for tag in $(echo "${{ steps.meta.outputs.tags }}"); do
-            echo "Creating manifest for ${tag}"
-
-            IMAGES=()
-            if [ "$BUILD_AMD64" = "true" ]; then
-              IMAGES+=("${tag}-amd64")
-            fi
-            if [ "$BUILD_ARM64" = "true" ]; then
-              IMAGES+=("${tag}-arm64")
-            fi
-
-            if [ ${#IMAGES[@]} -eq 0 ]; then
-              echo "::error::No architectures enabled for manifest creation"
-              exit 1
-            fi
-
-            docker buildx imagetools create --tag "$tag" "${IMAGES[@]}"
-          done
-
-      - name: Overwrite GitHub release notes
-        if: startsWith(github.ref, 'refs/tags/v') && env.REGISTRY == ''
-        uses: actions/github-script@v6
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const tag = context.ref.replace('refs/tags/', '');
-
-            const { data: releases } = await github.rest.repos.listReleases({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-            });
-
-            const release = releases.find(r => r.tag_name === tag);
-            if (!release) {
-              core.setFailed(`Release with tag ${tag} not found`);
-              return;
-            }
-
-            const baseTag = releases[1]?.tag_name || '';
-            if (!baseTag) {
-              core.setFailed('Could not determine a base tag (no previous release found)');
-              return;
-            }
-
-            const { data: comparison } = await github.rest.repos.compareCommits({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              base: baseTag,
-              head: tag,
-            });
-
-            const commits = comparison.commits.filter(c => c.parents.length === 1);
-
-            const changelog = commits
-              .map(c => `- ${c.sha.substring(0, 7)} ${c.commit.message.split('\n')[0]}`)
-              .join('\n');
-
-            if (!changelog) {
-              core.setFailed('No commits found for the changelog');
-              return;
-            }
-
-            const existingBody = (release.body || '').trim();
-
-            const separator = `\n\n---\n\n## Commits since ${baseTag}\n\n`;
-            const newBody = existingBody
-              ? `${existingBody}${separator}${changelog}`
-              : `## Commits since ${baseTag}\n\n${changelog}`;
-
-            await github.rest.repos.updateRelease({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              release_id: release.id,
-              body: newBody,
-            });
 
   dockerhub-sync-readme:
-    name: DockerHub README sync
+    needs: push-build
+    if: github.event_name != 'pull_request'
     runs-on: ubuntu-latest
-    permissions:
-      contents: read
-    if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.update_readme == 'true')
     steps:
       - name: Sync README
         uses: docker://lsiodev/readme-sync:latest
-        if: env.REGISTRY == ''
         env:
           DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
           DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/native-build.yml

@@ -1,225 +0,0 @@
-name: native-build
-
-on:
-  workflow_dispatch:
-    inputs:
-      tag:
-        required: true
-        description: "Ref to build from (e.g. v1.0.0)"
-      useWorkflowSpec:
-        type: boolean
-        default: false
-        description: "Use app.spec from workflow branch instead of tag"
-  push:
-    tags:
-      - "v*"
-
-jobs:
-  build:
-    if: ${{ vars.REGISTRY == '' }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        include:
-          - os: ubuntu-latest
-            arch: amd64
-          - os: ubuntu-latest
-            arch: arm64
-          - os: macos-latest
-            arch: arm64
-          - os: windows-latest
-            arch: amd64
-          - os: windows-latest
-            arch: arm64
-
-    permissions:
-      packages: write
-      contents: write
-
-    env:
-      PYTHON_VERSION: "3.13"
-      BUN_VERSION: latest
-      NODE_VERSION: 22
-      TAG_NAME: ${{ github.event.inputs.tag || github.ref_name }}
-
-    steps:
-      - name: Checkout source repo
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ env.TAG_NAME }}
-
-      - name: Overwrite app.spec from workflow branch
-        if: ${{ github.event.inputs.useWorkflowSpec == 'true' }}
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.ref }}
-          sparse-checkout: |
-            app.spec
-          sparse-checkout-cone-mode: false
-          path: temp-spec
-
-      - name: Replace app.spec with version from workflow branch
-        if: ${{ github.event.inputs.useWorkflowSpec == 'true' }}
-        shell: bash
-        run: cp temp-spec/app.spec ./app.spec
-
-      - name: Update app/library/version.py
-        shell: bash
-        run: |
-          VERSION="${TAG_NAME}"
-          SHA=$(git rev-parse HEAD)
-          DATE=$(date -u +"%Y%m%d")
-          BRANCH=$(git branch --show-current)
-
-          if [ -z "${BRANCH}" ]; then
-            BRANCH="${TAG_NAME}"
-          fi
-
-          BRANCH=$(echo "${BRANCH}" | sed 's#/#-#g')
-
-          echo "APP_VERSION=${VERSION}" >> "$GITHUB_ENV"
-          echo "APP_SHA=${SHA}" >> "$GITHUB_ENV"
-          echo "APP_DATE=${DATE}" >> "$GITHUB_ENV"
-          echo "APP_BRANCH=${BRANCH}" >> "$GITHUB_ENV"
-
-          sed -i.bak \
-            -e "s/^APP_VERSION = \".*\"/APP_VERSION = \"${VERSION}\"/" \
-            -e "s/^APP_COMMIT_SHA = \".*\"/APP_COMMIT_SHA = \"${SHA}\"/" \
-            -e "s/^APP_BUILD_DATE = \".*\"/APP_BUILD_DATE = \"${DATE}\"/" \
-            -e "s/^APP_BRANCH = \".*\"/APP_BRANCH = \"${BRANCH}\"/" \
-            app/library/version.py
-
-          rm -f app/library/version.py.bak
-
-          echo "Updated version info:"
-          cat app/library/version.py
-
-      - name: Cache Python venv
-        id: cache-python
-        uses: actions/cache@v4
-        with:
-          path: .venv
-          key: uv-${{ runner.os }}-${{ matrix.arch }}-${{ env.PYTHON_VERSION }}-${{ hashFiles('uv.lock') }}
-          restore-keys: |
-            uv-${{ runner.os }}-${{ matrix.arch }}-${{ env.PYTHON_VERSION }}-
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-          architecture: "x64"
-
-      - name: Install Python dependencies
-        shell: bash
-        run: |
-          pip install uv
-          if [ ! -d .venv ]; then
-            uv venv --system-site-packages --relocatable
-          fi
-          uv sync --link-mode=copy --active --extra installer
-
-      - name: Cache bun installation
-        id: cache-bun
-        uses: actions/cache@v4
-        with:
-          path: ~/.bun/install/cache
-          key: ${{ runner.os }}-${{ matrix.arch }}-bun-${{ hashFiles('**/bun.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.arch }}-bun-
-
-      - name: Install bun
-        uses: oven-sh/setup-bun@v2
-        with:
-          bun-version: ${{ env.BUN_VERSION }}
-
-      - name: Install Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: ${{ env.NODE_VERSION }}
-
-      - name: Build frontend
-        working-directory: ui
-        shell: bash
-        run: |
-          bun install --production --prefer-offline --frozen-lockfile
-          bun run generate
-
-      - name: Clean build outputs (Unix)
-        if: runner.os != 'Windows'
-        shell: bash
-        run: rm -rf build dist release || true
-
-      - name: Clean build outputs (Windows)
-        if: runner.os == 'Windows'
-        shell: powershell
-        run: |
-          if (Test-Path build)   { Remove-Item -Recurse -Force build }
-          if (Test-Path dist)    { Remove-Item -Recurse -Force dist }
-          if (Test-Path release) { Remove-Item -Recurse -Force release }
-
-      - name: Build native binary (Windows)
-        if: matrix.os == 'windows-latest'
-        shell: powershell
-        run: |
-          .\.venv\Scripts\activate
-          uv run pyinstaller ./app.spec
-
-      - name: Build native binary (Linux/macOS)
-        if: matrix.os != 'windows-latest'
-        shell: bash
-        run: |
-          . .venv/bin/activate
-          uv run pyinstaller ./app.spec
-
-      - name: Upload artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: app-${{ runner.os }}-${{ matrix.arch }}-${{ env.TAG_NAME }}
-          path: dist/YTPTube/**
-
-      - name: Package artifact (Unix)
-        if: startsWith(env.TAG_NAME, 'v') && runner.os != 'Windows'
-        shell: bash
-        run: |
-          mkdir -p release
-          PKG_DIR="YTPTube-${{ env.TAG_NAME }}"
-          ZIP_NAME="ytptube-${{ runner.os }}-${{ matrix.arch }}-${{ env.TAG_NAME }}.zip"
-
-          if [ ! -f "dist/YTPTube/YTPTube" ]; then
-            echo "Missing dist/YTPTube/YTPTube"
-            find dist -maxdepth 3 -type f | head -n 200
-            exit 1
-          fi
-
-          rm -rf "dist/${PKG_DIR}"
-          mv "dist/YTPTube" "dist/${PKG_DIR}"
-
-          zip -yr "release/${ZIP_NAME}" "dist/${PKG_DIR}"
-
-      - name: Package artifact (Windows)
-        if: startsWith(env.TAG_NAME, 'v') && runner.os == 'Windows'
-        shell: powershell
-        run: |
-          $pkgDir = "YTPTube-${{ env.TAG_NAME }}"
-          $zipName = "ytptube-${{ runner.os }}-${{ matrix.arch }}-${{ env.TAG_NAME }}.zip"
-          New-Item -ItemType Directory -Force -Path release | Out-Null
-
-          if (!(Test-Path "dist\YTPTube\YTPTube.exe")) {
-            Write-Host "Missing dist\YTPTube\YTPTube.exe"
-            Get-ChildItem -Recurse dist | Select-Object -First 200 | Format-Table -AutoSize
-            exit 1
-          }
-
-          if (Test-Path "dist\$pkgDir") { Remove-Item -Recurse -Force "dist\$pkgDir" }
-          Move-Item "dist\YTPTube" "dist\$pkgDir"
-
-          Compress-Archive -Path "dist\$pkgDir" -DestinationPath "release\$zipName" -Force
-
-      - name: Upload to GitHub Release
-        if: startsWith(env.TAG_NAME, 'v')
-        uses: softprops/action-gh-release@v2
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ env.TAG_NAME }}
-          files: release/*.zip

.github/workflows/update-yt-dlp.yml

@@ -21,40 +21,26 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-        with:
-          ref: "dev"
-
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: "3.11"
-
+      - name: Update yt-dlp
-      - name: Install uv
-        run: pip install uv
-
-      - name: Sync dependencies
-        run: uv venv && . .venv/bin/activate && uv sync
-
-      - name: Check and update yt-dlp
         id: ytdlp_update
         run: |
-          . .venv/bin/activate
+          pip install pipenv
-          VER=$(uv pip list --outdated | awk '$1 == "yt-dlp" {print $3}')
+          pipenv sync
+          VER=`pipenv run pip list -o | awk '$1 == "yt-dlp" {print $3}'`
           if [ -n "$VER" ]; then
             echo "YTLDLP_VER=${VER}" >> "$GITHUB_OUTPUT"
-            uv pip install --upgrade yt-dlp[default]
+            pipenv update yt-dlp
-            uv sync --upgrade
-            UPDATED=true
-          else
-            UPDATED=false
           fi
-          echo "UPDATED=${UPDATED}" >> "$GITHUB_OUTPUT"
-
       - name: Create Pull Request
-        if: steps.ytdlp_update.outputs.UPDATED == 'true'
         uses: peter-evans/create-pull-request@v5
         with:
-          title: "[yt-dlp] automated update to ${{ steps.ytdlp_update.outputs.YTLDLP_VER }}"
+          title: "Update yt-dlp"
           commit-message: "Update yt-dlp to ${{ steps.ytdlp_update.outputs.YTLDLP_VER }}"
           body: "This is an automated request to update yt-dlp dependency to ${{ steps.ytdlp_update.outputs.YTLDLP_VER }}"
           delete-branch: true
+
+

.gitignore

@@ -1,11 +1,8 @@
 # compiled output
 /frontend/dist
-/ui/dist
-/ui/exported/CHANGELOG.json
 
 # dependencies
-**/node_modules/*
+/frontend/node_modules
-**/.nuxt/*
 
 # IDEs and editors
 /ui/.idea
@@ -33,10 +30,3 @@ var/*
 # Misc - python stuff
 __pycache__
 .venv
-.idea
-dist
-build
-version.txt
-test_impl.py
-./eslint.config.js
-.pytest_cache

.vscode/launch.json

@@ -1,69 +1,37 @@
 {
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "UI: Dev",
+      "name": "Node: App.Vue",
       "request": "launch",
       "runtimeArgs": [
         "run",
-        "dev",
+        "serve",
+        "--",
         "--port",
-        "8082"
+        "3000"
       ],
-      "runtimeExecutable": "bun",
+      "runtimeExecutable": "npm",
       "type": "node",
-      "cwd": "${workspaceFolder}/ui",
+      "cwd": "${workspaceFolder}/frontend",
       "env": {
-        "NUXT_API_URL": "http://localhost:8081/api/",
+        "VUE_APP_BASE_URL": "http://localhost:8081",
-        "NUXT_PUBLIC_WSS": ":8081/"
+      }
-      },
-      "console": "internalConsole",
-      "outputCapture": "std"
     },
     {
       "name": "Python: main.py",
       "type": "debugpy",
       "request": "launch",
       "program": "app/main.py",
-      "console": "integratedTerminal",
+      "console": "internalConsole",
-      "justMyCode": true,
-      "subProcess": true,
-      "env": {
-        "YTP_CONFIG_PATH": "${workspaceFolder}/var/config",
-        "YTP_DOWNLOAD_PATH": "${workspaceFolder}/var/downloads",
-        "YTP_TEMP_PATH": "${workspaceFolder}/var/tmp",
-        "PYDEVD_DISABLE_FILE_VALIDATION": "1",
-        "YTP_IGNORE_UI": "true",
-        "YTP_DEBUG": "true"
-      },
-    },
-    {
-      "name": "UI: Generate",
-      "request": "launch",
-      "runtimeArgs": [
-        "run",
-        "generate"
-      ],
-      "runtimeExecutable": "bun",
-      "type": "node",
-      "cwd": "${workspaceFolder}/ui",
-      "console": "internalConsole",
-      "outputCapture": "std",
-      "env": {
-        "APP_ENV": "production"
-      }
-    },
-    {
-      "name": "Python: main.py (Deep)",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "app/main.py",
-      "console": "internalConsole",
-      "justMyCode": false,
       "env": {
         "YTP_CONFIG_PATH": "${workspaceFolder}/var/config",
         "YTP_DOWNLOAD_PATH": "${workspaceFolder}/var/downloads",
         "YTP_TEMP_PATH": "${workspaceFolder}/var/tmp",
+        "YTP_URL_HOST": "http://localhost:8081",
         "YTP_LOG_LEVEL": "DEBUG"
       }
     },
@@ -71,7 +39,7 @@
       "name": "Python: Attach To Process",
       "type": "debugpy",
       "request": "attach",
-      "processId": "${command:pickProcess}"
+      "processId": "${command:pickProcess}",
     },
     {
       "name": "Python: Attach To Container",
@@ -88,6 +56,6 @@
         }
       ],
       "justMyCode": true
-    },
+    }
   ]
 }

.vscode/settings.json

@@ -2,33 +2,33 @@
   "files.exclude": {
     "**/__pycache__": true
   },
-  "editor.rulers": [120],
+  "autopep8.args": [
-  "cSpell.enabled": false,
+    "--max-line-length",
-  "css.styleSheets": ["ui/app/assets/css/*.css"],
+    "120",
-  "search.exclude": {
+    "--experimental"
-    "ui/.nuxt": true,
+  ],
-    "ui/.output": true,
+  "editor.rulers": [
-    "ui/dist": true,
+    120
-    "ui/exported": true,
+  ],
-    "ui/node_modules": true,
+  "cSpell.words": [
-    "ui/.out": true,
+    "aconvert",
-    "**/.venv": true
+    "aiocron",
-  },
+    "copyts",
-  "eslint.format.enable": false,
+    "dotenv",
-  "eslint.ignoreUntitled": true,
+    "finaldir",
-  "python.testing.unittestEnabled": false,
+    "getpid",
-  "python.testing.pytestEnabled": true,
+    "libcurl",
-  "python.testing.pytestArgs": ["app"],
+    "libx",
-  "oxc.path.oxfmt": "./ui/node_modules/.bin/oxfmt",
+    "mpegts",
-  "oxc.fmt.configPath": "./ui/.oxfmtrc.json",
+    "muxdelay",
-  "[typescript]": {
+    "nodesc",
-    "editor.defaultFormatter": "oxc.oxc-vscode",
+    "noprogress",
-    "editor.formatOnSave": true,
+    "postprocessor",
-    "editor.formatOnSaveMode": "file"
+    "preferredcodec",
-  },
+    "preferredquality",
-  "[vue]": {
+    "tmpfilename",
-    "editor.defaultFormatter": "oxc.oxc-vscode",
+    "vconvert",
-    "editor.formatOnSave": true,
+    "writedescription",
-    "editor.formatOnSaveMode": "file"
+    "xerror"
-  }
+  ],
 }

CONTRIBUTING.md

@@ -1,149 +0,0 @@
-# Contributing to YTPTube
-
-YTPTube is a **personal-first project**. While contributions are welcome, **all final decisions rest with the maintainer**.
-
-## Core Principles
-
-* **The maintainer has final say** on all changes and project direction.
-* This project is built **for personal use first**. Community contributions are secondary.
-* **All contributions require prior discussion and approval.**
-* **All pull requests must target the `dev` branch** never `master`.
-* Contributions must align with the project's goals, architecture, and coding standards.
-
-**Opening a PR without prior approval will result in immediate closure without review.**
-
----
-
-## Contribution Process
-
-### 1. Start a Discussion
-
-Before writing any code, **propose your idea** through one of these channels:
-
-* Open a **GitHub Issue** for bug fixes or small features
-* Start a **GitHub Discussion** for larger changes or questions
-
-**Include in your proposal:**
-* **What** you want to change
-* **Why** it's needed or beneficial
-  * Bug fixes and functional improvements are prioritized
-  * Performance enhancements with measurable impact
-  * Features that align with the project's core purpose
-  * **Not acceptable:** Personal UI/UX preferences, stylistic changes, or "I think it looks better" rationale
-* **How** you plan to implement it (high-level approach)
-* Any relevant context or use cases
-
-### 2. Wait for Approval
-
-* Only proceed after **explicit approval** from the maintainer to not waste effort.
-* The maintainer may suggest modifications or alternative approaches.
-* Not all proposals will be accepted this protects project coherence.
-
-### 3. Develop Your Changes
-
-**Branch from `dev`:**
-```bash
-git checkout dev
-git pull origin dev
-git checkout -b feature/descriptive-name
-```
-
-**Follow project standards:**
-* Match existing code style and conventions
-* Add or update tests for all changes:
-  * New features **MUST** include tests
-  * Bug fixes **MUST** include a regression test
-* Ensure all linting and tests pass
-* Keep changes focused and atomic
-
-### 4. Submit a Pull Request
-
-**Target the `dev` branch:**
-* Reference the approved issue or discussion number
-* Provide a clear description of what changed and why
-* List any breaking changes or migration steps
-* Ensure CI checks pass
-
-**PR template checklist:**
-- [ ] Discussed and approved beforehand
-- [ ] Targets `dev` branch
-- [ ] Tests added/updated and passing
-- [ ] Linting passes
-- [ ] Documentation updated (if needed)
-
----
-
-## Automatic Rejections
-
-The following will be **closed immediately without review**:
-
-* PRs opened without prior discussion and approval
-* PRs targeting `master` instead of `dev`
-* Large refactors or architectural changes without approval
-* Fully AI-generated code without meaningful human oversight
-* Changes that don't align with project goals or philosophy
-* PRs where the contributor cannot explain or justify the changes
-
----
-
-## AI-Assisted Development
-
-AI tools are **permitted** as development aids, but **you remain fully responsible** for all submitted code.
-
-### Acceptable Use
-
-AI-assisted code is welcome **when**:
-
-* You **fully understand** every line being submitted
-* The code **seamlessly integrates** with existing patterns and style
-* You have **reviewed, tested, and validated** the output yourself
-* **Comprehensive tests** are included (not AI-generated stubs)
-* The code is **indistinguishable in quality** from hand-written contributions
-* You can **explain and defend** design decisions in the PR
-
-**AI is a tool, not a substitute for understanding.**
-
-### Not Acceptable
-
-The following will be rejected:
-
-* Fully AI-generated PRs with minimal human review
-* Code that introduces new patterns or abstractions without approval
-* "Prompt-dump" output that doesn't match project conventions
-* Changes the contributor cannot explain or justify
-* AI-generated test suites that don't meaningfully validate behavior
-
-### Disclosure
-
-You are **not required** to disclose AI usage. However, if code quality suggests pure AI generation without human oversight, the PR will be closed.
-
-**You are accountable for correctness, maintainability, and alignment regardless of how the code was created.**
-
----
-
-## Testing Requirements
-
-All contributions must include appropriate tests:
-
-* **New features:** Full test coverage including edge cases
-* **Bug fixes:** Regression test that fails before the fix and passes after
-* **Refactors:** Existing tests must continue to pass
-* **Performance changes:** Benchmarks or performance tests when applicable
-
-Tests should be clear, maintainable, and actually validate the intended behavior.
-
----
-
-## Questions?
-
-* Check existing **Issues** and **Discussions** first
-* Join the project **Discord** for real-time discussion
-* Be patient, this is a personal project with limited maintenance time
-
----
-
-## License
-
-By contributing, you agree that your code will be licensed under the project's **MIT License**.
-
-Thank you for respecting this contribution process. It helps maintain project quality and the maintainer's sanity.

Dockerfile

@@ -1,86 +1,58 @@
-FROM node:lts-alpine AS node_builder
+FROM node:lts-alpine as npm_builder
+
+WORKDIR /ytptube
+COPY frontend ./
+RUN npm ci && npm run build
+
+FROM python:3.11-alpine as python_builder
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONFAULTHANDLER 1
+
+# Use sed to strip carriage-return characters from the entrypoint script (in case building on Windows)
+# Install dependencies
+RUN apk add --update coreutils curl gcc g++ musl-dev libffi-dev openssl-dev && pip install pipenv
 
 WORKDIR /app
-COPY ui ./
-ENV NODE_ENV=production
-RUN if [ ! -f "/app/exported/index.html" ]; then \
-  npm install -g bun && \
-  NODE_ENV=production bun install --frozen-lockfile --production && \
-  bun run generate; \
-  else echo "Skipping UI build, already built."; fi
 
-FROM python:3.13-bookworm AS python_builder
+COPY ./Pipfile* .
 
-ENV LANG=C.UTF-8
+RUN PIPENV_VENV_IN_PROJECT=1 pipenv install --deploy
-ENV LC_ALL=C.UTF-8
-ENV PYTHONDONTWRITEBYTECODE=1
-ENV PYTHONFAULTHANDLER=1
-ENV PIP_NO_CACHE_DIR=off
-ENV PIP_CACHE_DIR=/root/.cache/pip
-ENV UV_CACHE_DIR=/root/.cache/uv
-ENV DEBIAN_FRONTEND=noninteractive
-ENV UV_INSTALL_DIR=/usr/bin
 
-COPY --from=astral/uv:latest /uv /usr/bin/
+FROM python:3.11-alpine
-
-WORKDIR /opt/
-
-COPY ./pyproject.toml ./uv.lock ./
-RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \
-  --mount=type=cache,target=/root/.cache/uv,id=uv-cache \
-  uv venv --system-site-packages --relocatable ./python && \
-  VIRTUAL_ENV=/opt/python uv sync --no-dev --link-mode=copy --active
-
-FROM python:3.13-slim
 
 ARG TZ=UTC
 ARG USER_ID=1000
 ENV IN_CONTAINER=1
-ENV UMASK=0002
+ENV UMASK=022
 ENV YTP_CONFIG_PATH=/config
 ENV YTP_TEMP_PATH=/tmp
 ENV YTP_DOWNLOAD_PATH=/downloads
 ENV YTP_PORT=8081
-ENV XDG_CONFIG_HOME=/config
-ENV XDG_CACHE_HOME=/tmp
-ENV PYDEVD_DISABLE_FILE_VALIDATION=1
-ENV PYTHONDONTWRITEBYTECODE=1
-ENV PYTHONFAULTHANDLER=1
 
-ARG DEBIAN_FRONTEND=noninteractive
+# removed ffmpeg as 6.1.0 is broken with DASH protocal downloads
+COPY --from=mwader/static-ffmpeg:6.1.1 /ffmpeg /usr/bin/
+COPY --from=mwader/static-ffmpeg:6.1.1 /ffprobe /usr/bin/
 
-RUN sed -i -E '/^Suites:[[:space:]]*trixie[[:space:]]+trixie-updates$/ {n; s/^(Components:[[:space:]]*)main([[:space:]]*|$)/\1main contrib non-free\2/}' /etc/apt/sources.list.d/debian.sources && \
+RUN mkdir /config /downloads && ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone && \
-  mkdir /config /downloads && ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone && \
+  apk add --update --no-cache bash mkvtoolnix patch aria2 coreutils curl shadow sqlite tzdata libmagic && \
-  apt-get update && \
+  useradd -u ${USER_ID:-1000} -U -d /app -s /bin/bash app && \
-  ARCH="$(dpkg --print-architecture)" && \
+  rm -rf /var/cache/apk/*
-  EXTRA_PACKAGES="" && \
-  if [ "$ARCH" = "amd64" ]; then EXTRA_PACKAGES="intel-media-va-driver-non-free i965-va-driver libmfx-gen1.2"; fi && \
-  apt-get install -y --no-install-recommends locales procps \
-  bash mkvtoolnix patch aria2 curl ca-certificates xz-utils git sqlite3 tzdata file libmagic1 vainfo ${EXTRA_PACKAGES} \
-  && useradd -u ${USER_ID:-1000} -U -d /app -s /bin/bash app && \
-  sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
-  dpkg-reconfigure --frontend=noninteractive locales && \
-  update-locale LANG=en_US.UTF-8 \
-  mkdir -p /opt/bin && rm -rf /var/lib/apt/lists/*
 
 COPY entrypoint.sh /
-COPY --chown=app:app yt-dlp /opt/bin/yt-dlp
+
+RUN sed -i 's/\r$//g' /entrypoint.sh && chmod +x /entrypoint.sh
+
 COPY --chown=app:app ./app /app/app
-COPY --chown=app:app --from=node_builder /app/exported /app/ui/exported
+COPY --chown=app:app --from=npm_builder /ytptube/dist /app/frontend/dist
-COPY --chown=app:app --from=python_builder /opt/python /opt/python
+COPY --chown=app:app --from=python_builder /app/.venv /app/.venv
-COPY --from=ghcr.io/arabcoders/alpine-mp4box /usr/bin/mp4box /usr/bin/mp4box
-COPY --from=ghcr.io/arabcoders/jellyfin-ffmpeg /usr/bin/ffmpeg /usr/bin/ffmpeg
-COPY --from=ghcr.io/arabcoders/jellyfin-ffmpeg /usr/bin/ffprobe /usr/bin/ffprobe
-COPY --from=denoland/deno:latest /usr/bin/deno /usr/bin/deno
 COPY --chown=app:app ./healthcheck.sh /usr/local/bin/healthcheck
 
-ENV PATH="/opt/bin:/opt/python/bin:$PATH"
+ENV PATH="/app/.venv/bin:$PATH"
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
 
-RUN sed -i 's/\r$//g' /entrypoint.sh && chmod +x /entrypoint.sh && chown -R app:app /config /downloads && \
+RUN chown -R app:app /config /downloads && chmod +x /usr/local/bin/healthcheck
-  chmod +x /usr/local/bin/healthcheck /usr/bin/mp4box /usr/bin/ffmpeg /usr/bin/ffprobe /usr/bin/deno /opt/bin/yt-dlp
 
 VOLUME /config
 VOLUME /downloads
@@ -91,8 +63,10 @@ USER app
 
 WORKDIR /tmp
 
-HEALTHCHECK --interval=10s --timeout=20s --start-period=60s --retries=3 CMD [ "/usr/local/bin/healthcheck" ]
+HEALTHCHECK --interval=10s --timeout=20s --start-period=10s --retries=3 CMD [ "/usr/local/bin/healthcheck" ]
 
 ENTRYPOINT ["/entrypoint.sh"]
 
-CMD ["/opt/python/bin/python", "/app/app/main.py", "--ytp-process"]
+ENV PYDEVD_DISABLE_FILE_VALIDATION=1
+
+CMD ["/app/.venv/bin/python", "/app/app/main.py", "--ytptube-mp"]

FAQ.md

@@ -1,758 +0,0 @@
-# Environment variables
-
-Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line, 
-or the `environment:` section in `compose.yaml` file.
-
-<details>
-<summary>Click to expand</summary>
-
-| Environment Variable            | Description                                                         | Default               |
-| ------------------------------- | ------------------------------------------------------------------- | --------------------- |
-| TZ                              | The timezone to use for the application                             | `(not_set)`           |
-| YTP_OUTPUT_TEMPLATE             | The template for the filenames of the downloaded videos             | `%(title)s.%(ext)s`   |
-| YTP_DEFAULT_PRESET              | The default preset to use for the download                          | `default`             |
-| YTP_INSTANCE_TITLE              | The title of the instance                                           | `(not_set)`           |
-| YTP_FILE_LOGGING                | Whether to log to file                                              | `false`               |
-| YTP_DOWNLOAD_PATH               | Path to where the downloads will be saved                           | `/downloads`          |
-| YTP_MAX_WORKERS                 | The maximum number of workers to use for downloading                | `20`                  |
-| YTP_MAX_WORKERS_PER_EXTRACTOR   | The maximum number of concurrent downloads per extractor            | `2`                   |
-| YTP_AUTH_USERNAME               | Username for basic authentication                                   | `(not_set)`           |
-| YTP_AUTH_PASSWORD               | Password for basic authentication                                   | `(not_set)`           |
-| YTP_CONSOLE_ENABLED             | Whether to enable the console                                       | `false`               |
-| YTP_REMOVE_FILES                | Remove the actual file when clicking the remove button              | `false`               |
-| YTP_CONFIG_PATH                 | Path to where the config files will be stored.                      | `/config`             |
-| YTP_TEMP_PATH                   | Path to where tmp files are stored.                                 | `/tmp`                |
-| YTP_TEMP_KEEP                   | Whether to keep the Individual video temp directory or remove it    | `false`               |
-| YTP_HOST                        | Which IP address to bind to                                         | `0.0.0.0`             |
-| YTP_PORT                        | Which port to bind to                                               | `8081`                |
-| YTP_LOG_LEVEL                   | Log level                                                           | `info`                |
-| YTP_STREAMER_VCODEC             | The video encoding codec, default to GPU and fallback to software   | `""`                  |
-| YTP_STREAMER_ACODEC             | The audio codec to use for in-browser streaming                     | `aac`                 |
-| YTP_VAAPI_DEVICE                | The VAAPI device to use for hardware acceleration.                  | `/dev/dri/renderD128` |
-| YTP_ACCESS_LOG                  | Whether to log access to the web server                             | `true`                |
-| YTP_DEBUG                       | Whether to turn on debug mode                                       | `false`               |
-| YTP_DEBUGPY_PORT                | The port to use for the debugpy debugger                            | `5678`                |
-| YTP_EXTRACT_INFO_TIMEOUT        | The timeout for extracting video information                        | `70`                  |
-| YTP_PIP_PACKAGES                | A space separated list of pip packages to install                   | `(not_set)`           |
-| YTP_PIP_IGNORE_UPDATES          | Do not update the custom pip packages                               | `false`               |
-| YTP_PICTURES_BACKENDS           | A comma separated list of pictures urls to use                      | `(not_set)`           |
-| YTP_BROWSER_CONTROL_ENABLED     | Whether to enable the file browser actions                          | `false`               |
-| YTP_YTDLP_AUTO_UPDATE           | Whether to enable the auto update for yt-dlp                        | `true`                |
-| YTP_YTDLP_DEBUG                 | Whether to turn debug logging for the internal `yt-dlp` package     | `false`               |
-| YTP_YTDLP_VERSION               | The version of yt-dlp to use. Defaults to latest version            | `(not_set)`           |
-| YTP_BROWSER_URL                 | Remote browser endpoint for the bundled `browser` extractor         | `(not_set)`           |
-| YTP_FLARESOLVERR_URL            | FlareSolverr endpoint URL.                                          | `(not_set)`           |
-| YTP_FLARESOLVERR_MAX_TIMEOUT    | Max FlareSolverr challenge timeout in seconds                       | `120`                 |
-| YTP_FLARESOLVERR_CLIENT_TIMEOUT | HTTP client timeout (seconds) when calling FlareSolverr             | `120`                 |
-| YTP_FLARESOLVERR_CACHE_TTL      | The cache TTL (in seconds) for FlareSolverr solutions               | `600`                 |
-| YTP_BASE_PATH                   | Set this if you are serving YTPTube from sub-folder                 | `/`                   |
-| YTP_PREVENT_LIVE_PREMIERE       | Prevents the initial youtube premiere stream from being downloaded  | `false`               |
-| YTP_QUEUE_DISPLAY_LIMIT         | Max queued downloads returned to the UI. `0` = unlimited            | `100`                 |
-| YTP_LIVE_PREMIERE_BUFFER        | buffer time in minutes to add to video duration                     | `5`                   |
-| YTP_TASKS_HANDLER_TIMER         | The cron expression for the tasks handler timer                     | `15 */1 * * *`        |
-| YTP_TEMP_DISABLED               | Disable temp files handling.                                        | `false`               |
-| YTP_ALLOW_INTERNAL_URLS         | Allow requests to internal URLs                                     | `false`               |
-| YTP_SIMPLE_MODE                 | Switch default interface to Simple mode.                            | `false`               |
-| YTP_STATIC_UI_PATH              | Path to custom static UI files.                                     | `(not_set)`           |
-| YTP_AUTO_CLEAR_HISTORY_DAYS     | Number of days after which completed download history is cleared.   | `0`                   |
-| YTP_DEFAULT_PAGINATION          | The default number of items per page for history.                   | `50`                  |
-| YTP_TASK_HANDLER_RANDOM_DELAY   | The maximum random delay in seconds before starting a task handler. | `60`                  |
-| YTP_IGNORE_ARCHIVED_ITEMS       | Don't report archived items in the download history.                | `false`               |
-| YTP_CHECK_FOR_UPDATES           | Whether to check for application updates.                           | `true`                |
-| YTP_EXTRACT_INFO_CONCURRENCY    | The number of concurrent extract info operations.                   | `4`                   |
-| YTP_THUMB_CONCURRENCY           | The number of concurrent ffmpeg thumbnail generations allowed.      | `2`                   |
-| YTP_THUMB_GENERATE              | Enable ffmpeg thumbnail generation when no local thumbnail exists.  | `true`                |
-| YTP_THUMB_SIDECAR               | Save generated thumbnails next to media instead of temp cache.      | `false`               |
-| YTP_DISABLE_EXEC                | Strip some dangerous yt-dlp options.                                | `false`               |
-</details>
-
-> [!NOTE]
-> To raise the worker limit for a specific extractor, set an env variable using this format: `YTP_MAX_WORKERS_FOR_<EXTRACTOR_NAME>`
-> The extractor name must be uppercase. You can find the extractor name in the download logs. This value cannot be 
-> higher than `YTP_MAX_WORKERS`; higher values are ignored.
->
-> `YTP_SIMPLE_MODE=true` only applies when the browser has no saved layout choice yet. Users can still choose a layout in 
-> WebUI Settings. `/?simple=1` forces and saves Simple for that browser.
-> 
-> `YTP_AUTO_CLEAR_HISTORY_DAYS`  `0` days means no automatic clearing of the download history. lowest value that will 
-> trigger the clearing is `1` day. This setting will **NOT** delete the downloaded files, it will only clear the 
-> history from the database.
-
-# Browser extensions & bookmarklets
-
-## Simple bookmarklet
-
-```javascript
-javascript:(() => { const url = "https://ytp.example.org"; const preset = "default"; const mUrl = new URL(url);mUrl.pathname="/api/history/add";mUrl.searchParams.set("url",document.location.href);mUrl.searchParams.set("preset",preset);fetch(mUrl,{method: "GET"}).then(j => j.json()).then(json =>alert(json.message)).catch(err =>alert(err)); })()
-```
-
-Change the the variable `url` and `preset` variables to match your YTPTube instance and preset name.
-
-> [!NOTE]
-> The bookmarklet should be served from https page, otherwise, some browsers will block the request. for mixed content.
-
-## Browser stores
-
-- For Firefox via [Firefox Store](https://addons.mozilla.org/en-US/firefox/addon/ytptube-extension/)
-- For Chrome/Chromium Browsers via [Chrome Store](https://chromewebstore.google.com/detail/ytptube-extension/kiepfnpeflemfokokgjiaelddchglfil)
-
-## iOS Shortcuts
-
-You can download [Add To YTPTube](https://www.icloud.com/shortcuts/6df61c97d97b4e539c9100999ba39dd4) shortcut and use it to send links to your YTPTube instance.
-You have to edit the shortcut and replace the following:
-
-- `https://ytp.example.org` with your YTPTube instance.
-- `username:password` replace this with your own username & password or leave it empty if you don't have authentication enabled.
-
-This shortcut is powerful, as it's allow you to select your preset on the fly pulled directly from your instance.
-Combined with the new and powerful presets system, you could add presets for specific websites that need cookies,
-and use that preset to download directly from your iOS device.
-
-### Advanced iOS Shortcut
-
-This shortcut [YTPTube To Media](https://www.icloud.com/shortcuts/4dc579382f254635ad5785424055f173) is more advanced, as it's parses
-the `yt-dlp` output and attempt to download the media directly to your iOS device. It doesn't always work, but it's a good
-starting point for those who want to download media directly to their iOS device. We provide no support for this use case
-other than the shortcut itself. this shortcut missing support for parsing the http_headers, it's only parse the cookies.
-
-# Authentication
-
-To enable basic authentication, set the `YTP_AUTH_USERNAME` and `YTP_AUTH_PASSWORD` environment variables. And restart 
-the container. This will prompt the user to enter the username and password before accessing the web interface/API.
-As this is a simple basic authentication, if your browser doesn't show the prompt, you can use the following URL
-
-`http://username:password@your_ytptube_url:port`
-
-# Security recommendations
-
-YTPTube is designed for LAN and home-lab use behind a firewall or reverse proxy. The web interface and API are
-unauthenticated by default because in a trusted network, auth adds friction without meaningful benefit. However,
-if you expose YTPTube to the internet directly or via port forwarding **YOU MUST enable authentication**.
-
-### Without auth, anyone who can reach the API can:
-
-- Download arbitrary content through your IP and server.
-- Delete or modify your downloaded files and database.
-- Run arbitrary `yt-dlp` options, including `--exec`, which executes shell commands inside the container.
-
-This is not a vulnerability, it's the intended design. The `cli` field passes options directly to `yt-dlp`,
-a tool that by design can execute commands. Auth is the mechanism that controls who gets to use that power.
-
-**If you expose YTPTube to untrusted networks**, do one of the following:
-
-1. **Enable authentication** — set both `YTP_AUTH_USERNAME` and `YTP_AUTH_PASSWORD`.
-2. **Put it behind a reverse proxy** with its own authentication layer (see [Run behind reverse proxy](#run-behind-reverse-proxy)).
-3. **Keep it on a private network** with no public exposure.
-
-YTPTube already gates other powerful features behind explicit opt-in: the built-in terminal, file browser actions and internal
-URL requests for example. The `cli` field is no different, its power is by design, and access control is your responsibility.
-
-> [!NOTE]
-> If you choose to run without authentication but still want to reduce at least some impact, you can set
-> `YTP_DISABLE_EXEC=true`. This strips some dangerous options at run time. However, understand that this is not a 
-> substitute for auth an unauthenticated API is still fully open for all other operations.
-
-# I cant download anything
-
-If you are receiving errors like:
-- "OSError: [Errno 5] I/O error"
-- "OSError: [Errno 18] Cross-device link: '/tmp/random_id/name.webm' -> '/downloads/name.webm'
-- "Operation not permitted: '/downloads/name.webm'
-
-This indicates an error with your mounts and how they interact with the container. So, the basic solution is to do the following:
-
-```yaml
-services:
-  ytptube:
-    user: "${UID:-1000}:${UID:-1000}" # change this to your user id and group id, for example: "1000:1000"
-    image: ghcr.io/arabcoders/ytptube:latest
-    container_name: ytptube
-    restart: unless-stopped
-    environment:
-      - YTP_TEMP_PATH=/downloads/tmp
-      - YTP_DOWNLOAD_PATH=/downloads/files
-    ports:
-      - "8081:8081"
-    volumes:
-      - ./config:/config:rw
-      - ./downloads:/downloads:rw
-```
-
-Then run the following command to create the necessary directories and start the container:
-
-```bash
-mkdir -p ./config && mkdir -p ./downloads/{tmp,files} && docker compose -f compose.yaml up -d
-```
-
-Reference: [Issue #363](https://github.com/arabcoders/ytptube/issues/363)
-
-# I want to use link with playlist but only download the video not all the videos in the playlist?
-
-Simply create a preset, and in the `Command options for yt-dlp` field set `--no-playlist`. Then select the preset 
-whenever the link includes a playlist id.
-
-> [!NOTE]
-> You can also do the same via advanced options `Command options for yt-dlp` field, but presets are more convenient.
-
-# Install specific yt-dlp version?
-
-You can force specific version of `yt-dlp` by setting the `YTP_YTDLP_VERSION` environment variable for example
-
-```env
-YTP_YTDLP_VERSION=2025.07.21 or master or nightly
-```
-
-Then restart the container to apply the changes.
-
-# Custom output template placeholders
-
-YTPTube supports custom `ytp_*` placeholders in `yt-dlp` output template via the following syntax `%(ytp_*:<args>)s`.
-
-## Currently available extra placeholders are:
-
-- `ytp_random`: random mixed letters and digits,
-  - `N` A number is required to specify the length of the random string, for example `%(ytp_random:8)s` will generate a random string of 8 characters. 
-  - if the args followed by `:s` it will generate random letters only, if followed by `:d` it will generate random digits only.
-
-## Examples of the custom placeholders in action:
-
-- Template: `%(title)s [%(ytp_random:8)s].%(ext)s`
-  - Example result: `My Video [A7k2Pq9Z].mp4`
-- Template: `%(uploader)s/%(ytp_random:6:d)s - %(title)s.%(ext)s`
-  - Example result: `MyChannel/483920 - My Video.mp4`
-- Template: `%(playlist)s/%(ytp_random:10:s)s/%(title)s.%(ext)s`
-  - Example result: `Favorites/QwErTyUiOp/My Video.mp4`
-
-> [!NOTE]
-> `%(ytp_` placeholders are a YTPTube extension and not avaliable via console or directly via yt-dlp.
-
-# How can I monitor sites without RSS feeds?
-
-YTPTube includes a **generic task handler** that turns JSON definitions into site-specific scrapers. You can use it
-to watch pages that do not expose RSS or public APIs and automatically enqueue new links into the download queue.
-
-1. Create definition via the WebUI > tasks > Definitions.
-2. Create task that reference same url click on inspect to see the results. Make sure it uses a preset that enables 
-   a download archive (`--download-archive`).
-3. When the task handler run, the handler scans the definitions, picks the first definition whose `match` rule covers
-   the task URL, fetches the page, extracts items, and queues the unseen ones.
-
-### Definition schema
-
-Each definition must contain a single JSON object with the following keys:
-
-```json5
-{
-  "name": "example",                  // Friendly identifier shown in logs
-  "match_url": [
-    "https://example.com/articles/*", // Glob strings
-    "https://example.com/post/[0-9]+" // Regex strings
-  ],
-  "engine": {                         // Optional, defaults to HTTPX
-    "type": "httpx",                  // "httpx" (default) or "selenium"
-    "options": {
-      "url": "http://selenium:4444/wd/hub", // Selenium-only: remote hub URL
-      "arguments": ["--headless", "--disable-gpu"],
-      "wait_for": { "type": "css", "expression": ".article" },
-      "wait_timeout": 15,
-      "page_load_timeout": 60
-    }
-  },
-  "request": {                      // Optional HTTP settings
-    "method": "GET",                // GET or POST
-    "url": "https://example.com/articles/latest", // Override the task URL if needed
-    "headers": { "User-Agent": "MyAgent/1.0" },
-    "params": { "page": 1 },
-    "data": null,
-    "json_data": null,
-    "timeout": 30
-  },
-  "response": {                      // Optional: how to interpret the body
-    "type": "html"                   // "html" (default) or "json"
-  },
-  "parse": {
-    "items": {                        // Optional container for per-item extraction
-      "selector": ".columns .card",   // Defaults to CSS; set "type": "xpath" to use XPath
-      "fields": {
-        "link": {                     // Required inside fields: the per-item URL
-          "type": "css",
-          "expression": ".card-header a",
-          "attribute": "href"
-        },
-        "title": {
-          "type": "css",
-          "expression": ".card-header a",
-          "attribute": "text"
-        },
-        "poet": {
-          "type": "css",
-          "expression": "footer .card-footer-item:first-child a",
-          "attribute": "text"
-        }
-      }
-    },
-    "page_title": {                    // Optional global field outside the container
-      "type": "css",
-      "expression": "title",
-      "attribute": "text"
-    }
-  }
-}
-```
-
-For JSON endpoints, switch the response format and use `jsonpath` selectors:
-
-```json5
-{
-  ...
-  "response": { "type": "json" },
-  "parse": {
-    "items": {
-      "type": "jsonpath",
-      "selector": "items",
-      "fields": {
-        "link": { "type": "jsonpath", "expression": "url" },
-        "title": { "type": "jsonpath", "expression": "title" }
-      }
-    }
-  }
-}
-```
-
-### Parsing rules
-
-- Every definition must provide a `link` field either at the top level or inside `parse.items.fields`. Other fields are optional metadata attached to the queued item.
-- CSS and XPath rules may specify `attribute`:
-  - `text` / `inner_text` applies `normalize-space()`.
-  - `html` / `outer_html` returns the raw HTML fragment.
-  - Any other value reads that attribute from the element. When omitted, the handler uses text and, for `link`, falls
-    back to `href` automatically.
-- Regex rules scan the HTML fragment associated with the current scope (page-level or container). Set `attribute` to a named/numbered capture group or rely on the first group.
-- `post_filter` lets you run a final regex on the extracted value and pick a named (`value`) group.
-- When you declare `parse.items`, each matching container is processed independently so missing fields in one card do not shift values for the rest.
-- For JSON responses (`response.type = "json"`), set the container `type` and field `type` to `jsonpath` and supply [JMESPath](https://jmespath.org/) expressions. Relative values are resolved against each container object and converted to strings automatically.
-
-### Fetch engines
-
-- **httpx (default)**: supports custom headers, query params, JSON/body payloads, proxy inherited from the task preset,
-  and optional timeout.
-- **selenium**: uses a remote Chrome session. Provide the hub URL under `engine.options.url`; only Chrome is supported at
-  the moment. Optional keys: `arguments` (list or string), `wait_for` (type `css`/`xpath` + `expression`), `wait_timeout`,
-  and `page_load_timeout`.
-
-> [!NOTE]
-> A machine-readable schema is available at `app/schema/task_definition.json` if you want to validate your JSON with editors or CI tools.
-
-# How to generate POT tokens?
-
-You need a pot provider server we already have the extractor `bgutil-ytdlp-pot-provider` pre-installed in the container.
-You can simply do the following to enable the support for it.
-
-```yaml
-services:
-  ytptube:
-    user: "${UID:-1000}:${UID:-1000}" # change this to your user id and group id, for example: "1000:1000"
-    image: ghcr.io/arabcoders/ytptube:latest
-    container_name: ytptube
-    restart: unless-stopped
-    ports:
-      - "8081:8081"
-    volumes:
-      - ./config:/config:rw
-      - ./downloads:/downloads:rw
-    tmpfs:
-      - /tmp
-    depends_on:
-      - bgutil_provider
-  bgutil_provider:
-    init: true
-    image: brainicism/bgutil-ytdlp-pot-provider:latest
-    container_name: bgutil_provider
-    restart: unless-stopped    
-```
-
-Then simply create a new preset, and in the `Command options for yt-dlp` field set the following:
-
-```bash
---extractor-args "youtubepot-bgutilhttp:base_url=http://bgutil_provider:4416" 
-```
-
-For more information please visit [bgutil-ytdlp-pot-provider](https://github.com/Brainicism/bgutil-ytdlp-pot-provider) project.
-
-# Troubleshooting and submitting issues
-
-Before asking a question or submitting an issue for YTPTube, please remember that YTPTube is only a wrapper for 
-[yt-dlp](https://github.com/yt-dlp/yt-dlp). Any issues you might be experiencing with authentication to video websites, 
-postprocessing, permissions, other `yt-dlp options` configurations which seem not to work, or anything else that 
-concerns the workings of the underlying yt-dlp library, need not be opened on the YTPTube project.
-
-In order to debug and troubleshoot them, it's advised to try using the yt-dlp binary directly first, bypassing the UI, 
-and once that is working, importing the options that worked for you into a new `preset`.
-
-## Via HTTP
-
-If you have enabled the web terminal via `YTP_CONSOLE_ENABLED` environment variable, simply go to `Other > Terminal` use
- the yt-dlp command, the interface is jailed to the `yt-dlp` binary you can't access anything else. Or from download form 
- by clicking `advanced options` button than the yellow terminal icon `Run directly in console`.
-
-## Via CLI 
-
-Assuming your YTPTube container is called `ytptube`, run the following on your docker host to get a shell inside the container:
-
-```bash
-docker exec -ti ytptube bash
-cd /downloads
-yt-dlp ....
-```
-
-Once there, you can use the yt-dlp command freely.
-
-# Run behind reverse proxy.
-
-It's advisable to run YTPTube behind a reverse proxy, if better authentication and/or HTTPS support are required.
-
-## Caddy http server
-
-The following example Caddyfile gets a reverse proxy going behind [caddy](https://caddyserver.com).
-
-```caddyfile
-# If you are using sub-domain.
-# make sure to change "ytptube:8081" to the actual name of your YTPTube container/port.
-ytp.example.org {
-  reverse_proxy ytptube:8081
-}
-
-# If you are using sub-folder, for example: https://example.org/ytptube/
-# Also make sure to set the `YTP_BASE_PATH` environment variable to `/ytptube/`
-# make sure to change "ytptube:8081" to the actual name of your YTPTube container/port.
-example.org {
-  redir /ytptube /ytptube/
-  route /ytptube/* {
-    reverse_proxy ytptube:8081
-  }
-}
-```
-
-# How to autoload yt-dlp plugins?
-
-Loading yt-dlp plugins in YTPTube is quite simple, we already have everything setup for you. simply, create a folder 
-inside the `/config` directory named `yt-dlp` so, the path will be `/config/yt-dlp`. then follow 
-[yt-dlp plugins docs](https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#plugins) to know how to install the plugins.
-
-Once you have installed the plugins, restart the container and the plugins will be auto-loaded on demand.
-
-# How to load random backgrounds from WatchState or any other source?
-
-YTPTube can be configured to pull random background images from different sources, including `WatchState` which is another 
-project of mine, simply change the `YTP_PICTURES_BACKENDS` environment variable to the following url
-
-```env
-YTP_PICTURES_BACKENDS=https://watchstate.ip/v1/api/system/images/background?apikey=[api_key]
-```
-
-Where `[api_key]` is the api key you get from your WatchState instance.
-
-# How to use share folder or external storage as download target?
-
-You can simply mount the share folder as target for `/downloads` path, but in some cases you might face issues with permissions or
-cross-device link errors. To avoid these issues, you can mount the share folder as a named volume, and then mount the 
-named volume to `/downloads/smb` or `/downloads/nfs`.
-
-```yaml
-services:
-  ytptube:
-    user: "${UID:-1000}:${UID:-1000}" # change this to your user id and group id, for example: "1000:1000"
-    image: ghcr.io/arabcoders/ytptube:latest
-    container_name: ytptube
-    restart: unless-stopped
-    ports:
-      - "8081:8081"
-    volumes:
-      # Config must be mounted locally as read-write sqlite doesn't support network mounts.
-      - ./config:/config:rw
-      # Mount a local directory
-      - ./downloads:/downloads/local:rw
-      # Mount the NFS share
-      - nfs-data:/downloads/nfs:rw
-      # Mount the SMB share
-      - smb-data:/downloads/smb:rw
-    tmpfs:
-      - /tmp
-
-volumes:
-  nfs-data:
-    driver: local
-    driver_opts:
-      type: nfs
-      o: addr=10.0.0.3,rw,nfsvers=4 # <--- Change server IP and options
-      device: ":/exported/path" # <--- Remote NFS path
-  
-  smb-data:
-    driver: local
-    driver_opts:
-      type: cifs
-      o: username=my_username,password=my_password,vers=3.0,uid=1000,gid=1000,file_mode=0777,dir_mode=0777 # <--- Change options to fit your needs
-      device: "//10.0.0.3/public" # <--- Remote SMB path
-```
-
-If you prefer, you can bypass YTPTube `download_path` and set it to `/` and completely manage your own mounts. However,
-please be aware that the file browser feature will expose whatever `download_path` is set to. **So, if you set it to `/`, 
-the file browser will expose the entire container filesystem.**
-
-# The origin of the project.
-
-The project first started as a fork [meTube](https://github.com/alexta69/metube), since then it has been completely 
-rewritten and redesigned. The original project was a great starting point, but it didn't align with my vision for the 
-project and what i wanted to achieve with it.
-
-# How to use hardware acceleration for video transcoding?
-
-As the container is rootless, we cannot do the necessary changes to the container to enable hardware acceleration.
-However, We do have the drivers and ffmpeg already installed and the CPU transcoding should work regardless. To enable
-hardware acceleration You need to alter your `compose.yaml` file to mount the necessary devices to the container. Here
-is an example of how to do it for debian based systems.
-
-```yaml
-services:
- ytptube:
-    ........ # see above for the rest of the configuration
-    devices:
-      # mount the dri devices to the container if you only have one gpu you can simply do the following
-      - /dev/dri:/dev/dri                       
-      # Otherwise, selectively mount the devices you need.
-      - /dev/dri/card0      # Intel GPU device
-      - /dev/dri/renderD128 # Intel GPU render node
-    group_add:
-      # Add the necessary groups to the container to access the gpu devices.
-      - 44   # it might be different on your system.                                 
-      - 105  # it might be different on your system.
-```
-
-This setup should work for at VAAPI encoding in `x86_64` containers.
-
-> [!NOTE]
-> Your `video`, `render` group id might be different from mine, you can run the follow command in docker host server to get the group ids for both groups.
-
-```bash
-cat /etc/group | grep -E 'render|video'
-
-video:x:44:your_docker_username
-render:x:105:your_docker_username
-```
-
-In my docker host the group id for `video` is `44` and for `render` is `105`. change what needed in the `compose.yaml`
-file to match your setup.
-
-If for some reason the initial test for GPU encoding fails, YTPTube will fallback to software encoding. You can force
-software encoding by setting the `YTP_STREAMER_VCODEC` environment variable to `libx264`. If you want to force GPU encoding, set the
-`YTP_STREAMER_VCODEC` environment variable to one of the supported GPU codecs, for example `h264_vaapi` or `h264_nvenc` depending on your GPU.
-For more information about the supported codecs, please refer to the [SegmentEncoders.py](app/library/SegmentEncoders.py) file.
-
-If GPU encoding fails and software encoding is used, you will have to restart the container to try GPU encoding again. 
-as we only test for GPU encoding once on first video stream.
-
-# Allowing internal URLs requests
-
-By default, YTPTube prevents requests to internal resources. However, if you want to allow requests to internal URLs, you can set the `YTP_ALLOW_INTERNAL_URLS` environment variable to `true`. This will allow requests to internal URLs. 
-
-We do not recommend enabling this option unless you know what you are doing, as it can expose your internal network to 
-potential security risks. This should only be used if it's truly needed.
-
-# How to setup CI on Gitea?
-
-The docker container builder already support self-hosted repositories like Gitea, you simply need to define two things at your repository settings.
-
-1. Create a secret named `GIT_TOKEN` and set it to your Gitea personal access token.
-2. Create a variable named `REGISTRY` and set it to your docker registry, for example `gitea.domain.org`.
-
-Thats it, the `main.yml` will now disable the docker/github container registries, and use your Gitea repository instead. It will follow the usual
-naming, your container name will be named `REGISTRY/ytptube` and the tags will be the same as the ones used in the github registry.
-
-Unfortunately, the `native-builder.yml` workflow doesn't support self-hosted repositories at the moment.
-
-# Getting No space left on device error
-
-If you encounter this error: `OSError: [Errno 28] No space left on device` This indicates that either 
-the `/tmp` or `/downloads` directory has run out of available space.
-
-This issue commonly occurs when:
-
-- `/tmp` is mounted as `tmpfs` (memory-based storage)
-- Your system has limited RAM
-- You're downloading large video files
-
-Since videos are temporarily stored in `/tmp` before being moved to the final download location, memory-based storage 
-may be insufficient for large downloads.
-
-To fix the issue, modify your `compose.yaml` to use a disk-based directory for temporary files:
-
-```yaml
-services:
-  ytptube:
-    user: "${UID:-1000}:${UID:-1000}"
-    image: ghcr.io/arabcoders/ytptube:latest
-    container_name: ytptube
-    restart: unless-stopped
-    ports:
-      - "8081:8081"
-    volumes:
-      - ./config:/config:rw
-      - ./downloads:/downloads/local:rw
-      - ./temp:/tmp:rw
-```
-
-> [!NOTE]
-> Replace the `tmpfs` mount with a local directory volume (`./temp:/tmp:rw`). This allows temporary files to use disk space instead of RAM.
-
-After making the changes, restart your container. This should resolve the "No space left on device" 
-error during download.
-
-# How to prevent loading screen during YouTube premieres?
-
-Depending on how you look at it, YTPTube live download implementation is rather great and fast. However, during YouTube 
-premieres, usually streams will contain a loading screen of say, 1-5 minutes before the actual video content starts 
-playing. By default we wait for 5min + the duration of the video before starting the download to ensure we get the full video without
-the loading screen. However, you can override the behavior by setting the following environment variable:
-
-```env
-YTP_LIVE_PREMIERE_BUFFER=10
-```
-
-Where `YTP_LIVE_PREMIERE_BUFFER` is the buffer time in minutes to add to the video duration before the download starts. 
-This will help in case the premiere has a longer loading screen than usual.
-
-# How to bypass CF challenges?
-
-You need to setup [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) and then set the `YTP_FLARESOLVERR_URL` 
-environment variable to point to your FlareSolverr instance. For example:
-
-```yaml
-services:
-  ytptube:
-    user: "${UID:-1000}:${UID:-1000}" # change this to your user id and group id, for example: "1000:1000"
-    image: ghcr.io/arabcoders/ytptube:latest
-    container_name: ytptube
-    restart: unless-stopped
-    environment:
-      - YTP_FLARESOLVERR_URL=http://flaresolverr:8191/v1
-    ports:
-      - "8081:8081"
-    volumes:
-      - ./config:/config:rw
-      - ./downloads:/downloads:rw
-    tmpfs:
-      - /tmp
-    depends_on:
-      - flaresolverr
-  flaresolverr:
-    image: flaresolverr/flaresolverr:latest
-    container_name: flaresolverr
-    restart: unless-stopped    
-```
-
-For more information please visit [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr) project.
-
-# How to use the browser extractor?
-
-YTPTube ships with a bundled `browser` extractor plugin for `yt-dlp`. It opens the target page in a remote browser,
-waits for the page network activity, then tries to extract media urls from the captured requests and media elements.
-
-This is mainly useful for sites where the regular extractor or generic page parsing does not expose the final media url,
-but a real browser session does.
-
-## Environment variables
-
-You can define the remote browser endpoint globally with the following environment variable.
-
-### Selenium example
-
-```env
-YTP_BROWSER_URL=selenium+http://selenium:4444/wd/hub
-```
-
-### Playwright example
-
-For a native Playwright server, point to the Playwright websocket endpoint:
-
-```env
-YTP_BROWSER_URL=playwright+ws://playwright:3000/
-```
-
-### Playwright CDP example
-
-To connect Playwright over the Chrome DevTools Protocol, use either the shorthand `playwright+cdp://` form or an explicit transport form:
-
-```env
-YTP_BROWSER_URL=playwright+cdp://chrome:9222/
-# same as playwright+cdp+http://chrome:9222/
-```
-
-- `YTP_BROWSER_URL` is the full backend selector and endpoint in one value.
-- Supported forms are:
-  - `selenium+http://...`
-  - `selenium+https://...`
-  - `playwright+ws://...`
-  - `playwright+wss://...`
-  - `playwright+cdp://...` which is treated as `playwright+cdp+http://...`
-  - `playwright+cdp+http://...`
-  - `playwright+cdp+https://...`
-  - `playwright+cdp+ws://...`
-  - `playwright+cdp+wss://...`
-
-## yt-dlp usage
-
-If you want to set the browser extractor options directly on the yt-dlp side, you can also use `--extractor-args` with `generic:url=...`:
-
-### Selenium example
-
-```bash
---use-extractors "generic" --extractor-args "generic:url=selenium+http://selenium:4444/wd/hub"
-```
-
-### Playwright example
-
-```bash
---use-extractors "generic" --extractor-args "generic:url=playwright+ws://playwright:3000/"
-```
-
-### Playwright CDP example
-
-```bash
---use-extractors "generic" --extractor-args "generic:url=playwright+cdp://chrome:9222/"
-```
-
-The explicit `--extractor-args` value takes priority over `YTP_BROWSER_URL`.
-
-## Example compose setup
-
-```yaml
-services:
-  ytptube:
-    user: "${UID:-1000}:${UID:-1000}"
-    image: ghcr.io/arabcoders/ytptube:latest
-    container_name: ytptube
-    restart: unless-stopped
-    environment:
-      - YTP_BROWSER_URL=selenium+http://selenium:4444/wd/hub # or playwright+ws://playwright:3000/ or playwright+cdp://chrome:9222/
-    ports:
-      - "8081:8081"
-    volumes:
-      - ./config:/config:rw
-      - ./downloads:/downloads:rw
-
-  selenium:
-    image: selenium/standalone-chrome:latest
-    container_name: selenium
-    restart: unless-stopped
-    shm_size: 2gb
-    
-  playwright:
-    image: mcr.microsoft.com/playwright:v1.59.0-noble
-    container_name: playwright
-    restart: unless-stopped
-    command: /bin/sh -c "npx -y playwright@1.59.0 run-server --port 3000 --host 0.0.0.0"
-```
-
-> [!NOTE]
-> The browser extractor is slower than the normal extractor flow and should only be used when a site actually needs a real browser session.
-> playwright require same version for both the server and the client, so make sure to use the same version in the container and in your local environment if you want to test it locally.

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2025 ArabCoders
+Copyright (c) 2024 ArabCoders
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Pipfile

@@ -0,0 +1,20 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+python-socketio = "~=5.0"
+aiohttp = "*"
+yt-dlp = "*"
+caribou = "*"
+coloredlogs = "*"
+aiocron = "*"
+python-dotenv = "*"
+python-magic = "*"
+debugpy = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.11"

Pipfile.lock

@@ -0,0 +1,919 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "969599ea205de43f64d64331fc3307bb0d9be1dc1830c57b51f93c607d2bbe1f"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.11"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "aiocron": {
+            "hashes": [
+                "sha256:48546513faf2eb7901e65a64eba7b653c80106ed00ed9ca3419c3d10b6555a01",
+                "sha256:b6313214c311b62aa2220e872b94139b648631b3103d062ef29e5d3230ddce6d"
+            ],
+            "index": "pypi",
+            "version": "==1.8"
+        },
+        "aiohttp": {
+            "hashes": [
+                "sha256:017a21b0df49039c8f46ca0971b3a7fdc1f56741ab1240cb90ca408049766168",
+                "sha256:039df344b45ae0b34ac885ab5b53940b174530d4dd8a14ed8b0e2155b9dddccb",
+                "sha256:055ce4f74b82551678291473f66dc9fb9048a50d8324278751926ff0ae7715e5",
+                "sha256:06a9b2c8837d9a94fae16c6223acc14b4dfdff216ab9b7202e07a9a09541168f",
+                "sha256:07b837ef0d2f252f96009e9b8435ec1fef68ef8b1461933253d318748ec1acdc",
+                "sha256:0ed621426d961df79aa3b963ac7af0d40392956ffa9be022024cd16297b30c8c",
+                "sha256:0fa43c32d1643f518491d9d3a730f85f5bbaedcbd7fbcae27435bb8b7a061b29",
+                "sha256:1f5a71d25cd8106eab05f8704cd9167b6e5187bcdf8f090a66c6d88b634802b4",
+                "sha256:1f5cd333fcf7590a18334c90f8c9147c837a6ec8a178e88d90a9b96ea03194cc",
+                "sha256:27468897f628c627230dba07ec65dc8d0db566923c48f29e084ce382119802bc",
+                "sha256:298abd678033b8571995650ccee753d9458dfa0377be4dba91e4491da3f2be63",
+                "sha256:2c895a656dd7e061b2fd6bb77d971cc38f2afc277229ce7dd3552de8313a483e",
+                "sha256:361a1026c9dd4aba0109e4040e2aecf9884f5cfe1b1b1bd3d09419c205e2e53d",
+                "sha256:363afe77cfcbe3a36353d8ea133e904b108feea505aa4792dad6585a8192c55a",
+                "sha256:38a19bc3b686ad55804ae931012f78f7a534cce165d089a2059f658f6c91fa60",
+                "sha256:38f307b41e0bea3294a9a2a87833191e4bcf89bb0365e83a8be3a58b31fb7f38",
+                "sha256:3e59c23c52765951b69ec45ddbbc9403a8761ee6f57253250c6e1536cacc758b",
+                "sha256:4b4af9f25b49a7be47c0972139e59ec0e8285c371049df1a63b6ca81fdd216a2",
+                "sha256:504b6981675ace64c28bf4a05a508af5cde526e36492c98916127f5a02354d53",
+                "sha256:50fca156d718f8ced687a373f9e140c1bb765ca16e3d6f4fe116e3df7c05b2c5",
+                "sha256:522a11c934ea660ff8953eda090dcd2154d367dec1ae3c540aff9f8a5c109ab4",
+                "sha256:52df73f14ed99cee84865b95a3d9e044f226320a87af208f068ecc33e0c35b96",
+                "sha256:595f105710293e76b9dc09f52e0dd896bd064a79346234b521f6b968ffdd8e58",
+                "sha256:59c26c95975f26e662ca78fdf543d4eeaef70e533a672b4113dd888bd2423caa",
+                "sha256:5bce0dc147ca85caa5d33debc4f4d65e8e8b5c97c7f9f660f215fa74fc49a321",
+                "sha256:5eafe2c065df5401ba06821b9a054d9cb2848867f3c59801b5d07a0be3a380ae",
+                "sha256:5ed3e046ea7b14938112ccd53d91c1539af3e6679b222f9469981e3dac7ba1ce",
+                "sha256:5fe9ce6c09668063b8447f85d43b8d1c4e5d3d7e92c63173e6180b2ac5d46dd8",
+                "sha256:648056db9a9fa565d3fa851880f99f45e3f9a771dd3ff3bb0c048ea83fb28194",
+                "sha256:69361bfdca5468c0488d7017b9b1e5ce769d40b46a9f4a2eed26b78619e9396c",
+                "sha256:6b0e029353361f1746bac2e4cc19b32f972ec03f0f943b390c4ab3371840aabf",
+                "sha256:6b88f9386ff1ad91ace19d2a1c0225896e28815ee09fc6a8932fded8cda97c3d",
+                "sha256:770d015888c2a598b377bd2f663adfd947d78c0124cfe7b959e1ef39f5b13869",
+                "sha256:7943c414d3a8d9235f5f15c22ace69787c140c80b718dcd57caaade95f7cd93b",
+                "sha256:7cf5c9458e1e90e3c390c2639f1017a0379a99a94fdfad3a1fd966a2874bba52",
+                "sha256:7f46acd6a194287b7e41e87957bfe2ad1ad88318d447caf5b090012f2c5bb528",
+                "sha256:82e6aa28dd46374f72093eda8bcd142f7771ee1eb9d1e223ff0fa7177a96b4a5",
+                "sha256:835a55b7ca49468aaaac0b217092dfdff370e6c215c9224c52f30daaa735c1c1",
+                "sha256:84871a243359bb42c12728f04d181a389718710129b36b6aad0fc4655a7647d4",
+                "sha256:8aacb477dc26797ee089721536a292a664846489c49d3ef9725f992449eda5a8",
+                "sha256:8e2c45c208c62e955e8256949eb225bd8b66a4c9b6865729a786f2aa79b72e9d",
+                "sha256:90842933e5d1ff760fae6caca4b2b3edba53ba8f4b71e95dacf2818a2aca06f7",
+                "sha256:938a9653e1e0c592053f815f7028e41a3062e902095e5a7dc84617c87267ebd5",
+                "sha256:939677b61f9d72a4fa2a042a5eee2a99a24001a67c13da113b2e30396567db54",
+                "sha256:9d3c9b50f19704552f23b4eaea1fc082fdd82c63429a6506446cbd8737823da3",
+                "sha256:a6fe5571784af92b6bc2fda8d1925cccdf24642d49546d3144948a6a1ed58ca5",
+                "sha256:a78ed8a53a1221393d9637c01870248a6f4ea5b214a59a92a36f18151739452c",
+                "sha256:ab40e6251c3873d86ea9b30a1ac6d7478c09277b32e14745d0d3c6e76e3c7e29",
+                "sha256:abf151955990d23f84205286938796c55ff11bbfb4ccfada8c9c83ae6b3c89a3",
+                "sha256:acef0899fea7492145d2bbaaaec7b345c87753168589cc7faf0afec9afe9b747",
+                "sha256:b40670ec7e2156d8e57f70aec34a7216407848dfe6c693ef131ddf6e76feb672",
+                "sha256:b791a3143681a520c0a17e26ae7465f1b6f99461a28019d1a2f425236e6eedb5",
+                "sha256:b955ed993491f1a5da7f92e98d5dad3c1e14dc175f74517c4e610b1f2456fb11",
+                "sha256:ba39e9c8627edc56544c8628cc180d88605df3892beeb2b94c9bc857774848ca",
+                "sha256:bca77a198bb6e69795ef2f09a5f4c12758487f83f33d63acde5f0d4919815768",
+                "sha256:c3452ea726c76e92f3b9fae4b34a151981a9ec0a4847a627c43d71a15ac32aa6",
+                "sha256:c46956ed82961e31557b6857a5ca153c67e5476972e5f7190015018760938da2",
+                "sha256:c7c8b816c2b5af5c8a436df44ca08258fc1a13b449393a91484225fcb7545533",
+                "sha256:cd73265a9e5ea618014802ab01babf1940cecb90c9762d8b9e7d2cc1e1969ec6",
+                "sha256:dad46e6f620574b3b4801c68255492e0159d1712271cc99d8bdf35f2043ec266",
+                "sha256:dc9b311743a78043b26ffaeeb9715dc360335e5517832f5a8e339f8a43581e4d",
+                "sha256:df822ee7feaaeffb99c1a9e5e608800bd8eda6e5f18f5cfb0dc7eeb2eaa6bbec",
+                "sha256:e083c285857b78ee21a96ba1eb1b5339733c3563f72980728ca2b08b53826ca5",
+                "sha256:e5e46b578c0e9db71d04c4b506a2121c0cb371dd89af17a0586ff6769d4c58c1",
+                "sha256:e99abf0bba688259a496f966211c49a514e65afa9b3073a1fcee08856e04425b",
+                "sha256:ee43080e75fc92bf36219926c8e6de497f9b247301bbf88c5c7593d931426679",
+                "sha256:f033d80bc6283092613882dfe40419c6a6a1527e04fc69350e87a9df02bbc283",
+                "sha256:f1088fa100bf46e7b398ffd9904f4808a0612e1d966b4aa43baa535d1b6341eb",
+                "sha256:f56455b0c2c7cc3b0c584815264461d07b177f903a04481dfc33e08a89f0c26b",
+                "sha256:f59dfe57bb1ec82ac0698ebfcdb7bcd0e99c255bd637ff613760d5f33e7c81b3",
+                "sha256:f7217af2e14da0856e082e96ff637f14ae45c10a5714b63c77f26d8884cf1051",
+                "sha256:f734e38fd8666f53da904c52a23ce517f1b07722118d750405af7e4123933511",
+                "sha256:f95511dd5d0e05fd9728bac4096319f80615aaef4acbecb35a990afebe953b0e",
+                "sha256:fdd215b7b7fd4a53994f238d0f46b7ba4ac4c0adb12452beee724ddd0743ae5d",
+                "sha256:feeb18a801aacb098220e2c3eea59a512362eb408d4afd0c242044c33ad6d542",
+                "sha256:ff30218887e62209942f91ac1be902cc80cddb86bf00fbc6783b7a43b2bea26f"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==3.9.3"
+        },
+        "aiosignal": {
+            "hashes": [
+                "sha256:54cd96e15e1649b75d6c87526a6ff0b6c1b0dd3459f43d9ca11d48c339b68cfc",
+                "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.3.1"
+        },
+        "argparse": {
+            "hashes": [
+                "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4",
+                "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314"
+            ],
+            "version": "==1.4.0"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30",
+                "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==23.2.0"
+        },
+        "bidict": {
+            "hashes": [
+                "sha256:03069d763bc387bbd20e7d49914e75fc4132a41937fa3405417e1a5a2d006d71",
+                "sha256:5dae8d4d79b552a71cbabc7deb25dfe8ce710b17ff41711e13010ead2abfc3e5"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.23.1"
+        },
+        "brotli": {
+            "hashes": [
+                "sha256:03d20af184290887bdea3f0f78c4f737d126c74dc2f3ccadf07e54ceca3bf208",
+                "sha256:0541e747cce78e24ea12d69176f6a7ddb690e62c425e01d31cc065e69ce55b48",
+                "sha256:069a121ac97412d1fe506da790b3e69f52254b9df4eb665cd42460c837193354",
+                "sha256:0b63b949ff929fbc2d6d3ce0e924c9b93c9785d877a21a1b678877ffbbc4423a",
+                "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128",
+                "sha256:11d00ed0a83fa22d29bc6b64ef636c4552ebafcef57154b4ddd132f5638fbd1c",
+                "sha256:141bd4d93984070e097521ed07e2575b46f817d08f9fa42b16b9b5f27b5ac088",
+                "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9",
+                "sha256:1ab4fbee0b2d9098c74f3057b2bc055a8bd92ccf02f65944a241b4349229185a",
+                "sha256:1ae56aca0402a0f9a3431cddda62ad71666ca9d4dc3a10a142b9dce2e3c0cda3",
+                "sha256:224e57f6eac61cc449f498cc5f0e1725ba2071a3d4f48d5d9dffba42db196438",
+                "sha256:22fc2a8549ffe699bfba2256ab2ed0421a7b8fadff114a3d201794e45a9ff578",
+                "sha256:23032ae55523cc7bccb4f6a0bf368cd25ad9bcdcc1990b64a647e7bbcce9cb5b",
+                "sha256:2333e30a5e00fe0fe55903c8832e08ee9c3b1382aacf4db26664a16528d51b4b",
+                "sha256:2954c1c23f81c2eaf0b0717d9380bd348578a94161a65b3a2afc62c86467dd68",
+                "sha256:2de9d02f5bda03d27ede52e8cfe7b865b066fa49258cbab568720aa5be80a47d",
+                "sha256:30924eb4c57903d5a7526b08ef4a584acc22ab1ffa085faceb521521d2de32dd",
+                "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409",
+                "sha256:38025d9f30cf4634f8309c6874ef871b841eb3c347e90b0851f63d1ded5212da",
+                "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50",
+                "sha256:3d7954194c36e304e1523f55d7042c59dc53ec20dd4e9ea9d151f1b62b4415c0",
+                "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180",
+                "sha256:43ce1b9935bfa1ede40028054d7f48b5469cd02733a365eec8a329ffd342915d",
+                "sha256:4d4a848d1837973bf0f4b5e54e3bec977d99be36a7895c61abb659301b02c112",
+                "sha256:4ed11165dd45ce798d99a136808a794a748d5dc38511303239d4e2363c0695dc",
+                "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265",
+                "sha256:524f35912131cc2cabb00edfd8d573b07f2d9f21fa824bd3fb19725a9cf06327",
+                "sha256:587ca6d3cef6e4e868102672d3bd9dc9698c309ba56d41c2b9c85bbb903cdb95",
+                "sha256:5b3cc074004d968722f51e550b41a27be656ec48f8afaeeb45ebf65b561481dd",
+                "sha256:5eeb539606f18a0b232d4ba45adccde4125592f3f636a6182b4a8a436548b914",
+                "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0",
+                "sha256:5fb2ce4b8045c78ebbc7b8f3c15062e435d47e7393cc57c25115cfd49883747a",
+                "sha256:6172447e1b368dcbc458925e5ddaf9113477b0ed542df258d84fa28fc45ceea7",
+                "sha256:6c3020404e0b5eefd7c9485ccf8393cfb75ec38ce75586e046573c9dc29967a0",
+                "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451",
+                "sha256:7905193081db9bfa73b1219140b3d315831cbff0d8941f22da695832f0dd188f",
+                "sha256:7c4855522edb2e6ae7fdb58e07c3ba9111e7621a8956f481c68d5d979c93032e",
+                "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248",
+                "sha256:7f4bf76817c14aa98cc6697ac02f3972cb8c3da93e9ef16b9c66573a68014f91",
+                "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724",
+                "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966",
+                "sha256:890b5a14ce214389b2cc36ce82f3093f96f4cc730c1cffdbefff77a7c71f2a97",
+                "sha256:89f4988c7203739d48c6f806f1e87a1d96e0806d44f0fba61dba81392c9e474d",
+                "sha256:8dadd1314583ec0bf2d1379f7008ad627cd6336625d6679cf2f8e67081b83acf",
+                "sha256:901032ff242d479a0efa956d853d16875d42157f98951c0230f69e69f9c09bac",
+                "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951",
+                "sha256:919e32f147ae93a09fe064d77d5ebf4e35502a8df75c29fb05788528e330fe74",
+                "sha256:929811df5462e182b13920da56c6e0284af407d1de637d8e536c5cd00a7daf60",
+                "sha256:949f3b7c29912693cee0afcf09acd6ebc04c57af949d9bf77d6101ebb61e388c",
+                "sha256:a090ca607cbb6a34b0391776f0cb48062081f5f60ddcce5d11838e67a01928d1",
+                "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8",
+                "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d",
+                "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc",
+                "sha256:a469274ad18dc0e4d316eefa616d1d0c2ff9da369af19fa6f3daa4f09671fd61",
+                "sha256:a599669fd7c47233438a56936988a2478685e74854088ef5293802123b5b2460",
+                "sha256:a743e5a28af5f70f9c080380a5f908d4d21d40e8f0e0c8901604d15cfa9ba751",
+                "sha256:a77def80806c421b4b0af06f45d65a136e7ac0bdca3c09d9e2ea4e515367c7e9",
+                "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1",
+                "sha256:ae15b066e5ad21366600ebec29a7ccbc86812ed267e4b28e860b8ca16a2bc474",
+                "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2",
+                "sha256:c8146669223164fc87a7e3de9f81e9423c67a79d6b3447994dfb9c95da16e2d6",
+                "sha256:c8fd5270e906eef71d4a8d19b7c6a43760c6abcfcc10c9101d14eb2357418de9",
+                "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2",
+                "sha256:cdad5b9014d83ca68c25d2e9444e28e967ef16e80f6b436918c700c117a85467",
+                "sha256:cdbc1fc1bc0bff1cef838eafe581b55bfbffaed4ed0318b724d0b71d4d377619",
+                "sha256:ceb64bbc6eac5a140ca649003756940f8d6a7c444a68af170b3187623b43bebf",
+                "sha256:d0c5516f0aed654134a2fc936325cc2e642f8a0e096d075209672eb321cff408",
+                "sha256:d143fd47fad1db3d7c27a1b1d66162e855b5d50a89666af46e1679c496e8e579",
+                "sha256:d192f0f30804e55db0d0e0a35d83a9fead0e9a359a9ed0285dbacea60cc10a84",
+                "sha256:db85ecf4e609a48f4b29055f1e144231b90edc90af7481aa731ba2d059226b1b",
+                "sha256:de6551e370ef19f8de1807d0a9aa2cdfdce2e85ce88b122fe9f6b2b076837e59",
+                "sha256:e1140c64812cb9b06c922e77f1c26a75ec5e3f0fb2bf92cc8c58720dec276752",
+                "sha256:e6a904cb26bfefc2f0a6f240bdf5233be78cd2488900a2f846f3c3ac8489ab80",
+                "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0",
+                "sha256:e93dfc1a1165e385cc8239fab7c036fb2cd8093728cbd85097b284d7b99249a2",
+                "sha256:efa8b278894b14d6da122a72fefcebc28445f2d3f880ac59d46c90f4c13be9a3",
+                "sha256:f0d8a7a6b5983c2496e364b969f0e526647a06b075d034f3297dc66f3b360c64",
+                "sha256:f296c40e23065d0d6650c4aefe7470d2a25fffda489bcc3eb66083f3ac9f6643",
+                "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e",
+                "sha256:f733d788519c7e3e71f0855c96618720f5d3d60c3cb829d8bbb722dddce37985",
+                "sha256:fce1473f3ccc4187f75b4690cfc922628aed4d3dd013d047f95a9b3919a86596",
+                "sha256:fd5f17ff8f14003595ab414e45fce13d073e0762394f957182e69035c9f3d7c2",
+                "sha256:fdc3ff3bfccdc6b9cc7c342c03aa2400683f0cb891d46e94b64a197910dc4064"
+            ],
+            "markers": "implementation_name == 'cpython'",
+            "version": "==1.1.0"
+        },
+        "caribou": {
+            "hashes": [
+                "sha256:5ca6e6e6ad7d3175137c68d809e203fbd931f79163a5613808a789449fef7863"
+            ],
+            "index": "pypi",
+            "version": "==0.3.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f",
+                "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2024.2.2"
+        },
+        "charset-normalizer": {
+            "hashes": [
+                "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027",
+                "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087",
+                "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786",
+                "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8",
+                "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09",
+                "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185",
+                "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574",
+                "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e",
+                "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519",
+                "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898",
+                "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269",
+                "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3",
+                "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f",
+                "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6",
+                "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8",
+                "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a",
+                "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73",
+                "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc",
+                "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714",
+                "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2",
+                "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc",
+                "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce",
+                "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d",
+                "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e",
+                "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6",
+                "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269",
+                "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96",
+                "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d",
+                "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a",
+                "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4",
+                "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77",
+                "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d",
+                "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0",
+                "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed",
+                "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068",
+                "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac",
+                "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25",
+                "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8",
+                "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab",
+                "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26",
+                "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2",
+                "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db",
+                "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f",
+                "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5",
+                "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99",
+                "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c",
+                "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d",
+                "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811",
+                "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa",
+                "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a",
+                "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03",
+                "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b",
+                "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04",
+                "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c",
+                "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001",
+                "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458",
+                "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389",
+                "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99",
+                "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985",
+                "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537",
+                "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238",
+                "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f",
+                "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d",
+                "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796",
+                "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a",
+                "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143",
+                "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8",
+                "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c",
+                "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5",
+                "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5",
+                "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711",
+                "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4",
+                "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6",
+                "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c",
+                "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7",
+                "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4",
+                "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b",
+                "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae",
+                "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12",
+                "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c",
+                "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae",
+                "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8",
+                "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887",
+                "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b",
+                "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4",
+                "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f",
+                "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5",
+                "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33",
+                "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519",
+                "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"
+            ],
+            "markers": "python_full_version >= '3.7.0'",
+            "version": "==3.3.2"
+        },
+        "coloredlogs": {
+            "hashes": [
+                "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934",
+                "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==15.0.1"
+        },
+        "croniter": {
+            "hashes": [
+                "sha256:4cb064ce2d8f695b3b078be36ff50115cf8ac306c10a7e8653ee2a5b534673d7",
+                "sha256:d199b2ec3ea5e82988d1f72022433c5f9302b3b3ea9e6bfd6a1518f6ea5e700a"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.0.1"
+        },
+        "debugpy": {
+            "hashes": [
+                "sha256:016a9fcfc2c6b57f939673c874310d8581d51a0fe0858e7fac4e240c5eb743cb",
+                "sha256:0de56aba8249c28a300bdb0672a9b94785074eb82eb672db66c8144fff673146",
+                "sha256:1a9fe0829c2b854757b4fd0a338d93bc17249a3bf69ecf765c61d4c522bb92a8",
+                "sha256:28acbe2241222b87e255260c76741e1fbf04fdc3b6d094fcf57b6c6f75ce1242",
+                "sha256:3a79c6f62adef994b2dbe9fc2cc9cc3864a23575b6e387339ab739873bea53d0",
+                "sha256:3bda0f1e943d386cc7a0e71bfa59f4137909e2ed947fb3946c506e113000f741",
+                "sha256:3ebb70ba1a6524d19fa7bb122f44b74170c447d5746a503e36adc244a20ac539",
+                "sha256:58911e8521ca0c785ac7a0539f1e77e0ce2df753f786188f382229278b4cdf23",
+                "sha256:6df9aa9599eb05ca179fb0b810282255202a66835c6efb1d112d21ecb830ddd3",
+                "sha256:7a3afa222f6fd3d9dfecd52729bc2e12c93e22a7491405a0ecbf9e1d32d45b39",
+                "sha256:7eb7bd2b56ea3bedb009616d9e2f64aab8fc7000d481faec3cd26c98a964bcdd",
+                "sha256:92116039b5500633cc8d44ecc187abe2dfa9b90f7a82bbf81d079fcdd506bae9",
+                "sha256:a2e658a9630f27534e63922ebf655a6ab60c370f4d2fc5c02a5b19baf4410ace",
+                "sha256:bfb20cb57486c8e4793d41996652e5a6a885b4d9175dd369045dad59eaacea42",
+                "sha256:caad2846e21188797a1f17fc09c31b84c7c3c23baf2516fed5b40b378515bbf0",
+                "sha256:d915a18f0597ef685e88bb35e5d7ab968964b7befefe1aaea1eb5b2640b586c7",
+                "sha256:dda73bf69ea479c8577a0448f8c707691152e6c4de7f0c4dec5a4bc11dee516e",
+                "sha256:e38beb7992b5afd9d5244e96ad5fa9135e94993b0c551ceebf3fe1a5d9beb234",
+                "sha256:edcc9f58ec0fd121a25bc950d4578df47428d72e1a0d66c07403b04eb93bcf98",
+                "sha256:efd3fdd3f67a7e576dd869c184c5dd71d9aaa36ded271939da352880c012e703",
+                "sha256:f696d6be15be87aef621917585f9bb94b1dc9e8aced570db1b8a6fc14e8f9b42",
+                "sha256:fd97ed11a4c7f6d042d320ce03d83b20c3fb40da892f994bc041bbc415d7a099"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==1.8.1"
+        },
+        "frozenlist": {
+            "hashes": [
+                "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7",
+                "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98",
+                "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad",
+                "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5",
+                "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae",
+                "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e",
+                "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a",
+                "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701",
+                "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d",
+                "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6",
+                "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6",
+                "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106",
+                "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75",
+                "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868",
+                "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a",
+                "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0",
+                "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1",
+                "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826",
+                "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec",
+                "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6",
+                "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950",
+                "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19",
+                "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0",
+                "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8",
+                "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a",
+                "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09",
+                "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86",
+                "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c",
+                "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5",
+                "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b",
+                "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b",
+                "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d",
+                "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0",
+                "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea",
+                "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776",
+                "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a",
+                "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897",
+                "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7",
+                "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09",
+                "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9",
+                "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe",
+                "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd",
+                "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742",
+                "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09",
+                "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0",
+                "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932",
+                "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1",
+                "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a",
+                "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49",
+                "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d",
+                "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7",
+                "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480",
+                "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89",
+                "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e",
+                "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b",
+                "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82",
+                "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb",
+                "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068",
+                "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8",
+                "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b",
+                "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb",
+                "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2",
+                "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11",
+                "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b",
+                "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc",
+                "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0",
+                "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497",
+                "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17",
+                "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0",
+                "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2",
+                "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439",
+                "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5",
+                "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac",
+                "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825",
+                "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887",
+                "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced",
+                "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.4.1"
+        },
+        "h11": {
+            "hashes": [
+                "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d",
+                "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.14.0"
+        },
+        "humanfriendly": {
+            "hashes": [
+                "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477",
+                "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==10.0"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca",
+                "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==3.6"
+        },
+        "multidict": {
+            "hashes": [
+                "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556",
+                "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c",
+                "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29",
+                "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b",
+                "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8",
+                "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7",
+                "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd",
+                "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40",
+                "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6",
+                "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3",
+                "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c",
+                "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9",
+                "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5",
+                "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae",
+                "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442",
+                "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9",
+                "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc",
+                "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c",
+                "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea",
+                "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5",
+                "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50",
+                "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182",
+                "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453",
+                "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e",
+                "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600",
+                "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733",
+                "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda",
+                "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241",
+                "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461",
+                "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e",
+                "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e",
+                "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b",
+                "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e",
+                "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7",
+                "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386",
+                "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd",
+                "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9",
+                "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf",
+                "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee",
+                "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5",
+                "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a",
+                "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271",
+                "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54",
+                "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4",
+                "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496",
+                "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb",
+                "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319",
+                "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3",
+                "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f",
+                "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527",
+                "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed",
+                "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604",
+                "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef",
+                "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8",
+                "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5",
+                "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5",
+                "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626",
+                "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c",
+                "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d",
+                "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c",
+                "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc",
+                "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc",
+                "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b",
+                "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38",
+                "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450",
+                "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1",
+                "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f",
+                "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3",
+                "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755",
+                "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226",
+                "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a",
+                "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046",
+                "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf",
+                "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479",
+                "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e",
+                "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1",
+                "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a",
+                "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83",
+                "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929",
+                "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93",
+                "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a",
+                "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c",
+                "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44",
+                "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89",
+                "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba",
+                "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e",
+                "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da",
+                "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24",
+                "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423",
+                "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==6.0.5"
+        },
+        "mutagen": {
+            "hashes": [
+                "sha256:719fadef0a978c31b4cf3c956261b3c58b6948b32023078a2117b1de09f0fc99",
+                "sha256:edd96f50c5907a9539d8e5bba7245f62c9f520aef333d13392a79a4f70aca719"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.47.0"
+        },
+        "pycryptodomex": {
+            "hashes": [
+                "sha256:0daad007b685db36d977f9de73f61f8da2a7104e20aca3effd30752fd56f73e1",
+                "sha256:108e5f1c1cd70ffce0b68739c75734437c919d2eaec8e85bffc2c8b4d2794305",
+                "sha256:19764605feea0df966445d46533729b645033f134baeb3ea26ad518c9fdf212c",
+                "sha256:1be97461c439a6af4fe1cf8bf6ca5936d3db252737d2f379cc6b2e394e12a458",
+                "sha256:25cd61e846aaab76d5791d006497134602a9e451e954833018161befc3b5b9ed",
+                "sha256:2a47bcc478741b71273b917232f521fd5704ab4b25d301669879e7273d3586cc",
+                "sha256:59af01efb011b0e8b686ba7758d59cf4a8263f9ad35911bfe3f416cee4f5c08c",
+                "sha256:5dcac11031a71348faaed1f403a0debd56bf5404232284cf8c761ff918886ebc",
+                "sha256:62a5ec91388984909bb5398ea49ee61b68ecb579123694bffa172c3b0a107079",
+                "sha256:645bd4ca6f543685d643dadf6a856cc382b654cc923460e3a10a49c1b3832aeb",
+                "sha256:653b29b0819605fe0898829c8ad6400a6ccde096146730c2da54eede9b7b8baa",
+                "sha256:69138068268127cd605e03438312d8f271135a33140e2742b417d027a0539427",
+                "sha256:6e186342cfcc3aafaad565cbd496060e5a614b441cacc3995ef0091115c1f6c5",
+                "sha256:76bd15bb65c14900d98835fcd10f59e5e0435077431d3a394b60b15864fddd64",
+                "sha256:7805830e0c56d88f4d491fa5ac640dfc894c5ec570d1ece6ed1546e9df2e98d6",
+                "sha256:7a710b79baddd65b806402e14766c721aee8fb83381769c27920f26476276c1e",
+                "sha256:7a7a8f33a1f1fb762ede6cc9cbab8f2a9ba13b196bfaf7bc6f0b39d2ba315a43",
+                "sha256:82ee7696ed8eb9a82c7037f32ba9b7c59e51dda6f105b39f043b6ef293989cb3",
+                "sha256:88afd7a3af7ddddd42c2deda43d53d3dfc016c11327d0915f90ca34ebda91499",
+                "sha256:8af1a451ff9e123d0d8bd5d5e60f8e3315c3a64f3cdd6bc853e26090e195cdc8",
+                "sha256:8ee606964553c1a0bc74057dd8782a37d1c2bc0f01b83193b6f8bb14523b877b",
+                "sha256:91852d4480a4537d169c29a9d104dda44094c78f1f5b67bca76c29a91042b623",
+                "sha256:9c682436c359b5ada67e882fec34689726a09c461efd75b6ea77b2403d5665b7",
+                "sha256:bc3ee1b4d97081260d92ae813a83de4d2653206967c4a0a017580f8b9548ddbc",
+                "sha256:bca649483d5ed251d06daf25957f802e44e6bb6df2e8f218ae71968ff8f8edc4",
+                "sha256:c39778fd0548d78917b61f03c1fa8bfda6cfcf98c767decf360945fe6f97461e",
+                "sha256:cbe71b6712429650e3883dc81286edb94c328ffcd24849accac0a4dbcc76958a",
+                "sha256:d00fe8596e1cc46b44bf3907354e9377aa030ec4cd04afbbf6e899fc1e2a7781",
+                "sha256:d3584623e68a5064a04748fb6d76117a21a7cb5eaba20608a41c7d0c61721794",
+                "sha256:e48217c7901edd95f9f097feaa0388da215ed14ce2ece803d3f300b4e694abea",
+                "sha256:f2e497413560e03421484189a6b65e33fe800d3bd75590e6d78d4dfdb7accf3b",
+                "sha256:ff5c9a67f8a4fba4aed887216e32cbc48f2a6fb2673bb10a99e43be463e15913"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==3.20.0"
+        },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
+                "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.8.2"
+        },
+        "python-dotenv": {
+            "hashes": [
+                "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca",
+                "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==1.0.1"
+        },
+        "python-engineio": {
+            "hashes": [
+                "sha256:979859bff770725b75e60353d7ae53b397e8b517d05ba76733b404a3dcca3e4c",
+                "sha256:e87459c15638e567711fd156e6f9c4a402668871bed79523f0ecfec744729ec7"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==4.9.0"
+        },
+        "python-magic": {
+            "hashes": [
+                "sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b",
+                "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==0.4.27"
+        },
+        "python-socketio": {
+            "hashes": [
+                "sha256:bbcbd758ed8c183775cb2853ba001361e2fa018babf5cbe11a5b77e91c2ec2a2",
+                "sha256:f1a0228b8b1fbdbd93fbbedd821ebce0ef54b2b5bf6e98fcf710deaa7c574259"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==5.11.1"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812",
+                "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"
+            ],
+            "version": "==2024.1"
+        },
+        "requests": {
+            "hashes": [
+                "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
+                "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.31.0"
+        },
+        "simple-websocket": {
+            "hashes": [
+                "sha256:17d2c72f4a2bd85174a97e3e4c88b01c40c3f81b7b648b0cc3ce1305968928c8",
+                "sha256:1d5bf585e415eaa2083e2bcf02a3ecf91f9712e7b3e6b9fa0b461ad04e0837bc"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.0.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.16.0"
+        },
+        "tzlocal": {
+            "hashes": [
+                "sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8",
+                "sha256:8d399205578f1a9342816409cc1e46a93ebd5755e39ea2d85334bea911bf0e6e"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==5.2"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d",
+                "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2.2.1"
+        },
+        "websockets": {
+            "hashes": [
+                "sha256:00700340c6c7ab788f176d118775202aadea7602c5cc6be6ae127761c16d6b0b",
+                "sha256:0bee75f400895aef54157b36ed6d3b308fcab62e5260703add87f44cee9c82a6",
+                "sha256:0e6e2711d5a8e6e482cacb927a49a3d432345dfe7dea8ace7b5790df5932e4df",
+                "sha256:12743ab88ab2af1d17dd4acb4645677cb7063ef4db93abffbf164218a5d54c6b",
+                "sha256:1a9d160fd080c6285e202327aba140fc9a0d910b09e423afff4ae5cbbf1c7205",
+                "sha256:1bf386089178ea69d720f8db6199a0504a406209a0fc23e603b27b300fdd6892",
+                "sha256:1df2fbd2c8a98d38a66f5238484405b8d1d16f929bb7a33ed73e4801222a6f53",
+                "sha256:1e4b3f8ea6a9cfa8be8484c9221ec0257508e3a1ec43c36acdefb2a9c3b00aa2",
+                "sha256:1f38a7b376117ef7aff996e737583172bdf535932c9ca021746573bce40165ed",
+                "sha256:23509452b3bc38e3a057382c2e941d5ac2e01e251acce7adc74011d7d8de434c",
+                "sha256:248d8e2446e13c1d4326e0a6a4e9629cb13a11195051a73acf414812700badbd",
+                "sha256:25eb766c8ad27da0f79420b2af4b85d29914ba0edf69f547cc4f06ca6f1d403b",
+                "sha256:27a5e9964ef509016759f2ef3f2c1e13f403725a5e6a1775555994966a66e931",
+                "sha256:2c71bd45a777433dd9113847af751aae36e448bc6b8c361a566cb043eda6ec30",
+                "sha256:2cb388a5bfb56df4d9a406783b7f9dbefb888c09b71629351cc6b036e9259370",
+                "sha256:2d225bb6886591b1746b17c0573e29804619c8f755b5598d875bb4235ea639be",
+                "sha256:2e5fc14ec6ea568200ea4ef46545073da81900a2b67b3e666f04adf53ad452ec",
+                "sha256:363f57ca8bc8576195d0540c648aa58ac18cf85b76ad5202b9f976918f4219cf",
+                "sha256:3c6cc1360c10c17463aadd29dd3af332d4a1adaa8796f6b0e9f9df1fdb0bad62",
+                "sha256:3d829f975fc2e527a3ef2f9c8f25e553eb7bc779c6665e8e1d52aa22800bb38b",
+                "sha256:3e3aa8c468af01d70332a382350ee95f6986db479ce7af14d5e81ec52aa2b402",
+                "sha256:3f61726cae9f65b872502ff3c1496abc93ffbe31b278455c418492016e2afc8f",
+                "sha256:423fc1ed29f7512fceb727e2d2aecb952c46aa34895e9ed96071821309951123",
+                "sha256:46e71dbbd12850224243f5d2aeec90f0aaa0f2dde5aeeb8fc8df21e04d99eff9",
+                "sha256:4d87be612cbef86f994178d5186add3d94e9f31cc3cb499a0482b866ec477603",
+                "sha256:5693ef74233122f8ebab026817b1b37fe25c411ecfca084b29bc7d6efc548f45",
+                "sha256:5aa9348186d79a5f232115ed3fa9020eab66d6c3437d72f9d2c8ac0c6858c558",
+                "sha256:5d873c7de42dea355d73f170be0f23788cf3fa9f7bed718fd2830eefedce01b4",
+                "sha256:5f6ffe2c6598f7f7207eef9a1228b6f5c818f9f4d53ee920aacd35cec8110438",
+                "sha256:604428d1b87edbf02b233e2c207d7d528460fa978f9e391bd8aaf9c8311de137",
+                "sha256:6350b14a40c95ddd53e775dbdbbbc59b124a5c8ecd6fbb09c2e52029f7a9f480",
+                "sha256:6e2df67b8014767d0f785baa98393725739287684b9f8d8a1001eb2839031447",
+                "sha256:6e96f5ed1b83a8ddb07909b45bd94833b0710f738115751cdaa9da1fb0cb66e8",
+                "sha256:6e9e7db18b4539a29cc5ad8c8b252738a30e2b13f033c2d6e9d0549b45841c04",
+                "sha256:70ec754cc2a769bcd218ed8d7209055667b30860ffecb8633a834dde27d6307c",
+                "sha256:7b645f491f3c48d3f8a00d1fce07445fab7347fec54a3e65f0725d730d5b99cb",
+                "sha256:7fa3d25e81bfe6a89718e9791128398a50dec6d57faf23770787ff441d851967",
+                "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b",
+                "sha256:8572132c7be52632201a35f5e08348137f658e5ffd21f51f94572ca6c05ea81d",
+                "sha256:87b4aafed34653e465eb77b7c93ef058516cb5acf3eb21e42f33928616172def",
+                "sha256:8e332c210b14b57904869ca9f9bf4ca32f5427a03eeb625da9b616c85a3a506c",
+                "sha256:9893d1aa45a7f8b3bc4510f6ccf8db8c3b62120917af15e3de247f0780294b92",
+                "sha256:9edf3fc590cc2ec20dc9d7a45108b5bbaf21c0d89f9fd3fd1685e223771dc0b2",
+                "sha256:9fdf06fd06c32205a07e47328ab49c40fc1407cdec801d698a7c41167ea45113",
+                "sha256:a02413bc474feda2849c59ed2dfb2cddb4cd3d2f03a2fedec51d6e959d9b608b",
+                "sha256:a1d9697f3337a89691e3bd8dc56dea45a6f6d975f92e7d5f773bc715c15dde28",
+                "sha256:a571f035a47212288e3b3519944f6bf4ac7bc7553243e41eac50dd48552b6df7",
+                "sha256:ab3d732ad50a4fbd04a4490ef08acd0517b6ae6b77eb967251f4c263011a990d",
+                "sha256:ae0a5da8f35a5be197f328d4727dbcfafa53d1824fac3d96cdd3a642fe09394f",
+                "sha256:b067cb952ce8bf40115f6c19f478dc71c5e719b7fbaa511359795dfd9d1a6468",
+                "sha256:b2ee7288b85959797970114deae81ab41b731f19ebcd3bd499ae9ca0e3f1d2c8",
+                "sha256:b81f90dcc6c85a9b7f29873beb56c94c85d6f0dac2ea8b60d995bd18bf3e2aae",
+                "sha256:ba0cab91b3956dfa9f512147860783a1829a8d905ee218a9837c18f683239611",
+                "sha256:baa386875b70cbd81798fa9f71be689c1bf484f65fd6fb08d051a0ee4e79924d",
+                "sha256:bbe6013f9f791944ed31ca08b077e26249309639313fff132bfbf3ba105673b9",
+                "sha256:bea88d71630c5900690fcb03161ab18f8f244805c59e2e0dc4ffadae0a7ee0ca",
+                "sha256:befe90632d66caaf72e8b2ed4d7f02b348913813c8b0a32fae1cc5fe3730902f",
+                "sha256:c3181df4583c4d3994d31fb235dc681d2aaad744fbdbf94c4802485ececdecf2",
+                "sha256:c4e37d36f0d19f0a4413d3e18c0d03d0c268ada2061868c1e6f5ab1a6d575077",
+                "sha256:c588f6abc13f78a67044c6b1273a99e1cf31038ad51815b3b016ce699f0d75c2",
+                "sha256:cbe83a6bbdf207ff0541de01e11904827540aa069293696dd528a6640bd6a5f6",
+                "sha256:d554236b2a2006e0ce16315c16eaa0d628dab009c33b63ea03f41c6107958374",
+                "sha256:dbcf72a37f0b3316e993e13ecf32f10c0e1259c28ffd0a85cee26e8549595fbc",
+                "sha256:dc284bbc8d7c78a6c69e0c7325ab46ee5e40bb4d50e494d8131a07ef47500e9e",
+                "sha256:dff6cdf35e31d1315790149fee351f9e52978130cef6c87c4b6c9b3baf78bc53",
+                "sha256:e469d01137942849cff40517c97a30a93ae79917752b34029f0ec72df6b46399",
+                "sha256:eb809e816916a3b210bed3c82fb88eaf16e8afcf9c115ebb2bacede1797d2547",
+                "sha256:ed2fcf7a07334c77fc8a230755c2209223a7cc44fc27597729b8ef5425aa61a3",
+                "sha256:f44069528d45a933997a6fef143030d8ca8042f0dfaad753e2906398290e2870",
+                "sha256:f764ba54e33daf20e167915edc443b6f88956f37fb606449b4a5b10ba42235a5",
+                "sha256:fc4e7fa5414512b481a2483775a8e8be7803a35b30ca805afa4998a84f9fd9e8",
+                "sha256:ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==12.0"
+        },
+        "wsproto": {
+            "hashes": [
+                "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065",
+                "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736"
+            ],
+            "markers": "python_full_version >= '3.7.0'",
+            "version": "==1.2.0"
+        },
+        "yarl": {
+            "hashes": [
+                "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51",
+                "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce",
+                "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559",
+                "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0",
+                "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81",
+                "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc",
+                "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4",
+                "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c",
+                "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130",
+                "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136",
+                "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e",
+                "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec",
+                "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7",
+                "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1",
+                "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455",
+                "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099",
+                "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129",
+                "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10",
+                "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142",
+                "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98",
+                "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa",
+                "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7",
+                "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525",
+                "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c",
+                "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9",
+                "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c",
+                "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8",
+                "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b",
+                "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf",
+                "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23",
+                "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd",
+                "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27",
+                "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f",
+                "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece",
+                "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434",
+                "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec",
+                "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff",
+                "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78",
+                "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d",
+                "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863",
+                "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53",
+                "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31",
+                "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15",
+                "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5",
+                "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b",
+                "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57",
+                "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3",
+                "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1",
+                "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f",
+                "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad",
+                "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c",
+                "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7",
+                "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2",
+                "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b",
+                "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2",
+                "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b",
+                "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9",
+                "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be",
+                "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e",
+                "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984",
+                "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4",
+                "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074",
+                "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2",
+                "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392",
+                "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91",
+                "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541",
+                "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf",
+                "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572",
+                "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66",
+                "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575",
+                "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14",
+                "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5",
+                "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1",
+                "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e",
+                "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551",
+                "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17",
+                "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead",
+                "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0",
+                "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe",
+                "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234",
+                "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0",
+                "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7",
+                "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34",
+                "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42",
+                "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385",
+                "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78",
+                "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be",
+                "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958",
+                "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749",
+                "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.9.4"
+        },
+        "yt-dlp": {
+            "hashes": [
+                "sha256:a11862e57721b0a0f0883dfeb5a4d79ba213a2d4c45e1880e9fd70f8e6570c38",
+                "sha256:c00d9a71d64472ad441bcaa1ec0c3797d6e60c9f934f270096a96fe51657e7b3"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==2023.12.30"
+        }
+    },
+    "develop": {}
+}

README.md

@@ -1,131 +1,281 @@
 # YTPTube
 
-![Build Status](https://github.com/arabcoders/ytptube/actions/workflows/main.yml/badge.svg)
+![Build Status](https://github.com/ArabCoders/ytptube/actions/workflows/main.yml/badge.svg)
-![MIT License](https://img.shields.io/github/license/arabcoders/ytptube.svg)
-![Docker Pull](https://img.shields.io/docker/pulls/arabcoders/ytptube.svg)
-![gchr Pull](https://ghcr-badge.elias.eu.org/shield/arabcoders/ytptube/ytptube)
 
-**YTPTube** is a web-based GUI for [yt-dlp](https://github.com/yt-dlp/yt-dlp), designed to make downloading videos from 
+Web GUI for [yt-dlp](https://github.com/yt-dlp/yt-dlp) with playlist & channel support.
-video platforms easier and user-friendly. It supports downloading playlists, channels, live streams and 
-includes features like scheduling downloads, sending notifications, and built-in video player.
 
-# Screenshots
+YTPTube started as a fork of [meTube](https://github.com/alexta69/metube), Since then it went under heavy changes, and it supports many new features.
-Example of the regular view interface.
-![Short screenshot](https://raw.githubusercontent.com/ArabCoders/ytptube/dev/sc_short.jpg)
-
-Example of the Simple mode interface.
-![Short screenshot](https://raw.githubusercontent.com/ArabCoders/ytptube/dev/sc_simple.jpg)
 
 # YTPTube Features.
+* A built in video player that can play any video file regardless of the format.
+* New `/add_batch` endpoint that allow multiple links to be sent.
+* Completely redesigned the frontend UI.
+* Switched out of binary file storage in favor of SQLite.
+* Handle live streams.
+* Support per link, `yt-dlp config` and `cookies`. and `output format`.
+* Tasks Runner. It allow you to queue channels for downloading using simple `json` file.
+* Webhook sender. It allow you to add webhook endpoints that receive events related to downloads using simple `json` file.
+* **Experimental** Multi-downloads support.
 
-* Multi-download support.
+### Tips
-* Random beautiful background.
+Your `yt-dlp` config should include the following options for optimal working conditions.
-* Handles live and upcoming streams.
-* A dual view mode for both technical and non-technical users.
-* Schedule channels or playlists to be downloaded automatically with support for creating custom download feeds from non-supported sites. See [Feeds documentation](FAQ.md#how-can-i-monitor-sites-without-rss-feeds).
-* Send notification to targets based on selected events. includes [Apprise](https://github.com/caronc/apprise?tab=readme-ov-file#readme) support.
-* Support per link options.
-* Support for limits per extractor and overall global limit.
-* Queue multiple URLs at once.
-* Powerful presets system for applying `yt-dlp` options. with a pre-made preset for media servers users.
-* A simple file browser.
-* A built in video player **with support for sidecar external subtitles**. `Require ffmpeg to be in PATH in non-docker setups`.
-* Basic authentication support.
-* Supports `curl-cffi`. See [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#impersonation). `In docker only`.
-* Bundled `pot provider plugin`. See [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide). `In docker only`.
-* Support using [flaresolverr/flaresolverr](https://github.com/flaresolverr/flaresolverr) to bypass Cloudflare protections for yt-dlp and internal http client. See [related FAQ](FAQ.md#how-to-bypass-cf-challenges).
-* Automatic updates for `yt-dlp` and custom `pip` packages. `In docker only`.
-* Conditions feature to apply custom options based on `yt-dlp` returned info.
-* Custom browser extensions, bookmarklets and iOS shortcuts to send links to YTPTube instance.
-* A bundled executable version for Windows, macOS and Linux. `MacOS version is untested`.
-* Use playwright or selenium for extractors that require a browser. see [related FAQ](FAQ.md#how-to-use-the-browser-extractor).
 
-Please read the [FAQ](FAQ.md) for more information.
+```json
+{
+    "windowsfilenames": true,
+    "live_from_start": true,
+    "format_sort": [
+        "codec:avc:m4a"
+    ]
+}
+```
+* Note, the `format_sort`, forces YouTube to use x264 instead of vp9 codec, you can ignore it if you want. i prefer the media in x264.
 
-# Installation
+[![Short screenshot](https://raw.githubusercontent.com/ArabCoders/ytptube/master/sc_short.png)](https://raw.githubusercontent.com/ArabCoders/ytptube/master/sc_full.png)
 
-> [!IMPORTANT]
+## Run using Docker
-> By default YTPTube runs without authentication. If you expose it to the internet, **enable auth**. See [security recommendations](FAQ.md#security-recommendations).
-
-## Run using docker command
 
 ```bash
-mkdir -p ./{config,downloads/{files,tmp}} && docker run -itd --rm --user "${UID}:${UID}" --name ytptube \
+docker run -d --name ytptube -p 8081:8081 -v ./config:/config:rw -v ./downloads:/downloads:rw ghcr.io/arabcoders/ytptube
--e YTP_TEMP_PATH=/downloads/tmp -e YTP_DOWNLOAD_PATH=/downloads/files \
--p 8081:8081 -v ./config:/config:rw -v ./downloads:/downloads:rw \
-ghcr.io/arabcoders/ytptube:latest
 ```
 
-## Run using podman
+## Run using docker-compose
-
-```bash
-mkdir -p ./{config,downloads/{files,tmp}} && podman run -itd --rm --userns=keep-id --name ytptube \
--e YTP_TEMP_PATH=/downloads/tmp -e YTP_DOWNLOAD_PATH=/downloads/files \
--p 8081:8081 -v ./config:/config:rw -v ./downloads:/downloads:rw \
-arabcoders/ytptube:latest
-```
-
-Then you can access the WebUI at `http://localhost:8081`.
-
-## Using compose file
-
-The following is an example of a `compose.yaml` file that can be used to run YTPTube.
 
 ```yaml
+version: "3.9"
 services:
   ytptube:
-    user: "${UID:-1000}:${UID:-1000}" # change this to your user id and group id.
+    user: "1000:1000"
-    # comment out the above line and uncomment the below line if you are using podman-compose.
+    image: ghcr.io/arabcoders/ytptube
-    #userns_mode: keep-id
-    image: ghcr.io/arabcoders/ytptube:latest
     container_name: ytptube
     restart: unless-stopped
-    environment:
-      - YTP_TEMP_PATH=/downloads/tmp
-      - YTP_DOWNLOAD_PATH=/downloads/files
     ports:
       - "8081:8081"
     volumes:
       - ./config:/config:rw
       - ./downloads:/downloads:rw
+    tmpfs:
+      - /tmp
 ```
 
-> [!IMPORTANT]
+## Configuration via environment variables
-> Make sure to change the `user` line to match your user id and group id in docker setups, or use `userns_mode: keep-id` in podman setups.
+
+Certain values can be set via environment variables, using the `-e` parameter on the docker command line, or the `environment:` section in docker-compose.
+
+* __YTP_CONFIG_PATH__: path to where the queue persistence files will be saved. Defaults to `/config` in the docker image, and `./var/config` otherwise.
+* __YTP_DOWNLOAD_PATH__: path to where the downloads will be saved. Defaults to `/downloads` in the docker image, and `./var/downloads` otherwise.
+* __YTP_TEMP_PATH__: path where intermediary download files will be saved. Defaults to `/tmp` in the docker image, and `./var/tmp` otherwise.
+* __YTP_TEMP_KEEP__: Whether to keep the Individual video temp directory or remove it. Defaults to `false`.
+* __YTP_URL_PREFIX__: base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`.
+* __YTP_OUTPUT_TEMPLATE__: the template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`. This will be the default for all downloads unless the request include output template.
+* __YTP_KEEP_ARCHIVE__: Whether to keep history of downloaded videos to prevent downloading same file multiple times. Defaults to `true`.
+* __YTP_YTDL_DEBUG__: Whether to turn debug logging for the internal `yt-dlp` package. Defaults to `false`.
+* __YTP_ALLOW_MANIFESTLESS__: Allow `yt-dlp` to download live streams videos which are yet to be processed by YouTube. Defaults to `false`
+* __YTP_HOST__: Which IP address to bind to. Defaults to `0.0.0.0`.
+* __YTP_PORT__: Which port to bind to. Defaults to `8081`.
+* __YTP_LOG_LEVEL__: Log level. Defaults to `info`.
+* __YTP_MAX_WORKERS__: How many works to use for downloads. Defaults to `1`.
+
+## Running behind a reverse proxy
+
+It's advisable to run YTPTube behind a reverse proxy, if authentication and/or HTTPS support are required.
+
+When running behind a reverse proxy which remaps the URL (i.e. serves YTPTube under a subdirectory and not under root), don't forget to set the `YTP_URL_PREFIX` environment variable to the correct value.
+
+### NGINX
+
+```nginx
+location /ytptube/ {
+  proxy_pass http://ytptube:8081;
+  proxy_http_version 1.1;
+  proxy_set_header Upgrade $http_upgrade;
+  proxy_set_header Connection "upgrade";
+  proxy_set_header Host $host;
+}
+```
+
+Note: the extra `proxy_set_header` directives are there to make WebSocket work.
+
+### Caddy
+
+The following example Caddyfile gets a reverse proxy going behind [caddy](https://caddyserver.com).
+
+```caddyfile
+example.com {
+  route /ytptube/* {
+    uri strip_prefix ytptube
+    reverse_proxy ytptube:8081
+  }
+}
+```
+
+## Updating yt-dlp
+
+The engine which powers the actual video downloads in YTPTube is [yt-dlp](https://github.com/yt-dlp/yt-dlp). Since video sites regularly change their layouts, frequent updates of yt-dlp are required to keep up.
+
+There's an automatic nightly build of YTPTube which looks for a new version of yt-dlp, and if one exists, the build pulls it and publishes an updated docker image. Therefore, in order to keep up with the changes, it's recommended that you update your YTPTube container regularly with the latest image.
+
+## Troubleshooting and submitting issues
+
+Before asking a question or submitting an issue for YTPTube, please remember that YTPTube is only a UI for [yt-dlp](https://github.com/yt-dlp/yt-dlp). Any issues you might be experiencing with authentication to video websites, postprocessing, permissions, other `yt-dlp options` configurations which seem not to work, or anything else that concerns the workings of the underlying yt-dlp library, need not be opened on the YTPTube project. In order to debug and troubleshoot them, it's advised to try using the yt-dlp binary directly first, bypassing the UI, and once that is working, importing the options that worked for you into `yt-dlp options` file.
+
+In order to test with the yt-dlp command directly, you can either download it and run it locally, or for a better simulation of its actual conditions, you can run it within the YTPTube container itself. Assuming your YTPTube container is called `YTPTube`, run the following on your Docker host to get a shell inside the container:
 
 ```bash
-mkdir -p ./{config,downloads/{files,tmp}} && docker compose -f compose.yaml up -d
+docker exec -ti ytptube sh
+cd /downloads
 ```
 
-Then you can access the WebUI at `http://localhost:8081`.
+Once there, you can use the yt-dlp command freely.
 
-## Unraid
+## Building and running locally
 
-For `Unraid` users You can install the `Community Applications` plugin, and search for **ytptube** it comes 
+Make sure you have `node.js` and Python `3.11+` installed.
-pre-configured.
 
-# API Documentation
+```bash
+cd ytptube/frontend
+# install Vue and build the UI
+npm install
+npm run build
+# install python dependencies
+cd ..
+python -m venv .venv
+source .venv/bin/activate
+pip3 install pipenv
+pipenv install
+# run
+python app/main.py
+```
 
-For simple API documentation, you can refer to the [API documentation](API.md).
+A Docker image can be built locally (it will build the UI too):
 
-# Disclaimer
+```bash
+docker build . -t ytptube
+```
 
-This project is not affiliated with yt-dlp or any other service.
+### ytdlp.json File
 
-This is a personal project designed to make downloading videos from the internet more convenient for me. It is not 
+The `config/ytdlp.json`, is a json file which can be used to alter the default `yt-dlp` config settings. For example these are the options i personally use,
-intended for piracy or any unlawful use. This project was built primarily for my own use and preferences.
 
-AI-assisted tools are used in this project. If you are uncomfortable with this, you should not use this project.
+```json5
+{
+  // Make the final filename windows compatible.
+  "windowsfilenames": true,
+  // Write subtitles if the stream has them.
+  "writesubtitles": true,
+  // Write info.json file for each download. It can be used by many tools to generate info etc.
+  "writeinfojson": true,
+  // Write thumbnail if available.
+  "writethumbnail": true,
+  // Do not download automatically generated subtitles. 
+  "writeautomaticsub": false,
+  // MP4 is limited with the codecs we use, so "mkv" make sense.
+  "merge_output_format": "mkv",
+  // Record live stream from the start.
+  "live_from_start": true,
+  // For YouTube try to force H264 video codec & AAC audio.
+  "format_sort": [
+      "codec:avc:m4a"
+  ],
+  // Your Choice of subtitle languages to download.
+  "subtitleslangs": [ "en", "ar" ],
+  // postprocessors to run on the file
+  "postprocessors": [
+      // this processor convert the downloaded thumbnail to JPG.
+      {
+          "key": "FFmpegThumbnailsConvertor",
+          "format": "jpg"
+      },
+      // This processor convert subtitles to SRT
+      {
+          "key": "FFmpegSubtitlesConvertor",
+          "format": "srt"
+      },
+      // This processor embed metadata & info.json file into the final MKV file.
+      {
+          "key": "FFmpegMetadata",
+          "add_infojson": true,
+          "add_metadata": true
+      },
+      // This process embed subtitles into the final file if it doesn't subtitles embedded.
+      {
+          "key": "FFmpegEmbedSubtitle",
+          "already_have_subtitle": false
+      }
+  ]
+}
+``` 
 
-Contributions are welcome, but I may decline changes that do not interest me or do not align with my vision for this 
+### tasks.json File
-project. Unsolicited pull requests will be closed. For suggestions or feature requests, please open a discussion or 
+
-join the Discord server.
+The `config/tasks.json`, is a json file, which can be used to queue URLs for downloading, it's mainly useful if you follow specific channels and you want it downloaded automatically, The schema for the file is as the following, Only the `URL` key is required.
+
+```json5
+[
+  {
+    // (URL: string) **REQUIRED**, URL to the content.
+    "url": "", 
+    // (Name: string) Optional field. Mainly used for logging. If omitted, random GUID will be shown.
+    "name": "My super secret channel", 
+    // (Timer: string) Optional field. Using regular cronjob timer, if the field is omitted, it will run every hour in random minute.
+    "timer": "1 */1 * * *", 
+    // (yt-dlp cookies: object) Optional field. A JSON cookies exported by flagCookies.
+    "ytdlp_cookies": {}, 
+    // (yt-dlp config: object) Optional field. A JSON yt-dlp config.
+    "ytdlp_config": {},
+    // (Output Template: string) Optional field. A File output format,
+    "output_template": "",
+    // (Folder: string) Optional field. Where to store the downloads relative to the main download path.
+    "folder":"",
+    // (Format: string) Optional field. Format as specified in Web GUI. Defaults to "any".
+    "format": "",
+    // (Quality: string), Optional field. Quality as specified in Web GUI. Defaults to "best".
+    "quality": "", 
+  },
+  {
+   // (URL: string) **REQUIRED**, URL to the content.
+   "url": "https://..." // This is valid config, it will queue the channel for downloading every hour at random minute.
+  },
+  ...
+]
+```
+
+The task runner is doing what you are doing when you click the add button on the WebGUI, this just fancy way to automate that.
+
+**WARNING**: We strongly advice turning on `YTP_KEEP_ARCHIVE` option. Otherwise, you will keep re-downloading the items, and you will eventually get banned from the source or or you will waste space, bandwidth re-downloading content over and over.
+
+### webhooks.json File
+
+The `config/webhooks.json`, is a json file, which can be used to add webhook endpoints that would receive events related to the downloads.
+
+```json5
+[
+  {
+    // (name: string) - REQUIRED - The webhook name.
+    "name": "my very smart webhook receiver",
+    // (on: array) - OPTIONAL - List of accepted events, if left empty it will send all events.
+    // Allowed events ["added", "completed", "error", "not_live" ] you can choose one or all of them.
+    "on": [ "added", "completed", "error", "not_live" ],
+    "request":{
+      // (url: string) - REQUIRED-  The webhook url
+      "url": "https://mysecert.webhook.com/endpoint", 
+      // (method: string) - OPTIONAL - The request method, it can be POST or PUT
+      "method": "POST",
+      // (headers: dictionary) - OPTIONAL - Extra headers to include.
+      "headers":{
+        "Authorization": "Bearer my_secret_token"
+      }
+  }
+  ...
+]
+```
 
 # Social contact
 
-If you have short or quick questions, you are free to join my [discord server](https://discord.gg/G3GpVR8xpb) and ask
+If you have short or quick questions, you are free to join my [discord server](https://discord.gg/haUXHJyj6Y) and ask
 the question. keep in mind it's solo project, as such it might take me a bit of time to reply.
 
 # Donation 
 
 If you feel like donating and appreciate my work, you can do so by donating to children charity. For example [Make-A-Wish](https://worldwish.org). 
+I Personally don't need the money, but I do appreciate the gesture. Making a child happy is more worthwhile.

app.spec

@@ -1,112 +0,0 @@
-import importlib.metadata
-import os
-import re
-import sys
-import tomllib
-
-block_cipher = None
-
-MANUAL_MAP = {
-    "bgutil_ytdlp_pot_provider": ["yt_dlp_plugins"],
-    "python-dotenv": ["dotenv"],
-}
-
-
-def top_level_modules(dist_name: str) -> list[str]:
-    manual: list[str] | None = MANUAL_MAP.get(dist_name) or MANUAL_MAP.get(dist_name.replace("-", "_"))
-    if manual:
-        return manual
-
-    try:
-        dist = importlib.metadata.distribution(dist_name)
-
-        top_level: list[str] = []
-        for f in dist.files or []:
-            if f.name == "top_level.txt":
-                txt = (dist.locate_file(f)).read_text().splitlines()
-                top_level.extend([line.strip() for line in txt if line.strip()])
-                break
-
-        if not top_level:
-            for f in dist.files or []:
-                if f.parent == "" and f.suffix == "" and "." not in f.name:
-                    top_level.append(f.name)  # noqa: PERF401
-
-        if top_level:
-            return sorted(set(top_level))
-    except Exception:
-        pass
-
-    return [dist_name.replace("-", "_")]
-
-
-def parse_pyproject(path: str = "pyproject.toml") -> set[str]:
-    with open(path, "rb") as f:
-        data = tomllib.load(f)
-
-    reqs = set(data["project"]["dependencies"])
-    for extra in data["project"].get("optional-dependencies", {}).values():
-        reqs.update(extra)
-
-    return {re.split(r"[ ;<>=]", r, 1)[0] for r in reqs}  # noqa: B034
-
-
-dist_names = parse_pyproject()
-hidden: list[str] = []
-
-for dist in dist_names:
-    if sys.platform != "win32" and dist in ("python-magic-bin", "tzdata"):
-        continue
-    if sys.platform == "win32" and dist == "python-magic":
-        continue
-    hidden.extend(top_level_modules(dist))
-
-hidden += [
-    "engineio.async_drivers.aiohttp",
-    "socketio.async_drivers.aiohttp",
-    "socketio",  # python-socketio top-level
-    "engineio",
-    "aiohttp",
-    "dotenv",
-    "app",
-]
-
-hidden = sorted(set(hidden))
-
-a = Analysis(  # noqa: F821 # type: ignore
-    ["app/local.py"],
-    pathex=[os.getcwd()],
-    binaries=[],
-    datas=[
-        ("ui/exported", "ui/exported"),
-        ("app/", "app/"),
-    ],
-    hiddenimports=hidden,
-    hookspath=[],
-    runtime_hooks=[],
-    excludes=[],  # do NOT exclude libstdc++.so.6
-    cipher=block_cipher,
-)
-
-pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher)  # type: ignore # noqa: F821
-
-exe = EXE(  # noqa: F821 # type: ignore
-    pyz,
-    a.scripts,
-    name="YTPTube",
-    debug=False,
-    strip=False,
-    console=True,
-    icon="ui/public/favicon.ico",
-    exclude_binaries=True,
-)
-
-coll = COLLECT(  # noqa: F821 # type: ignore
-    exe,
-    a.binaries,
-    a.zipfiles,
-    a.datas,
-    strip=False,
-    upx=False,
-    name="YTPTube",
-)

app/AsyncPool.py

@@ -0,0 +1,187 @@
+import asyncio
+from datetime import datetime, timezone
+
+
+class Terminator:
+    pass
+
+
+class AsyncPool:
+    def __init__(self, loop, num_workers: int, name: str, logger, worker_co, load_factor: int = 1,
+                 job_accept_duration: int = None, max_task_time: int = None, return_futures: bool = False,
+                 raise_on_join: bool = False, log_every_n: int = None, expected_total=None):
+        """
+        This class will create `num_workers` asyncio tasks to work against a queue of
+        `num_workers * load_factor` items of back-pressure (IOW we will block after such
+        number of items of work is in the queue).  `worker_co` will be called
+        against each item retrieved from the queue. If any exceptions are raised out of
+        worker_co, self.exceptions will be set to True.
+
+        :param loop: asyncio loop to use
+        :param num_workers: number of async tasks which will pull from the internal queue
+        :param name: name of the worker pool (used for logging)
+        :param logger: logger to use
+        :param worker_co: async coroutine to call when an item is retrieved from the queue
+        :param load_factor: multiplier used for number of items in queue
+        :param job_accept_duration: maximum number of seconds from first push to last push before a TimeoutError will be thrown.
+                Set to None for no limit.  Note this does not get reset on aenter/aexit.
+        :param max_task_time: maximum time allowed for each task before a CancelledError is raised in the task.
+            Set to None for no limit.
+        :param return_futures: set to reture to return a future for each `push` (imposes CPU overhead)
+        :param raise_on_join: raise on join if any exceptions have occurred, default is False
+        :param log_every_n: (optional) set to number of `push`s each time a log statement should be printed (default does not print every-n pushes)
+        :param expected_total: (optional) expected total number of jobs (used for `log_event_n` logging)
+
+        :return: instance of AsyncWorkerPool
+        """
+        loop = loop if loop else asyncio.get_event_loop()
+        self._loop = loop
+        self._num_workers = num_workers
+        self._logger = logger
+        self._queue = asyncio.Queue(num_workers * load_factor)
+        self._workers = None
+        self._exceptions = False
+        self._job_accept_duration = job_accept_duration
+        self._first_push_dt = None
+        self._max_task_time = max_task_time
+        self._return_futures = return_futures
+        self._raise_on_join = raise_on_join
+        self._name = name
+        self._worker_co = worker_co
+        self._total_queued = 0
+        self._log_every_n = log_every_n
+        self._expected_total = expected_total
+        self._active_workers = 0
+
+    async def _worker_loop(self):
+        while True:
+            got_obj = False
+            future = None
+
+            try:
+                item = await self._queue.get()
+                got_obj = True
+
+                if item.__class__ is Terminator:
+                    break
+
+                self._active_workers += 1
+                future, args, kwargs = item
+                # the wait_for will cancel the task (task sees CancelledError) and raises a TimeoutError from here
+                # so be wary of catching TimeoutErrors in this loop
+                result = await asyncio.wait_for(self._worker_co(*args, **kwargs), self._max_task_time)
+
+                if future:
+                    future.set_result(result)
+            except (KeyboardInterrupt, MemoryError, SystemExit) as e:
+                if future:
+                    future.set_exception(e)
+                self._exceptions = True
+                raise
+            except BaseException as e:
+                self._exceptions = True
+
+                if future:
+                    # don't log the failure when the client is receiving the future
+                    future.set_exception(e)
+                else:
+                    self._logger.exception('Worker call failed')
+            finally:
+                self._active_workers -= 1
+                if got_obj:
+                    self._queue.task_done()
+
+    @property
+    def exceptions(self):
+        return self._exceptions
+
+    @property
+    def total_queued(self):
+        return self._total_queued
+
+    def has_open_workers(self) -> bool:
+        """
+        :return: True if there are open workers.
+        """
+        return self._active_workers < self._num_workers
+
+    def get_available_workers(self) -> int:
+        """
+        :return: number of available workers.
+        """
+        return self._num_workers - self._active_workers
+
+    async def __aenter__(self):
+        self.start()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        await self.join()
+
+    async def push(self, *args, **kwargs) -> asyncio.Future:
+        """
+        Method to push work to `worker_co` passed initially to `__init__`.
+
+        :param args: position arguments to be passed to `worker_co`
+        :param kwargs: keyword arguments to be passed to `worker_co`
+
+        :return: future of result.
+        """
+        if self._first_push_dt is None:
+            self._first_push_dt = self._time()
+
+        if self._job_accept_duration is not None and (self._time() - self._first_push_dt) > self._job_accept_duration:
+            raise TimeoutError(f"Max life time of {self._job_accept_duration}s exceeded for {self._name} pool.")
+
+        future = asyncio.futures.Future(loop=self._loop) if self._return_futures else None
+        await self._queue.put((future, args, kwargs))
+        self._total_queued += 1
+
+        if self._log_every_n is not None and (self._total_queued % self._log_every_n) == 0:
+            self._logger.info(f"pushed {self._total_queued}/{self._expected_total} items to {self._name} pool.")
+
+        self._logger.debug(f"'{self._name}' pool has received a new job. {args} {kwargs}")
+
+        return future
+
+    def start(self):
+        """ Will start up worker pool and reset exception state """
+        assert self._workers is None
+        self._exceptions = False
+
+        self._workers = []
+        for _ in range(self._num_workers):
+            self._workers.append(
+                asyncio.ensure_future(
+                    coro_or_future=self._worker_loop(),
+                    loop=self._loop
+                )
+            )
+
+    async def join(self):
+        # no-op if workers aren't running
+        if not self._workers:
+            return
+
+        self._logger.info(f'Joining {self._name}')
+
+        # The Terminators will kick each worker from being blocked against the _queue.get() and allow
+        # each one to exit
+        for _ in range(self._num_workers):
+            await self._queue.put(Terminator())
+
+        try:
+            await asyncio.gather(*self._workers)
+            self._workers = None
+        except:
+            self._logger.exception(f'Exception joining {self._name}')
+            raise
+        finally:
+            self._logger.info(f'Completed {self._name}')
+
+        if self._exceptions and self._raise_on_join:
+            raise Exception(f"Exception occurred in {self._name} pool")
+
+    def _time(self):
+        # utcnow returns a naive datetime, so we have to set the timezone manually <sigh>
+        return datetime.utcnow().replace(tzinfo=timezone.utc)

app/Config.py

@@ -0,0 +1,171 @@
+import json
+import logging
+import os
+import re
+import sys
+import coloredlogs
+from version import APP_VERSION
+from dotenv import load_dotenv
+
+
+class Config:
+    __instance = None
+    config_path: str = '.'
+    download_path: str = '.'
+    temp_path: str = '{download_path}'
+    temp_keep: bool = False
+    db_file: str = '{config_path}/ytptube.db'
+
+    url_host: str = ''
+    url_prefix: str = ''
+    url_socketio: str = '{url_prefix}socket.io'
+
+    output_template: str = '%(title)s.%(ext)s'
+    output_template_chapter: str = '%(title)s - %(section_number)s %(section_title)s.%(ext)s'
+
+    ytdl_options: dict | str = {}
+    ytdl_debug: bool = False
+
+    host: str = '0.0.0.0'
+    port: int = 8081
+
+    keep_archive: bool = True
+
+    base_path: str = ''
+
+    log_level: str = 'info'
+
+    allow_manifestless: bool = False
+
+    max_workers: int = 1
+
+    version: str = APP_VERSION
+
+    debug: bool = False
+
+    new_version_available: bool = False
+
+    _int_vars: tuple = ('port', 'max_workers',)
+    _immutable: tuple = ('version', '__instance', 'ytdl_options', 'new_version_available')
+    _boolean_vars: tuple = ('keep_archive', 'ytdl_debug', 'debug', 'temp_keep', 'allow_manifestless',)
+
+    @staticmethod
+    def get_instance():
+        """ Static access method. """
+        return Config() if not Config.__instance else Config.__instance
+
+    def __init__(self):
+        """ Virtually private constructor. """
+        if Config.__instance is not None:
+            raise Exception("This class is a singleton!. Use Config.getInstance() instead.")
+        else:
+            Config.__instance = self
+
+        baseDefaultPath: str = os.path.dirname(__file__)
+
+        self.temp_path = os.environ.get('YTP_TEMP_PATH', None) or os.path.join(baseDefaultPath, 'var', 'tmp')
+        self.config_path = os.environ.get('YTP_CONFIG_PATH', None) or os.path.join(baseDefaultPath, 'var', 'config')
+        self.download_path = os.environ.get(
+            'YTP_DOWNLOAD_PATH', None) or os.path.join(
+            baseDefaultPath, 'var', 'downloads')
+
+        envFile: str = os.path.join(self.config_path, '.env')
+
+        if os.path.exists(envFile):
+            logging.info(f'Loading environment variables from [{envFile}].')
+            load_dotenv(envFile)
+
+        for k, v in self._getAttributes().items():
+            if k.startswith('_'):
+                continue
+
+            # If the variable declared as immutable, set it to the default value.
+            if k in self._immutable:
+                setattr(self, k, v)
+                continue
+
+            lookUpKey: str = f'YTP_{k}'.upper()
+            setattr(self, k, os.environ[lookUpKey] if lookUpKey in os.environ else v)
+
+        for k, v in self.__dict__.items():
+            if k.startswith('_') or k in self._immutable:
+                continue
+
+            if isinstance(v, str) and '{' in v and '}' in v:
+                for key in re.findall(r'\{.*?\}', v):
+                    localKey: str = key[1:-1]
+                    if localKey not in self.__dict__:
+                        logging.error(f'Config variable "{k}" had non existing config reference "{key}"')
+                        sys.exit(1)
+
+                    v = v.replace(key, getattr(self, localKey))
+
+                setattr(self, k, v)
+
+            if k in self._boolean_vars:
+                if str(v).lower() not in (True, False, 'true', 'false', 'on', 'off', '1', '0'):
+                    raise ValueError(f'Config variable "{k}" is set to a non-boolean value "{v}".')
+
+                setattr(self, k, str(v).lower() in (True, 'true', 'on', '1'))
+
+            if k in self._int_vars:
+                setattr(self, k, int(v))
+
+        if not self.url_prefix.endswith('/'):
+            self.url_prefix += '/'
+
+        numeric_level = getattr(logging, self.log_level.upper(), None)
+        if not isinstance(numeric_level, int):
+            raise ValueError(f"Invalid log level: {self.log_level}")
+
+        coloredlogs.install(
+            level=numeric_level,
+            fmt="%(asctime)s [%(name)s] [%(levelname)-5.5s] %(message)s",
+            datefmt='%H:%M:%S'
+        )
+
+        LOG = logging.getLogger('config')
+
+        if self.debug:
+            try:
+                import debugpy
+                debugpy.listen(("0.0.0.0", 5678))
+                LOG.info("starting debugpy server on [0.0.0.0:5678]")
+            except ImportError:
+                LOG.error("debugpy not found, please install it with 'pip install debugpy'")
+
+        optsFile: str = os.path.join(self.config_path, 'ytdlp.json')
+        if os.path.exists(optsFile) and os.path.getsize(optsFile) > 0:
+            LOG.info(f'Loading yt-dlp custom options from "{optsFile}"')
+
+            try:
+                with open(optsFile) as json_data:
+                    opts = json.load(json_data)
+                assert isinstance(opts, dict)
+                self.ytdl_options.update(opts)
+            except (json.decoder.JSONDecodeError, AssertionError) as e:
+                LOG.error(f'JSON error in "{optsFile}": {e}')
+                sys.exit(1)
+        else:
+            LOG.info(f'No custom yt-dlp options found in "{self.config_path}"')
+
+        if self.keep_archive:
+            LOG.info(f'keep archive: {self.keep_archive}')
+            self.ytdl_options['download_archive'] = os.path.join(
+                self.config_path, 'archive.log')
+
+        LOG.info(f'Keep temp: {self.temp_keep}')
+
+    def _getAttributes(self) -> dict:
+        attrs: dict = {}
+        vClass: str = self.__class__
+
+        for attribute in vClass.__dict__.keys():
+            if attribute.startswith('_'):
+                continue
+
+            value = getattr(vClass, attribute)
+            if not callable(value):
+                attrs[attribute] = value
+
+        return attrs

app/DataStore.py

@@ -0,0 +1,144 @@
+from collections import OrderedDict
+import copy
+from datetime import datetime, timezone
+from email.utils import formatdate
+import json
+from sqlite3 import Connection
+from Utils import calcDownloadPath
+from Config import Config
+from Download import Download
+from ItemDTO import ItemDTO
+
+
+class DataStore:
+    """
+    Persistent queue.
+    """
+    type: str = None
+    dict: OrderedDict[str, Download] = None
+    config: Config = None
+
+    connection: Connection
+
+    def __init__(self, type: str, connection: Connection):
+        self.dict = OrderedDict()
+        self.type = type
+        self.config = Config.get_instance()
+        self.connection = connection
+
+    def load(self) -> None:
+        for id, item in self.saved_items():
+            self.dict.update({id: Download(
+                info=item,
+                download_dir=calcDownloadPath(basePath=self.config.download_path, folder=item.folder),
+                temp_dir=self.config.temp_path,
+                output_template_chapter=self.config.output_template_chapter,
+                default_ytdl_opts=self.config.ytdl_options)
+            })
+
+    def exists(self, key: str = None, url: str = None) -> bool:
+        if not key and not url:
+            raise KeyError('key or url must be provided.')
+
+        if key and key in self.dict:
+            return True
+
+        for i in self.dict:
+            if (key and self.dict[i].info._id == key) or (url and self.dict[i].info.url == url):
+                return True
+
+        return False
+
+    def get(self, key: str, url: str = None) -> Download:
+        if not key and not url:
+            raise KeyError('key or url must be provided.')
+
+        for i in self.dict:
+            if (key and self.dict[i].info._id == key) or (url and self.dict[i].info.url == url):
+                return self.dict[i]
+
+        raise KeyError(f'{key=} or {url=} not found.')
+
+    def items(self) -> list[tuple[str, Download]]:
+        return self.dict.items()
+
+    def saved_items(self) -> list[tuple[str, ItemDTO]]:
+        items: list[tuple[str, ItemDTO]] = []
+
+        cursor = self.connection.execute(
+            f'SELECT "id", "data", "created_at" FROM "history" WHERE "type" = ? ORDER BY "created_at" ASC',
+            (self.type,)
+        )
+
+        for row in cursor:
+            rowDate = datetime.strptime(row['created_at'], '%Y-%m-%d %H:%M:%S')
+            data: dict = json.loads(row['data'])
+            key: str = data.pop('_id')
+            item: ItemDTO = ItemDTO(**data)
+            item._id = key
+            item.datetime = formatdate(rowDate.replace(tzinfo=timezone.utc).timestamp())
+            items.append((row['id'], item))
+
+        return items
+
+    def put(self, value: Download) -> Download:
+        self.dict.update({value.info._id: value})
+        self._updateStoreItem(self.type, value.info)
+
+        return self.dict[value.info._id]
+
+    def delete(self, key: str) -> None:
+        self.dict.pop(key, None)
+        self._deleteStoreItem(key)
+
+    def next(self) -> tuple[str, Download]:
+        return next(iter(self.dict.items()))
+
+    def empty(self):
+        return not bool(self.dict)
+
+    def hasDownloads(self):
+        if 0 == len(self.dict):
+            return False
+
+        for key in self.dict:
+            if self.dict[key].started() is False:
+                return True
+
+        return False
+
+    def getNextDownload(self) -> Download:
+        for key in self.dict:
+            if self.dict[key].started() is False and self.dict[key].is_canceled() is False:
+                return self.dict[key]
+
+        return None
+
+    def _updateStoreItem(self, type: str, item: ItemDTO) -> None:
+        sqlStatement = """
+        INSERT INTO "history" ("id", "type", "url", "data")
+        VALUES (?, ?, ?, ?)
+        ON CONFLICT DO UPDATE SET "type" = ?, "url" = ?, "data" = ?, created_at = ?
+        """
+
+        stored = copy.deepcopy(item)
+
+        if hasattr(stored, 'datetime'):
+            try:
+                delattr(stored, 'datetime')
+            except AttributeError:
+                pass
+
+        if hasattr(stored, 'live_in') and stored.status == 'finished':
+            try:
+                delattr(stored, 'live_in')
+            except AttributeError:
+                pass
+
+        self.connection.execute(sqlStatement.strip(), (
+            stored._id, type, stored.url, stored.json(), type, stored.url,
+            stored.json(), datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
+        ))
+
+    def _deleteStoreItem(self, key: str) -> None:
+        self.connection.execute('DELETE FROM "history" WHERE "type" = ? AND "id" = ?', (self.type, key,))

app/Download.py

@@ -0,0 +1,292 @@
+import asyncio
+import json
+import logging
+import multiprocessing
+import os
+import re
+import shutil
+import yt_dlp
+from Utils import Notifier, get_format, get_opts, jsonCookie, mergeConfig
+from ItemDTO import ItemDTO
+from Config import Config
+import hashlib
+
+LOG = logging.getLogger('download')
+
+
+class Download:
+    """
+    Download task.
+    """
+    id: str = None
+    manager = None
+    download_dir: str = None
+    temp_dir: str = None
+    output_template: str = None
+    output_template_chapter: str = None
+    format: str = None
+    ytdl_opts: dict = None
+    info: ItemDTO = None
+    default_ytdl_opts: dict = None
+    debug: bool = False
+    tempPath: str = None
+    notifier: Notifier = None
+    canceled: bool = False
+    is_live: bool = False
+
+    bad_live_options: list = [
+        "concurrent_fragment_downloads",
+        "fragment_retries",
+        "skip_unavailable_fragments",
+    ]
+    "Bad yt-dlp options which are known to cause issues with live stream and post manifestless mode."
+
+    _ytdlp_fields: tuple = (
+        'tmpfilename',
+        'filename',
+        'status',
+        'msg',
+        'total_bytes',
+        'total_bytes_estimate',
+        'downloaded_bytes',
+        'speed',
+        'eta',
+    )
+    "Fields to be extracted from yt-dlp progress hook."
+
+    tempKeep: bool = False
+    "Keep temp directory after download."
+
+    def __init__(
+        self,
+        info: ItemDTO,
+        download_dir: str,
+        temp_dir: str,
+        output_template_chapter: str,
+        default_ytdl_opts: dict,
+        debug: bool = False
+    ):
+        self.download_dir = download_dir
+        self.temp_dir = temp_dir
+        self.output_template_chapter = output_template_chapter
+        self.output_template = info.output_template
+        self.format = get_format(info.format, info.quality)
+        self.ytdl_opts = get_opts(info.format, info.quality, info.ytdlp_config if info.ytdlp_config else {})
+        self.info = info
+        self.id = info._id
+        self.default_ytdl_opts = default_ytdl_opts
+        self.debug = debug
+
+        self.canceled = False
+        self.tmpfilename = None
+        self.status_queue = None
+        self.proc = None
+        self.loop = None
+        self.notifier = None
+        self.max_workers = int(Config.get_instance().max_workers)
+        self.tempKeep = bool(Config.get_instance().temp_keep)
+        self.is_live = bool(info.is_live) or info.live_in is not None
+        self.is_manifestless = 'is_manifestless' in self.info.options and self.info.options['is_manifestless'] is True
+
+    def _progress_hook(self, data: dict):
+        dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields}
+        self.status_queue.put({'id': self.id, **dataDict})
+
+    def _postprocessor_hook(self, data: dict):
+        if data.get('postprocessor') != 'MoveFiles' or data.get('status') != 'finished':
+            return
+
+        if '__finaldir' in data['info_dict']:
+            filename = os.path.join(data['info_dict']['__finaldir'], os.path.basename(data['info_dict']['filepath']))
+        else:
+            filename = data['info_dict']['filepath']
+
+        self.status_queue.put({
+            'id': self.id,
+            'status': 'finished',
+            'filename': filename
+        })
+
+    def _download(self):
+        try:
+            params: dict = {
+                'color': 'no_color',
+                'format': self.format,
+                'paths': {
+                    'home': self.download_dir,
+                    'temp': self.tempPath,
+                },
+                'outtmpl': {
+                    'default': self.output_template,
+                    'chapter': self.output_template_chapter
+                },
+                'noprogress': True,
+                'break_on_existing': True,
+                'ignoreerrors': False,
+                'progress_hooks': [self._progress_hook],
+                'postprocessor_hooks': [self._postprocessor_hook],
+                **mergeConfig(self.default_ytdl_opts, self.ytdl_opts),
+            }
+
+            params['ignoreerrors'] = False
+
+            if self.debug:
+                params['verbose'] = True
+                params['noprogress'] = False
+
+            if self.info.ytdlp_cookies:
+                try:
+                    data = jsonCookie(json.loads(self.info.ytdlp_cookies))
+                    if not data:
+                        LOG.warning(
+                            f'The cookie string that was provided for {self.info.title} is empty or not in expected spec.')
+                    with open(os.path.join(self.tempPath, f'cookie_{self.info._id}.txt'), 'w') as f:
+                        f.write(data)
+
+                    params['cookiefile'] = f.name
+                except ValueError as e:
+                    LOG.error(
+                        f'Invalid cookies: was provided for {self.info.title} - {str(e)}')
+
+            if self.is_live or self.is_manifestless:
+                hasDeletedOptions = False
+                deletedOpts: list = []
+                for opt in self.bad_live_options:
+                    if opt in params:
+                        params.pop(opt, None)
+                        hasDeletedOptions = True
+                        deletedOpts.append(opt)
+
+                if hasDeletedOptions:
+                    LOG.warning(
+                        f'Live stream detected for [{self.info.title}], The following opts [{deletedOpts=}] have been deleted which are known to cause issues with live stream and post stream manifestless mode.')
+
+            LOG.info(f'Downloading {os.getpid()=} id="{self.info.id}" title="{self.info.title}".')
+
+            ret = yt_dlp.YoutubeDL(params=params).download([self.info.url])
+
+            self.status_queue.put({'id': self.id, 'status': 'finished' if ret == 0 else 'error'})
+        except Exception as exc:
+            self.status_queue.put({
+                'id': self.id,
+                'status': 'error',
+                'msg': str(exc),
+                'error': str(exc)
+            })
+
+        LOG.info(f'Finished {os.getpid()=} id="{self.info.id}" title="{self.info.title}".')
+
+    async def start(self, notifier: Notifier):
+        self.manager = multiprocessing.Manager() if self.manager is None else self.manager
+
+        self.status_queue = self.manager.Queue()
+        self.loop = asyncio.get_running_loop()
+        self.notifier = notifier
+
+        # Create temp dir for each download.
+        self.tempPath = os.path.join(self.temp_dir, hashlib.shake_256(self.info.id.encode('utf-8')).hexdigest(5))
+
+        if not os.path.exists(self.tempPath):
+            os.makedirs(self.tempPath, exist_ok=True)
+
+        self.proc = multiprocessing.Process(target=self._download)
+        self.proc.start()
+        self.info.status = 'preparing'
+        await self.notifier.updated(self.info)
+        asyncio.create_task(self.progress_update())
+        return await self.loop.run_in_executor(None, self.proc.join)
+
+    def started(self) -> bool:
+        return self.proc is not None
+
+    def cancel(self):
+        self.kill()
+        self.canceled = True
+
+    def close(self):
+        if self.started():
+            self.proc.close()
+
+        self.delete_temp()
+
+    def running(self) -> bool:
+        return self.started() and self.proc.is_alive()
+
+    def is_canceled(self) -> bool:
+        return self.canceled
+
+    def kill(self):
+        if self.running():
+            LOG.info(f'Killing download process: {self.proc.ident}')
+            self.proc.kill()
+
+    def delete_temp(self):
+        if self.tempKeep is True or not self.tempPath:
+            return
+
+        if self.info.status != 'finished' and self.is_live:
+            LOG.warning(
+                f'Keeping live temp directory [{self.tempPath}], as the reported status is not finished [{self.info.status}].')
+            return
+
+        if not os.path.exists(self.tempPath):
+            return
+
+        if self.tempPath == self.temp_dir:
+            LOG.warning(
+                f'Attempted to delete video temp directory: {self.tempPath}, but it is the same as main temp directory.')
+            return
+
+        LOG.info(f'Deleting Temp directory: {self.tempPath}')
+        shutil.rmtree(self.tempPath, ignore_errors=True)
+
+    async def progress_update(self):
+        """
+        Update status of download task and notify the client.
+        """
+        while True:
+            status = await self.loop.run_in_executor(None, self.status_queue.get)
+            if status.get('id') != self.id or len(status) < 2:
+                return
+
+            if self.debug:
+                LOG.debug(f'Status Update: {self.info._id=} {status=}')
+
+            if isinstance(status, str):
+                await self.notifier.updated(self.info)
+                return
+
+            self.tmpfilename = status.get('tmpfilename')
+
+            if 'filename' in status:
+                self.info.filename = os.path.relpath(status.get('filename'), self.download_dir)
+
+                # Set correct file extension for thumbnails
+                if self.info.format == 'thumbnail':
+                    self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename)
+
+            self.info.status = status.get('status', self.info.status)
+            self.info.msg = status.get('msg')
+
+            if self.info.status == 'error' and 'error' in status:
+                self.info.error = status.get('error')
+                await self.notifier.error(self.info, self.info.error)
+
+            if 'downloaded_bytes' in status:
+                total = status.get('total_bytes') or status.get('total_bytes_estimate')
+                if total:
+                    self.info.percent = status['downloaded_bytes'] / total * 100
+                    self.info.total_bytes = total
+
+            self.info.speed = status.get('speed')
+            self.info.eta = status.get('eta')
+
+            if self.info.status == 'finished' and 'filename' in status and self.info.format != 'thumbnail':
+                try:
+                    self.info.file_size = os.path.getsize(status.get('filename'))
+                except FileNotFoundError:
+                    LOG.warning(f'File not found: {status.get("filename")}')
+                    self.info.file_size = None
+                    pass
+
+            await self.notifier.updated(self.info)

app/DownloadQueue.py

@@ -0,0 +1,413 @@
+import asyncio
+from email.utils import formatdate
+import logging
+import os
+import time
+import yt_dlp
+from sqlite3 import Connection
+from Config import Config
+from Download import Download
+from ItemDTO import ItemDTO
+from DataStore import DataStore
+from Utils import Notifier, ObjectSerializer, calcDownloadPath, ExtractInfo, isDownloaded, mergeConfig
+from AsyncPool import AsyncPool
+from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
+LOG = logging.getLogger('DownloadQueue')
+TYPE_DONE: str = 'done'
+TYPE_QUEUE: str = 'queue'
+
+
+class DownloadQueue:
+    config: Config = None
+    notifier: Notifier = None
+    serializer: ObjectSerializer = None
+    queue: DataStore = None
+    done: DataStore = None
+    event: asyncio.Event = None
+
+    def __init__(self, notifier: Notifier, connection: Connection, serializer: ObjectSerializer):
+        self.config = Config.get_instance()
+        self.notifier = notifier
+        self.serializer = serializer
+        self.done = DataStore(type=TYPE_DONE, connection=connection)
+        self.queue = DataStore(type=TYPE_QUEUE, connection=connection)
+        self.done.load()
+        self.queue.load()
+
+    async def initialize(self):
+        self.event = asyncio.Event()
+        LOG.info(f'Using {self.config.max_workers} workers for downloading.')
+
+        asyncio.create_task(
+            self.__download_pool() if self.config.max_workers > 1 else self.__download()
+        )
+
+    async def __add_entry(
+        self,
+        entry: dict,
+        quality: str,
+        format: str,
+        folder: str,
+        ytdlp_config: dict = {},
+        ytdlp_cookies: str = '',
+        output_template: str = '',
+        already=None
+    ):
+        if not entry:
+            return {'status': 'error', 'msg': 'Invalid/empty data was given.'}
+
+        options: dict = {}
+
+        error: str = None
+        live_in: str = None
+
+        eventType = entry.get('_type') or 'video'
+        if eventType == 'playlist':
+            entries = entry['entries']
+            playlist_index_digits = len(str(len(entries)))
+            results = []
+            for index, etr in enumerate(entries, start=1):
+                etr['playlist'] = entry.get('id')
+                etr['playlist_index'] = '{{0:0{0:d}d}}'.format(playlist_index_digits).format(index)
+                for property in ('id', 'title', 'uploader', 'uploader_id'):
+                    if property in entry:
+                        etr[f'playlist_{property}'] = entry.get(property)
+
+                results.append(
+                    await self.__add_entry(
+                        entry=etr,
+                        quality=quality,
+                        format=format,
+                        folder=folder,
+                        ytdlp_config=ytdlp_config,
+                        ytdlp_cookies=ytdlp_cookies,
+                        output_template=output_template,
+                        already=already
+                    )
+                )
+
+            if any(res['status'] == 'error' for res in results):
+                return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}
+
+            return {'status': 'ok'}
+        elif (eventType == 'video' or eventType.startswith('url')) and 'id' in entry and 'title' in entry:
+            # check if the video is live stream.
+            if 'live_status' in entry and entry.get('live_status') == 'is_upcoming':
+                if 'release_timestamp' in entry and entry.get('release_timestamp'):
+                    live_in = formatdate(entry.get('release_timestamp'))
+                else:
+                    error = 'Live stream not yet started. And no date is set.'
+            else:
+                error = entry.get('msg', None)
+
+            LOG.debug(f"entry: {entry.get('id', None)} - {entry.get('webpage_url', None)} - {entry.get('url', None)}")
+
+            if self.done.exists(key=entry['id'], url=entry.get('webpage_url') or entry.get('url')):
+                item = self.done.get(key=entry['id'], url=entry.get('webpage_url') or entry['url'])
+
+                LOG.debug(f'Item [{item.info.title}] already downloaded. Removing from history.')
+
+                await self.clear([item.info._id])
+
+            try:
+                item = self.queue.get(key=entry.get('id'), url=entry.get('webpage_url') or entry.get('url'))
+                if item is not None:
+                    err_message = f'Item [{item.info.id}: {item.info.title}] already in download queue.'
+                    LOG.info(err_message)
+                    return {'status': 'error', 'msg': err_message}
+            except KeyError:
+                pass
+
+            is_manifestless = entry.get('is_manifestless', False)
+            options.update({'is_manifestless': is_manifestless})
+
+            dl = ItemDTO(
+                id=entry.get('id'),
+                title=entry.get('title'),
+                url=entry.get('webpage_url') or entry.get('url'),
+                quality=quality,
+                format=format,
+                folder=folder,
+                ytdlp_cookies=ytdlp_cookies,
+                ytdlp_config=ytdlp_config,
+                output_template=output_template if output_template else self.config.output_template,
+                datetime=formatdate(time.time()),
+                error=error,
+                is_live=entry.get('is_live', None) or 'is_live' == entry.get('live_status', None) or live_in,
+                live_in=live_in,
+                options=options
+            )
+
+            try:
+                download_dir = calcDownloadPath(basePath=self.config.download_path, folder=folder)
+            except Exception as e:
+                LOG.exception(e)
+                return {'status': 'error', 'msg': str(e)}
+
+            output_chapter = self.config.output_template_chapter
+
+            for property, value in entry.items():
+                if property.startswith("playlist"):
+                    dl.output_template = dl.output_template.replace(f"%({property})s", str(value))
+
+            dlInfo: Download = Download(
+                info=dl,
+                download_dir=download_dir,
+                temp_dir=self.config.temp_path,
+                output_template_chapter=output_chapter,
+                default_ytdl_opts=self.config.ytdl_options,
+                debug=bool(self.config.ytdl_debug)
+            )
+
+            if dlInfo.info.live_in:
+                dlInfo.info.status = 'not_live'
+                itemDownload = self.done.put(dlInfo)
+                NotifyEvent = 'completed'
+            elif self.config.allow_manifestless is False and is_manifestless is True:
+                dlInfo.info.status = 'error'
+                dlInfo.info.error = 'Video is in Post-Live Manifestless mode.'
+                itemDownload = self.done.put(dlInfo)
+                NotifyEvent = 'completed'
+            else:
+                NotifyEvent = 'added'
+                itemDownload = self.queue.put(dlInfo)
+                self.event.set()
+
+            await self.notifier.emit(NotifyEvent, itemDownload.info)
+
+            return {
+                'status': 'ok'
+            }
+        elif eventType.startswith('url'):
+            return await self.add(
+                entry=entry.get('url'),
+                quality=quality,
+                format=format,
+                folder=folder,
+                ytdlp_config=ytdlp_config,
+                ytdlp_cookies=ytdlp_cookies,
+                output_template=output_template,
+                already=already
+            )
+
+        return {
+            'status': 'error',
+            'msg': f'Unsupported resource "{eventType}"'
+        }
+
+    async def add(
+        self,
+        url: str,
+        quality: str,
+        format: str,
+        folder: str,
+        ytdlp_config: dict = {},
+        ytdlp_cookies: str = '',
+        output_template: str = '',
+        already=None
+    ):
+        ytdlp_config = ytdlp_config if ytdlp_config else {}
+
+        LOG.info(f'Adding {url=} {quality=} {format=} {folder=} {output_template=} {ytdlp_cookies=} {ytdlp_config=}')
+
+        already = set() if already is None else already
+        if url in already:
+            LOG.info('recursion detected, skipping')
+            return {'status': 'ok'}
+        else:
+            already.add(url)
+        try:
+            with ThreadPoolExecutor(thread_name_prefix='extract_info') as pool:
+                LOG.debug(f'extracting info from {url=}')
+                entry = await asyncio.get_running_loop().run_in_executor(
+                    pool,
+                    ExtractInfo,
+                    mergeConfig(self.config.ytdl_options, ytdlp_config),
+                    url,
+                    bool(self.config.ytdl_debug)
+                )
+
+                if not entry:
+                    if not self.config.keep_archive:
+                        return {
+                            'status': 'error',
+                            'msg': 'No metadata, most likely video has been downloaded before.' if self.config.keep_archive else 'Unable to extract info check logs.'
+                        }
+
+                    LOG.debug(f'No metadata, Rechecking with archive disabled. {url=}')
+                    entry = await asyncio.get_running_loop().run_in_executor(
+                        pool,
+                        ExtractInfo,
+                        mergeConfig(self.config.ytdl_options, ytdlp_config),
+                        url,
+                        bool(self.config.ytdl_debug),
+                        True
+                    )
+
+                    if not entry:
+                        return {'status': 'error', 'msg': 'Unable to extract info check logs.'}
+
+                    if self.isDownloaded(entry):
+                        LOG.info(f'[{entry.get("id")}: {entry.get("title")}]: has been downloaded already.')
+                        return {
+                            'status': 'error',
+                            'msg': f'[{entry.get("id")}: {entry.get("title")}]: has been downloaded already.'
+                        }
+
+            if self.isDownloaded(entry):
+                raise yt_dlp.utils.ExistingVideoReached()
+
+            LOG.debug(f'extract_info length: {len(entry)}')
+        except yt_dlp.utils.ExistingVideoReached:
+            return {'status': 'error', 'msg': 'Video has been downloaded already and recorded in archive.log file.'}
+        except yt_dlp.utils.YoutubeDLError as exc:
+            return {'status': 'error', 'msg': str(exc)}
+
+        return await self.__add_entry(
+            entry=entry,
+            quality=quality,
+            format=format,
+            folder=folder,
+            ytdlp_config=ytdlp_config,
+            ytdlp_cookies=ytdlp_cookies,
+            output_template=output_template,
+            already=already,
+        )
+
+    async def cancel(self, ids):
+        for id in ids:
+            try:
+                item = self.queue.get(key=id)
+            except KeyError as e:
+                LOG.warning(f'Requested cancel for non-existent download {id=}. {str(e)}')
+                continue
+
+            if item.started() is True:
+                LOG.info(f'Canceling {id=} {item.info.title=}')
+                item.cancel()
+            else:
+                item.close()
+                LOG.info(f'Deleting from queue {id=} {item.info.title=}')
+                self.queue.delete(id)
+                await self.notifier.canceled(id)
+                self.done.put(item)
+                await self.notifier.completed(item)
+
+        return {'status': 'ok'}
+
+    async def clear(self, ids):
+        for id in ids:
+
+            try:
+                item = self.done.get(key=id)
+            except KeyError as e:
+                LOG.warning(f'Requested delete for non-existent download {id=}. {str(e)}')
+                continue
+
+            LOG.info(f'Deleting completed download {id=} {item.info.title=}')
+            self.done.delete(id)
+            await self.notifier.cleared(id)
+
+        return {'status': 'ok'}
+
+    def get(self) -> dict[str, list[dict[str, ItemDTO]]]:
+        items = {'queue': {}, 'done': {}}
+
+        for k, v in self.queue.saved_items():
+            items['queue'][k] = v
+
+        for k, v in self.done.saved_items():
+            items['done'][k] = v
+
+        for k, v in self.queue.items():
+            if not k in items['queue']:
+                items['queue'][k] = v.info
+
+        for k, v in self.done.items():
+            if not k in items['done']:
+                items['done'][k] = v.info
+
+        return items
+
+    async def __download_pool(self):
+        async with AsyncPool(
+            loop=asyncio.get_running_loop(),
+            num_workers=self.config.max_workers,
+            worker_co=self.__downloadFile,
+            name='download_pool',
+            logger=logging.getLogger('WorkerPool'),
+        ) as executor:
+            lastLog = time.time()
+
+            while True:
+                while True:
+                    if executor.has_open_workers() is True:
+                        break
+                    if time.time() - lastLog > 120:
+                        lastLog = time.time()
+                        LOG.info(f'Waiting for worker to be free.')
+                    await asyncio.sleep(1)
+
+                LOG.debug(f"Has '{executor.get_available_workers()}' free workers.")
+
+                while not self.queue.hasDownloads():
+                    LOG.info('Waiting for item to download.')
+                    await self.event.wait()
+                    self.event.clear()
+                    LOG.debug(f"Cleared wait event.")
+
+                entry = self.queue.getNextDownload()
+                await asyncio.sleep(0.2)
+
+                if entry is None:
+                    continue
+
+                LOG.debug(f"Pushing {entry=} to executor.")
+
+                if entry.started() is False and entry.is_canceled() is False:
+                    await executor.push(id=entry.info._id, entry=entry)
+                    LOG.debug(f"Pushed {entry=} to executor.")
+                    await asyncio.sleep(1)
+
+    async def __download(self):
+        while True:
+            while self.queue.empty():
+                LOG.info('Waiting for item to download.')
+                await self.event.wait()
+                self.event.clear()
+
+            id, entry = self.queue.next()
+            await self.__downloadFile(id, entry)
+
+    async def __downloadFile(self, id: str, entry: Download):
+        LOG.info(f'Queuing {id=} - {entry.info.title=} - {entry.info.url=} - {entry.info.folder=}.')
+
+        await entry.start(self.notifier)
+
+        if entry.info.status != 'finished':
+            if entry.tmpfilename and os.path.isfile(entry.tmpfilename):
+                try:
+                    os.remove(entry.tmpfilename)
+                except:
+                    pass
+
+            entry.info.status = 'error'
+
+        entry.close()
+
+        if self.queue.exists(key=id):
+
+            self.queue.delete(key=id)
+
+            if entry.is_canceled() is True:
+                await self.notifier.canceled(id)
+                entry.info.status = 'canceled'
+                entry.info.error = 'Canceled by user.'
+
+            self.done.put(value=entry)
+            await self.notifier.completed(entry.info)
+
+        self.event.set()
+
+    def isDownloaded(self, info: dict) -> bool:
+        return self.config.keep_archive and isDownloaded(self.config.ytdl_options.get('download_archive', None), info)

app/ItemDTO.py

@@ -0,0 +1,42 @@
+from email.utils import formatdate
+import json
+import time
+from dataclasses import dataclass, field
+import uuid
+
+
+@dataclass(kw_only=True)
+class ItemDTO:
+    _id: int = field(default_factory=lambda: str(uuid.uuid4()), init=False)
+
+    error: str = None
+    id: str
+    title: str
+    url: str
+    quality: str
+    format: str
+    folder: str
+    status: str = None
+    ytdlp_cookies: str = None
+    ytdlp_config: dict = field(default_factory=dict)
+    output_template: str = None
+    timestamp: float = time.time_ns()
+    is_live: bool = None
+    datetime: str = field(default_factory=lambda: str(formatdate(time.time())))
+    live_in: str = None
+    file_size: int = None
+    options: dict = field(default_factory=dict)
+
+    # yt-dlp injected fields.
+    tmpfilename: str = None
+    filename: str = None
+    total_bytes: int = None
+    total_bytes_estimate: int = None
+    downloaded_bytes: int = None
+    msg: str = None
+    percent: int = None
+    speed: str = None
+    eta: str = None
+
+    def json(self) -> str:
+        return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4)

app/Utils.py

@@ -0,0 +1,320 @@
+import asyncio
+import copy
+from datetime import datetime, timezone
+import json
+import logging
+import os
+from typing import Any
+import yt_dlp
+from socketio import AsyncServer
+from Webhooks import Webhooks
+
+LOG = logging.getLogger('Utils')
+
+AUDIO_FORMATS: tuple = ('m4a', 'mp3', 'opus', 'wav')
+IGNORED_KEYS: tuple = ('cookiefile', 'paths', 'outtmpl', 'progress_hooks', 'postprocessor_hooks',)
+
+
+def get_format(format: str, quality: str) -> str:
+    """
+    Returns format for download
+
+    Args:
+      format (str): format selected
+      quality (str): quality selected
+
+    Raises:
+      Exception: unknown quality, unknown format
+
+    Returns:
+      dl_format: Formatted download string
+    """
+    format = format or "any"
+
+    if format.startswith("custom:"):
+        return format[7:]
+
+    if format == "thumbnail":
+        # Quality is irrelevant in this case since we skip the download
+        return "bestaudio/best"
+
+    if format in AUDIO_FORMATS:
+        # Audio quality needs to be set post-download, set in opts
+        return "bestaudio/best"
+
+    if format in ('mp4', 'any'):
+        if quality == 'audio':
+            return "bestaudio/best"
+
+        videoFormat, audioFormat = ('[ext=mp4]', '[ext=m4a]') if format == "mp4" else ('', '')
+
+        videoResolution = f'[height<={quality}]' if quality != 'best' else ''
+
+        videoCombo = videoResolution + videoFormat
+
+        return f'bestvideo{videoCombo}+bestaudio{audioFormat}/best{videoCombo}'
+
+    raise Exception(f"Unknown format {format}")
+
+
+def get_opts(format: str, quality: str, ytdl_opts: dict) -> dict:
+    """
+    Returns extra download options
+    Mostly postprocessing options
+
+    Args:
+      format (str): format selected
+      quality (str): quality of format selected (needed for some formats)
+      ytdl_opts (dict): current options selected
+
+    Returns:
+      ytdl_opts: Extra options
+    """
+    opts = copy.deepcopy(ytdl_opts)
+    if not opts:
+        opts: dict = {
+            "postprocessors": [],
+        }
+
+    postprocessors = []
+
+    if format in AUDIO_FORMATS:
+        postprocessors.append({
+            "key": "FFmpegExtractAudio",
+            "preferredcodec": format,
+            "preferredquality": 0 if quality == "best" else quality,
+        })
+
+        # Audio formats without thumbnail
+        if format not in ("wav") and "writethumbnail" not in opts:
+            opts["writethumbnail"] = True
+            postprocessors.append({"key": "FFmpegThumbnailsConvertor", "format": "jpg", "when": "before_dl"})
+            postprocessors.append({"key": "FFmpegMetadata"})
+            postprocessors.append({"key": "EmbedThumbnail"})
+
+    if format == "thumbnail":
+        opts["skip_download"] = True
+        opts["writethumbnail"] = True
+        postprocessors.append({"key": "FFmpegThumbnailsConvertor", "format": "jpg", "when": "before_dl"})
+
+    opts["postprocessors"] = postprocessors + (opts["postprocessors"] if "postprocessors" in opts else [])
+    return opts
+
+
+def getVideoInfo(url: str, ytdlp_opts: dict = None) -> (Any | dict[str, Any] | None):
+    params: dict = {
+        'quiet': True,
+        'color': 'no_color',
+        'extract_flat': True,
+    }
+
+    if ytdlp_opts:
+        params = {**params, **ytdlp_opts}
+
+    return yt_dlp.YoutubeDL().extract_info(url, download=False)
+
+
+def calcDownloadPath(basePath: str, folder: str = None, createPath: bool = True) -> str:
+    """Calculates download path and prevents directory traversal.
+
+    Returns:
+        Download path with base directory factored in.
+    """
+    if not folder:
+        return basePath
+
+    realBasePath = os.path.realpath(basePath)
+    download_path = os.path.realpath(os.path.join(basePath, folder))
+
+    if not download_path.startswith(realBasePath):
+        raise Exception(f'Folder "{folder}" must resolve inside the base download directory "{realBasePath}".')
+
+    if not os.path.isdir(download_path) and createPath:
+        os.makedirs(download_path, exist_ok=True)
+
+    return download_path
+
+
+def ExtractInfo(config: dict, url: str, debug: bool = False, forceLookup: bool = False) -> dict:
+    params: dict = {
+        'color': 'no_color',
+        'extract_flat': True,
+        'skip_download': True,
+        'ignoreerrors': True,
+        'ignore_no_formats_error': True,
+        **config,
+    }
+
+    # Remove keys that are not needed for info extraction as those keys generate files when used with extract_info.
+    for key in ('writeinfojson', 'writethumbnail', 'writedescription', 'writeautomaticsub',):
+        if key in params:
+            params.pop(key)
+
+    if debug:
+        params['verbose'] = True
+        params['logger'] = logging.getLogger('YTPTube-ytdl')
+    else:
+        params['quiet'] = True
+
+    if forceLookup and 'download_archive' in params:
+        params.pop('download_archive')
+
+    return yt_dlp.YoutubeDL(params=params).extract_info(url, download=False)
+
+
+def mergeDict(source: dict, destination: dict) -> dict:
+    """ Merge data from source into destination """
+    destination_copy = copy.deepcopy(destination)
+
+    for key, value in source.items():
+        destination_key_value = destination_copy.get(key)
+        if isinstance(value, dict) and isinstance(destination_key_value, dict):
+            destination_copy[key] = mergeDict(source=value, destination=destination_copy.setdefault(key, {}))
+        elif isinstance(value, list) and isinstance(destination_key_value, list):
+            destination_copy[key] = destination_key_value + value
+        else:
+            destination_copy[key] = value
+
+    return destination_copy
+
+
+def mergeConfig(config: dict, new_config: dict) -> dict:
+    """ Merge user provided config into default config """
+
+    ignored_keys: tuple = (
+        'cookiefile',
+        'download_archive'
+        'paths',
+        'outtmpl',
+        'progress_hooks',
+        'postprocessor_hooks',
+    )
+
+    for key in ignored_keys:
+        if key in new_config:
+            del new_config[key]
+
+    return mergeDict(new_config, config)
+
+
+def isDownloaded(archive_file: str, info: dict) -> bool:
+    if not info or not archive_file or not os.path.exists(archive_file):
+        return False
+
+    try:
+        id = yt_dlp.YoutubeDL()._make_archive_id(info)
+        if not id:
+            return False
+    except Exception as e:
+        LOG.error(f'Error generating archive id: {e}')
+        return False
+
+    with open(archive_file, 'r') as f:
+        for line in f.readlines():
+            if id in line:
+                return True
+
+    return False
+
+
+def jsonCookie(cookies: dict[dict[str, any]]) -> str | None:
+    """Converts JSON cookies to Netscape cookies
+
+       Returns None if no cookies are found, otherwise returns a string of cookies in Netscape format.
+    """
+
+    netscapeCookies = "# Netscape HTTP Cookie File\n# https://curl.haxx.se/docs/http-cookies.html\n# This file was generated by libcurl! Edit at your own risk.\n\n"
+    hasCookies: bool = False
+
+    for domain in cookies:
+        if not isinstance(cookies[domain], dict):
+            continue
+
+        for subDomain in cookies[domain]:
+            if not isinstance(cookies[domain][subDomain], dict):
+                continue
+
+            for cookie in cookies[domain][subDomain]:
+                if not isinstance(cookies[domain][subDomain][cookie], dict):
+                    continue
+
+                cookieDict = cookies[domain][subDomain][cookie]
+
+                if 0 == int(cookieDict['expirationDate']):
+                    cookieDict['expirationDate']: float = datetime.now(timezone.utc).timestamp() + (86400 * 1000)
+
+                hasCookies = True
+                netscapeCookies += "\t".join([
+                    cookieDict['domain'] if str(cookieDict['domain']).startswith('.') else '.' + cookieDict['domain'],
+                    'TRUE',
+                    cookieDict['path'],
+                    'TRUE' if cookieDict['secure'] else 'FALSE',
+                    str(int(cookieDict['expirationDate'])),
+                    cookieDict['name'],
+                    cookieDict['value']
+                ])+"\n"
+
+    return netscapeCookies if hasCookies else None
+
+
+class ObjectSerializer(json.JSONEncoder):
+    """
+    This class is used to serialize objects to JSON.
+    The only difference between this and the default JSONEncoder is that this one
+    will call the __dict__ method of an object if it exists.
+    """
+
+    def default(self, obj):
+        return obj.__dict__ if isinstance(obj, object) else json.JSONEncoder.default(self, obj)
+
+
+class Notifier:
+    '''
+    This class is used to send events to the frontend.
+    '''
+
+    sio: AsyncServer = None
+    "SocketIO server instance"
+
+    serializer: ObjectSerializer = None
+    "Serializer used to serialize objects to JSON"
+
+    webhooks: Webhooks = None
+    "Send webhooks events."
+
+    webhooks_allowed_events: tuple = (
+        'added', 'completed', 'error', 'not_live'
+    )
+    "Events that are allowed to be sent to webhooks."
+
+    def __init__(self, sio: AsyncServer, serializer: ObjectSerializer, webhooks: Webhooks = None):
+        self.sio = sio
+        self.serializer = serializer
+        self.webhooks = webhooks
+
+    async def added(self, dl: dict):
+        await self.emit('added', dl)
+
+    async def updated(self, dl: dict):
+        await self.emit('updated', dl)
+
+    async def completed(self, dl: dict):
+        await self.emit('completed', dl)
+
+    async def canceled(self, id: str):
+        await self.emit('canceled', id)
+
+    async def cleared(self, id: str):
+        await self.emit('cleared', id)
+
+    async def error(self, dl: dict, message: str):
+        await self.emit('error', (dl, message))
+
+    async def emit(self, event: str, data):
+        tasks = []
+        tasks.append(self.sio.emit(event, self.serializer.encode(data)))
+
+        if self.webhooks and event in self.webhooks_allowed_events:
+            tasks.append(self.webhooks.send(event, data))
+
+        await asyncio.gather(*tasks)

app/Webhooks.py

@@ -0,0 +1,71 @@
+import asyncio
+import json
+import logging
+import os
+from ItemDTO import ItemDTO
+from aiohttp import client
+
+LOG = logging.getLogger('Webhooks')
+
+
+class Webhooks:
+    targets: list[dict] = {}
+
+    def __init__(self, file: str):
+        if os.path.exists(file):
+            self.load(file)
+
+    def load(self, file: str):
+        try:
+            if os.path.getsize(file) < 2:
+                raise Exception(f'file is empty.')
+
+            LOG.info(f'Loading webhooks from {file}')
+            with open(file, 'r') as f:
+                self.targets = json.load(f)
+        except Exception as e:
+            LOG.error(f'Error loading webhooks from {file}: {e}')
+            pass
+
+    async def send(self, event: str, item: ItemDTO) -> list[dict]:
+        if len(self.targets) == 0:
+            return []
+
+        if not isinstance(item, ItemDTO):
+            return []
+
+        tasks = []
+
+        for target in self.targets:
+            allowed_events = target.get('on') if 'on' in target else []
+            if len(allowed_events) > 0 and event not in allowed_events:
+                continue
+
+            tasks.append(self.__send(event, target, item))
+
+        return await asyncio.gather(*tasks)
+
+    async def __send(self, event: str, target: dict, item: ItemDTO) -> dict:
+        req: dict = target.get('request')
+        try:
+            LOG.info(f"Sending {event=} {item.id=} to [{target.get('name')}]")
+            async with client.ClientSession() as session:
+                headers = req.get('headers', {}) if 'headers' in req else {}
+                async with session.request(method=req.get('method', 'POST'), url=req.get('url'), json=item.__dict__, headers=headers) as response:
+                    respData = {
+                        'url': req.get('url'),
+                        'status': response.status,
+                        'text': await response.text()
+                    }
+                    msg = f"[{target.get('name')}] Response to [{event=} {item.id=}] [status: {response.status}]."
+                    if respData.get('text'):
+                        msg += f" [Body: {respData.get('text','??')}]"
+                    LOG.info(msg)
+
+                    return respData
+        except Exception as e:
+            return {
+                'url': req.get('url'),
+                'status': 500,
+                'text': str(e)
+            }

app/__init__.py

@@ -1,13 +0,0 @@
-import os
-import sys
-from pathlib import Path
-
-if base_dir := os.environ.get("YTP_CONFIG_PATH"):
-    base_dir = Path(base_dir)
-    user_site = base_dir / f"python{sys.version_info.major}.{sys.version_info.minor}-packages"
-
-    if not user_site.exists():
-        user_site.mkdir(parents=True, exist_ok=True)
-
-    if user_site.is_dir() and str(user_site) not in sys.path:
-        sys.path.insert(0, str(user_site))

app/conftest.py

@@ -1,25 +0,0 @@
-from __future__ import annotations
-
-import os
-import tempfile
-
-from app.tests.helpers import cleanup_test_run_root, get_test_run_root, get_test_system_temp_root
-
-
-def pytest_configure(config) -> None:
-    temp_root = get_test_system_temp_root()
-    for env_name in ("TMPDIR", "TEMP", "TMP"):
-        os.environ[env_name] = str(temp_root)
-
-    tempfile.tempdir = None
-
-    if getattr(config.option, "basetemp", None) is None:
-        config.option.basetemp = str(get_test_run_root() / "pytest")
-
-    os.environ["YTP_FILE_LOGGING"] = "false"
-
-
-def pytest_unconfigure(config) -> None:
-    del config
-    os.environ.pop("YTP_FILE_LOGGING", None)
-    cleanup_test_run_root()

app/features/conditions/__init__.py

@@ -1 +0,0 @@
-"""Conditions Feature"""

app/features/conditions/deps.py

@@ -1,5 +0,0 @@
-from app.features.conditions.repository import ConditionsRepository
-
-
-def get_conditions_repo() -> ConditionsRepository:
-    return ConditionsRepository.get_instance()

app/features/conditions/migration.py

@@ -1,118 +0,0 @@
-from __future__ import annotations
-
-import json
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, cast
-
-from app.features.conditions.models import ConditionModel
-from app.features.core.migration import Migration as FeatureMigration
-from app.library.config import Config
-from app.library.log import get_logger
-
-if TYPE_CHECKING:
-    from app.features.conditions.repository import ConditionsRepository
-
-LOG = get_logger()
-
-
-class Migration(FeatureMigration):
-    name: str = "conditions"
-
-    def __init__(self, repo: ConditionsRepository, config: Config | None = None):
-        self._config: Config = config or Config.get_instance()
-        super().__init__(config=self._config)
-        self._repo: ConditionsRepository = repo
-        self._source_file: Path = Path(self._config.config_path) / "conditions.json"
-
-    async def should_run(self) -> bool:
-        return self._source_file.exists()
-
-    async def migrate(self) -> None:
-        if await self._repo.count() > 0:
-            LOG.warning("Conditions already exist in the database; skipping migration.")
-            await self._move_file(self._source_file)
-            return
-
-        try:
-            items: list[dict] | None = json.loads(self._source_file.read_text())
-        except Exception as exc:
-            LOG.exception(
-                "Failed to read conditions migration file '%s'. Ignoring.",
-                self._source_file,
-                extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
-            )
-            await self._move_file(self._source_file)
-            return
-
-        if items is None:
-            LOG.warning("No conditions found in %s; skipping migration.", self._source_file)
-            await self._move_file(self._source_file)
-            return
-
-        inserted = 0
-        seen_names: dict[str, int] = {}
-        for index, item in enumerate(items):
-            if not (normalized := await self._normalize(item, index, seen_names)):
-                continue
-            try:
-                await self._repo.create(normalized)
-                inserted += 1
-            except Exception as exc:
-                LOG.exception(
-                    "Failed to insert condition '%s'.",
-                    normalized.name,
-                    extra={"condition_name": normalized.name, "exception_type": type(exc).__name__},
-                )
-
-        LOG.info("Migrated %s condition(s) from %s.", inserted, self._source_file)
-        await self._move_file(self._source_file)
-
-    async def _normalize(self, item: Any, index: int, seen_names: dict[str, int]) -> ConditionModel | None:
-        if not isinstance(item, dict):
-            LOG.warning("Skipping condition at index %s due to invalid type.", index)
-            return None
-
-        assert isinstance(item, dict)
-        name: str | None = item.get("name")
-        if not name or not isinstance(name, str):
-            LOG.warning("Skipping condition at index %s due to missing name.", index)
-            return None
-
-        normalized_name = name.strip()
-        if not normalized_name:
-            LOG.warning("Skipping condition at index %s due to empty name.", index)
-            return None
-
-        name = self._unique_name(normalized_name, seen_names)
-
-        extras = item.get("extras") if isinstance(item.get("extras"), dict) else {}
-        extras = cast("dict[str, Any]", extras)
-        filter_value: str | None = item.get("filter")
-        if (not filter_value or not isinstance(filter_value, str)) and len(extras) == 0:
-            LOG.warning("Skipping condition '%s' due to missing filter.", name)
-            return None
-
-        cli: str | None = item.get("cli")
-        if not isinstance(cli, str):
-            cli = ""
-
-        enabled_value = item.get("enabled")
-        enabled: bool = enabled_value if isinstance(enabled_value, bool) else True
-        priority_value = item.get("priority")
-        priority: int = priority_value if isinstance(priority_value, int) else 0
-        if isinstance(priority, bool) or 0 > priority:
-            priority = 0
-
-        description_value = item.get("description")
-        description: str = description_value if isinstance(description_value, str) else ""
-
-        return ConditionModel(
-            id=index,
-            name=name,
-            filter=filter_value,
-            cli=cli,
-            extras=extras,
-            enabled=bool(enabled),
-            priority=priority,
-            description=description,
-        )

app/features/conditions/models.py

@@ -1,28 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime  # noqa: TC003
-
-from sqlalchemy import JSON, Boolean, Index, Integer, String, Text
-from sqlalchemy.orm import Mapped, mapped_column
-
-from app.features.core.models import Base, UTCDateTime, utcnow
-
-
-class ConditionModel(Base):
-    __tablename__: str = "conditions"
-    __table_args__: tuple[Index, ...] = (
-        Index("ix_conditions_name", "name"),
-        Index("ix_conditions_enabled", "enabled"),
-        Index("ix_conditions_priority", "priority"),
-    )
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
-    name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
-    filter: Mapped[str] = mapped_column(Text, nullable=False)
-    cli: Mapped[str] = mapped_column(Text, nullable=False, default="")
-    enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
-    priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-    description: Mapped[str] = mapped_column(Text, nullable=False, default="")
-    extras: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
-    created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, nullable=False)
-    updated_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, onupdate=utcnow, nullable=False)

app/features/conditions/repository.py

@@ -1,192 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Any
-
-from app.features.conditions.migration import Migration
-from app.library.Singleton import Singleton
-
-if TYPE_CHECKING:
-    from collections.abc import Callable, Iterable
-    from contextlib import AbstractAsyncContextManager
-
-    from sqlalchemy.engine.result import Result
-    from sqlalchemy.ext.asyncio import AsyncSession
-    from sqlalchemy.sql.elements import ColumnElement
-    from sqlalchemy.sql.selectable import Select
-
-    SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
-
-from sqlalchemy import delete, func, or_, select
-
-from app.features.conditions.models import ConditionModel
-from app.features.core.deps import get_session
-from app.library.log import get_logger
-
-LOG = get_logger()
-
-
-def _model_from_payload(payload: dict[str, Any]) -> ConditionModel:
-    model = ConditionModel()
-    for key, value in payload.items():
-        if not hasattr(model, key):
-            msg = f"'{key}' is an invalid keyword argument for ConditionModel"
-            raise TypeError(msg)
-        setattr(model, key, value)
-    return model
-
-
-def _coerce_model(payload: ConditionModel | dict[str, Any]) -> ConditionModel:
-    if isinstance(payload, ConditionModel):
-        return payload
-    return _model_from_payload(payload)
-
-
-class ConditionsRepository(metaclass=Singleton):
-    def __init__(self, session: SessionFactory | None = None) -> None:
-        self._migrated = False
-        self.session: SessionFactory = session or get_session
-
-    async def run_migrations(self) -> None:
-        if self._migrated:
-            return
-
-        self._migrated = True
-        await Migration(repo=self, config=None).run()
-
-    @staticmethod
-    def get_instance() -> ConditionsRepository:
-        return ConditionsRepository()
-
-    async def all(self) -> list[ConditionModel]:
-        async with self.session() as session:
-            result: Result[tuple[ConditionModel]] = await session.execute(
-                select(ConditionModel).order_by(ConditionModel.priority.desc(), ConditionModel.name.asc())
-            )
-            return list(result.scalars().all())
-
-    async def list_paginated(self, page: int, per_page: int) -> tuple[list[ConditionModel], int, int, int]:
-        async with self.session() as session:
-            total: int = await self.count()
-            total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1
-
-            if page > total_pages and total > 0:
-                page = total_pages
-
-            query: Select[tuple[ConditionModel]] = (
-                select(ConditionModel)
-                .order_by(ConditionModel.priority.desc(), ConditionModel.name.asc())
-                .limit(per_page)
-                .offset((page - 1) * per_page)
-            )
-            result: Result[tuple[ConditionModel]] = await session.execute(query)
-            return list(result.scalars().all()), total, page, total_pages
-
-    async def count(self) -> int:
-        async with self.session() as session:
-            result: Result[tuple[int]] = await session.execute(select(func.count()).select_from(ConditionModel))
-            return int(result.scalar_one())
-
-    async def get(self, identifier: int | str) -> ConditionModel | None:
-        async with self.session() as session:
-            if not identifier:
-                return None
-
-            if isinstance(identifier, int):
-                clause: ColumnElement[bool] = ConditionModel.id == identifier
-            elif isinstance(identifier, str) and identifier.isdigit():
-                clause: ColumnElement[bool] = or_(
-                    ConditionModel.id == int(identifier), ConditionModel.name == identifier
-                )
-            else:
-                clause: ColumnElement[bool] = ConditionModel.name == identifier
-
-            result: Result[tuple[ConditionModel]] = await session.execute(select(ConditionModel).where(clause).limit(1))
-            return result.scalar_one_or_none()
-
-    async def get_by_name(self, name: str, exclude_id: int | None = None) -> ConditionModel | None:
-        async with self.session() as session:
-            query: Select[tuple[ConditionModel]] = select(ConditionModel).where(ConditionModel.name == name)
-            if exclude_id is not None:
-                query = query.where(ConditionModel.id != exclude_id)
-
-            result: Result[tuple[ConditionModel]] = await session.execute(query.limit(1))
-            return result.scalar_one_or_none()
-
-    async def create(self, payload: ConditionModel | dict[str, Any]) -> ConditionModel:
-        async with self.session() as session:
-            model = _coerce_model(payload)
-            if model.id is not None:
-                model.id = None  # ty: ignore
-
-            if await self.get_by_name(name=model.name) is not None:
-                msg: str = f"Condition with name '{model.name}' already exists."
-                raise ValueError(msg)
-
-            session.add(model)
-            await session.commit()
-            await session.refresh(model)
-            return model
-
-    async def update(self, identifier: int | str, payload: dict[str, Any]) -> ConditionModel:
-        async with self.session() as session:
-            if isinstance(identifier, int):
-                clause: ColumnElement[bool] = ConditionModel.id == identifier
-            elif isinstance(identifier, str) and identifier.isdigit():
-                clause: ColumnElement[bool] = or_(
-                    ConditionModel.id == int(identifier), ConditionModel.name == identifier
-                )
-            else:
-                clause: ColumnElement[bool] = ConditionModel.name == identifier
-
-            result: Result[tuple[ConditionModel]] = await session.execute(select(ConditionModel).where(clause).limit(1))
-            if not (model := result.scalar_one_or_none()):
-                msg: str = f"Condition '{identifier}' not found."
-                raise KeyError(msg)
-
-            payload.pop("id", None)
-            payload.pop("created_at", None)
-            payload.pop("updated_at", None)
-
-            if "name" in payload and await self.get_by_name(name=payload["name"], exclude_id=model.id) is not None:
-                msg: str = f"Condition with name '{payload['name']}' already exists."
-                raise ValueError(msg)
-
-            for key, value in payload.items():
-                if hasattr(model, key):
-                    setattr(model, key, value)
-
-            await session.commit()
-            await session.refresh(model)
-            return model
-
-    async def delete(self, identifier: int | str) -> ConditionModel:
-        async with self.session() as session:
-            if isinstance(identifier, int):
-                clause: ColumnElement[bool] = ConditionModel.id == identifier
-            elif isinstance(identifier, str) and identifier.isdigit():
-                clause: ColumnElement[bool] = or_(
-                    ConditionModel.id == int(identifier), ConditionModel.name == identifier
-                )
-            else:
-                clause: ColumnElement[bool] = ConditionModel.name == identifier
-
-            result: Result[tuple[ConditionModel]] = await session.execute(select(ConditionModel).where(clause).limit(1))
-            if not (model := result.scalar_one_or_none()):
-                msg: str = f"Condition '{identifier}' not found."
-                raise KeyError(msg)
-
-            await session.delete(model)
-            await session.commit()
-            return model
-
-    async def replace_all(self, items: Iterable[dict[str, Any] | ConditionModel]) -> list[ConditionModel]:
-        async with self.session() as session:
-            try:
-                await session.execute(delete(ConditionModel))
-                models: list[ConditionModel] = [_coerce_model(item) for item in items]
-                session.add_all(models)
-                await session.commit()
-            except Exception:
-                await session.rollback()
-                raise
-            return models

app/features/conditions/router.py

@@ -1,361 +0,0 @@
-import asyncio
-from collections import OrderedDict
-from typing import Any
-
-from aiohttp import web
-from aiohttp.web import Request, Response
-from pydantic import ValidationError
-
-from app.features.conditions.schemas import Condition, ConditionList, ConditionPatch
-from app.features.conditions.service import Conditions
-from app.features.core.schemas import CEAction, CEFeature, ConfigEvent, Pagination
-from app.features.core.utils import build_pagination, format_validation_errors, normalize_pagination
-from app.library.cache import Cache
-from app.library.config import Config
-from app.library.encoder import Encoder
-from app.library.Events import EventBus, Events
-from app.library.log import get_logger
-from app.library.router import route
-from app.library.Utils import validate_url
-
-LOG = get_logger()
-
-
-def _model(model: Any) -> Condition:
-    return Condition.model_validate(model)
-
-
-def _serialize(model: Any) -> dict:
-    return _model(model).model_dump()
-
-
-@route("GET", "api/conditions/", name="conditions_list")
-async def conditions_list(request: Request, encoder: Encoder) -> Response:
-    """
-    Get the conditions
-
-    Args:
-        request (Request): The request object.
-        encoder (Encoder): The encoder instance.
-
-    Returns:
-        Response: The response object.
-
-    """
-    repo = Conditions.get_instance()._repo
-
-    page, per_page = normalize_pagination(request)
-    items, total, current_page, total_pages = await repo.list_paginated(page, per_page)
-    return web.json_response(
-        data=ConditionList(
-            items=[_model(model) for model in items],
-            pagination=Pagination.model_validate(build_pagination(total, current_page, per_page, total_pages)),
-        ),
-        status=web.HTTPOk.status_code,
-        dumps=encoder.encode,
-    )
-
-
-@route("POST", "api/conditions/test/", name="condition_test")
-async def conditions_test(request: Request, encoder: Encoder, cache: Cache, config: Config) -> Response:
-    """
-    Test condition against URL.
-
-    Args:
-        request (Request): The request object.
-        encoder (Encoder): The encoder instance.
-        cache (Cache): The cache instance.
-        config (Config): The configuration instance.
-
-    Returns:
-        Response: The response object
-
-    """
-    params = await request.json()
-
-    if not isinstance(params, dict):
-        return web.json_response(
-            {"error": "Invalid request body expecting dict."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    if not (url := params.get("url")):
-        return web.json_response({"error": "url is required."}, status=web.HTTPBadRequest.status_code)
-
-    if not (cond := params.get("condition")):
-        return web.json_response({"error": "condition is required."}, status=web.HTTPBadRequest.status_code)
-
-    try:
-        await asyncio.to_thread(validate_url, url, config.allow_internal_urls)
-    except ValueError as e:
-        return web.json_response({"error": str(e)}, status=web.HTTPBadRequest.status_code)
-
-    try:
-        preset: str = params.get("preset", config.default_preset)
-        key: str = cache.hash(url + str(preset))
-        if not cache.has(key):
-            from app.features.ytdlp.extractor import fetch_info
-            from app.features.ytdlp.ytdlp_opts import YTDLPOpts
-
-            (data, _) = await fetch_info(
-                config=YTDLPOpts.get_instance().preset(name=preset).get_all(),
-                url=url,
-                debug=False,
-                no_archive=True,
-                follow_redirect=True,
-                sanitize_info=True,
-            )
-
-            if not data:
-                return web.json_response(
-                    data={"error": f"Failed to extract info from '{url!s}'."},
-                    status=web.HTTPBadRequest.status_code,
-                )
-            cache.set(key=key, value=data, ttl=600)
-        else:
-            data = cache.get(key)
-    except Exception as e:
-        LOG.exception(
-            "Failed to extract video info for condition check '%s'.",
-            cond,
-            extra={
-                "route": "conditions.match",
-                "condition": cond,
-                "url": url,
-                "preset": preset,
-                "exception_type": type(e).__name__,
-            },
-        )
-        return web.json_response(
-            data={"error": f"Failed to extract video info. '{e!s}'"},
-            status=web.HTTPInternalServerError.status_code,
-        )
-
-    if not isinstance(data, dict):
-        return web.json_response(
-            data={"error": "Failed to extract video info."},
-            status=web.HTTPInternalServerError.status_code,
-        )
-
-    try:
-        from app.features.ytdlp.mini_filter import match_str
-
-        status: bool = match_str(cond, data)
-    except Exception as e:
-        LOG.exception(
-            "Failed to evaluate condition '%s'.",
-            cond,
-            extra={
-                "route": "conditions.match",
-                "condition": cond,
-                "url": url,
-                "preset": preset,
-                "exception_type": type(e).__name__,
-            },
-        )
-        return web.json_response(
-            data={"error": str(e)},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    return web.json_response(
-        data={
-            "status": status,
-            "condition": cond,
-            "data": OrderedDict(sorted(data.items(), key=lambda item: len(str(item[1])))),
-        },
-        status=web.HTTPOk.status_code,
-        dumps=encoder.encode,
-    )
-
-
-@route("POST", "api/conditions/", name="condition_add")
-async def conditions_add(request: Request, encoder: Encoder, notify: EventBus) -> Response:
-    """
-    Add Condition.
-
-    Args:
-        request (Request): The request object.
-        encoder (Encoder): The encoder instance.
-        notify (EventBus): The event bus instance.
-
-    Returns:
-        Response: The response object
-
-    """
-    data = await request.json()
-
-    if not isinstance(data, dict):
-        return web.json_response(
-            {"error": "Invalid request body expecting list with dicts."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    try:
-        item: Condition = Condition.model_validate(data)
-    except ValidationError as exc:
-        return web.json_response(
-            data={"error": "Failed to validate condition.", "detail": format_validation_errors(exc)},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    try:
-        saved = _serialize(await Conditions.get_instance().save(item=item.model_dump()))
-    except ValueError as exc:
-        return web.json_response({"error": str(exc)}, status=web.HTTPBadRequest.status_code)
-
-    notify.emit(
-        Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.CONDITIONS, action=CEAction.CREATE, data=saved)
-    )
-    return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode)
-
-
-@route("GET", r"api/conditions/{id:\d+}", name="condition_get")
-async def conditions_get(request: Request, encoder: Encoder) -> Response:
-    """
-    Get the conditions
-
-    Args:
-        request (Request): The request object.
-        encoder (Encoder): The encoder instance.
-
-    Returns:
-        Response: The response object.
-
-    """
-    if not (id := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    if not (model := await Conditions.get_instance().get(id)):
-        return web.json_response({"error": "Condition not found"}, status=web.HTTPNotFound.status_code)
-
-    return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode)
-
-
-@route("DELETE", r"api/conditions/{id:\d+}", name="condition_delete")
-async def conditions_delete(request: Request, encoder: Encoder, notify: EventBus) -> Response:
-    """
-    Delete Condition.
-
-    Args:
-        request (Request): The request object.
-        encoder (Encoder): The encoder instance.
-        notify (EventBus): The event bus instance.
-
-    Returns:
-        Response: The response object.
-
-    """
-    if not (id := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    try:
-        deleted = _serialize(await Conditions.get_instance()._repo.delete(id))
-        notify.emit(
-            Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.CONDITIONS, action=CEAction.DELETE, data=deleted)
-        )
-        return web.json_response(data=deleted, status=web.HTTPOk.status_code, dumps=encoder.encode)
-    except KeyError as exc:
-        return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code)
-
-
-@route("PATCH", r"api/conditions/{id:\d+}", name="condition_patch")
-async def conditions_patch(request: Request, encoder: Encoder, notify: EventBus) -> Response:
-    """
-    Patch Condition.
-
-    Args:
-        request (Request): The request object.
-        encoder (Encoder): The encoder instance.
-        notify (EventBus): The event bus instance.
-
-    Returns:
-        Response: The response object.
-
-    """
-    if not (id := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    if not (model := await Conditions.get_instance().get(id)):
-        return web.json_response({"error": "Condition not found"}, status=web.HTTPNotFound.status_code)
-
-    data = await request.json()
-
-    if not isinstance(data, dict):
-        return web.json_response(
-            {"error": "Invalid request body expecting list with dicts."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    service = Conditions.get_instance()
-
-    try:
-        validated = ConditionPatch.model_validate(data)
-    except ValidationError as exc:
-        return web.json_response(
-            data={"error": "Failed to validate condition.", "detail": format_validation_errors(exc)},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    if validated.name and await service._repo.get_by_name(validated.name, exclude_id=model.id):
-        return web.json_response(
-            data={"error": f"Condition with name '{validated.name}' already exists."},
-            status=web.HTTPConflict.status_code,
-        )
-
-    updated = _serialize(await service._repo.update(model.id, validated.model_dump(exclude_unset=True)))
-    notify.emit(
-        Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.CONDITIONS, action=CEAction.UPDATE, data=updated)
-    )
-    return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode)
-
-
-@route("PUT", r"api/conditions/{id:\d+}", name="condition_update")
-async def conditions_update(request: Request, encoder: Encoder, notify: EventBus) -> Response:
-    """
-    Update Condition.
-
-    Args:
-        request (Request): The request object.
-        encoder (Encoder): The encoder instance.
-        notify (EventBus): The event bus instance.
-
-    Returns:
-        Response: The response object.
-
-    """
-    if not (id := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    if not (model := await Conditions.get_instance().get(id)):
-        return web.json_response({"error": "Condition not found"}, status=web.HTTPNotFound.status_code)
-
-    data = await request.json()
-
-    if not isinstance(data, dict):
-        return web.json_response(
-            {"error": "Invalid request body expecting list with dicts."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    service = Conditions.get_instance()
-
-    try:
-        validated = Condition.model_validate(data)
-    except ValidationError as exc:
-        return web.json_response(
-            data={"error": "Failed to validate condition.", "detail": format_validation_errors(exc)},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    if validated.name and await service._repo.get_by_name(validated.name, exclude_id=model.id):
-        return web.json_response(
-            data={"error": f"Condition with name '{validated.name}' already exists."},
-            status=web.HTTPConflict.status_code,
-        )
-
-    updated = _serialize(await service._repo.update(model.id, validated.model_dump(exclude_unset=True)))
-    notify.emit(
-        Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.CONDITIONS, action=CEAction.UPDATE, data=updated)
-    )
-    return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode)

app/features/conditions/schemas.py

@@ -1,93 +0,0 @@
-from __future__ import annotations
-
-from typing import Any
-
-from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
-
-from app.features.core.schemas import Pagination
-from app.features.core.utils import parse_int
-from app.features.ytdlp.mini_filter import match_str
-
-
-class Condition(BaseModel):
-    model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True)
-
-    id: int | None = None
-    name: str = Field(min_length=1)
-    filter: str = ""
-    cli: str = ""
-    extras: dict[str, Any] = Field(default_factory=dict)
-    enabled: bool = True
-    priority: int = 0
-    description: str = ""
-
-    @field_validator("filter")
-    @classmethod
-    def _validate_filter(cls, value: str) -> str:
-        try:
-            match_str(value, {})
-        except Exception as exc:
-            msg: str = f"Invalid filter. '{exc!s}'."
-            raise ValueError(msg) from exc
-        return value
-
-    @field_validator("cli")
-    @classmethod
-    def _validate_cli(cls, value: str) -> str:
-        if not value:
-            return ""
-        try:
-            from app.features.ytdlp.utils import arg_converter
-
-            arg_converter(args=value)
-        except ModuleNotFoundError:
-            return value
-        except Exception as exc:
-            msg: str = f"Invalid command options for yt-dlp. '{exc!s}'."
-            raise ValueError(msg) from exc
-        return value
-
-    @field_validator("priority", mode="before")
-    @classmethod
-    def _normalize_priority(cls, value: Any) -> int:
-        return parse_int(value, field="Priority", minimum=0)
-
-    @field_validator("enabled", mode="before")
-    @classmethod
-    def _validate_enabled(cls, value: Any) -> bool:
-        if not isinstance(value, bool):
-            msg: str = "Enabled must be a boolean."
-            raise ValueError(msg)
-        return value
-
-    @model_validator(mode="after")
-    def _validate_cli_or_extras(self) -> Condition:
-        if not self.cli and not self.extras:
-            msg: str = "Either cli or extras must be set."
-            raise ValueError(msg)
-        return self
-
-
-class ConditionPatch(Condition):
-    """
-    Model for patching Condition fields. All fields are optional.
-    """
-
-    name: str | None = None
-    filter: str | None = None
-    cli: str | None = None
-    extras: dict[str, Any] | None = None
-    enabled: bool | None = None
-    priority: int | None = None
-    description: str | None = None
-
-    @model_validator(mode="after")
-    def _validate_cli_or_extras(self) -> ConditionPatch:
-        return self
-
-
-class ConditionList(BaseModel):
-    model_config = ConfigDict(from_attributes=True)
-
-    items: list[Condition] = Field(default_factory=list)
-    pagination: Pagination

app/features/conditions/service.py

@@ -1,195 +0,0 @@
-from collections.abc import Iterable
-
-from aiohttp import web
-
-from app.features.conditions.models import ConditionModel
-from app.features.conditions.repository import ConditionsRepository
-from app.features.conditions.schemas import Condition
-from app.features.ytdlp.mini_filter import match_str
-from app.library.Events import EventBus, Events
-from app.library.log import get_logger
-from app.library.Singleton import Singleton
-
-LOG = get_logger()
-
-
-def _ignored_identifiers(ignore_conditions: Iterable[str | int | float] | None) -> tuple[set[str], bool]:
-    ignored: set[str] = set()
-    ignore_all = False
-
-    if not ignore_conditions:
-        return ignored, ignore_all
-
-    for value in ignore_conditions:
-        if isinstance(value, bool) or not isinstance(value, (str, int, float)):
-            continue
-
-        identifier = str(value).strip()
-        if not identifier:
-            continue
-
-        if "*" == identifier:
-            ignore_all = True
-            continue
-
-        ignored.add(identifier)
-
-    return ignored, ignore_all
-
-
-class Conditions(metaclass=Singleton):
-    def __init__(self):
-        self._repo: ConditionsRepository = ConditionsRepository.get_instance()
-
-    @staticmethod
-    def get_instance() -> "Conditions":
-        return Conditions()
-
-    async def on_shutdown(self, _: web.Application):
-        pass
-
-    def attach(self, _: web.Application):
-        async def handle_event(_, __):
-            await self._repo.run_migrations()
-
-        EventBus.get_instance().subscribe(Events.STARTED, handle_event, "ConditionsRepository.run_migrations")
-
-    async def get_all(self) -> list[ConditionModel]:
-        return await self._repo.all()
-
-    async def save(self, item: ConditionModel | dict) -> ConditionModel:
-        """
-        Save the item.
-
-        Args:
-            item (Condition|dict): The items to save.
-
-        Returns:
-            ConditionModel: The current instance.
-
-        """
-        try:
-            if not isinstance(item, ConditionModel):
-                item: ConditionModel = ConditionModel(**item)
-        except Exception as exc:
-            msg: str = f"Failed to parse item. '{exc!s}'"
-            raise ValueError(msg) from exc
-
-        try:
-            repo = self._repo
-            if item.id is None or 0 == item.id:
-                model = await repo.create(item)
-            else:
-                model = await repo.update(item.id, Condition.model_validate(item).model_dump())
-        except KeyError as exc:
-            raise ValueError(str(exc)) from exc
-
-        return model
-
-    async def has(self, identifier: str) -> bool:
-        """
-        Check if the item exists by id or name.
-
-        Args:
-            identifier (str): The id or name of the preset.
-
-        Returns:
-            bool: True if the item exists, False otherwise.
-
-        """
-        return await self.get(identifier) is not None
-
-    async def get(self, identifier: str) -> ConditionModel | None:
-        """
-        Get the item by id or name.
-
-        Args:
-            identifier (str): The id or name of the preset.
-
-        Returns:
-            ConditionModel|None: The item if found, None otherwise.
-
-        """
-        repo = self._repo
-        return await repo.get(identifier)
-
-    async def match(
-        self, info: dict, ignore_conditions: Iterable[str | int | float] | None = None
-    ) -> ConditionModel | None:
-        """
-        Check if any condition matches the info dict.
-
-        Args:
-            info (dict): The info dict to check.
-            ignore_conditions (Iterable[str | int | float] | None): Condition ids or names to skip for this match.
-
-        Returns:
-            Condition|None: The condition if found, None otherwise.
-
-        """
-        if not info or not isinstance(info, dict) or len(info) < 1:
-            return None
-
-        ignored_identifiers, ignore_all = _ignored_identifiers(ignore_conditions)
-        if ignore_all:
-            return None
-
-        repo = self._repo
-        items: list[ConditionModel] = await repo.all()
-        if len(items) < 1:
-            return None
-
-        for item in sorted(items, key=lambda x: x.priority, reverse=True):
-            if not item.enabled:
-                continue
-
-            if str(item.id) in ignored_identifiers or item.name in ignored_identifiers:
-                continue
-
-            if not item.filter:
-                LOG.error(
-                    "Filter is empty for '%s'.", item.name, extra={"condition_id": item.id, "condition_name": item.name}
-                )
-                continue
-
-            try:
-                if not match_str(item.filter, info):
-                    continue
-
-                LOG.debug(
-                    "Matched '%s: %s' with filter '%s'.",
-                    item.id,
-                    item.name,
-                    item.filter,
-                    extra={"condition_id": item.id, "condition_name": item.name, "filter": item.filter},
-                )
-                return item
-            except Exception as e:
-                LOG.exception(
-                    "Failed to evaluate condition '%s'.",
-                    item.name,
-                    extra={"condition_id": item.id, "condition_name": item.name, "exception_type": type(e).__name__},
-                )
-                continue
-
-        return None
-
-    async def single_match(self, identifier: int | str, info: dict) -> ConditionModel | None:
-        """
-        Check if condition matches the info dict.
-
-        Args:
-            identifier (int|str): The id or name of the item.
-            info (dict): The info dict to check.
-
-        Returns:
-            ConditionModel|None: The condition if found, None otherwise.
-
-        """
-        if not info or not isinstance(info, dict) or len(info) < 1:
-            return None
-
-        if not (item := await self.get(str(identifier))) or not item.enabled or not item.filter:
-            return None
-
-        return item if match_str(item.filter, info) else None

app/features/conditions/tests/test_conditions.py

@@ -1,246 +0,0 @@
-"""Tests for ConditionsRepository."""
-
-from __future__ import annotations
-
-import pytest
-import pytest_asyncio
-from types import SimpleNamespace
-import pytest
-from typing import Any
-from aiohttp import web
-from aiohttp.test_utils import make_mocked_request
-
-from app.features.conditions.router import conditions_test
-from app.library.config import Config
-from app.library.encoder import Encoder
-from app.features.conditions.repository import ConditionsRepository
-from app.library.sqlite_store import SqliteStore
-from app.tests.helpers import make_in_memory_db_path
-
-
-@pytest_asyncio.fixture
-async def repo():
-    ConditionsRepository._reset_singleton()
-    SqliteStore._reset_singleton()
-
-    store = SqliteStore(db_path=make_in_memory_db_path("conditions-repository"))
-    await store.get_connection()
-
-    # Create repository
-    repository = ConditionsRepository.get_instance()
-
-    yield repository
-
-    # Cleanup - close connections properly
-    await store.close()
-
-    # Reset singletons
-    ConditionsRepository._reset_singleton()
-    SqliteStore._reset_singleton()
-
-
-def _json_request(path: str, payload: object) -> web.Request:
-    mock_request: Any = make_mocked_request("POST", path)
-
-    async def _json() -> object:
-        return payload
-
-    mock_request.json = _json
-    return mock_request
-
-
-class TestAllowInternalUrlsScope:
-    def setup_method(self) -> None:
-        Config._reset_singleton()
-
-    @pytest.mark.asyncio
-    async def test_rejects_internal_url(self) -> None:
-        config = Config.get_instance()
-        config.allow_internal_urls = False
-        encoder = Encoder()
-        cache = SimpleNamespace(hash=lambda value: value, has=lambda _key: False, set=lambda **_kwargs: None)
-
-        request = _json_request(
-            "/api/conditions/test/",
-            {"url": "http://127.0.0.1/test", "condition": "title", "preset": config.default_preset},
-        )
-
-        response = await conditions_test(request, encoder, cache, config)
-
-        assert response.status == web.HTTPBadRequest.status_code
-        body = response.body
-        assert isinstance(body, bytes)
-        assert b"internal urls" in body.lower()
-
-
-class TestConditionsRepository:
-    """Test suite for ConditionsRepository database operations."""
-
-    @pytest.mark.asyncio
-    async def test_create_condition(self, repo):
-        """Create condition with valid data."""
-        data = {
-            "name": "Test Condition",
-            "filter": "duration > 60",
-            "cli": "--format best",
-            "enabled": True,
-            "priority": 10,
-            "description": "Test description",
-            "extras": {"key": "value"},
-        }
-
-        model = await repo.create(data)
-
-        assert model.id is not None, "Should generate ID for new condition"
-        assert model.name == "Test Condition", "Should store name correctly"
-        assert model.filter == "duration > 60", "Should store filter correctly"
-        assert model.cli == "--format best", "Should store CLI correctly"
-        assert model.enabled is True, "Should store enabled flag correctly"
-        assert model.priority == 10, "Should store priority correctly"
-        assert model.description == "Test description", "Should store description correctly"
-        assert model.extras == {"key": "value"}, "Should store extras as dict"
-
-    @pytest.mark.asyncio
-    async def test_create_with_defaults(self, repo):
-        """Create condition with minimal data uses defaults."""
-        data = {
-            "name": "Minimal",
-            "filter": "duration > 30",
-        }
-
-        model = await repo.create(data)
-
-        assert model.cli == "", "Should default CLI to empty string"
-        assert model.enabled is True, "Should default enabled to True"
-        assert model.priority == 0, "Should default priority to 0"
-        assert model.description == "", "Should default description to empty"
-        assert model.extras == {}, "Should default extras to empty dict"
-
-    @pytest.mark.asyncio
-    async def test_get_by_id(self, repo):
-        """Get condition by integer ID."""
-        created = await repo.create({"name": "Get Test", "filter": "duration > 40"})
-
-        retrieved = await repo.get(created.id)
-
-        assert retrieved is not None, "Should retrieve created condition"
-        assert retrieved.id == created.id, "Should retrieve correct condition by ID"
-        assert retrieved.name == "Get Test", "Should match created condition name"
-
-    @pytest.mark.asyncio
-    async def test_get_by_name(self, repo):
-        """Get condition by string name."""
-        await repo.create({"name": "Named Test", "filter": "duration > 50"})
-
-        retrieved = await repo.get("Named Test")
-
-        assert retrieved is not None, "Should retrieve by name"
-        assert retrieved.name == "Named Test", "Should match condition name"
-
-    @pytest.mark.asyncio
-    async def test_get_nonexistent(self, repo):
-        """Get nonexistent condition returns None."""
-        result = await repo.get(99999)
-        assert result is None, "Should return None for nonexistent ID"
-
-        result = await repo.get("nonexistent")
-        assert result is None, "Should return None for nonexistent name"
-
-    @pytest.mark.asyncio
-    async def test_update_condition(self, repo):
-        """Update existing condition."""
-        created = await repo.create({"name": "Update Test", "filter": "duration > 60"})
-
-        updated = await repo.update(
-            created.id,
-            {
-                "name": "Updated Name",
-                "priority": 5,
-                "extras": {"updated": True},
-            },
-        )
-
-        assert updated.name == "Updated Name", "Should update name"
-        assert updated.priority == 5, "Should update priority"
-        assert updated.extras == {"updated": True}, "Should update extras"
-        assert updated.filter == "duration > 60", "Should preserve unchanged filter"
-
-    @pytest.mark.asyncio
-    async def test_update_nonexistent_raises(self, repo):
-        """Update nonexistent condition raises KeyError."""
-        with pytest.raises(KeyError, match="not found"):
-            await repo.update(99999, {"name": "Should Fail"})
-
-    @pytest.mark.asyncio
-    async def test_delete_condition(self, repo):
-        """Delete existing condition."""
-        created = await repo.create({"name": "Delete Test", "filter": "duration > 70"})
-
-        deleted = await repo.delete(created.id)
-
-        assert deleted.id == created.id, "Should return deleted condition"
-
-        result = await repo.get(created.id)
-        assert result is None, "Deleted condition should not be retrievable"
-
-    @pytest.mark.asyncio
-    async def test_delete_nonexistent_raises(self, repo):
-        """Delete nonexistent condition raises KeyError."""
-        with pytest.raises(KeyError, match="not found"):
-            await repo.delete(99999)
-
-    @pytest.mark.asyncio
-    async def test_list_paginated(self, repo):
-        """List paginated returns correct subset."""
-        for i in range(5):
-            await repo.create({"name": f"Item {i}", "filter": "duration > 10", "priority": i})
-
-        items, total, page, total_pages = await repo.list_paginated(page=1, per_page=2)
-
-        assert len(items) == 2, "Should return 2 items per page"
-        assert total == 5, "Should report total count of 5"
-        assert page == 1, "Should be on page 1"
-        assert total_pages == 3, "Should have 3 pages total"
-
-    @pytest.mark.asyncio
-    async def test_list_ordering(self, repo):
-        """List orders by priority desc then name asc."""
-        await repo.create({"name": "B", "filter": "test", "priority": 1})
-        await repo.create({"name": "A", "filter": "test", "priority": 1})
-        await repo.create({"name": "C", "filter": "test", "priority": 2})
-
-        items = await repo.all()
-
-        assert items[0].name == "C", "Highest priority should be first"
-        assert items[1].name == "A", "Same priority sorted alphabetically"
-        assert items[2].name == "B", "Same priority sorted alphabetically"
-
-    @pytest.mark.asyncio
-    async def test_get_by_name_excludes_id(self, repo):
-        """Get by name can exclude specific ID."""
-        first = await repo.create({"name": "Duplicate", "filter": "test"})
-
-        result = await repo.get_by_name("Duplicate", exclude_id=first.id)
-        assert result is None, "Should not find when excluding only match"
-
-        result = await repo.get_by_name("Duplicate", exclude_id=None)
-        assert result is not None, "Should find without exclusion"
-
-    @pytest.mark.asyncio
-    async def test_replace_all(self, repo):
-        """Replace all conditions atomically."""
-        await repo.create({"name": "Old 1", "filter": "test"})
-        await repo.create({"name": "Old 2", "filter": "test"})
-
-        new_items = [
-            {"name": "New 1", "filter": "duration > 10"},
-            {"name": "New 2", "filter": "duration > 20"},
-        ]
-
-        result = await repo.replace_all(new_items)
-
-        assert len(result) == 2, "Should create 2 new conditions"
-
-        all_items = await repo.all()
-        assert len(all_items) == 2, "Should only have new conditions"
-        assert all_items[0].name in ["New 1", "New 2"], "Should only have new items"

app/features/core/deps.py

@@ -1,12 +0,0 @@
-from collections.abc import AsyncGenerator
-from contextlib import asynccontextmanager
-
-from sqlalchemy.ext.asyncio import AsyncSession
-
-from app.library.sqlite_store import SqliteStore
-
-
-@asynccontextmanager
-async def get_session() -> AsyncGenerator[AsyncSession]:
-    async with SqliteStore.get_instance().sessionmaker()() as session:
-        yield session

app/features/core/migration.py

@@ -1,71 +0,0 @@
-from __future__ import annotations
-
-import abc
-import time
-from pathlib import Path
-from typing import TYPE_CHECKING
-
-from app.library.log import get_logger
-
-if TYPE_CHECKING:
-    from app.library.config import Config
-
-LOG = get_logger()
-
-
-class Migration(abc.ABC):
-    name: str = ""
-
-    def __init__(self, config: Config):
-        self._migrated_dir: Path = Path(config.config_path) / "migrated"
-
-    async def run(self) -> bool:
-        if not await self.should_run():
-            return False
-
-        self._migrated_dir.mkdir(parents=True, exist_ok=True)
-        try:
-            await self.migrate()
-        except Exception as exc:
-            LOG.exception(
-                "Feature migration '%s' failed.",
-                self.name,
-                extra={"feature": self.name, "exception_type": type(exc).__name__},
-            )
-            return False
-
-        return True
-
-    @abc.abstractmethod
-    async def should_run(self) -> bool:
-        raise NotImplementedError
-
-    @abc.abstractmethod
-    async def migrate(self) -> None:
-        raise NotImplementedError
-
-    async def _move_file(self, source: Path) -> Path:
-        destination: Path = self._migrated_dir / source.name
-        if destination.exists():
-            timestamp = int(time.time())
-            destination: Path = self._migrated_dir / f"{source.stem}_{timestamp}{source.suffix}"
-
-        source.rename(destination)
-        return destination
-
-    def _unique_name(self, name: str, seen_names: dict[str, int]) -> str:
-        base = name
-        count = seen_names.get(base, 0)
-        if count == 0:
-            seen_names[base] = 1
-            return base
-
-        suffix = count + 1
-        new_name = f"{base} ({suffix})"
-        while new_name in seen_names:
-            suffix += 1
-            new_name = f"{base} ({suffix})"
-
-        seen_names[base] = suffix
-        seen_names[new_name] = 1
-        return new_name

app/features/core/models.py

@@ -1,41 +0,0 @@
-from __future__ import annotations
-
-from datetime import UTC, datetime
-
-from sqlalchemy import DateTime as SQLADateTime
-from sqlalchemy import Dialect, TypeDecorator
-from sqlalchemy.orm import DeclarativeBase
-
-
-def utcnow() -> datetime:
-    """
-    Return current UTC time as timezone-aware datetime.
-
-    This is the canonical way to get current UTC time for database fields.
-    """
-    return datetime.now(tz=UTC)
-
-
-class UTCDateTime(TypeDecorator):
-    impl = SQLADateTime
-    cache_ok = True
-
-    def process_bind_param(self, value: datetime | None, dialect: Dialect) -> datetime | None:
-        """Convert datetime to UTC before storing."""
-        _ = dialect
-        if value is not None:
-            if value.tzinfo is None:
-                return value.replace(tzinfo=UTC)
-            return value.astimezone(UTC).replace(tzinfo=None)
-        return value
-
-    def process_result_value(self, value: datetime | None, dialect: Dialect) -> datetime | None:
-        """Ensure datetime is timezone-aware (UTC) when loading."""
-        _ = dialect
-        if value is not None and value.tzinfo is None:
-            return value.replace(tzinfo=UTC)
-        return value
-
-
-class Base(DeclarativeBase):
-    pass

app/features/core/schemas.py

@@ -1,44 +0,0 @@
-from __future__ import annotations
-
-from enum import Enum
-from typing import Any
-
-from pydantic import BaseModel, Field
-
-
-class Pagination(BaseModel):
-    page: int
-    per_page: int
-    total: int
-    total_pages: int
-    has_next: bool
-    has_prev: bool
-
-
-class CEFeature(str, Enum):
-    PRESETS = "presets"
-    DL_FIELDS = "dl_fields"
-    CONDITIONS = "conditions"
-    NOTIFICATIONS = "notifications"
-    TASKS = "tasks"
-    TASKS_DEFINITIONS = "tasks_definitions"
-
-    def __str__(self) -> str:
-        return self.value
-
-
-class CEAction(str, Enum):
-    CREATE = "create"
-    UPDATE = "update"
-    DELETE = "delete"
-    REPLACE = "replace"
-
-    def __str__(self) -> str:
-        return self.value
-
-
-class ConfigEvent(BaseModel):
-    feature: CEFeature
-    action: CEAction
-    data: dict[str, Any] | list[dict[str, Any]]
-    extras: dict[str, Any] = Field(default_factory=dict)

app/features/core/utils.py

@@ -1,98 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Any
-
-if TYPE_CHECKING:
-    from aiohttp.web import Request
-    from pydantic import ValidationError
-
-
-def parse_int(value: Any, *, field: str, minimum: int | None = None) -> int:
-    try:
-        parsed = int(value)
-    except Exception as exc:
-        msg = f"{field} must be an integer."
-        raise ValueError(msg) from exc
-
-    if minimum is not None and parsed < minimum:
-        msg = f"{field} must be >= {minimum}."
-        raise ValueError(msg)
-
-    return parsed
-
-
-def normalize_pagination(request: Request, page: int | None = None, per_page: int | None = None) -> tuple[int, int]:
-    if page is None:
-        page = int(request.query.get("page", 1))
-
-    if per_page is None:
-        per_page = int(request.query.get("per_page", 0))
-
-    if 0 == per_page:
-        from app.library.config import Config
-
-        items_per_page: int = int(Config.get_instance().default_pagination)
-        per_page = items_per_page // 2 if items_per_page > 50 else items_per_page
-
-    if page < 1:
-        msg = "page must be >= 1."
-        raise ValueError(msg)
-
-    if per_page < 1 or per_page > 1000:
-        msg = "per_page must be between 1 and 1000."
-        raise ValueError(msg)
-
-    return int(page), int(per_page)
-
-
-def build_pagination(total: int, page: int, per_page: int, total_pages: int) -> dict:
-    return {
-        "page": page,
-        "per_page": per_page,
-        "total": total,
-        "total_pages": total_pages,
-        "has_next": page < total_pages,
-        "has_prev": page > 1,
-    }
-
-
-def format_validation_errors(exc: ValidationError) -> list[dict[str, Any]]:
-    """
-    Format Pydantic ValidationError.
-
-    Args:
-        exc: The ValidationError from Pydantic
-
-    Returns:
-        List of dicts with loc, msg, and type
-
-    """
-    return [
-        {
-            "loc": list(error.get("loc", [])),
-            "msg": error.get("msg", "Validation error"),
-            "type": error.get("type", "value_error"),
-        }
-        for error in exc.errors()
-    ]
-
-
-def gen_random(length: int = 16) -> str:
-    import secrets
-    import string
-
-    if length < 1:
-        msg = "length must be >= 1"
-        raise ValueError(msg)
-
-    middle_alphabet = string.ascii_letters + string.digits + "-_"
-    edge_alphabet = string.ascii_letters + string.digits
-
-    if 1 == length:
-        return secrets.choice(edge_alphabet)
-
-    return (
-        secrets.choice(edge_alphabet)
-        + "".join(secrets.choice(middle_alphabet) for _ in range(length - 2))
-        + secrets.choice(edge_alphabet)
-    )

app/features/dl_fields/__init__.py

@@ -1 +0,0 @@
-"""DL Fields Feature"""

app/features/dl_fields/deps.py

@@ -1,5 +0,0 @@
-from app.features.dl_fields.repository import DLFieldsRepository
-
-
-def get_dl_fields_repo() -> DLFieldsRepository:
-    return DLFieldsRepository.get_instance()

app/features/dl_fields/migration.py

@@ -1,121 +0,0 @@
-from __future__ import annotations
-
-import json
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, cast
-
-from app.features.core.migration import Migration as FeatureMigration
-from app.features.dl_fields.schemas import DLField
-from app.library.config import Config
-from app.library.log import get_logger
-
-if TYPE_CHECKING:
-    from app.features.dl_fields.repository import DLFieldsRepository
-
-LOG = get_logger()
-
-
-class Migration(FeatureMigration):
-    name: str = "dl_fields"
-
-    def __init__(self, repo: DLFieldsRepository, config: Config | None = None):
-        self._config: Config = config or Config.get_instance()
-        super().__init__(config=self._config)
-        self._repo: DLFieldsRepository = repo
-        self._source_file: Path = Path(self._config.config_path) / "dl_fields.json"
-
-    async def should_run(self) -> bool:
-        return self._source_file.exists()
-
-    async def migrate(self) -> None:
-        if await self._repo.count() > 0:
-            LOG.warning("DL fields already exist in the database; skipping migration.")
-            await self._move_file(self._source_file)
-            return
-
-        try:
-            items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text())
-        except Exception as exc:
-            LOG.exception(
-                "Failed to read download fields migration file '%s'. Ignoring.",
-                self._source_file,
-                extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
-            )
-            await self._move_file(self._source_file)
-            return
-
-        if items is None:
-            LOG.warning("No dl fields found in %s; skipping migration.", self._source_file)
-            await self._move_file(self._source_file)
-            return
-
-        inserted = 0
-        seen_names: dict[str, int] = {}
-        for index, item in enumerate(items):
-            if not (normalized := self._normalize(item, index, seen_names)):
-                continue
-            try:
-                await self._repo.create(normalized)
-                inserted += 1
-            except Exception as exc:
-                LOG.exception(
-                    "Failed to insert download field '%s'.",
-                    normalized["name"],
-                    extra={"field_name": normalized["name"], "exception_type": type(exc).__name__},
-                )
-
-        LOG.info("Migrated %s dl field(s) from %s.", inserted, self._source_file)
-        await self._move_file(self._source_file)
-
-    def _normalize(self, item: Any, index: int, seen_names: dict[str, int]) -> dict[str, Any] | None:
-        if not isinstance(item, dict):
-            LOG.warning("Skipping dl field at index %s due to invalid type.", index)
-            return None
-
-        assert isinstance(item, dict)
-
-        name: str | None = item.get("name")
-        if not name or not isinstance(name, str):
-            LOG.warning("Skipping dl field at index %s due to missing name.", index)
-            return None
-
-        normalized_name = name.strip()
-        if not normalized_name:
-            LOG.warning("Skipping dl field at index %s due to empty name.", index)
-            return None
-
-        name = self._unique_name(normalized_name, seen_names)
-        description: str = item.get("description") if isinstance(item.get("description"), str) else ""
-        field_value: str | None = item.get("field")
-        if not field_value or not isinstance(field_value, str):
-            LOG.warning("Skipping dl field '%s' due to missing field value.", name)
-            return None
-
-        kind: str = item.get("kind") if isinstance(item.get("kind"), str) else "text"
-        icon: str = item.get("icon") if isinstance(item.get("icon"), str) else ""
-        value: str = item.get("value") if isinstance(item.get("value"), str) else ""
-        extras = item.get("extras") if isinstance(item.get("extras"), dict) else {}
-        extras = cast("dict[str, Any]", extras)
-
-        order_value: int = 0
-        if isinstance(item.get("order"), int) and not isinstance(item.get("order"), bool):
-            order_value = item.get("order", 0)
-
-        payload = {
-            "name": name,
-            "description": description,
-            "field": field_value,
-            "kind": kind,
-            "icon": icon,
-            "order": order_value,
-            "value": value,
-            "extras": extras,
-        }
-
-        try:
-            DLField.model_validate(payload)
-        except Exception as exc:
-            LOG.warning("Skipping dl field '%s' due to validation error: %s", name, exc)
-            return None
-
-        return payload

app/features/dl_fields/models.py

@@ -1,29 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime  # noqa: TC003
-
-from sqlalchemy import JSON, Index, Integer, String, Text
-from sqlalchemy.orm import Mapped, mapped_column
-
-from app.features.core.models import Base, UTCDateTime, utcnow
-
-
-class DLFieldModel(Base):
-    __tablename__: str = "dl_fields"
-    __table_args__: tuple[Index, ...] = (
-        Index("ix_dl_fields_name", "name"),
-        Index("ix_dl_fields_order", "order"),
-        Index("ix_dl_fields_kind", "kind"),
-    )
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
-    name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
-    description: Mapped[str] = mapped_column(Text, nullable=False, default="")
-    field: Mapped[str] = mapped_column(String(255), nullable=False)
-    kind: Mapped[str] = mapped_column(String(50), nullable=False, default="text")
-    icon: Mapped[str] = mapped_column(String(255), nullable=False, default="")
-    order: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-    value: Mapped[str] = mapped_column(Text, nullable=False, default="")
-    extras: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
-    created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, nullable=False)
-    updated_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, onupdate=utcnow, nullable=False)

app/features/dl_fields/repository.py

@@ -1,190 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Any, cast
-
-from sqlalchemy import delete, func, or_, select
-
-from app.features.core.deps import get_session
-from app.features.dl_fields.migration import Migration
-from app.features.dl_fields.models import DLFieldModel
-from app.library.log import get_logger
-from app.library.Singleton import Singleton
-
-if TYPE_CHECKING:
-    from collections.abc import Callable, Iterable
-    from contextlib import AbstractAsyncContextManager
-
-    from sqlalchemy.engine.result import Result
-    from sqlalchemy.ext.asyncio import AsyncSession
-    from sqlalchemy.sql.elements import ColumnElement
-    from sqlalchemy.sql.selectable import Select
-
-    SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
-
-LOG = get_logger()
-
-
-def _model_from_payload(payload: dict[str, Any]) -> DLFieldModel:
-    model = DLFieldModel()
-    for key, value in payload.items():
-        if not hasattr(model, key):
-            msg = f"'{key}' is an invalid keyword argument for DLFieldModel"
-            raise TypeError(msg)
-        setattr(model, key, value)
-    return model
-
-
-def _coerce_model(payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
-    if isinstance(payload, DLFieldModel):
-        return payload
-    return _model_from_payload(payload)
-
-
-class DLFieldsRepository(metaclass=Singleton):
-    def __init__(self, session: SessionFactory | None = None) -> None:
-        self._migrated = False
-        self.session: SessionFactory = session or get_session
-
-    async def run_migrations(self) -> None:
-        if self._migrated:
-            return
-
-        self._migrated = True
-        await Migration(repo=self).run()
-
-    @staticmethod
-    def get_instance() -> DLFieldsRepository:
-        return DLFieldsRepository()
-
-    async def all(self) -> list[DLFieldModel]:
-        async with self.session() as session:
-            result: Result[tuple[DLFieldModel]] = await session.execute(
-                select(DLFieldModel).order_by(DLFieldModel.order.asc(), DLFieldModel.name.asc())
-            )
-            return list(result.scalars().all())
-
-    async def list_paginated(self, page: int, per_page: int) -> tuple[list[DLFieldModel], int, int, int]:
-        async with self.session() as session:
-            total: int = await self.count()
-            total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1
-
-            if page > total_pages and total > 0:
-                page = total_pages
-
-            query: Select[tuple[DLFieldModel]] = (
-                select(DLFieldModel)
-                .order_by(DLFieldModel.order.asc(), DLFieldModel.name.asc())
-                .limit(per_page)
-                .offset((page - 1) * per_page)
-            )
-            result: Result[tuple[DLFieldModel]] = await session.execute(query)
-            return list(result.scalars().all()), total, page, total_pages
-
-    async def count(self) -> int:
-        async with self.session() as session:
-            result: Result[tuple[int]] = await session.execute(select(func.count()).select_from(DLFieldModel))
-            return int(result.scalar_one())
-
-    async def get(self, identifier: int | str) -> DLFieldModel | None:
-        async with self.session() as session:
-            if not identifier:
-                return None
-
-            if isinstance(identifier, int):
-                clause: ColumnElement[bool] = DLFieldModel.id == identifier
-            elif isinstance(identifier, str) and identifier.isdigit():
-                clause = or_(DLFieldModel.id == int(identifier), DLFieldModel.name == identifier)
-            else:
-                clause = DLFieldModel.name == identifier
-
-            result: Result[tuple[DLFieldModel]] = await session.execute(select(DLFieldModel).where(clause).limit(1))
-            return result.scalar_one_or_none()
-
-    async def get_by_name(self, name: str, exclude_id: int | None = None) -> DLFieldModel | None:
-        async with self.session() as session:
-            query: Select[tuple[DLFieldModel]] = select(DLFieldModel).where(DLFieldModel.name == name)
-            if exclude_id is not None:
-                query = query.where(DLFieldModel.id != exclude_id)
-
-            result: Result[tuple[DLFieldModel]] = await session.execute(query.limit(1))
-            return result.scalar_one_or_none()
-
-    async def create(self, payload: DLFieldModel | dict[str, Any]) -> DLFieldModel:
-        async with self.session() as session:
-            model = _coerce_model(payload)
-
-            if await self.get_by_name(name=model.name) is not None:
-                msg: str = f"DL field with name '{model.name}' already exists."
-                raise ValueError(msg)
-
-            session.add(model)
-            await session.commit()
-            await session.refresh(model)
-            return model
-
-    async def update(self, identifier: int | str, payload: dict[str, Any]) -> DLFieldModel:
-        async with self.session() as session:
-            if isinstance(identifier, int):
-                clause: ColumnElement[bool] = DLFieldModel.id == identifier
-            elif isinstance(identifier, str) and identifier.isdigit():
-                clause = or_(DLFieldModel.id == int(identifier), DLFieldModel.name == identifier)
-            else:
-                clause = DLFieldModel.name == identifier
-
-            result: Result[tuple[DLFieldModel]] = await session.execute(select(DLFieldModel).where(clause).limit(1))
-            model: DLFieldModel | None = result.scalar_one_or_none()
-
-            if not model:
-                msg: str = f"DL field '{identifier}' not found."
-                raise KeyError(msg)
-
-            payload.pop("id", None)
-
-            if "name" in payload and await self.get_by_name(name=payload["name"], exclude_id=model.id) is not None:
-                msg = f"DL field with name '{payload['name']}' already exists."
-                raise ValueError(msg)
-
-            for key, value in payload.items():
-                if hasattr(model, key):
-                    setattr(model, key, value)
-
-            await session.commit()
-            await session.refresh(model)
-            return model
-
-    async def delete(self, identifier: int | str) -> DLFieldModel:
-        async with self.session() as session:
-            if isinstance(identifier, int):
-                clause: ColumnElement[bool] = DLFieldModel.id == identifier
-            elif isinstance(identifier, str) and identifier.isdigit():
-                clause = or_(DLFieldModel.id == int(identifier), DLFieldModel.name == identifier)
-            else:
-                clause = DLFieldModel.name == identifier
-
-            result: Result[tuple[DLFieldModel]] = await session.execute(select(DLFieldModel).where(clause).limit(1))
-            model = result.scalar_one_or_none()
-
-            if not model:
-                msg: str = f"DL field '{identifier}' not found."
-                raise KeyError(msg)
-
-            await session.delete(model)
-            await session.commit()
-            return model
-
-    async def replace_all(self, items: Iterable[dict[str, Any] | DLFieldModel]) -> list[DLFieldModel]:
-        async with self.session() as session:
-            try:
-                await session.execute(delete(DLFieldModel))
-                models: list[DLFieldModel] = []
-                for item in items:
-                    if isinstance(item, dict):
-                        models.append(DLFieldModel(**cast("dict[str, Any]", item)))
-                    else:
-                        models.append(item)
-                session.add_all(models)
-                await session.commit()
-            except Exception:
-                await session.rollback()
-                raise
-            return models

app/features/dl_fields/router.py

@@ -1,176 +0,0 @@
-from typing import Any
-
-from aiohttp import web
-from aiohttp.web import Request, Response
-from pydantic import ValidationError
-
-from app.features.core.schemas import CEAction, CEFeature, ConfigEvent, Pagination
-from app.features.core.utils import build_pagination, format_validation_errors, normalize_pagination
-from app.features.dl_fields.schemas import DLField, DLFieldList, DLFieldPatch
-from app.features.dl_fields.service import DLFields
-from app.library.encoder import Encoder
-from app.library.Events import EventBus, Events
-from app.library.log import get_logger
-from app.library.router import route
-
-LOG = get_logger()
-
-
-def _model(model: Any) -> DLField:
-    return DLField.model_validate(model)
-
-
-def _serialize(model: Any) -> dict:
-    return _model(model).model_dump()
-
-
-@route("GET", "api/dl_fields/", "dl_fields")
-async def dl_fields_list(request: Request, encoder: Encoder) -> Response:
-    repo = DLFields.get_instance()._repo
-    page, per_page = normalize_pagination(request)
-    items, total, current_page, total_pages = await repo.list_paginated(page, per_page)
-    return web.json_response(
-        data=DLFieldList(
-            items=[_model(model) for model in items],
-            pagination=Pagination.model_validate(build_pagination(total, current_page, per_page, total_pages)),
-        ),
-        status=web.HTTPOk.status_code,
-        dumps=encoder.encode,
-    )
-
-
-@route("POST", "api/dl_fields/", "dl_fields_add")
-async def dl_fields_add(request: Request, encoder: Encoder, notify: EventBus) -> Response:
-    data = await request.json()
-
-    if not isinstance(data, dict):
-        return web.json_response(
-            {"error": "Invalid request body expecting dict."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    try:
-        item: DLField = DLField.model_validate(data)
-    except ValidationError as exc:
-        return web.json_response(
-            data={"error": "Failed to validate dl field.", "detail": format_validation_errors(exc)},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    try:
-        saved = _serialize(await DLFields.get_instance().save(item=item.model_dump()))
-    except ValueError as exc:
-        return web.json_response({"error": str(exc)}, status=web.HTTPBadRequest.status_code)
-
-    notify.emit(Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.DL_FIELDS, action=CEAction.CREATE, data=saved))
-
-    return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode)
-
-
-@route("GET", r"api/dl_fields/{id:\d+}", "dl_fields_get")
-async def dl_fields_get(request: Request, encoder: Encoder) -> Response:
-    if not (identifier := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    if not (model := await DLFields.get_instance().get(identifier)):
-        return web.json_response({"error": "DL field not found"}, status=web.HTTPNotFound.status_code)
-
-    return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode)
-
-
-@route("DELETE", r"api/dl_fields/{id:\d+}", "dl_fields_delete")
-async def dl_fields_delete(request: Request, encoder: Encoder, notify: EventBus) -> Response:
-    if not (identifier := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    try:
-        deleted = _serialize(await DLFields.get_instance()._repo.delete(identifier))
-        notify.emit(
-            Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.DL_FIELDS, action=CEAction.DELETE, data=deleted)
-        )
-        return web.json_response(
-            data=deleted,
-            status=web.HTTPOk.status_code,
-            dumps=encoder.encode,
-        )
-    except KeyError as exc:
-        return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code)
-
-
-@route("PATCH", r"api/dl_fields/{id:\d+}", "dl_fields_patch")
-async def dl_fields_patch(request: Request, encoder: Encoder, notify: EventBus) -> Response:
-    if not (identifier := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    if not (model := await DLFields.get_instance().get(identifier)):
-        return web.json_response({"error": "DL field not found"}, status=web.HTTPNotFound.status_code)
-
-    data = await request.json()
-
-    if not isinstance(data, dict):
-        return web.json_response(
-            {"error": "Invalid request body expecting dict."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    try:
-        validated = DLFieldPatch.model_validate(data)
-    except ValidationError as exc:
-        return web.json_response(
-            data={"error": "Failed to validate dl field.", "detail": format_validation_errors(exc)},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    if validated.name and await DLFields.get_instance()._repo.get_by_name(validated.name, exclude_id=model.id):
-        return web.json_response(
-            data={"error": f"DL field with name '{validated.name}' already exists."},
-            status=web.HTTPConflict.status_code,
-        )
-
-    updated = _serialize(await DLFields.get_instance()._repo.update(model.id, validated.model_dump(exclude_unset=True)))
-    notify.emit(
-        Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.DL_FIELDS, action=CEAction.UPDATE, data=updated)
-    )
-    return web.json_response(
-        data=updated,
-        status=web.HTTPOk.status_code,
-        dumps=encoder.encode,
-    )
-
-
-@route("PUT", r"api/dl_fields/{id:\d+}", "dl_fields_update")
-async def dl_fields_update(request: Request, encoder: Encoder, notify: EventBus) -> Response:
-    if not (identifier := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    if not (model := await DLFields.get_instance().get(identifier)):
-        return web.json_response({"error": "DL field not found"}, status=web.HTTPNotFound.status_code)
-
-    data = await request.json()
-
-    if not isinstance(data, dict):
-        return web.json_response(
-            {"error": "Invalid request body expecting dict."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    try:
-        validated = DLField.model_validate(data)
-    except ValidationError as exc:
-        return web.json_response(
-            data={"error": "Failed to validate dl field.", "detail": format_validation_errors(exc)},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    if validated.name and await DLFields.get_instance()._repo.get_by_name(validated.name, exclude_id=model.id):
-        return web.json_response(
-            data={"error": f"DL field with name '{validated.name}' already exists."},
-            status=web.HTTPConflict.status_code,
-        )
-
-    updated = _serialize(await DLFields.get_instance()._repo.update(model.id, validated.model_dump(exclude_unset=True)))
-    notify.emit(
-        Events.CONFIG_UPDATE, data=ConfigEvent(feature=CEFeature.DL_FIELDS, action=CEAction.UPDATE, data=updated)
-    )
-
-    return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode)

app/features/dl_fields/schemas.py

@@ -1,101 +0,0 @@
-from __future__ import annotations
-
-from enum import Enum
-from typing import Any
-
-from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
-
-from app.features.core.schemas import Pagination
-from app.features.core.utils import parse_int
-
-
-class FieldType(str, Enum):
-    STRING = "string"
-    TEXT = "text"
-    BOOL = "bool"
-
-
-class DLField(BaseModel):
-    model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True)
-
-    id: int | None = None
-    name: str = Field(min_length=1)
-    description: str = ""
-    field: str = Field(min_length=1)
-    kind: FieldType = FieldType.TEXT
-    icon: str = ""
-    order: int = 0
-    value: str = ""
-    extras: dict[str, Any] = Field(default_factory=dict)
-
-    @field_validator("field")
-    @classmethod
-    def _validate_field(cls, value: str) -> str:
-        if not value.startswith("--"):
-            msg: str = "Field must start with '--'."
-            raise ValueError(msg)
-        if " " in value:
-            msg = "Field must not contain spaces."
-            raise ValueError(msg)
-        return value
-
-    @field_validator("order", mode="before")
-    @classmethod
-    def _normalize_order(cls, value: Any) -> int:
-        return parse_int(value, field="Order", minimum=0)
-
-    @model_validator(mode="after")
-    def _normalize_extras(self) -> DLField:
-        if not isinstance(self.extras, dict):
-            msg: str = "Extras must be a dictionary."
-            raise ValueError(msg)
-        return self
-
-
-class DLFieldPatch(BaseModel):
-    model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True)
-
-    name: str | None = None
-    description: str | None = None
-    field: str | None = None
-    kind: FieldType | None = None
-    icon: str | None = None
-    order: int | None = None
-    value: str | None = None
-    extras: dict[str, Any] | None = None
-
-    @field_validator("field")
-    @classmethod
-    def _validate_field(cls, value: str | None) -> str | None:
-        if value is None:
-            return value
-        if not value.startswith("--"):
-            msg: str = "Field must start with '--'."
-            raise ValueError(msg)
-        if " " in value:
-            msg = "Field must not contain spaces."
-            raise ValueError(msg)
-        return value
-
-    @field_validator("order", mode="before")
-    @classmethod
-    def _normalize_order(cls, value: Any) -> int | None:
-        if value is None:
-            return None
-        return parse_int(value, field="Order", minimum=0)
-
-    @model_validator(mode="after")
-    def _normalize_extras(self) -> DLFieldPatch:
-        if self.extras is None:
-            return self
-        if not isinstance(self.extras, dict):
-            msg: str = "Extras must be a dictionary."
-            raise ValueError(msg)
-        return self
-
-
-class DLFieldList(BaseModel):
-    model_config = ConfigDict(from_attributes=True)
-
-    items: list[DLField] = Field(default_factory=list)
-    pagination: Pagination

app/features/dl_fields/service.py

@@ -1,62 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Any
-
-from app.features.dl_fields.models import DLFieldModel
-from app.features.dl_fields.repository import DLFieldsRepository
-from app.features.dl_fields.schemas import DLField
-from app.library.Events import EventBus, Events
-from app.library.log import get_logger
-from app.library.Singleton import Singleton
-
-if TYPE_CHECKING:
-    from aiohttp import web
-
-LOG = get_logger()
-
-
-class DLFields(metaclass=Singleton):
-    def __init__(self):
-        self._repo: DLFieldsRepository = DLFieldsRepository.get_instance()
-
-    @staticmethod
-    def get_instance() -> DLFields:
-        return DLFields()
-
-    async def on_shutdown(self, _: web.Application) -> None:
-        pass
-
-    def attach(self, _: web.Application) -> None:
-        async def handle_event(_, __):
-            await self._repo.run_migrations()
-
-        EventBus.get_instance().subscribe(Events.STARTED, handle_event, "DLFieldsRepository.run_migrations")
-
-    async def get_all(self) -> list[DLFieldModel]:
-        return await self._repo.all()
-
-    async def get_all_serialized(self) -> list[dict[str, Any]]:
-        items = await self._repo.all()
-        return [DLField.model_validate(item).model_dump() for item in items]
-
-    async def save(self, item: DLField | dict) -> DLFieldModel:
-        try:
-            if not isinstance(item, DLField):
-                item = DLField.model_validate(item)
-        except Exception as exc:
-            msg: str = f"Failed to parse item. '{exc!s}'"
-            raise ValueError(msg) from exc
-
-        try:
-            repo = self._repo
-            if item.id is None or 0 == item.id:
-                model = await repo.create(item.model_dump(exclude_unset=True))
-            else:
-                model = await repo.update(item.id, item.model_dump(exclude_unset=True))
-        except KeyError as exc:
-            raise ValueError(str(exc)) from exc
-
-        return model
-
-    async def get(self, identifier: int | str) -> DLFieldModel | None:
-        return await self._repo.get(identifier)

app/features/dl_fields/tests/test_dl_fields.py

@@ -1,74 +0,0 @@
-"""Tests for dl_fields feature service."""
-
-from __future__ import annotations
-
-import pytest
-import pytest_asyncio
-
-from app.features.dl_fields.repository import DLFieldsRepository
-from app.features.dl_fields.service import DLFields
-from app.library.sqlite_store import SqliteStore
-from app.tests.helpers import make_in_memory_db_path
-
-
-@pytest_asyncio.fixture
-async def repo():
-    """Provide a fresh repository instance with initialized database for each test."""
-    DLFieldsRepository._reset_singleton()
-    DLFields._reset_singleton()
-    SqliteStore._reset_singleton()
-
-    store = SqliteStore(db_path=make_in_memory_db_path("dl-fields-service"))
-    await store.get_connection()
-
-    repository = DLFieldsRepository.get_instance()
-
-    yield repository
-
-    await store.close()
-
-    DLFieldsRepository._reset_singleton()
-    DLFields._reset_singleton()
-    SqliteStore._reset_singleton()
-
-
-class TestDLFieldsService:
-    """Test suite for DLFields service methods."""
-
-    @pytest.mark.asyncio
-    async def test_save_creates_field(self, repo):
-        """Save should create a new dl field when ID is missing."""
-        service = DLFields.get_instance()
-        payload = {
-            "name": "quality",
-            "description": "Video quality",
-            "field": "--format",
-            "kind": "string",
-            "order": 1,
-            "extras": {"options": ["best", "worst"]},
-        }
-
-        model = await service.save(payload)
-
-        assert model.id is not None, "Should create new dl field"
-        assert model.name == "quality", "Should store name correctly"
-
-    @pytest.mark.asyncio
-    async def test_get_all_serialized(self, repo):
-        """Get all serialized returns list of dictionaries."""
-        service = DLFields.get_instance()
-        await service.save(
-            {
-                "name": "audio_only",
-                "description": "Audio",
-                "field": "--extract-audio",
-                "kind": "bool",
-                "order": 2,
-            }
-        )
-
-        items = await service.get_all_serialized()
-
-        assert len(items) == 1, "Should return one dl field"
-        assert items[0]["name"] == "audio_only", "Should serialize name"
-        assert isinstance(items[0]["id"], int), "Should serialize integer ID"

app/features/dl_fields/tests/test_dl_fields_repository.py

@@ -1,248 +0,0 @@
-"""Tests for DLFieldsRepository."""
-
-from __future__ import annotations
-
-import pytest
-import pytest_asyncio
-
-from app.features.dl_fields.repository import DLFieldsRepository
-from app.library.sqlite_store import SqliteStore
-from app.tests.helpers import make_in_memory_db_path
-
-
-@pytest_asyncio.fixture
-async def repo():
-    """Provide a fresh repository instance with initialized database for each test."""
-    DLFieldsRepository._reset_singleton()
-    SqliteStore._reset_singleton()
-
-    store = SqliteStore(db_path=make_in_memory_db_path("dl-fields-repository"))
-    await store.get_connection()
-
-    repository = DLFieldsRepository.get_instance()
-
-    yield repository
-
-    await store.close()
-
-    DLFieldsRepository._reset_singleton()
-    SqliteStore._reset_singleton()
-
-
-class TestDLFieldsRepository:
-    """Test suite for DLFieldsRepository database operations."""
-
-    @pytest.mark.asyncio
-    async def test_create_field(self, repo):
-        """Create field with valid data."""
-        data = {
-            "name": "quality",
-            "description": "Video quality setting",
-            "field": "--format",
-            "kind": "string",
-            "icon": "fa-video",
-            "order": 1,
-            "value": "best",
-            "extras": {"options": ["best", "worst"]},
-        }
-
-        model = await repo.create(data)
-
-        assert model.id is not None, "Should generate ID for new field"
-        assert model.name == "quality", "Should store name correctly"
-        assert model.description == "Video quality setting", "Should store description correctly"
-        assert model.field == "--format", "Should store field correctly"
-        assert model.kind == "string", "Should store kind correctly"
-        assert model.icon == "fa-video", "Should store icon correctly"
-        assert model.order == 1, "Should store order correctly"
-        assert model.value == "best", "Should store value correctly"
-        assert model.extras == {"options": ["best", "worst"]}, "Should store extras as dict"
-
-    @pytest.mark.asyncio
-    async def test_create_with_defaults(self, repo):
-        """Create field with minimal data uses defaults."""
-        data = {
-            "name": "minimal",
-            "description": "Minimal",
-            "field": "--minimal",
-            "kind": "text",
-        }
-
-        model = await repo.create(data)
-
-        assert model.icon == "", "Should default icon to empty string"
-        assert model.order == 0, "Should default order to 0"
-        assert model.value == "", "Should default value to empty string"
-        assert model.extras == {}, "Should default extras to empty dict"
-
-    @pytest.mark.asyncio
-    async def test_get_by_id(self, repo):
-        """Get field by integer ID."""
-        created = await repo.create(
-            {
-                "name": "get_test",
-                "description": "Get test",
-                "field": "--get-test",
-                "kind": "text",
-            }
-        )
-
-        retrieved = await repo.get(created.id)
-
-        assert retrieved is not None, "Should retrieve created field"
-        assert retrieved.id == created.id, "Should retrieve correct field by ID"
-        assert retrieved.name == "get_test", "Should match created field name"
-
-    @pytest.mark.asyncio
-    async def test_get_by_name(self, repo):
-        """Get field by string name."""
-        await repo.create(
-            {
-                "name": "named_test",
-                "description": "Named test",
-                "field": "--named-test",
-                "kind": "text",
-            }
-        )
-
-        retrieved = await repo.get("named_test")
-
-        assert retrieved is not None, "Should retrieve by name"
-        assert retrieved.name == "named_test", "Should match field name"
-
-    @pytest.mark.asyncio
-    async def test_get_nonexistent(self, repo):
-        """Get nonexistent field returns None."""
-        result = await repo.get(99999)
-        assert result is None, "Should return None for nonexistent ID"
-
-        result = await repo.get("nonexistent")
-        assert result is None, "Should return None for nonexistent name"
-
-    @pytest.mark.asyncio
-    async def test_update_field(self, repo):
-        """Update existing field."""
-        created = await repo.create(
-            {
-                "name": "update_test",
-                "description": "Update test",
-                "field": "--update-test",
-                "kind": "text",
-            }
-        )
-
-        updated = await repo.update(
-            created.id,
-            {
-                "name": "updated_name",
-                "order": 5,
-                "extras": {"updated": True},
-            },
-        )
-
-        assert updated.name == "updated_name", "Should update name"
-        assert updated.order == 5, "Should update order"
-        assert updated.extras == {"updated": True}, "Should update extras"
-        assert updated.field == "--update-test", "Should preserve unchanged field"
-
-    @pytest.mark.asyncio
-    async def test_update_nonexistent_raises(self, repo):
-        """Update nonexistent field raises KeyError."""
-        with pytest.raises(KeyError, match="not found"):
-            await repo.update(99999, {"name": "should_fail"})
-
-    @pytest.mark.asyncio
-    async def test_delete_field(self, repo):
-        """Delete existing field."""
-        created = await repo.create(
-            {
-                "name": "delete_test",
-                "description": "Delete test",
-                "field": "--delete-test",
-                "kind": "text",
-            }
-        )
-
-        deleted = await repo.delete(created.id)
-
-        assert deleted.id == created.id, "Should return deleted field"
-
-        result = await repo.get(created.id)
-        assert result is None, "Deleted field should not be retrievable"
-
-    @pytest.mark.asyncio
-    async def test_delete_nonexistent_raises(self, repo):
-        """Delete nonexistent field raises KeyError."""
-        with pytest.raises(KeyError, match="not found"):
-            await repo.delete(99999)
-
-    @pytest.mark.asyncio
-    async def test_list_paginated(self, repo):
-        """List paginated returns correct subset."""
-        for i in range(5):
-            await repo.create(
-                {
-                    "name": f"item_{i}",
-                    "description": "desc",
-                    "field": f"--item-{i}",
-                    "kind": "text",
-                    "order": i,
-                }
-            )
-
-        items, total, page, total_pages = await repo.list_paginated(page=1, per_page=2)
-
-        assert len(items) == 2, "Should return 2 items per page"
-        assert total == 5, "Should report total count of 5"
-        assert page == 1, "Should be on page 1"
-        assert total_pages == 3, "Should have 3 pages total"
-
-    @pytest.mark.asyncio
-    async def test_list_ordering(self, repo):
-        """List orders by order asc then name asc."""
-        await repo.create({"name": "b", "description": "b", "field": "--b", "kind": "text", "order": 1})
-        await repo.create({"name": "a", "description": "a", "field": "--a", "kind": "text", "order": 1})
-        await repo.create({"name": "c", "description": "c", "field": "--c", "kind": "text", "order": 0})
-
-        items = await repo.all()
-
-        assert items[0].name == "c", "Lowest order should be first"
-        assert items[1].name == "a", "Same order sorted alphabetically"
-        assert items[2].name == "b", "Same order sorted alphabetically"
-
-    @pytest.mark.asyncio
-    async def test_get_by_name_excludes_id(self, repo):
-        """Get by name can exclude specific ID."""
-        first = await repo.create(
-            {
-                "name": "duplicate",
-                "description": "duplicate",
-                "field": "--duplicate",
-                "kind": "text",
-            }
-        )
-
-        result = await repo.get_by_name("duplicate", exclude_id=first.id)
-        assert result is None, "Should not find when excluding only match"
-
-        result = await repo.get_by_name("duplicate", exclude_id=None)
-        assert result is not None, "Should find without exclusion"
-
-    @pytest.mark.asyncio
-    async def test_replace_all(self, repo):
-        """Replace all fields atomically."""
-        await repo.create({"name": "old_1", "description": "old", "field": "--old-1", "kind": "text"})
-        await repo.create({"name": "old_2", "description": "old", "field": "--old-2", "kind": "text"})
-
-        new_items = [
-            {"name": "new_1", "description": "new", "field": "--new-1", "kind": "text"},
-            {"name": "new_2", "description": "new", "field": "--new-2", "kind": "text"},
-        ]
-
-        result = await repo.replace_all(new_items)
-
-        assert len(result) == 2, "Should create 2 new fields"
-
-        all_items = await repo.all()
-        assert len(all_items) == 2, "Should only have new fields"
-        assert {item.name for item in all_items} == {"new_1", "new_2"}, "Should only have new items"

app/features/notifications/deps.py

@@ -1,5 +0,0 @@
-from app.features.notifications.repository import NotificationsRepository
-
-
-def get_notifications_repo() -> NotificationsRepository:
-    return NotificationsRepository.get_instance()

app/features/notifications/migration.py

@@ -1,187 +0,0 @@
-from __future__ import annotations
-
-import json
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, cast
-
-from app.features.core.migration import Migration as FeatureMigration
-from app.features.notifications.schemas import NotificationEvents
-from app.features.presets.service import Presets
-from app.library.config import Config
-from app.library.log import get_logger
-
-if TYPE_CHECKING:
-    from app.features.notifications.repository import NotificationsRepository
-
-LOG = get_logger()
-
-
-class Migration(FeatureMigration):
-    name: str = "notifications"
-
-    def __init__(self, repo: NotificationsRepository, config: Config | None = None):
-        self._config: Config = config or Config.get_instance()
-        super().__init__(config=self._config)
-        self._repo: NotificationsRepository = repo
-        self._source_file: Path = Path(self._config.config_path) / "notifications.json"
-
-    async def should_run(self) -> bool:
-        return self._source_file.exists()
-
-    async def migrate(self) -> None:
-        if await self._repo.count() > 0:
-            LOG.warning("Notification targets already exist in the database; skipping migration.")
-            await self._move_file(self._source_file)
-            return
-
-        try:
-            items: list[dict] | None = json.loads(self._source_file.read_text())
-        except Exception as exc:
-            LOG.exception(
-                "Failed to read notifications migration file '%s'. Ignoring.",
-                self._source_file,
-                extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
-            )
-            await self._move_file(self._source_file)
-            return
-
-        if items is None:
-            LOG.warning("No notification targets found in %s; skipping migration.", self._source_file)
-            await self._move_file(self._source_file)
-            return
-
-        inserted = 0
-        seen_names: dict[str, int] = {}
-        for index, item in enumerate(items):
-            normalized = self._normalize(item, index, seen_names)
-            if not normalized:
-                continue
-            try:
-                await self._repo.create(normalized)
-                inserted += 1
-            except Exception as exc:
-                LOG.exception(
-                    "Failed to insert notification target '%s'.",
-                    normalized.get("name"),
-                    extra={"target_name": normalized.get("name"), "exception_type": type(exc).__name__},
-                )
-
-        LOG.info("Migrated %s notification target(s) from %s.", inserted, self._source_file)
-        await self._move_file(self._source_file)
-
-    def _normalize(self, item: Any, index: int, seen_names: dict[str, int]) -> dict[str, Any] | None:
-        if not isinstance(item, dict):
-            LOG.warning("Skipping notification at index %s due to invalid type.", index)
-            return None
-
-        assert isinstance(item, dict)
-
-        name: str | None = item.get("name")
-        if not name or not isinstance(name, str):
-            LOG.warning("Skipping notification at index %s due to missing name.", index)
-            return None
-
-        normalized_name = name.strip()
-        if not normalized_name:
-            LOG.warning("Skipping notification at index %s due to empty name.", index)
-            return None
-
-        name = self._unique_name(normalized_name, seen_names)
-        request = item.get("request") if isinstance(item.get("request"), dict) else {}
-        request = cast("dict[str, Any]", request)
-        url: str | None = request.get("url")
-        if not url or not isinstance(url, str):
-            LOG.warning("Skipping notification '%s' due to missing request url.", name)
-            return None
-
-        raw_method = request.get("method")
-        method = raw_method.upper() if isinstance(raw_method, str) else "POST"
-        if method not in {"POST", "PUT"}:
-            LOG.warning("Skipping notification '%s' due to invalid method '%s'.", name, method)
-            return None
-
-        raw_type = request.get("type")
-        req_type = raw_type.lower() if isinstance(raw_type, str) else "json"
-        if req_type not in {"json", "form"}:
-            LOG.warning("Skipping notification '%s' due to invalid type '%s'.", name, req_type)
-            return None
-
-        data_key = request.get("data_key") if isinstance(request.get("data_key"), str) else "data"
-        if not data_key:
-            data_key = "data"
-
-        headers = self._normalize_headers(request.get("headers"))
-
-        enabled_value = item.get("enabled")
-        enabled: bool = enabled_value if isinstance(enabled_value, bool) else True
-        events = self._normalize_events(item.get("on"))
-        if events is None:
-            LOG.warning("Skipping notification '%s' due to invalid events.", name)
-            return None
-
-        presets = self._normalize_presets(item.get("presets"))
-        if presets is None:
-            LOG.warning("Skipping notification '%s' due to invalid presets.", name)
-            return None
-
-        return {
-            "name": name,
-            "on": events,
-            "presets": presets,
-            "enabled": enabled,
-            "request_url": url,
-            "request_method": method,
-            "request_type": req_type,
-            "request_data_key": data_key,
-            "request_headers": headers,
-        }
-
-    def _normalize_events(self, events: Any) -> list[str] | None:
-        if events is None:
-            return []
-        if not isinstance(events, list):
-            return []
-
-        allowed = set(NotificationEvents.get_events().values())
-        valid = [event for event in events if event in allowed]
-        invalid = [event for event in events if event not in allowed]
-
-        if len(invalid) > 0 and len(valid) < 1:
-            return None
-
-        if len(invalid) > 0:
-            LOG.warning("Dropping invalid notification events: %s", ", ".join(invalid))
-
-        return valid
-
-    def _normalize_presets(self, presets: Any) -> list[str] | None:
-        if presets is None:
-            return []
-        if not isinstance(presets, list):
-            return []
-
-        allowed = {preset.name for preset in Presets.get_instance().get_all()}
-        valid = [preset for preset in presets if preset in allowed]
-        invalid = [preset for preset in presets if preset not in allowed]
-
-        if len(invalid) > 0 and len(valid) < 1:
-            return None
-
-        if len(invalid) > 0:
-            LOG.warning("Dropping invalid notification presets: %s", ", ".join(invalid))
-
-        return valid
-
-    def _normalize_headers(self, headers: Any) -> list[dict[str, str]]:
-        if not isinstance(headers, list):
-            return []
-
-        normalized: list[dict[str, str]] = []
-        for header in headers:
-            if not isinstance(header, dict):
-                continue
-            key = str(header.get("key", "")).strip()
-            value = str(header.get("value", "")).strip()
-            if key and value:
-                normalized.append({"key": key, "value": value})
-        return normalized

app/features/notifications/models.py

@@ -1,39 +0,0 @@
-from __future__ import annotations
-
-from datetime import datetime  # noqa: TC003
-
-from sqlalchemy import JSON, Boolean, Index, Integer, String, Text
-from sqlalchemy import Enum as SQLAEnum
-from sqlalchemy.orm import Mapped, mapped_column
-
-from app.features.core.models import Base, UTCDateTime, utcnow
-from app.features.notifications.schemas import NotificationRequestMethod, NotificationRequestType
-
-
-class NotificationModel(Base):
-    __tablename__: str = "notifications"
-    __table_args__: tuple[Index, ...] = (
-        Index("ix_notifications_name", "name"),
-        Index("ix_notifications_enabled", "enabled"),
-    )
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
-    name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
-    on: Mapped[list[str]] = mapped_column(JSON, nullable=False, default=list)
-    presets: Mapped[list[str]] = mapped_column(JSON, nullable=False, default=list)
-    enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
-    request_url: Mapped[str] = mapped_column(Text, nullable=False)
-    request_method: Mapped[NotificationRequestMethod] = mapped_column(
-        SQLAEnum(NotificationRequestMethod, name="notification_request_method", native_enum=False),
-        nullable=False,
-        default=NotificationRequestMethod.POST,
-    )
-    request_type: Mapped[NotificationRequestType] = mapped_column(
-        SQLAEnum(NotificationRequestType, name="notification_request_type", native_enum=False),
-        nullable=False,
-        default=NotificationRequestType.JSON,
-    )
-    request_data_key: Mapped[str] = mapped_column(String(255), nullable=False, default="data")
-    request_headers: Mapped[list[dict]] = mapped_column(JSON, nullable=False, default=list)
-    created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, nullable=False)
-    updated_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, onupdate=utcnow, nullable=False)

app/features/notifications/repository.py

@@ -1,181 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Any
-
-from sqlalchemy import func, or_, select
-
-from app.features.core.deps import get_session
-from app.features.notifications.migration import Migration
-from app.features.notifications.models import NotificationModel
-from app.library.log import get_logger
-from app.library.Singleton import Singleton
-
-if TYPE_CHECKING:
-    from collections.abc import Callable
-    from contextlib import AbstractAsyncContextManager
-
-    from sqlalchemy.engine.result import Result
-    from sqlalchemy.ext.asyncio import AsyncSession
-    from sqlalchemy.sql.elements import ColumnElement
-    from sqlalchemy.sql.selectable import Select
-
-    SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
-
-LOG = get_logger()
-
-
-def _model_from_payload(payload: dict[str, Any]) -> NotificationModel:
-    model = NotificationModel()
-    for key, value in payload.items():
-        if not hasattr(model, key):
-            msg = f"'{key}' is an invalid keyword argument for NotificationModel"
-            raise TypeError(msg)
-        setattr(model, key, value)
-    return model
-
-
-def _coerce_model(payload: NotificationModel | dict[str, Any]) -> NotificationModel:
-    if isinstance(payload, NotificationModel):
-        return payload
-    return _model_from_payload(payload)
-
-
-class NotificationsRepository(metaclass=Singleton):
-    def __init__(self, session: SessionFactory | None = None) -> None:
-        self._migrated = False
-        self.session: SessionFactory = session or get_session
-
-    async def run_migrations(self) -> None:
-        if self._migrated:
-            return
-
-        self._migrated = True
-        await Migration(repo=self, config=None).run()
-
-    @staticmethod
-    def get_instance() -> NotificationsRepository:
-        return NotificationsRepository()
-
-    async def all(self) -> list[NotificationModel]:
-        async with self.session() as session:
-            result: Result[tuple[NotificationModel]] = await session.execute(
-                select(NotificationModel).order_by(NotificationModel.name.asc())
-            )
-            return list(result.scalars().all())
-
-    async def list_paginated(self, page: int, per_page: int) -> tuple[list[NotificationModel], int, int, int]:
-        async with self.session() as session:
-            total: int = await self.count()
-            total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1
-
-            if page > total_pages and total > 0:
-                page = total_pages
-
-            query: Select[tuple[NotificationModel]] = (
-                select(NotificationModel)
-                .order_by(NotificationModel.name.asc())
-                .limit(per_page)
-                .offset((page - 1) * per_page)
-            )
-            result: Result[tuple[NotificationModel]] = await session.execute(query)
-            return list(result.scalars().all()), total, page, total_pages
-
-    async def count(self) -> int:
-        async with self.session() as session:
-            result: Result[tuple[int]] = await session.execute(select(func.count()).select_from(NotificationModel))
-            return int(result.scalar_one())
-
-    async def get(self, identifier: int | str) -> NotificationModel | None:
-        async with self.session() as session:
-            if not identifier:
-                return None
-
-            if isinstance(identifier, int):
-                clause: ColumnElement[bool] = NotificationModel.id == identifier
-            elif isinstance(identifier, str) and identifier.isdigit():
-                clause = or_(NotificationModel.id == int(identifier), NotificationModel.name == identifier)
-            else:
-                clause = NotificationModel.name == identifier
-
-            result: Result[tuple[NotificationModel]] = await session.execute(
-                select(NotificationModel).where(clause).limit(1)
-            )
-            return result.scalar_one_or_none()
-
-    async def get_by_name(self, name: str, exclude_id: int | None = None) -> NotificationModel | None:
-        async with self.session() as session:
-            query: Select[tuple[NotificationModel]] = select(NotificationModel).where(NotificationModel.name == name)
-            if exclude_id is not None:
-                query = query.where(NotificationModel.id != exclude_id)
-
-            result: Result[tuple[NotificationModel]] = await session.execute(query.limit(1))
-            return result.scalar_one_or_none()
-
-    async def create(self, payload: NotificationModel | dict[str, Any]) -> NotificationModel:
-        async with self.session() as session:
-            model = _coerce_model(payload)
-            if model.id is not None:
-                model.id = None  # ty: ignore
-
-            if await self.get_by_name(name=model.name) is not None:
-                msg: str = f"Notification target with name '{model.name}' already exists."
-                raise ValueError(msg)
-
-            session.add(model)
-            await session.commit()
-            await session.refresh(model)
-            return model
-
-    async def update(self, identifier: int | str, payload: dict[str, Any]) -> NotificationModel:
-        async with self.session() as session:
-            if isinstance(identifier, int):
-                clause: ColumnElement[bool] = NotificationModel.id == identifier
-            elif isinstance(identifier, str) and identifier.isdigit():
-                clause = or_(NotificationModel.id == int(identifier), NotificationModel.name == identifier)
-            else:
-                clause = NotificationModel.name == identifier
-
-            result: Result[tuple[NotificationModel]] = await session.execute(
-                select(NotificationModel).where(clause).limit(1)
-            )
-            model: NotificationModel | None = result.scalar_one_or_none()
-
-            if not model:
-                msg: str = f"Notification target '{identifier}' not found."
-                raise KeyError(msg)
-
-            payload.pop("id", None)
-
-            if "name" in payload and await self.get_by_name(name=payload["name"], exclude_id=model.id) is not None:
-                msg = f"Notification target with name '{payload['name']}' already exists."
-                raise ValueError(msg)
-
-            for key, value in payload.items():
-                if hasattr(model, key):
-                    setattr(model, key, value)
-
-            await session.commit()
-            await session.refresh(model)
-            return model
-
-    async def delete(self, identifier: int | str) -> NotificationModel:
-        async with self.session() as session:
-            if isinstance(identifier, int):
-                clause: ColumnElement[bool] = NotificationModel.id == identifier
-            elif isinstance(identifier, str) and identifier.isdigit():
-                clause = or_(NotificationModel.id == int(identifier), NotificationModel.name == identifier)
-            else:
-                clause = NotificationModel.name == identifier
-
-            result: Result[tuple[NotificationModel]] = await session.execute(
-                select(NotificationModel).where(clause).limit(1)
-            )
-            model = result.scalar_one_or_none()
-
-            if not model:
-                msg: str = f"Notification target '{identifier}' not found."
-                raise KeyError(msg)
-
-            await session.delete(model)
-            await session.commit()
-            return model

app/features/notifications/router.py

@@ -1,218 +0,0 @@
-from typing import Any
-
-from aiohttp import web
-from aiohttp.web import Request, Response
-from pydantic import ValidationError
-
-from app.features.core.schemas import CEAction, CEFeature, ConfigEvent, Pagination
-from app.features.core.utils import build_pagination, format_validation_errors, normalize_pagination
-from app.features.notifications.schemas import Notification, NotificationEvents, NotificationList, NotificationPatch
-from app.features.notifications.service import Notifications
-from app.library.encoder import Encoder
-from app.library.Events import EventBus, Events
-from app.library.log import get_logger
-from app.library.router import route
-
-LOG = get_logger()
-
-
-def _model(model: Any) -> Notification:
-    return Notifications.get_instance().model_to_schema(model)
-
-
-def _serialize(model: Any) -> dict:
-    return _model(model).model_dump()
-
-
-@route("GET", "api/notifications/", "notifications_list")
-async def notifications_list(request: Request, encoder: Encoder) -> Response:
-    page, per_page = normalize_pagination(request)
-    items, total, current_page, total_pages = await Notifications.get_instance().list_paginated(page, per_page)
-    return web.json_response(
-        data=NotificationList(
-            items=[_model(model) for model in items],
-            pagination=Pagination.model_validate(build_pagination(total, current_page, per_page, total_pages)),
-        ),
-        status=web.HTTPOk.status_code,
-        dumps=encoder.encode,
-    )
-
-
-@route("GET", "api/notifications/events/", "notifications_events")
-async def notifications_events(encoder: Encoder) -> Response:
-    return web.json_response(
-        data={"events": list(NotificationEvents.get_events().values())},
-        status=web.HTTPOk.status_code,
-        dumps=encoder.encode,
-    )
-
-
-@route("POST", "api/notifications/test/", "notification_test")
-async def notification_test(encoder: Encoder, notify: EventBus) -> Response:
-    notify.emit(Events.TEST, title="Test Notification", message="This is a test notification.")
-    return web.json_response(data={}, status=web.HTTPOk.status_code, dumps=encoder.encode)
-
-
-@route("POST", "api/notifications/", "notification_add")
-async def notifications_add(request: Request, encoder: Encoder, notify: EventBus) -> Response:
-    data = await request.json()
-
-    if not isinstance(data, dict):
-        return web.json_response(
-            {"error": "Invalid request body expecting dict."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    try:
-        item: Notification = Notification.model_validate(data)
-    except ValidationError as exc:
-        return web.json_response(
-            data={"error": "Failed to validate notification.", "detail": format_validation_errors(exc)},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    try:
-        saved = _serialize(await Notifications.get_instance().create(item))
-    except ValueError as exc:
-        return web.json_response({"error": str(exc)}, status=web.HTTPBadRequest.status_code)
-
-    notify.emit(
-        Events.CONFIG_UPDATE,
-        data=ConfigEvent(feature=CEFeature.NOTIFICATIONS, action=CEAction.CREATE, data=saved),
-    )
-    return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode)
-
-
-@route("GET", r"api/notifications/{id:\d+}", "notification_get")
-async def notifications_get(request: Request, encoder: Encoder) -> Response:
-    if not (identifier := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    if not (model := await Notifications.get_instance().get(identifier)):
-        return web.json_response({"error": "Notification not found"}, status=web.HTTPNotFound.status_code)
-
-    return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode)
-
-
-@route("DELETE", r"api/notifications/{id:\d+}", "notification_delete")
-async def notifications_delete(request: Request, encoder: Encoder, notify: EventBus) -> Response:
-    if not (identifier := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    try:
-        deleted = _serialize(await Notifications.get_instance().delete(identifier))
-        notify.emit(
-            Events.CONFIG_UPDATE,
-            data=ConfigEvent(feature=CEFeature.NOTIFICATIONS, action=CEAction.DELETE, data=deleted),
-        )
-        return web.json_response(
-            data=deleted,
-            status=web.HTTPOk.status_code,
-            dumps=encoder.encode,
-        )
-    except KeyError as exc:
-        return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code)
-
-
-@route("PATCH", r"api/notifications/{id:\d+}", "notification_patch")
-async def notifications_patch(request: Request, encoder: Encoder, notify: EventBus) -> Response:
-    if not (identifier := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    if not (model := await Notifications.get_instance().get(identifier)):
-        return web.json_response({"error": "Notification not found"}, status=web.HTTPNotFound.status_code)
-
-    data = await request.json()
-
-    if not isinstance(data, dict):
-        return web.json_response(
-            {"error": "Invalid request body expecting dict."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    try:
-        validated = NotificationPatch.model_validate(data)
-    except ValidationError as exc:
-        return web.json_response(
-            data={"error": "Failed to validate notification.", "detail": format_validation_errors(exc)},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    current = _model(model).model_dump()
-    if validated.name is not None:
-        current["name"] = validated.name
-    if validated.on is not None:
-        current["on"] = validated.on
-    if validated.presets is not None:
-        current["presets"] = validated.presets
-    if validated.enabled is not None:
-        current["enabled"] = validated.enabled
-
-    if validated.request is not None:
-        request_payload = current.get("request", {})
-        if validated.request.url is not None:
-            request_payload["url"] = validated.request.url
-        if validated.request.method is not None:
-            request_payload["method"] = validated.request.method
-        if validated.request.type is not None:
-            request_payload["type"] = validated.request.type
-        if validated.request.headers is not None:
-            request_payload["headers"] = [header.model_dump() for header in validated.request.headers]
-        if validated.request.data_key is not None:
-            request_payload["data_key"] = validated.request.data_key
-        current["request"] = request_payload
-
-    try:
-        updated = _serialize(await Notifications.get_instance().update(model.id, current))
-    except ValueError as exc:
-        return web.json_response({"error": str(exc)}, status=web.HTTPBadRequest.status_code)
-    except KeyError as exc:
-        return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code)
-
-    notify.emit(
-        Events.CONFIG_UPDATE,
-        data=ConfigEvent(feature=CEFeature.NOTIFICATIONS, action=CEAction.UPDATE, data=updated),
-    )
-    return web.json_response(
-        data=updated,
-        status=web.HTTPOk.status_code,
-        dumps=encoder.encode,
-    )
-
-
-@route("PUT", r"api/notifications/{id:\d+}", "notification_update")
-async def notifications_update(request: Request, encoder: Encoder, notify: EventBus) -> Response:
-    if not (identifier := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    if not (model := await Notifications.get_instance().get(identifier)):
-        return web.json_response({"error": "Notification not found"}, status=web.HTTPNotFound.status_code)
-
-    data = await request.json()
-
-    if not isinstance(data, dict):
-        return web.json_response(
-            {"error": "Invalid request body expecting dict."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    try:
-        validated = Notification.model_validate(data)
-    except ValidationError as exc:
-        return web.json_response(
-            data={"error": "Failed to validate notification.", "detail": format_validation_errors(exc)},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    try:
-        updated = _serialize(await Notifications.get_instance().update(model.id, validated))
-    except ValueError as exc:
-        return web.json_response({"error": str(exc)}, status=web.HTTPBadRequest.status_code)
-    except KeyError as exc:
-        return web.json_response({"error": str(exc)}, status=web.HTTPNotFound.status_code)
-
-    notify.emit(
-        Events.CONFIG_UPDATE,
-        data=ConfigEvent(feature=CEFeature.NOTIFICATIONS, action=CEAction.UPDATE, data=updated),
-    )
-    return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode)

app/features/notifications/schemas.py

@@ -1,146 +0,0 @@
-from __future__ import annotations
-
-from enum import Enum
-from typing import Any
-
-from pydantic import BaseModel, ConfigDict, Field, field_validator
-
-from app.features.core.schemas import Pagination
-from app.library.Events import Events
-
-
-class NotificationEvents:
-    TEST: str = Events.TEST
-
-    ITEM_ADDED: str = Events.ITEM_ADDED
-    ITEM_COMPLETED: str = Events.ITEM_COMPLETED
-    ITEM_CANCELLED: str = Events.ITEM_CANCELLED
-    ITEM_DELETED: str = Events.ITEM_DELETED
-    ITEM_BULK_DELETED: str = Events.ITEM_BULK_DELETED
-    ITEM_PAUSED: str = Events.ITEM_PAUSED
-    ITEM_RESUMED: str = Events.ITEM_RESUMED
-    ITEM_MOVED: str = Events.ITEM_MOVED
-
-    PAUSED: str = Events.PAUSED
-    RESUMED: str = Events.RESUMED
-
-    LOG_INFO: str = Events.LOG_INFO
-    LOG_SUCCESS: str = Events.LOG_SUCCESS
-    LOG_WARNING: str = Events.LOG_WARNING
-    LOG_ERROR: str = Events.LOG_ERROR
-
-    TASK_DISPATCHED: str = Events.TASK_DISPATCHED
-
-    @staticmethod
-    def get_events() -> dict[str, str]:
-        return {k: v for k, v in vars(NotificationEvents).items() if not k.startswith("__") and not callable(v)}
-
-    @staticmethod
-    def events() -> list[str]:
-        return [
-            getattr(NotificationEvents, ev)
-            for ev in dir(NotificationEvents)
-            if not ev.startswith("_") and not callable(getattr(NotificationEvents, ev))
-        ]
-
-    @staticmethod
-    def is_valid(event: str) -> bool:
-        return event in NotificationEvents.get_events().values()
-
-
-class NotificationRequestType(str, Enum):
-    JSON = "json"
-    FORM = "form"
-
-    def __str__(self) -> str:
-        return self.value
-
-
-class NotificationRequestMethod(str, Enum):
-    POST = "POST"
-    PUT = "PUT"
-
-    def __str__(self) -> str:
-        return self.value
-
-
-class NotificationRequestHeader(BaseModel):
-    model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True)
-
-    key: str = Field(min_length=1)
-    value: str = Field(min_length=1)
-
-
-class NotificationRequest(BaseModel):
-    model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True)
-
-    type: NotificationRequestType = NotificationRequestType.JSON
-    method: NotificationRequestMethod = NotificationRequestMethod.POST
-    url: str = Field(min_length=1)
-    headers: list[NotificationRequestHeader] = Field(default_factory=list)
-    data_key: str = Field(default="data", min_length=1)
-
-    @field_validator("method", mode="before")
-    @classmethod
-    def _normalize_method(cls, value: Any) -> Any:
-        if value is None:
-            return value
-        if isinstance(value, NotificationRequestMethod):
-            return value
-        return str(value).upper()
-
-    @field_validator("type", mode="before")
-    @classmethod
-    def _normalize_type(cls, value: Any) -> Any:
-        if value is None:
-            return value
-        if isinstance(value, NotificationRequestType):
-            return value
-        return str(value).lower()
-
-
-class NotificationRequestPatch(BaseModel):
-    model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True)
-
-    type: NotificationRequestType | None = None
-    method: NotificationRequestMethod | None = None
-    url: str | None = None
-    headers: list[NotificationRequestHeader] | None = None
-    data_key: str | None = None
-
-
-class Notification(BaseModel):
-    model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True)
-
-    id: int | None = None
-    name: str = Field(min_length=1)
-    on: list[str] = Field(default_factory=list)
-    presets: list[str] = Field(default_factory=list)
-    enabled: bool = True
-    request: NotificationRequest
-
-    @field_validator("on", "presets", mode="before")
-    @classmethod
-    def _normalize_list(cls, value: Any) -> list[str]:
-        if value is None:
-            return []
-        if isinstance(value, list):
-            return [str(item).strip() for item in value if str(item).strip()]
-        return [str(value).strip()]
-
-
-class NotificationPatch(BaseModel):
-    model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True)
-
-    name: str | None = None
-    on: list[str] | None = None
-    presets: list[str] | None = None
-    enabled: bool | None = None
-    request: NotificationRequestPatch | None = None
-
-
-class NotificationList(BaseModel):
-    model_config = ConfigDict(from_attributes=True)
-
-    items: list[Notification] = Field(default_factory=list)
-    pagination: Pagination

app/features/notifications/service.py

@@ -1,363 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-from pathlib import Path
-from typing import TYPE_CHECKING, Any, cast
-
-from app.features.notifications.models import NotificationModel
-from app.features.notifications.repository import NotificationsRepository
-from app.features.notifications.schemas import (
-    Notification,
-    NotificationEvents,
-    NotificationRequest,
-    NotificationRequestHeader,
-    NotificationRequestType,
-)
-from app.features.presets.schemas import Preset
-from app.features.presets.service import Presets
-from app.library.BackgroundWorker import BackgroundWorker
-from app.library.config import Config
-from app.library.encoder import Encoder
-from app.library.Events import Event, EventBus, Events
-from app.library.httpx_client import async_client
-from app.library.ItemDTO import Item, ItemDTO
-from app.library.log import get_logger
-from app.library.Singleton import Singleton
-
-if TYPE_CHECKING:
-    from collections.abc import Awaitable
-
-    import httpx
-    from aiohttp import web
-
-LOG = get_logger()
-
-
-class Notifications(metaclass=Singleton):
-    def __init__(
-        self,
-        repo: NotificationsRepository | None = None,
-        client: httpx.AsyncClient | None = None,
-        encoder: Encoder | None = None,
-        config: Config | None = None,
-        background_worker: BackgroundWorker | None = None,
-    ) -> None:
-        self._repo: NotificationsRepository = repo or NotificationsRepository.get_instance()
-
-        config = config or Config.get_instance()
-
-        self._debug: bool = config.debug
-        self._version: str = config.app_version
-        self._client: httpx.AsyncClient = client or async_client()
-        self._encoder: Encoder = encoder or Encoder()
-        self._offload: BackgroundWorker = background_worker or BackgroundWorker.get_instance()
-
-    @staticmethod
-    def get_instance() -> Notifications:
-        return Notifications()
-
-    async def on_shutdown(self, _: web.Application) -> None:
-        pass
-
-    def attach(self, _: web.Application) -> None:
-        async def handle_event(_, __):
-            await self._repo.run_migrations()
-
-        EventBus.get_instance().subscribe(Events.STARTED, handle_event, "NotificationsRepository.run_migrations")
-        EventBus.get_instance().subscribe(NotificationEvents.events(), self.emit, f"{type(self).__name__}.emit")
-
-    async def all(self) -> list[NotificationModel]:
-        return await self._repo.all()
-
-    async def list_paginated(self, page: int, per_page: int) -> tuple[list[NotificationModel], int, int, int]:
-        return await self._repo.list_paginated(page, per_page)
-
-    async def get(self, identifier: int | str) -> NotificationModel | None:
-        return await self._repo.get(identifier)
-
-    async def create(self, item: Notification | dict) -> NotificationModel:
-        if not isinstance(item, Notification):
-            item = Notification.model_validate(item)
-        normalized = self._normalize(item)
-        payload = self._payload_from_schema(normalized)
-        return await self._repo.create(payload)
-
-    async def update(self, identifier: int | str, payload: Notification | dict) -> NotificationModel:
-        if not isinstance(payload, Notification):
-            payload = Notification.model_validate(payload)
-        normalized = self._normalize(payload)
-        update_payload = self._payload_from_schema(normalized)
-        return await self._repo.update(identifier, update_payload)
-
-    async def delete(self, identifier: int | str) -> NotificationModel:
-        return await self._repo.delete(identifier)
-
-    async def send(self, ev: Event, wait: bool = True) -> list[dict | Awaitable[dict]]:
-        targets = await self._repo.all()
-        if len(targets) < 1:
-            return []
-
-        if not isinstance(ev.data, (ItemDTO, Item, dict)):
-            LOG.debug("Received invalid item type '%s' with event '%s'.", type(ev.data), ev.event)
-            return []
-
-        tasks: list[Awaitable[dict]] = []
-        apprise_targets: list[Notification] = []
-
-        for target_model in targets:
-            target = self.model_to_schema(target_model)
-            if not target.enabled:
-                continue
-
-            if len(target.on) > 0 and ev.event not in target.on and NotificationEvents.TEST != ev.event:
-                continue
-
-            if NotificationEvents.TEST != ev.event and not self._check_preset(target, ev):
-                continue
-
-            if not target.request.url.startswith("http"):
-                apprise_targets.append(target)
-            else:
-                tasks.append(self._send(target, ev))
-
-        if len(apprise_targets) > 0:
-            tasks.append(self._apprise(apprise_targets, ev))
-
-        if wait:
-            return await asyncio.gather(*tasks)
-
-        return cast("list[dict | Awaitable[dict]]", tasks)
-
-    def emit(self, e: Event, _, **__) -> None:
-        if not NotificationEvents.is_valid(e.event):
-            return
-
-        self._offload.submit(self.send, e)
-
-    def _normalize(self, item: Notification) -> Notification:
-        if item.enabled is not None and not isinstance(item.enabled, bool):
-            msg: str = "Enabled must be a boolean."
-            raise ValueError(msg)
-
-        item.on = self._filter_events(item.on)
-        item.presets = self._filter_presets(item.presets)
-        item.request.headers = [
-            NotificationRequestHeader(key=header.key, value=header.value)
-            for header in item.request.headers
-            if header.key and header.value
-        ]
-        return item
-
-    def _filter_events(self, events: list[str]) -> list[str]:
-        if not events:
-            return []
-
-        allowed = set(NotificationEvents.get_events().values())
-        valid = [event for event in events if event in allowed]
-        invalid = [event for event in events if event not in allowed]
-
-        if len(invalid) > 0 and len(valid) < 1:
-            msg: str = f"Invalid notification target. Invalid events '{', '.join(invalid)}' found."
-            raise ValueError(msg)
-
-        if len(invalid) > 0:
-            LOG.warning("Dropping invalid notification events: %s", ", ".join(invalid))
-
-        return valid
-
-    def _filter_presets(self, presets: list[str]) -> list[str]:
-        if not presets:
-            return []
-
-        all_presets: list[Preset] = Presets.get_instance().get_all()
-        allowed = {preset.name for preset in all_presets}
-        valid = [preset for preset in presets if preset in allowed]
-        invalid = [preset for preset in presets if preset not in allowed]
-
-        if len(invalid) > 0 and len(valid) < 1:
-            msg: str = f"Invalid notification target. Invalid presets '{', '.join(invalid)}' found."
-            raise ValueError(msg)
-
-        if len(invalid) > 0:
-            LOG.warning("Dropping invalid notification presets: %s", ", ".join(invalid))
-
-        return valid
-
-    def model_to_schema(self, model: NotificationModel) -> Notification:
-        headers: list[NotificationRequestHeader] = []
-        if isinstance(model.request_headers, list):
-            for header in model.request_headers:
-                if not isinstance(header, dict):
-                    continue
-                key = str(header.get("key", "")).strip()
-                value = str(header.get("value", "")).strip()
-                if key and value:
-                    headers.append(NotificationRequestHeader(key=key, value=value))
-
-        return Notification(
-            id=model.id,
-            name=model.name,
-            on=list(model.on or []),
-            presets=list(model.presets or []),
-            enabled=model.enabled,
-            request=NotificationRequest.model_validate(
-                {
-                    "type": model.request_type,
-                    "method": model.request_method,
-                    "url": model.request_url,
-                    "headers": headers,
-                    "data_key": model.request_data_key,
-                }
-            ),
-        )
-
-    def _payload_from_schema(self, item: Notification) -> dict[str, Any]:
-        return {
-            "name": item.name,
-            "on": list(item.on),
-            "presets": list(item.presets),
-            "enabled": item.enabled,
-            "request_url": item.request.url,
-            "request_method": str(item.request.method),
-            "request_type": str(item.request.type),
-            "request_data_key": item.request.data_key,
-            "request_headers": [header.model_dump() for header in item.request.headers],
-        }
-
-    def _check_preset(self, target: Notification, ev: Event) -> bool:
-        if len(target.presets) < 1:
-            return True
-
-        if not isinstance(ev.data, (Item, ItemDTO, dict)):
-            return False
-
-        preset_name: str | None = None
-
-        if isinstance(ev.data, Item):
-            preset_name = ev.data.preset
-
-        if isinstance(ev.data, ItemDTO):
-            preset_name = ev.data.preset
-
-        if isinstance(ev.data, dict):
-            preset_name = ev.data.get("preset", None)
-
-        if not preset_name:
-            return False
-
-        return preset_name in target.presets
-
-    async def _apprise(self, targets: list[Notification], ev: Event) -> dict:
-        if not targets:
-            return {}
-
-        import apprise
-
-        try:
-            notify = apprise.Apprise(debug=self._debug)
-            apr_config = Path(Config.get_instance().apprise_config)
-            if apr_config.exists():
-                apprise_file = apprise.AppriseConfig()
-                apprise_file.add(str(apr_config))
-                notify.add(apprise_file)
-
-            for target in targets:
-                notify.add(target.request.url)
-
-            status = await notify.async_notify(
-                body=ev.message or self._encoder.encode(ev.serialize()),
-                title=ev.title or f"YTPTube Event: {ev.event}",
-            )
-
-            if not status:
-                msg = "Apprise failed to send notification."
-                self._raise_apprise_error(msg)
-        except Exception as exc:
-            LOG.exception(
-                "Failed to send Apprise notification for event '%s'.",
-                ev.event,
-                extra={
-                    "event_id": ev.id,
-                    "event": ev.event,
-                    "target_count": len(targets),
-                    "targets": [t.name for t in targets],
-                    "exception_type": type(exc).__name__,
-                },
-            )
-            return {"error": str(exc), "event": ev.event, "id": ev.id, "targets": [t.name for t in targets]}
-
-        return {}
-
-    @staticmethod
-    def _raise_apprise_error(msg: str) -> None:
-        raise RuntimeError(msg)
-
-    async def _send(self, target: Notification, ev: Event) -> dict:
-        try:
-            LOG.info("Sending notification event '%s: %s' to '%s'.", ev.event, ev.id, target.name)
-
-            headers: dict[str, str] = {
-                "User-Agent": f"YTPTube/{self._version}",
-                "X-Event-Id": ev.id,
-                "X-Event": ev.event,
-                "Content-Type": "application/json"
-                if NotificationRequestType.JSON == target.request.type
-                else "application/x-www-form-urlencoded",
-            }
-
-            if len(target.request.headers) > 0:
-                headers.update({h.key: h.value for h in target.request.headers if h.key and h.value})
-
-            payload_data: dict[str, Any] = ev.serialize()
-
-            if "data" != target.request.data_key:
-                payload_data[target.request.data_key] = payload_data["data"]
-                payload_data.pop("data", None)
-
-            if NotificationRequestType.FORM == target.request.type:
-                payload_data[target.request.data_key] = self._encoder.encode(payload_data[target.request.data_key])
-                data_payload: dict[str, Any] | None = payload_data
-                content_payload: str | None = None
-            else:
-                data_payload = None
-                content_payload = self._encoder.encode(payload_data)
-
-            response = await self._client.request(
-                method=str(target.request.method).upper(),
-                url=target.request.url,
-                headers=headers,
-                data=data_payload,
-                content=content_payload,
-            )
-
-            resp_data: dict[str, Any] = {
-                "url": target.request.url,
-                "status": response.status_code,
-                "text": response.text,
-            }
-
-            msg: str = (
-                f"Notification target '{target.name}' Responded to event '{ev.event}: {ev.id}' "
-                f"with status '{response.status_code}'."
-            )
-            if self._debug and resp_data.get("text"):
-                msg += f" body '{resp_data.get('text', '??')}'."
-
-            LOG.info(msg)
-
-            return resp_data
-        except Exception as exc:
-            LOG.exception(
-                "Failed to send notification event '%s: %s' to '%s'.",
-                ev.event,
-                ev.id,
-                target.name,
-                extra={
-                    "event_id": ev.id,
-                    "event": ev.event,
-                    "target_name": target.name,
-                    "url": target.request.url,
-                    "exception_type": type(exc).__name__,
-                },
-            )
-            return {"url": target.request.url, "status": 500, "text": str(ev)}

app/features/notifications/tests/test_notifications_repository.py

@@ -1,152 +0,0 @@
-"""Tests for NotificationsRepository."""
-
-from __future__ import annotations
-
-import pytest
-import pytest_asyncio
-
-from app.features.notifications.repository import NotificationsRepository
-from app.library.sqlite_store import SqliteStore
-from app.tests.helpers import make_in_memory_db_path
-
-
-@pytest_asyncio.fixture
-async def repo():
-    """Provide a fresh repository instance with initialized database for each test."""
-    NotificationsRepository._reset_singleton()
-    SqliteStore._reset_singleton()
-
-    store = SqliteStore(db_path=make_in_memory_db_path("notifications-repository"))
-    await store.get_connection()
-
-    repository = NotificationsRepository.get_instance()
-
-    yield repository
-
-    await store.close()
-
-    NotificationsRepository._reset_singleton()
-    SqliteStore._reset_singleton()
-
-
-class TestNotificationsRepository:
-    """Test suite for NotificationsRepository database operations."""
-
-    @pytest.mark.asyncio
-    async def test_list_empty(self, repo):
-        """List returns empty when no notifications exist."""
-        notifications = await repo.all()
-        assert notifications == [], "Should return empty list when no notifications exist"
-
-    @pytest.mark.asyncio
-    async def test_create_notification(self, repo):
-        """Create notification with valid data."""
-        payload = {
-            "name": "Webhook",
-            "on": ["item_completed"],
-            "presets": [],
-            "enabled": True,
-            "request_url": "https://example.com/webhook",
-            "request_method": "POST",
-            "request_type": "json",
-            "request_data_key": "data",
-            "request_headers": [{"key": "Authorization", "value": "Bearer token"}],
-        }
-
-        model = await repo.create(payload)
-
-        assert model.id is not None, "Should generate ID for new notification"
-        assert model.name == "Webhook", "Should store name correctly"
-        assert model.request_url == "https://example.com/webhook", "Should store request url"
-        assert model.request_method == "POST", "Should store request method"
-
-    @pytest.mark.asyncio
-    async def test_get_by_id(self, repo):
-        """Get notification by integer ID."""
-        created = await repo.create(
-            {
-                "name": "Get Test",
-                "on": [],
-                "presets": [],
-                "enabled": True,
-                "request_url": "https://example.com",
-                "request_method": "POST",
-                "request_type": "json",
-                "request_data_key": "data",
-                "request_headers": [],
-            }
-        )
-
-        retrieved = await repo.get(created.id)
-
-        assert retrieved is not None, "Should retrieve created notification"
-        assert retrieved.id == created.id, "Should match created notification id"
-
-    @pytest.mark.asyncio
-    async def test_update_notification(self, repo):
-        """Update existing notification."""
-        created = await repo.create(
-            {
-                "name": "Update Test",
-                "on": [],
-                "presets": [],
-                "enabled": True,
-                "request_url": "https://example.com",
-                "request_method": "POST",
-                "request_type": "json",
-                "request_data_key": "data",
-                "request_headers": [],
-            }
-        )
-
-        updated = await repo.update(created.id, {"name": "Updated Name", "enabled": False})
-
-        assert updated.name == "Updated Name", "Should update name"
-        assert updated.enabled is False, "Should update enabled flag"
-
-    @pytest.mark.asyncio
-    async def test_delete_notification(self, repo):
-        """Delete existing notification."""
-        created = await repo.create(
-            {
-                "name": "Delete Test",
-                "on": [],
-                "presets": [],
-                "enabled": True,
-                "request_url": "https://example.com",
-                "request_method": "POST",
-                "request_type": "json",
-                "request_data_key": "data",
-                "request_headers": [],
-            }
-        )
-
-        deleted = await repo.delete(created.id)
-
-        assert deleted.id == created.id, "Should return deleted notification"
-        assert await repo.get(created.id) is None, "Deleted notification should not be retrievable"
-
-    @pytest.mark.asyncio
-    async def test_list_paginated(self, repo):
-        """List paginated returns correct subset."""
-        for i in range(4):
-            await repo.create(
-                {
-                    "name": f"Item {i}",
-                    "on": [],
-                    "presets": [],
-                    "enabled": True,
-                    "request_url": "https://example.com",
-                    "request_method": "POST",
-                    "request_type": "json",
-                    "request_data_key": "data",
-                    "request_headers": [],
-                }
-            )
-
-        items, total, page, total_pages = await repo.list_paginated(page=1, per_page=2)
-
-        assert len(items) == 2, "Should return 2 items per page"
-        assert total == 4, "Should report total count of 4"
-        assert page == 1, "Should be on page 1"
-        assert total_pages == 2, "Should have 2 pages total"

app/features/presets/__init__.py

@@ -1 +0,0 @@
-"""Presets feature module."""

app/features/presets/defaults.py

@@ -1,94 +0,0 @@
-from datetime import UTC, datetime
-
-DEFAULT_PRESET_UPDATED_AT: datetime = datetime(2026, 1, 25, tzinfo=UTC)
-
-DEFAULT_PRESETS: list[dict[str, object]] = [
-    {
-        "name": "default",
-        "default": True,
-        "cli": "--socket-timeout 30 --download-archive %(archive_file)s",
-        "description": "Default preset for yt-dlp. It will download whatever yt-dlp decides is the best quality for the video and audio.",
-        "folder": "",
-        "template": "",
-        "cookies": "",
-        "priority": 0,
-        "updated_at": DEFAULT_PRESET_UPDATED_AT,
-    },
-    {
-        "name": "mobile",
-        "cli": '--socket-timeout 30 --download-archive %(archive_file)s\n-t mp4 --merge-output-format mp4 --add-chapters --remux-video mp4 \n--embed-metadata --embed-thumbnail \n--postprocessor-args "-movflags +faststart"',
-        "default": True,
-        "description": "This preset is designed for mobile devices. It will download the best quality video and audio in mp4 format, merge them, and add chapters, metadata, and thumbnail.",
-        "folder": "",
-        "template": "",
-        "cookies": "",
-        "priority": 0,
-        "updated_at": DEFAULT_PRESET_UPDATED_AT,
-    },
-    {
-        "name": "1080p",
-        "cli": "--socket-timeout 30 --download-archive %(archive_file)s\n-S vcodec:h264 --format 'bv[height<=1080][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]'",
-        "default": True,
-        "description": "Download the best quality video and audio in mp4 format for 1080p resolution.",
-        "folder": "",
-        "template": "",
-        "cookies": "",
-        "priority": 0,
-        "updated_at": DEFAULT_PRESET_UPDATED_AT,
-    },
-    {
-        "name": "720p",
-        "cli": "--socket-timeout 30 --download-archive %(archive_file)s\n-S vcodec:h264 --format 'bv[height<=720][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]'",
-        "default": True,
-        "description": "Download the best quality video and audio in mp4 format for 720p resolution.",
-        "folder": "",
-        "template": "",
-        "cookies": "",
-        "priority": 0,
-        "updated_at": DEFAULT_PRESET_UPDATED_AT,
-    },
-    {
-        "name": "audio_only",
-        "cli": "--socket-timeout 30 --download-archive %(archive_file)s\n--extract-audio --add-chapters --embed-metadata --embed-thumbnail --format 'bestaudio/best'",
-        "default": True,
-        "description": "This preset is designed to download only the audio of the video. It will extract the audio, add chapters, metadata, and thumbnail.",
-        "folder": "",
-        "template": "",
-        "cookies": "",
-        "priority": 0,
-        "updated_at": DEFAULT_PRESET_UPDATED_AT,
-    },
-    {
-        "name": "info_reader_plugin",
-        "description": 'This preset generate specific filename format and metadata to work with yt-dlp info reader plugins for jellyfin/emby and plex and to make play state sync work for WatchState.\n\nThere is one more step you need to do via Other > Terminal if you have it enabled or directly from container shell\n\nyt-dlp -I0 --write-info-json --write-thumbnail --convert-thumbnails jpg --paths /downloads/youtube -o "%(channel|Unknown_title)s [%(channel_id|Unknown_id)s]/%(title).180B [%(channel_id|Unknown_id)s].%(ext)s" -- https://www.youtube.com/channel/UCClfFsWcT3N2I7VTXXyt84A\n\nChange the url to the channel you want to download.\n\nFor more information please visit \nhttps://github.com/arabcoders/watchstate/blob/master/FAQ.md#how-to-get-watchstate-working-with-youtube-contentlibrary',
-        "folder": "",
-        "template": "%(channel)s [%(channel_id|Unknown_id)s]/Season %(release_date>%Y,upload_date>%Y|Unknown)s/%(release_date>%Y%m%d,upload_date>%Y%m%d)s - %(title).180B [%(id)s].%(ext)s",
-        "cli": "--socket-timeout 30 --download-archive %(archive_file)s\n--windows-filenames --write-info-json --embed-info-json \n--convert-thumbnails jpg --write-thumbnail --embed-metadata",
-        "cookies": "",
-        "default": True,
-        "priority": 0,
-        "updated_at": DEFAULT_PRESET_UPDATED_AT,
-    },
-    {
-        "name": "nfo_maker_tv",
-        "description": "This preset generate specific filename format and metadata to work with media servers like jellyfin/emby and plex for episodes.",
-        "folder": "",
-        "template": "%(channel)s [%(channel_id|Unknown_id)s]/Season %(release_date>%Y,upload_date>%Y|Unknown)s/S%(release_date>%Y,upload_date>%Y)sE%(release_date>%m%d,upload_date>%m%d)s - %(title).100B [%(id)s].%(ext)s",
-        "cli": '--socket-timeout 30 --download-archive %(archive_file)s\n--use-postprocessor "NFOMakerPP:when=after_move" \n--windows-filenames --convert-thumbnails jpg --write-thumbnail',
-        "cookies": "",
-        "default": True,
-        "priority": 0,
-        "updated_at": DEFAULT_PRESET_UPDATED_AT,
-    },
-    {
-        "name": "nfo_maker_movie",
-        "description": "This preset generate specific filename format and metadata to work with media servers like jellyfin/emby and plex for movies.",
-        "folder": "",
-        "template": "%(channel)s [%(channel_id|Unknown_id)s]/Season %(release_date>%Y,upload_date>%Y|Unknown)s/S%(release_date>%Y,upload_date>%Y)sE%(release_date>%m%d,upload_date>%m%d)s - %(title).100B [%(id)s].%(ext)s",
-        "cli": '--socket-timeout 30 --download-archive %(archive_file)s\n--use-postprocessor "NFOMakerPP:when=after_move;mode=movie" \n--windows-filenames --convert-thumbnails jpg --write-thumbnail',
-        "cookies": "",
-        "default": True,
-        "priority": 0,
-        "updated_at": DEFAULT_PRESET_UPDATED_AT,
-    },
-]

app/features/presets/deps.py

@@ -1,10 +0,0 @@
-from app.features.presets.repository import PresetsRepository
-from app.features.presets.service import Presets
-
-
-def get_presets_repo() -> PresetsRepository:
-    return PresetsRepository.get_instance()
-
-
-def get_presets_service() -> Presets:
-    return Presets.get_instance()

app/features/presets/migration.py

@@ -1,121 +0,0 @@
-from __future__ import annotations
-
-import json
-from pathlib import Path
-from typing import TYPE_CHECKING, Any
-
-from app.features.core.migration import Migration as FeatureMigration
-from app.features.presets.schemas import Preset
-from app.features.presets.utils import preset_name
-from app.library.config import Config
-from app.library.log import get_logger
-
-if TYPE_CHECKING:
-    from app.features.presets.repository import PresetsRepository
-
-LOG = get_logger()
-
-
-class Migration(FeatureMigration):
-    name: str = "presets"
-
-    def __init__(self, repo: PresetsRepository, config: Config | None = None) -> None:
-        self._config: Config = config or Config.get_instance()
-        super().__init__(config=self._config)
-        self._repo: PresetsRepository = repo
-        self._source_file: Path = Path(self._config.config_path) / "presets.json"
-
-    async def should_run(self) -> bool:
-        return self._source_file.exists()
-
-    async def migrate(self) -> None:
-        try:
-            items: list[dict[str, Any]] | None = json.loads(self._source_file.read_text())
-        except Exception as exc:
-            LOG.exception(
-                "Failed to read presets migration file '%s'. Ignoring.",
-                self._source_file,
-                extra={"file_path": str(self._source_file), "exception_type": type(exc).__name__},
-            )
-            await self._move_file(self._source_file)
-            return
-
-        if not items:
-            LOG.warning("No presets found in %s; skipping migration.", self._source_file)
-            await self._move_file(self._source_file)
-            return
-
-        inserted = 0
-        seen_names: dict[str, int] = {preset_name(preset.name): 1 for preset in await self._repo.all()}
-
-        for index, item in enumerate(items):
-            if not (normalized := self._normalize(item, index, seen_names)):
-                continue
-            try:
-                await self._repo.create(normalized)
-                inserted += 1
-            except Exception as exc:
-                LOG.exception(
-                    "Failed to insert preset '%s'.",
-                    normalized["name"],
-                    extra={"preset": normalized["name"], "exception_type": type(exc).__name__},
-                )
-
-        LOG.info("Migrated %s preset(s) from %s.", inserted, self._source_file)
-        await self._move_file(self._source_file)
-
-    def _normalize(self, item: Any, index: int, seen_names: dict[str, int]) -> dict[str, Any] | None:
-        if not isinstance(item, dict):
-            LOG.warning("Skipping preset at index %s due to invalid type.", index)
-            return None
-
-        name: str | None = item.get("name")
-        if not name or not isinstance(name, str):
-            LOG.warning("Skipping preset at index %s due to missing name.", index)
-            return None
-
-        if not (name := preset_name(name)):
-            LOG.warning("Skipping preset at index %s due to empty name.", index)
-            return None
-
-        name = self._unique_name(name, seen_names)
-
-        description: str = item.get("description") if isinstance(item.get("description"), str) else ""
-        folder: str = item.get("folder") if isinstance(item.get("folder"), str) else ""
-        template: str = item.get("template") if isinstance(item.get("template"), str) else ""
-        cookies: str = item.get("cookies") if isinstance(item.get("cookies"), str) else ""
-        cli: str = item.get("cli") if isinstance(item.get("cli"), str) else ""
-        priority = 0
-        if isinstance(item.get("priority"), int) and not isinstance(item.get("priority"), bool):
-            priority = int(item.get("priority"))
-
-        if not cli and isinstance(item.get("format"), str):
-            cli = f"--format {item.get('format')}"
-
-        if cli:
-            try:
-                from app.features.ytdlp.utils import arg_converter
-
-                arg_converter(args=cli, level=True)
-            except Exception as exc:
-                LOG.warning("Skipping preset '%s' due to invalid CLI: %s", name, exc)
-                return None
-
-        payload = {
-            "name": name,
-            "description": description,
-            "folder": folder,
-            "template": template,
-            "cookies": cookies,
-            "cli": cli,
-            "default": False,
-            "priority": priority,
-        }
-
-        try:
-            Preset.model_validate(payload)
-        except Exception as exc:
-            LOG.warning("Skipping preset '%s' due to validation error: %s", name, exc)
-            return None
-
-        return payload

app/features/presets/models.py

@@ -1,27 +0,0 @@
-from datetime import datetime
-
-from sqlalchemy import Boolean, Index, Integer, String, Text
-from sqlalchemy.orm import Mapped, mapped_column
-
-from app.features.core.models import Base, UTCDateTime, utcnow
-
-
-class PresetModel(Base):
-    __tablename__: str = "presets"
-    __table_args__: tuple[Index, ...] = (
-        Index("ix_presets_name", "name"),
-        Index("ix_presets_is_default", "is_default"),
-        Index("ix_presets_priority", "priority"),
-    )
-
-    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
-    name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
-    description: Mapped[str] = mapped_column(Text, nullable=False, default="")
-    folder: Mapped[str] = mapped_column(Text, nullable=False, default="")
-    template: Mapped[str] = mapped_column(Text, nullable=False, default="")
-    cookies: Mapped[str] = mapped_column(Text, nullable=False, default="")
-    cli: Mapped[str] = mapped_column(Text, nullable=False, default="")
-    default: Mapped[bool] = mapped_column("is_default", Boolean, nullable=False, default=False)
-    priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-    created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, nullable=False)
-    updated_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow, onupdate=utcnow, nullable=False)

app/features/presets/repository.py

@@ -1,334 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING, Any
-
-from sqlalchemy import func, or_, select
-
-from app.features.core.deps import get_session
-from app.features.core.schemas import CEFeature, ConfigEvent
-from app.features.presets.migration import Migration
-from app.features.presets.models import PresetModel
-from app.features.presets.utils import preset_name, seed_defaults
-from app.library.config import Config
-from app.library.Events import Event, EventBus, Events
-from app.library.log import get_logger
-from app.library.Services import Services
-from app.library.Singleton import Singleton
-
-if TYPE_CHECKING:
-    from collections.abc import Callable
-    from contextlib import AbstractAsyncContextManager
-
-    from aiohttp import web
-    from sqlalchemy.engine.result import Result
-    from sqlalchemy.ext.asyncio import AsyncSession
-    from sqlalchemy.sql.elements import ColumnElement
-    from sqlalchemy.sql.selectable import Select
-
-    SessionFactory = Callable[[], AbstractAsyncContextManager[AsyncSession]]
-
-LOG = get_logger()
-
-
-def _payload_data(payload: PresetModel | dict[str, Any]) -> dict[str, Any]:
-    if isinstance(payload, dict):
-        data: dict[str, Any] = {}
-        for key, value in payload.items():
-            if not isinstance(key, str):
-                msg = "Preset payload keys must be strings."
-                raise TypeError(msg)
-            data[key] = value
-        return data
-
-    return {
-        "name": payload.name,
-        "description": payload.description,
-        "folder": payload.folder,
-        "template": payload.template,
-        "cookies": payload.cookies,
-        "cli": payload.cli,
-        "default": payload.default,
-        "priority": payload.priority,
-        "created_at": payload.created_at,
-        "updated_at": payload.updated_at,
-    }
-
-
-class PresetsRepository(metaclass=Singleton):
-    SORT_FIELDS: dict[str, Any] = {
-        "id": PresetModel.id,
-        "name": PresetModel.name,
-        "priority": PresetModel.priority,
-        "default": PresetModel.default,
-        "created_at": PresetModel.created_at,
-        "updated_at": PresetModel.updated_at,
-    }
-    SORT_DIRECTIONS: tuple[str, str] = ("asc", "desc")
-    DEFAULT_SORT_ORDER: tuple[tuple[str, str], ...] = (("priority", "desc"), ("name", "asc"))
-    FIELD_DEFAULT_DIRECTIONS: dict[str, str] = {
-        "id": "asc",
-        "name": "asc",
-        "priority": "desc",
-        "default": "desc",
-        "created_at": "desc",
-        "updated_at": "desc",
-    }
-
-    def __init__(self, session: SessionFactory | None = None) -> None:
-        self._migrated = False
-        self.session: SessionFactory = session or get_session
-
-    async def run_migrations(self) -> None:
-        if self._migrated:
-            return
-
-        self._migrated = True
-        await Migration(repo=self).run()
-
-    def attach(self, _: web.Application) -> None:
-        async def handle_event(_, __):
-            await seed_defaults(self)
-            await self.run_migrations()
-            await self._update_cache()
-            await self._ensure_default_preset()
-
-        async def handler(e: Event, __):
-            if isinstance(e.data, ConfigEvent) and CEFeature.PRESETS == e.data.feature:
-                LOG.debug("Refreshing presets cache due to configuration update.")
-                await self._update_cache()
-
-        Services.get_instance().add(PresetsRepository.__name__, self)
-        EventBus.get_instance().subscribe(
-            Events.STARTED, handle_event, f"{PresetsRepository.__name__}.run_migrations"
-        ).subscribe(Events.CONFIG_UPDATE, handler, "Presets.refresh_cache")
-
-    async def _update_cache(self) -> None:
-        from app.features.presets.service import Presets
-
-        await Presets.get_instance().refresh_cache(await self.all())
-
-    @staticmethod
-    def get_instance() -> PresetsRepository:
-        return PresetsRepository()
-
-    async def _ensure_default_preset(self) -> None:
-        config = Config.get_instance()
-        if not (default_name := preset_name(config.default_preset)):
-            config.default_preset = "default"
-            default_name = config.default_preset
-
-        if await self.get_by_name(default_name) is None:
-            LOG.error("Default preset '%s' not found, using 'default' preset.", default_name)
-            config.default_preset = "default"
-
-    async def all(self) -> list[PresetModel]:
-        async with self.session() as session:
-            result: Result[tuple[PresetModel]] = await session.execute(
-                select(PresetModel).order_by(PresetModel.priority.desc(), PresetModel.name.asc())
-            )
-            return list(result.scalars().all())
-
-    @classmethod
-    def parse_sorting(cls, sort: str | None = None, order: str | None = None) -> tuple[tuple[str, str], ...]:
-        sort_value = (sort or "").strip()
-        order_value = (order or "").strip()
-
-        if not sort_value and not order_value:
-            return cls.DEFAULT_SORT_ORDER
-
-        if order_value and not sort_value:
-            msg = "sort is required when order is provided."
-            raise ValueError(msg)
-
-        fields: list[str] = []
-        for raw_field in sort_value.split(","):
-            field = raw_field.strip().lower()
-            if not field:
-                continue
-            if field not in cls.SORT_FIELDS:
-                msg = f"sort must use supported fields: {', '.join(cls.SORT_FIELDS)}."
-                raise ValueError(msg)
-            if field not in fields:
-                fields.append(field)
-
-        if not fields:
-            msg = "sort must include at least one field."
-            raise ValueError(msg)
-
-        directions: list[str] = []
-        if order_value:
-            for raw_direction in order_value.split(","):
-                direction = raw_direction.strip().lower()
-                if not direction:
-                    continue
-                if direction not in cls.SORT_DIRECTIONS:
-                    msg = "order must be 'asc' or 'desc'."
-                    raise ValueError(msg)
-                directions.append(direction)
-
-            if not directions:
-                msg = "order must include at least one direction."
-                raise ValueError(msg)
-
-            if len(directions) not in {1, len(fields)}:
-                msg = "order must provide one direction or match the number of sort fields."
-                raise ValueError(msg)
-
-        if not directions:
-            return tuple((field, cls.FIELD_DEFAULT_DIRECTIONS.get(field, "asc")) for field in fields)
-
-        if len(directions) == 1:
-            return tuple((field, directions[0]) for field in fields)
-
-        return tuple((field, directions[index]) for index, field in enumerate(fields))
-
-    @classmethod
-    def _apply_sort_direction(cls, field: str, direction: str) -> Any:
-        column = cls.SORT_FIELDS[field]
-        return column.asc() if direction == "asc" else column.desc()
-
-    @classmethod
-    def _build_order_by(cls, sort: str | None = None, order: str | None = None) -> list[Any]:
-        sorting = cls.parse_sorting(sort, order)
-
-        order_by: list[Any] = [cls._apply_sort_direction(field, direction) for field, direction in sorting]
-
-        if all(field != "id" for field, _ in sorting):
-            order_by.append(PresetModel.id.asc())
-
-        return order_by
-
-    async def list_paginated(
-        self,
-        page: int,
-        per_page: int,
-        sort: str | None = None,
-        order: str | None = None,
-        exclude_defaults: bool = False,
-    ) -> tuple[list[PresetModel], int, int, int]:
-        order_by = self._build_order_by(sort, order)
-
-        async with self.session() as session:
-            total: int = await self.count(exclude_defaults=exclude_defaults)
-            total_pages: int = (total + per_page - 1) // per_page if total > 0 else 1
-
-            if page > total_pages and total > 0:
-                page = total_pages
-
-            query: Select[tuple[PresetModel]] = select(PresetModel)
-            if exclude_defaults:
-                query = query.where(PresetModel.default.is_(False))
-
-            query = query.order_by(*order_by).limit(per_page).offset((page - 1) * per_page)
-            result: Result[tuple[PresetModel]] = await session.execute(query)
-            return list(result.scalars().all()), total, page, total_pages
-
-    async def count(self, exclude_defaults: bool = False) -> int:
-        async with self.session() as session:
-            query = select(func.count()).select_from(PresetModel)
-            if exclude_defaults:
-                query = query.where(PresetModel.default.is_(False))
-
-            result: Result[tuple[int]] = await session.execute(query)
-            return int(result.scalar_one())
-
-    async def get(self, identifier: int | str) -> PresetModel | None:
-        async with self.session() as session:
-            if not identifier:
-                return None
-
-            if isinstance(identifier, int):
-                clause: ColumnElement[bool] = PresetModel.id == identifier
-            elif isinstance(identifier, str) and identifier.isdigit():
-                clause = or_(PresetModel.id == int(identifier), PresetModel.name == identifier)
-            else:
-                name = preset_name(identifier) if isinstance(identifier, str) else identifier
-                if not name:
-                    return None
-                clause = PresetModel.name == name
-
-            result: Result[tuple[PresetModel]] = await session.execute(select(PresetModel).where(clause).limit(1))
-            return result.scalar_one_or_none()
-
-    async def get_by_name(self, name: str, exclude_id: int | None = None) -> PresetModel | None:
-        async with self.session() as session:
-            if not (name := preset_name(name)):
-                return None
-
-            query: Select[tuple[PresetModel]] = select(PresetModel).where(PresetModel.name == name)
-            if None is not exclude_id:
-                query = query.where(PresetModel.id != exclude_id)
-
-            result: Result[tuple[PresetModel]] = await session.execute(query.limit(1))
-            return result.scalar_one_or_none()
-
-    async def create(self, payload: PresetModel | dict[str, Any]) -> PresetModel:
-        async with self.session() as session:
-            data = _payload_data(payload)
-
-            data.pop("id", None)
-            model = PresetModel(**data)
-
-            model.name = preset_name(model.name)
-
-            if await self.get_by_name(name=model.name) is not None:
-                msg: str = f"Preset with name '{model.name}' already exists."
-                raise ValueError(msg)
-
-            session.add(model)
-            await session.commit()
-            await session.refresh(model)
-            return model
-
-    async def update(self, identifier: int | str, payload: dict[str, Any]) -> PresetModel:
-        async with self.session() as session:
-            if isinstance(identifier, int):
-                clause: ColumnElement[bool] = PresetModel.id == identifier
-            elif isinstance(identifier, str) and identifier.isdigit():
-                clause = or_(PresetModel.id == int(identifier), PresetModel.name == identifier)
-            else:
-                clause = PresetModel.name == identifier
-
-            result: Result[tuple[PresetModel]] = await session.execute(select(PresetModel).where(clause).limit(1))
-            model: PresetModel | None = result.scalar_one_or_none()
-
-            if model is None:
-                msg: str = f"Preset '{identifier}' not found."
-                raise KeyError(msg)
-
-            payload.pop("id", None)
-            payload.pop("created_at", None)
-            payload.pop("updated_at", None)
-
-            for key, value in payload.items():
-                if hasattr(model, key):
-                    setattr(model, key, value)
-
-            normalized_name = preset_name(model.name)
-            model.name = normalized_name
-            if await self.get_by_name(name=normalized_name, exclude_id=model.id) is not None:
-                msg = f"Preset with name '{normalized_name}' already exists."
-                raise ValueError(msg)
-
-            await session.commit()
-            await session.refresh(model)
-            return model
-
-    async def delete(self, identifier: int | str) -> PresetModel:
-        async with self.session() as session:
-            if isinstance(identifier, int):
-                clause: ColumnElement[bool] = PresetModel.id == identifier
-            elif isinstance(identifier, str) and identifier.isdigit():
-                clause = or_(PresetModel.id == int(identifier), PresetModel.name == identifier)
-            else:
-                clause = PresetModel.name == preset_name(identifier)
-
-            result: Result[tuple[PresetModel]] = await session.execute(select(PresetModel).where(clause).limit(1))
-            model: PresetModel | None = result.scalar_one_or_none()
-            if model is None:
-                msg: str = f"Preset '{identifier}' not found."
-                raise KeyError(msg)
-
-            await session.delete(model)
-            await session.commit()
-            return model

app/features/presets/router.py

@@ -1,200 +0,0 @@
-from typing import Any
-
-from aiohttp import web
-from aiohttp.web import Request, Response
-from pydantic import ValidationError
-
-from app.features.core.schemas import CEAction, CEFeature, ConfigEvent, Pagination
-from app.features.core.utils import build_pagination, format_validation_errors, normalize_pagination
-from app.features.presets.repository import PresetsRepository
-from app.features.presets.schemas import Preset, PresetList, PresetPatch
-from app.library.encoder import Encoder
-from app.library.Events import EventBus, Events
-from app.library.router import route
-
-
-def _model(model: Any) -> Preset:
-    return Preset.model_validate(model)
-
-
-def _serialize(model: Any) -> dict:
-    return _model(model).model_dump()
-
-
-@route("GET", "api/presets/", "presets")
-async def presets_list(request: Request, encoder: Encoder, repo: PresetsRepository) -> Response:
-    try:
-        page, per_page = normalize_pagination(request)
-        items, total, current_page, total_pages = await repo.list_paginated(
-            page=page,
-            per_page=per_page,
-            sort=request.query.get("sort"),
-            order=request.query.get("order"),
-            exclude_defaults=bool(request.query.get("exclude_defaults", False)),
-        )
-    except ValueError as exc:
-        return web.json_response(data={"error": str(exc)}, status=web.HTTPBadRequest.status_code)
-
-    return web.json_response(
-        data=PresetList(
-            items=[_model(model) for model in items],
-            pagination=Pagination.model_validate(build_pagination(total, current_page, per_page, total_pages)),
-        ),
-        status=web.HTTPOk.status_code,
-        dumps=encoder.encode,
-    )
-
-
-@route("GET", r"api/presets/{id:\d+}", "presets_get")
-async def presets_get(request: Request, encoder: Encoder, repo: PresetsRepository) -> Response:
-    if not (identifier := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    if not (model := await repo.get(identifier)):
-        return web.json_response({"error": "Preset not found"}, status=web.HTTPNotFound.status_code)
-
-    return web.json_response(data=_serialize(model), status=web.HTTPOk.status_code, dumps=encoder.encode)
-
-
-@route("POST", "api/presets/", "presets_add")
-async def presets_add(request: Request, encoder: Encoder, notify: EventBus, repo: PresetsRepository) -> Response:
-    data = await request.json()
-
-    if not isinstance(data, dict):
-        return web.json_response(
-            {"error": "Invalid request body expecting dict."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    try:
-        item: Preset = Preset.model_validate(data)
-    except ValidationError as exc:
-        return web.json_response(
-            data={"error": "Failed to validate preset.", "detail": format_validation_errors(exc)},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    payload = item.model_dump(exclude_unset=True)
-    payload.pop("id", None)
-    payload["default"] = False
-
-    try:
-        saved = _serialize(await repo.create(payload))
-    except ValueError as exc:
-        return web.json_response({"error": str(exc)}, status=web.HTTPBadRequest.status_code)
-
-    notify.emit(
-        Events.CONFIG_UPDATE,
-        data=ConfigEvent(feature=CEFeature.PRESETS, action=CEAction.CREATE, data=saved),
-    )
-    return web.json_response(data=saved, status=web.HTTPOk.status_code, dumps=encoder.encode)
-
-
-@route("PATCH", r"api/presets/{id:\d+}", "presets_patch")
-async def presets_patch(request: Request, encoder: Encoder, notify: EventBus, repo: PresetsRepository) -> Response:
-    if not (identifier := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    if not (model := await repo.get(identifier)):
-        return web.json_response({"error": "Preset not found"}, status=web.HTTPNotFound.status_code)
-
-    if model.default:
-        return web.json_response(
-            {"error": "Default presets cannot be modified."}, status=web.HTTPBadRequest.status_code
-        )
-
-    data = await request.json()
-
-    if not isinstance(data, dict):
-        return web.json_response(
-            {"error": "Invalid request body expecting dict."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    try:
-        validated = PresetPatch.model_validate(data)
-    except ValidationError as exc:
-        return web.json_response(
-            data={"error": "Failed to validate preset.", "detail": format_validation_errors(exc)},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    if validated.name and await repo.get_by_name(validated.name, exclude_id=model.id):
-        return web.json_response(
-            data={"error": f"Preset with name '{validated.name}' already exists."},
-            status=web.HTTPConflict.status_code,
-        )
-
-    payload = validated.model_dump(exclude_unset=True)
-    payload.pop("default", None)
-    updated = _serialize(await repo.update(model.id, payload))
-    notify.emit(
-        Events.CONFIG_UPDATE,
-        data=ConfigEvent(feature=CEFeature.PRESETS, action=CEAction.UPDATE, data=updated),
-    )
-    return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode)
-
-
-@route("PUT", r"api/presets/{id:\d+}", "presets_update")
-async def presets_update(request: Request, encoder: Encoder, notify: EventBus, repo: PresetsRepository) -> Response:
-    if not (identifier := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    if not (model := await repo.get(identifier)):
-        return web.json_response({"error": "Preset not found"}, status=web.HTTPNotFound.status_code)
-
-    if model.default:
-        return web.json_response(
-            {"error": "Default presets cannot be modified."}, status=web.HTTPBadRequest.status_code
-        )
-
-    data = await request.json()
-
-    if not isinstance(data, dict):
-        return web.json_response(
-            {"error": "Invalid request body expecting dict."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    try:
-        validated = Preset.model_validate(data)
-    except ValidationError as exc:
-        return web.json_response(
-            data={"error": "Failed to validate preset.", "detail": format_validation_errors(exc)},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    if validated.name and await repo.get_by_name(validated.name, exclude_id=model.id):
-        return web.json_response(
-            data={"error": f"Preset with name '{validated.name}' already exists."},
-            status=web.HTTPConflict.status_code,
-        )
-
-    payload = validated.model_dump(exclude_unset=True)
-    payload.pop("default", None)
-    payload.pop("id", None)
-    updated = _serialize(await repo.update(model.id, payload))
-    notify.emit(
-        Events.CONFIG_UPDATE,
-        data=ConfigEvent(feature=CEFeature.PRESETS, action=CEAction.UPDATE, data=updated),
-    )
-    return web.json_response(data=updated, status=web.HTTPOk.status_code, dumps=encoder.encode)
-
-
-@route("DELETE", r"api/presets/{id:\d+}", "presets_delete")
-async def presets_delete(request: Request, encoder: Encoder, notify: EventBus, repo: PresetsRepository) -> Response:
-    if not (identifier := request.match_info.get("id")):
-        return web.json_response({"error": "ID required"}, status=web.HTTPBadRequest.status_code)
-
-    if not (model := await repo.get(identifier)):
-        return web.json_response({"error": "Preset not found"}, status=web.HTTPNotFound.status_code)
-
-    if model.default:
-        return web.json_response({"error": "Default presets cannot be deleted."}, status=web.HTTPBadRequest.status_code)
-
-    deleted = _serialize(await repo.delete(model.id))
-    notify.emit(
-        Events.CONFIG_UPDATE,
-        data=ConfigEvent(feature=CEFeature.PRESETS, action=CEAction.DELETE, data=deleted),
-    )
-    return web.json_response(data=deleted, status=web.HTTPOk.status_code, dumps=encoder.encode)

app/features/presets/schemas.py

@@ -1,97 +0,0 @@
-from __future__ import annotations
-
-from pathlib import Path
-from typing import Any
-
-from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator
-
-from app.features.core.schemas import Pagination
-from app.features.core.utils import parse_int
-from app.features.presets.utils import preset_name
-from app.library.config import Config
-from app.library.Utils import create_cookies_file
-
-
-class Preset(BaseModel):
-    model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True)
-
-    id: int | None = None
-    name: str = Field(min_length=1)
-    description: str = ""
-    folder: str = ""
-    template: str = ""
-    cookies: str = ""
-    cli: str = ""
-    default: bool = False
-    priority: int = 0
-
-    _cookies_file: Path | None = PrivateAttr(default=None)
-
-    @field_validator("name", mode="before")
-    @classmethod
-    def _normalize_name(cls, value: Any) -> str:
-        if not isinstance(value, str):
-            msg: str = "Name must be a string."
-            raise ValueError(msg)
-
-        if not (value := preset_name(value)):
-            msg = "Name cannot be empty."
-            raise ValueError(msg)
-        return value
-
-    @field_validator("priority", mode="before")
-    @classmethod
-    def _normalize_priority(cls, value: Any) -> int:
-        return parse_int(value, field="Priority", minimum=0)
-
-    @field_validator("cli", mode="before")
-    @classmethod
-    def _validate_cli(cls, value: Any) -> str:
-        if not value:
-            return ""
-        if not isinstance(value, str):
-            msg: str = "CLI must be a string."
-            raise ValueError(msg)
-
-        value = value.strip()
-        if not value:
-            return ""
-
-        try:
-            from app.features.ytdlp.utils import arg_converter
-
-            arg_converter(args=value, level=True)
-        except Exception as e:
-            msg = f"Invalid command options for yt-dlp: {e!s}"
-            raise ValueError(msg) from e
-        return value
-
-    def get_cookies_file(self, config: Config | None = None) -> Path | None:
-        if None is not self._cookies_file:
-            return self._cookies_file
-
-        if not self.cookies or self.id is None:
-            return None
-
-        config = config or Config.get_instance()
-        self._cookies_file = create_cookies_file(self.cookies, Path(config.config_path) / "cookies" / f"{self.id}.txt")
-        return self._cookies_file
-
-
-class PresetPatch(Preset):
-    model_config = ConfigDict(str_strip_whitespace=True, from_attributes=True)
-
-    name: str | None = None
-    description: str | None = None
-    folder: str | None = None
-    template: str | None = None
-    cookies: str | None = None
-    cli: str | None = None
-    priority: int | None = None
-
-
-class PresetList(BaseModel):
-    model_config = ConfigDict(from_attributes=True)
-
-    items: list[Preset] = Field(default_factory=list)
-    pagination: Pagination

app/features/presets/service.py

@@ -1,44 +0,0 @@
-from __future__ import annotations
-
-from app.features.presets.models import PresetModel
-from app.features.presets.repository import PresetsRepository
-from app.features.presets.schemas import Preset
-from app.features.presets.utils import preset_name
-from app.library.Services import Services
-from app.library.Singleton import Singleton
-
-
-class Presets(metaclass=Singleton):
-    def __init__(self, repo: PresetsRepository | None = None) -> None:
-        self._repo: PresetsRepository = repo or PresetsRepository.get_instance()
-        self._cache: list[tuple[int, str, Preset]] = []
-        Services.get_instance().add(type(self).__name__, self)
-
-    @staticmethod
-    def get_instance() -> Presets:
-        return Presets()
-
-    async def refresh_cache(self, items: list[PresetModel]) -> None:
-        presets = [Preset.model_validate(item) for item in items]
-        self._cache = [(preset.id if preset.id is not None else -1, preset.name, preset) for preset in presets]
-
-    def get_all(self) -> list[Preset]:
-        return [preset for _, _, preset in self._cache]
-
-    def get(self, identifier: int | str) -> Preset | None:
-        if not identifier:
-            return None
-
-        if isinstance(identifier, str) and not (identifier := preset_name(identifier)):
-            return None
-
-        for id, name, preset in self._cache:
-            if isinstance(identifier, int) and id == identifier:
-                return preset
-            if isinstance(identifier, str) and name == identifier:
-                return preset
-
-        return None
-
-    def has(self, identifier: int | str) -> bool:
-        return self.get(identifier) is not None

app/features/presets/tests/test_presets_repository.py

@@ -1,179 +0,0 @@
-from __future__ import annotations
-
-import json
-from unittest.mock import MagicMock
-
-import pytest
-import pytest_asyncio
-from aiohttp import web
-from aiohttp.web import Request
-
-from app.features.presets.repository import PresetsRepository
-from app.features.presets.router import presets_list
-from app.library.encoder import Encoder
-from app.library.sqlite_store import SqliteStore
-from app.tests.helpers import make_in_memory_db_path
-
-
-@pytest_asyncio.fixture
-async def repo():
-    PresetsRepository._reset_singleton()
-    SqliteStore._reset_singleton()
-
-    store = SqliteStore(db_path=make_in_memory_db_path("presets-repository"))
-    await store.get_connection()
-
-    repository = PresetsRepository.get_instance()
-    yield repository
-
-    await store.close()
-
-    PresetsRepository._reset_singleton()
-    SqliteStore._reset_singleton()
-
-
-class TestPresetsRepository:
-    @pytest.mark.asyncio
-    async def test_create_and_get(self, repo):
-        preset = await repo.create({"name": "Custom", "cli": "--format best"})
-
-        assert preset.id is not None, "Should generate ID for new preset"
-        assert preset.name == "custom", "Should normalize preset name"
-        assert preset.cli == "--format best", "Should store preset cli"
-
-        fetched = await repo.get(preset.id)
-        assert fetched is not None, "Should fetch preset by id"
-        assert fetched.name == "custom", "Should return matching preset"
-
-    @pytest.mark.asyncio
-    async def test_create_normalizes_spaces(self, repo):
-        preset = await repo.create({"name": "My Preset"})
-
-        assert preset.name == "my_preset", "Should normalize spaces to underscores"
-
-    @pytest.mark.asyncio
-    async def test_list_orders_by_priority_then_name(self, repo):
-        await repo.create({"name": "B", "priority": 1})
-        await repo.create({"name": "A", "priority": 1})
-        await repo.create({"name": "C", "priority": 2})
-
-        items = await repo.all()
-
-        assert items[0].name == "c", "Highest priority should be first"
-        assert items[1].name == "a", "Same priority should sort by name"
-        assert items[2].name == "b", "Same priority should sort by name"
-
-    @pytest.mark.asyncio
-    async def test_list_paginated(self, repo):
-        for i in range(5):
-            await repo.create({"name": f"Item {i}", "priority": i})
-
-        items, total, page, total_pages = await repo.list_paginated(page=1, per_page=2)
-
-        assert len(items) == 2, "Should return 2 items per page"
-        assert total == 5, "Should report total count"
-        assert page == 1, "Should be on page 1"
-        assert total_pages == 3, "Should have 3 pages total"
-        assert [item.priority for item in items] == [4, 3], "Should keep default priority-desc order"
-
-    @pytest.mark.asyncio
-    async def test_list_paginated_sorts_by_name_desc(self, repo):
-        await repo.create({"name": "Alpha", "priority": 1})
-        await repo.create({"name": "Gamma", "priority": 3})
-        await repo.create({"name": "Beta", "priority": 2})
-
-        items, _, _, _ = await repo.list_paginated(page=1, per_page=10, sort="name", order="desc")
-
-        assert [item.name for item in items] == ["gamma", "beta", "alpha"], "Should sort by requested field"
-
-    @pytest.mark.asyncio
-    async def test_list_paginated_excludes_defaults(self, repo):
-        await repo.create({"name": "System Default", "default": True, "priority": 10})
-        await repo.create({"name": "Custom Preset", "priority": 1})
-
-        items, total, page, total_pages = await repo.list_paginated(page=1, per_page=10, exclude_defaults=True)
-
-        assert [item.name for item in items] == ["custom_preset"], "Should exclude default presets"
-        assert total == 1, "Should count only custom presets"
-        assert page == 1, "Should keep current page when filtered results exist"
-        assert total_pages == 1, "Should compute pages from the filtered total"
-
-    @pytest.mark.asyncio
-    async def test_list_paginated_multi_sort(self, repo):
-        await repo.create({"name": "Charlie", "priority": 2})
-        await repo.create({"name": "Alpha", "priority": 1})
-        await repo.create({"name": "Bravo", "priority": 1})
-
-        items, _, _, _ = await repo.list_paginated(page=1, per_page=10, sort="priority,name", order="asc,desc")
-
-        assert [(item.priority, item.name) for item in items] == [
-            (1, "bravo"),
-            (1, "alpha"),
-            (2, "charlie"),
-        ], "Should support multiple sort fields and directions"
-
-    @pytest.mark.asyncio
-    async def test_list_paginated_bad_sort(self, repo):
-        with pytest.raises(ValueError, match="sort must use supported fields"):
-            await repo.list_paginated(page=1, per_page=10, sort="cli", order="asc")
-
-    @pytest.mark.asyncio
-    async def test_list_paginated_bad_order(self, repo):
-        with pytest.raises(ValueError, match="order must be 'asc' or 'desc'"):
-            await repo.list_paginated(page=1, per_page=10, sort="name", order="sideways")
-
-    @pytest.mark.asyncio
-    async def test_list_paginated_mismatched_order(self, repo):
-        with pytest.raises(ValueError, match="order must provide one direction or match the number of sort fields"):
-            await repo.list_paginated(page=1, per_page=10, sort="priority,name", order="asc,desc,asc")
-
-
-@pytest.mark.asyncio
-class TestPresetRoutes:
-    async def test_list_route_sort(self, repo):
-        await repo.create({"name": "Alpha", "priority": 1})
-        await repo.create({"name": "Bravo", "priority": 1})
-        await repo.create({"name": "Charlie", "priority": 2})
-
-        request = MagicMock(spec=Request)
-        request.query = {"page": "1", "per_page": "10", "sort": "priority,name", "order": "asc,desc"}
-
-        response = await presets_list(request, Encoder(), repo)
-        payload = json.loads(response.text)
-
-        assert response.status == web.HTTPOk.status_code, "Should return 200 for valid sorting"
-        assert [item["name"] for item in payload["items"]] == ["bravo", "alpha", "charlie"], "Should sort response"
-
-    async def test_list_route_bad_sort(self, repo):
-        request = MagicMock(spec=Request)
-        request.query = {"sort": "cli", "order": "asc"}
-
-        response = await presets_list(request, Encoder(), repo)
-        payload = json.loads(response.text)
-
-        assert response.status == web.HTTPBadRequest.status_code, "Should reject unsupported sort field"
-        assert "sort" in payload["error"], "Should explain invalid sort field"
-
-    async def test_list_route_bad_order(self, repo):
-        request = MagicMock(spec=Request)
-        request.query = {"sort": "name", "order": "sideways"}
-
-        response = await presets_list(request, Encoder(), repo)
-        payload = json.loads(response.text)
-
-        assert response.status == web.HTTPBadRequest.status_code, "Should reject unsupported sort direction"
-        assert "order" in payload["error"], "Should explain invalid sort direction"
-
-    async def test_list_route_exclude_defaults(self, repo):
-        await repo.create({"name": "System Default", "default": True, "priority": 10})
-        await repo.create({"name": "Custom Preset", "priority": 1})
-
-        request = MagicMock(spec=Request)
-        request.query = {"page": "1", "per_page": "10", "exclude_defaults": "true"}
-
-        response = await presets_list(request, Encoder(), repo)
-        payload = json.loads(response.text)
-
-        assert response.status == web.HTTPOk.status_code, "Should return 200 for valid default exclusion"
-        assert [item["name"] for item in payload["items"]] == ["custom_preset"], "Should exclude default presets"
-        assert payload["pagination"]["total"] == 1, "Should report filtered total"

app/features/presets/utils.py

@@ -1,60 +0,0 @@
-import re
-from datetime import UTC, datetime
-
-from app.library.log import get_logger
-
-NAME_WHITESPACE_PATTERN = re.compile(r"\s+")
-
-LOG = get_logger()
-
-
-async def seed_defaults(repo) -> None:
-    from .defaults import DEFAULT_PRESETS
-
-    for preset in DEFAULT_PRESETS:
-        try:
-            name = preset.get("name")
-            if not name or not isinstance(name, str):
-                LOG.warning("Skipping default preset with invalid name: %s", preset)
-                continue
-
-            if not (name := preset_name(name)):
-                LOG.warning("Skipping default preset with empty normalized name: %s", name)
-                continue
-
-            payload = {**preset}
-            payload.pop("id", None)
-            payload["default"] = True
-            payload["name"] = name
-
-            updated_at_value: datetime | None = None
-            updated_at_raw = payload.get("updated_at")
-            if isinstance(updated_at_raw, str):
-                updated_at_value = datetime.fromisoformat(updated_at_raw)
-            elif isinstance(updated_at_raw, datetime):
-                updated_at_value = updated_at_raw
-            if isinstance(updated_at_value, datetime) and updated_at_value.tzinfo is None:
-                updated_at_value = updated_at_value.replace(tzinfo=UTC)
-            payload["updated_at"] = updated_at_value
-
-            existing = await repo.get_by_name(name)
-            if None is existing:
-                await repo.create(payload)
-                continue
-
-            if existing.updated_at and updated_at_value and existing.updated_at >= updated_at_value:
-                if False is existing.default:
-                    await repo.update(existing.id, {"default": True})
-                continue
-
-            await repo.update(existing.id, payload)
-        except Exception as exc:
-            LOG.exception(
-                "Failed to seed default preset '%s'.",
-                preset.get("name"),
-                extra={"preset": preset.get("name"), "exception_type": type(exc).__name__},
-            )
-
-
-def preset_name(value: str) -> str:
-    return NAME_WHITESPACE_PATTERN.sub("_", value.strip().lower())

app/features/streaming/library/ffprobe.py

@@ -1,303 +0,0 @@
-"""
-Python wrapper for ffprobe command line tool. ffprobe must exist in the path.
-"""
-
-import asyncio
-import functools
-import json
-import operator
-import os
-import subprocess  # qa: ignore
-from pathlib import Path
-
-from app.features.streaming.types import FFProbeError
-from app.library.log import get_logger
-from app.library.Utils import timed_lru_cache
-
-LOG = get_logger()
-
-
-class FFStream:
-    """
-    An object representation of an individual stream in a multimedia file.
-    """
-
-    def __init__(self, json_data: dict):
-        for key, val in json_data.items():
-            setattr(self, key, val)
-
-        try:
-            self.__dict__["framerate"] = round(
-                functools.reduce(operator.truediv, map(int, self.__dict__.get("avg_frame_rate", "").split("/")))
-            )
-        except ValueError:
-            self.__dict__["framerate"] = None
-        except ZeroDivisionError:
-            self.__dict__["framerate"] = 0
-
-    def __repr__(self):
-        if "codec_long_name" not in self.__dict__:
-            self.__dict__["codec_long_name"] = self.__dict__.get("codec_name", "")
-
-        index = self.__dict__.get("index", "?")
-        codec_type = self.__dict__.get("codec_type", "unknown")
-        codec_long_name = self.__dict__.get("codec_long_name", "")
-
-        if self.is_video():
-            return f"<Stream: #{index} [{codec_type}] {codec_long_name}, {self.__dict__.get('framerate')}, ({self.__dict__.get('width')}x{self.__dict__.get('height')})>"
-
-        if self.is_audio():
-            return (
-                f"<Stream: #{index} [{codec_type}] {codec_long_name}, "
-                f"channels: {self.__dict__.get('channels')} ({self.__dict__.get('channel_layout')}), "
-                f"{self.__dict__.get('sample_rate')}Hz> "
-            )
-
-        if self.is_subtitle() or self.is_attachment():
-            return f"<Stream: #{index} [{codec_type}] {codec_long_name}>"
-
-        return f"<Stream: #{index} [{codec_type}]>"
-
-    def is_audio(self):
-        """
-        Is this stream labelled as an audio stream?
-        """
-        return self.__dict__.get("codec_type", None) == "audio"
-
-    def is_video(self):
-        """
-        Is the stream labelled as a video stream.
-        """
-        if self.__dict__.get("codec_type", None) != "video":
-            return False
-
-        return self.__dict__.get("codec_name", None) not in ["png", "mjpeg", "gif", "bmp", "tiff", "webp"]
-
-    def is_subtitle(self):
-        """
-        Is the stream labelled as a subtitle stream.
-        """
-        return self.__dict__.get("codec_type", None) == "subtitle"
-
-    def is_attachment(self):
-        """
-        Is the stream labelled as a attachment stream.
-        """
-        return self.__dict__.get("codec_type", None) == "attachment"
-
-    def frame_size(self):
-        """
-        Returns the pixel frame size as an integer tuple (width,height) if the stream is a video stream.
-        Returns None if it is not a video stream.
-        """
-        size = None
-        if self.is_video():
-            width = self.__dict__["width"]
-            height = self.__dict__["height"]
-
-            if width and height:
-                try:
-                    size = (int(width), int(height))
-                except ValueError as e:
-                    msg = f"None integer size {width}:{height}"
-                    raise FFProbeError(msg) from e
-        else:
-            return None
-
-        return size
-
-    def pixel_format(self):
-        """
-        Returns a string representing the pixel format of the video stream. e.g. yuv420p.
-        Returns none is it is not a video stream.
-        """
-        return self.__dict__.get("pix_fmt", None)
-
-    def frames(self):
-        """
-        Returns the length of a video stream in frames. Returns 0 if not a video stream.
-        """
-        if self.is_video() or self.is_audio():
-            try:
-                frame_count = int(self.__dict__.get("nb_frames", ""))
-            except ValueError as e:
-                msg = "None integer frame count"
-                raise FFProbeError(msg) from e
-        else:
-            frame_count = 0
-
-        return frame_count
-
-    def duration_seconds(self):
-        """
-        Returns the runtime duration of the video stream as a floating point number of seconds.
-        Returns 0.0 if not a video stream.
-        """
-        if self.is_video() or self.is_audio():
-            try:
-                duration = float(self.__dict__.get("duration", ""))
-            except ValueError as e:
-                msg = "None numeric duration"
-                raise FFProbeError(msg) from e
-        else:
-            duration = 0.0
-
-        return duration
-
-    def language(self):
-        """
-        Returns language tag of stream. e.g. eng
-        """
-        return self.__dict__.get("TAG:language", None)
-
-    def codec(self):
-        """
-        Returns a string representation of the stream codec.
-        """
-        return self.__dict__.get("codec_name", None)
-
-    def codec_description(self):
-        """
-        Returns a long representation of the stream codec.
-        """
-        return self.__dict__.get("codec_long_name", None)
-
-    def codec_tag(self):
-        """
-        Returns a short representative tag of the stream codec.
-        """
-        return self.__dict__.get("codec_tag_string", None)
-
-    def bit_rate(self):
-        """
-        Returns bit_rate as an integer in bps
-        """
-        try:
-            return int(self.__dict__.get("bit_rate", ""))
-        except ValueError as e:
-            msg = "None integer bit_rate"
-            raise FFProbeError(msg) from e
-
-
-class FFProbeResult:
-    def __init__(self):
-        self.metadata: dict = {}
-        self.video: list[FFStream] = []
-        self.audio: list[FFStream] = []
-        self.subtitle: list[FFStream] = []
-        self.attachment: list[FFStream] = []
-
-    @property
-    def is_video(self):
-        return self.has_video()
-
-    @property
-    def is_audio(self):
-        return self.has_audio()
-
-    def get(self, key: str, default=None):
-        return getattr(self, key) if hasattr(self, key) else default
-
-    def streams(self) -> list[FFStream]:
-        """List of all streams."""
-        return self.video + self.audio + self.subtitle + self.attachment
-
-    def has_video(self):
-        """Is there a video stream?"""
-        return len(self.video) > 0
-
-    def has_audio(self):
-        """Is there an audio stream?"""
-        return len(self.audio) > 0
-
-    def has_subtitle(self):
-        """Is there a subtitle stream?"""
-        return len(self.subtitle) > 0
-
-    def __repr__(self):
-        return "<FFprobe: {metadata}, {video}, {audio}, {subtitle}, {attachment}>".format(**vars(self))
-
-    def deserialize(self, data: dict):
-        self.metadata = data.get("metadata", {})
-        self.video = [FFStream(v) for v in data.get("video", [])]
-        self.audio = [FFStream(a) for a in data.get("audio", [])]
-        self.subtitle = [FFStream(s) for s in data.get("subtitle", [])]
-        self.attachment = [FFStream(a) for a in data.get("attachment", [])]
-
-    def serialize(self) -> dict:
-        return {
-            "metadata": self.metadata,
-            "video": [v.__dict__ for v in self.video],
-            "audio": [a.__dict__ for a in self.audio],
-            "subtitle": [s.__dict__ for s in self.subtitle],
-            "attachment": [a.__dict__ for a in self.attachment],
-            "is_video": self.has_video(),
-            "is_audio": self.has_audio(),
-        }
-
-
-@timed_lru_cache(ttl_seconds=300, max_size=128)
-async def ffprobe(file: Path | str) -> FFProbeResult:
-    """
-    Run ffprobe on a file and return the parsed data as a dictionary.
-
-    Args:
-        file (str): The path to the media file.
-
-    Returns:
-        dict: A dictionary containing the parsed data.
-
-    """
-    f = Path(file) if isinstance(file, str) else file
-
-    if not f.exists():
-        msg = f"No such media file '{file}'."
-        raise OSError(msg)
-
-    try:
-        proc = await asyncio.create_subprocess_exec(
-            "ffprobe",
-            "-h",
-            stdout=asyncio.subprocess.DEVNULL,
-            stderr=asyncio.subprocess.DEVNULL,
-            creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
-        )
-        await proc.wait()
-    except FileNotFoundError as e:
-        msg = "ffprobe not found."
-        raise OSError(msg) from e
-
-    args: list[str] = ["-v", "quiet", "-of", "json", "-show_streams", "-show_format", str(f)]
-
-    p = await asyncio.create_subprocess_exec(
-        "ffprobe",
-        *args,
-        stdout=asyncio.subprocess.PIPE,
-        stderr=asyncio.subprocess.PIPE,
-        creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
-    )
-
-    exitCode: int = await p.wait()
-
-    data, err = await p.communicate()
-    if 0 == exitCode:
-        parsed: dict = json.loads(data.decode("utf-8"))
-    else:
-        msg: str = f"ffprobe returned with non-0 exit code. '{err.decode('utf-8')}'"
-        raise FFProbeError(msg)
-
-    result = FFProbeResult()
-    result.metadata = parsed.get("format", {})
-
-    for raw_stream in parsed.get("streams", []):
-        stream = FFStream(raw_stream)
-        if stream.is_audio():
-            result.audio.append(stream)
-        elif stream.is_video():
-            result.video.append(stream)
-        elif stream.is_subtitle():
-            result.subtitle.append(stream)
-        elif stream.is_attachment():
-            result.attachment.append(stream)
-
-    return result

app/features/streaming/library/m3u8.py

@@ -1,84 +0,0 @@
-import math
-from pathlib import Path
-from urllib.parse import quote
-
-from app.features.streaming.library.ffprobe import ffprobe
-from app.features.streaming.types import StreamingError
-
-
-class M3u8:
-    duration: float = 6.000000
-
-    ok_vcodecs: tuple = ("h264", "x264", "avc")
-    ok_acodecs: tuple = ("aac", "m4a", "mp3")
-
-    def __init__(self, download_path: Path, url: str, segment_duration: float | None = None):
-        self.url = url
-        self.download_path: Path = download_path
-        self.duration = float(segment_duration) if segment_duration is not None else self.duration
-
-    async def make_stream(self, file: Path) -> str:
-        try:
-            ff = await ffprobe(file)
-        except UnicodeDecodeError:
-            pass
-
-        urlPath: str = str(file.relative_to(self.download_path).as_posix()).strip("/")
-
-        if "duration" not in ff.metadata:
-            error = f"Unable to get '{file}' play duration."
-            raise StreamingError(error)
-
-        duration: float = float(ff.metadata.get("duration"))
-
-        m3u8: list = []
-
-        m3u8.append("#EXTM3U")
-        m3u8.append("#EXT-X-VERSION:3")
-        m3u8.append(f"#EXT-X-TARGETDURATION:{int(self.duration)}")
-        m3u8.append("#EXT-X-MEDIA-SEQUENCE:0")
-        m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD")
-
-        segmentSize: str = f"{self.duration:.6f}"
-        splits: int = math.ceil(duration / self.duration)
-
-        segmentParams: dict = {}
-
-        for stream in ff.streams():
-            if stream.is_video() and stream.codec_name not in self.ok_vcodecs:
-                segmentParams["vc"] = 1
-            if stream.is_audio() and stream.codec_name not in self.ok_acodecs:
-                segmentParams["ac"] = 1
-
-        for i in range(splits):
-            if (i + 1) == splits:
-                segmentSize = f"{duration - (i * self.duration):.6f}"
-                segmentParams.update({"sd": segmentSize})
-
-            m3u8.append(f"#EXTINF:{segmentSize},")
-
-            url = f"{self.url}api/player/segments/{i}/{quote(urlPath)}.ts"
-            if len(segmentParams) > 0:
-                url += "?" + "&".join([f"{key}={value}" for key, value in segmentParams.items()])
-
-            m3u8.append(url)
-
-        m3u8.append("#EXT-X-ENDLIST")
-
-        return "\n".join(m3u8)
-
-    async def make_subtitle(self, file: Path, duration: float) -> str:
-        m3u8 = []
-
-        urlPath: str = str(file.relative_to(self.download_path).as_posix()).strip("/")
-
-        m3u8.append("#EXTM3U")
-        m3u8.append("#EXT-X-VERSION:3")
-        m3u8.append(f"#EXT-X-TARGETDURATION:{int(self.duration)}")
-        m3u8.append("#EXT-X-MEDIA-SEQUENCE:0")
-        m3u8.append("#EXT-X-PLAYLIST-TYPE:VOD")
-        m3u8.append(f"#EXTINF:{duration},")
-        m3u8.append(f"{self.url}api/player/subtitle/{quote(urlPath)}.vtt")
-        m3u8.append("#EXT-X-ENDLIST")
-
-        return "\n".join(m3u8)

app/features/streaming/library/playlist.py

@@ -1,48 +0,0 @@
-from pathlib import Path
-from urllib.parse import quote
-
-from app.features.streaming.library.ffprobe import ffprobe
-from app.features.streaming.types import StreamingError
-from app.library.Utils import get_file_sidecar
-
-
-class Playlist:
-    def __init__(self, download_path: Path, url: str):
-        self.url: str = url
-        "The base URL for the playlist."
-        self.download_path: Path = download_path
-        "The path where files are downloaded."
-
-    async def make(self, file: Path) -> str:
-        ref: Path = Path(str(file.relative_to(self.download_path)).strip("/"))
-
-        try:
-            ff = await ffprobe(file)
-        except UnicodeDecodeError:
-            pass
-
-        if "duration" not in ff.metadata:
-            msg = f"Unable to get '{ref}' duration."
-            raise StreamingError(msg)
-
-        playlist: list[str] = []
-        playlist.append("#EXTM3U")
-
-        subs: str = ""
-
-        duration: float = float(ff.metadata.get("duration"))
-        for sub_file in get_file_sidecar(file).get("subtitle", []):
-            lang: str = sub_file["lang"]
-            item: Path = sub_file["file"]
-            name: str = sub_file["name"]
-
-            subs = ',SUBTITLES="subs"'
-            url = f"{self.url}api/player/m3u8/subtitle/{quote(str(ref.with_name(item.name)))}.m3u8?duration={duration}"
-            playlist.append(
-                f'#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="subs",NAME="{name}",DEFAULT=NO,AUTOSELECT=NO,FORCED=NO,LANGUAGE="{lang}",URI="{url}"'
-            )
-
-        playlist.append(f"#EXT-X-STREAM-INF:PROGRAM-ID=1{subs}")
-        playlist.append(f"{self.url}api/player/m3u8/video/{quote(str(ref))}.m3u8")
-
-        return "\n".join(playlist)

app/features/streaming/library/segment_encoders.py

@@ -1,325 +0,0 @@
-from __future__ import annotations
-
-import os
-import shutil
-import subprocess
-import sys
-from functools import lru_cache
-from pathlib import Path
-from typing import Any, Protocol
-
-
-@lru_cache(maxsize=1)
-def detect_qsv_capabilities() -> dict[str, dict[str, bool]]:
-    """
-    Detects QSV encode capability for each relevant codec.
-    Returns a dict where keys are codec names (e.g. "h264", "hevc", "vp9") and
-    values are dicts with keys "full" and "lp" booleans.
-    """
-    vainfo = shutil.which("vainfo")
-    if not vainfo:
-        return {}
-
-    try:
-        result: subprocess.CompletedProcess[str] = subprocess.run(
-            [vainfo],
-            capture_output=True,
-            text=True,
-            check=False,
-        )
-        out: str = result.stdout + result.stderr
-    except Exception:
-        return {}
-
-    caps = {}
-    for line in out.splitlines():
-        line: str = line.strip()
-        parts: list[str] = line.split(":")
-        if len(parts) < 2:
-            continue
-        prof, ep_str = parts[0].strip(), parts[1].strip()
-        if "H264" in prof:
-            key = "h264"
-        elif "HEVC" in prof:
-            key = "hevc"
-        elif "VP9" in prof:
-            key = "vp9"
-        else:
-            continue
-        if key not in caps:
-            caps[key] = {"full": False, "lp": False}
-
-        if "EncSlice " in ep_str:
-            caps[key]["full"] = True
-
-        if "EncSliceLP" in ep_str:
-            caps[key]["lp"] = True
-
-    return caps
-
-
-@lru_cache(maxsize=1)
-def has_dri_devices() -> bool:
-    """
-    Check if there are any /dev/dri devices.
-
-    Returns:
-       bool: True if there are any /dev/dri devices, False otherwise.
-
-    """
-    try:
-        dri = Path("/dev/dri")
-        if not dri.exists() or not dri.is_dir():
-            return False
-
-        for _ in dri.iterdir():
-            return True
-
-        return False
-    except Exception:
-        return False
-
-
-@lru_cache(maxsize=1)
-def ffmpeg_encoders() -> set[str]:
-    """
-    Return a set of available ffmpeg encoders.
-
-    Returns:
-        set[str]: A set of available ffmpeg encoder names.
-
-    """
-    from app.library.config import SUPPORTED_CODECS
-
-    ffmpeg = shutil.which("ffmpeg")
-    if not ffmpeg:
-        return set()
-
-    try:
-        result: subprocess.CompletedProcess[str] = subprocess.run(
-            [ffmpeg, "-hide_banner", "-loglevel", "error", "-encoders"],
-            capture_output=True,
-            text=True,
-            check=False,
-            creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
-        )
-        out: str = (result.stdout or "") + (result.stderr or "")
-    except Exception:
-        return set()
-
-    encoders: set[str] = set()
-
-    if not out:
-        return encoders
-
-    for name in SUPPORTED_CODECS:
-        if name in out:
-            encoders.add(name)
-
-    return encoders
-
-
-def select_encoder(configured: str) -> str:
-    """
-    Select a concrete encoder.
-
-    Args:
-        configured (str): The configured encoder name, or empty for auto-detect.
-
-    Returns:
-        str: The selected concrete encoder name.
-
-    """
-    from app.library.config import SUPPORTED_CODECS
-
-    configured = (configured or "").strip()
-
-    avail: set[str] = ffmpeg_encoders()
-    if configured and configured in avail:
-        return configured
-
-    for name in SUPPORTED_CODECS:
-        if name in avail:
-            return name
-
-    return "libx264"
-
-
-class EncoderBuilder(Protocol):
-    def input_args(self, ctx: dict[str, Any] | None = None) -> list[str]:
-        """Encoder-specific input/global args that must appear before '-i'."""
-
-    def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]:
-        """Append encoder-specific video/output args and return the list."""
-
-
-class _BaseBuilder:
-    codec_name: str
-
-    def input_args(self, ctx: dict[str, Any] | None = None) -> list[str]:
-        _ = ctx
-        return []
-
-    def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]:
-        _ = ctx
-        return [*args, "-codec:v", self.codec_name]
-
-
-class SoftwareBuilder(_BaseBuilder):
-    codec_name = "libx264"
-
-    def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]:
-        return super().add_video_args(["-pix_fmt", "yuv420p", *args], ctx)
-
-
-class NvencBuilder(_BaseBuilder):
-    codec_name = "h264_nvenc"
-
-
-class AmfBuilder(_BaseBuilder):
-    codec_name = "h264_amf"
-
-
-class AppleVideoToolboxBuilder(_BaseBuilder):
-    codec_name = "h264_videotoolbox"
-
-
-class VaapiBuilder(_BaseBuilder):
-    codec_name = "h264_vaapi"
-
-    def input_args(self, ctx: dict[str, Any] | None = None) -> list[str]:
-        ctx = ctx or {}
-        is_linux: bool = bool(ctx.get("is_linux", sys.platform.startswith("linux")))
-        has_dri: bool = bool(ctx.get("has_dri", False))
-        device: str = ctx.get("vaapi_device", "/dev/dri/renderD128")
-        if is_linux and has_dri:
-            return ["-hwaccel", "vaapi", "-vaapi_device", str(device)]
-        return []
-
-    def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]:
-        ctx = ctx or {}
-        new_args: list[str] = list(args)
-        is_linux: bool = bool(ctx.get("is_linux", sys.platform.startswith("linux")))
-        has_dri: bool = bool(ctx.get("has_dri", False))
-        if is_linux and has_dri:
-            new_args += [
-                # Ensure frames are in VAAPI-compatible format and uploaded
-                "-vf",
-                "format=nv12,hwupload",
-                # Optional quality/preset flags similar to user's working config
-                "-crf",
-                "23",
-                "-preset:v",
-                "fast",
-                "-level",
-                "4.1",
-                "-profile:v",
-                "main",
-            ]
-        return super().add_video_args(new_args)
-
-
-class QsvBuilder(_BaseBuilder):
-    codec_name = "h264_qsv"
-
-    def input_args(self, ctx: dict[str, Any] | None = None) -> list[str]:
-        ctx = ctx or {}
-        args = []
-        is_linux: bool = bool(ctx.get("is_linux", sys.platform.startswith("linux")))
-        has_dri: bool = bool(ctx.get("has_dri", False))
-        device: str = ctx.get("vaapi_device", "/dev/dri/renderD128")
-        if is_linux and has_dri:
-            args: list[str] = [
-                "-init_hw_device",
-                f"qsv=hw:{device}",
-                "-hwaccel",
-                "qsv",
-                "-hwaccel_output_format",
-                "qsv",
-                "-filter_hw_device",
-                "hw",
-            ]
-
-        return args
-
-    def add_video_args(self, args: list[str], ctx: dict[str, Any] | None = None) -> list[str]:
-        ctx = ctx or {}
-        is_linux: bool = bool(ctx.get("is_linux", sys.platform.startswith("linux")))
-        has_dri: bool = bool(ctx.get("has_dri", False))
-        if is_linux and has_dri:
-            if ctx.get("qsv", {}).get("full", False):
-                return [
-                    *args,
-                    "-vf",
-                    "vpp_qsv=w=trunc(iw/2)*2:h=trunc(ih/2)*2:format=nv12",
-                    "-codec:v",
-                    self.codec_name,
-                    "-global_quality",
-                    "24",
-                    "-rc_mode",
-                    "cqp",
-                ]
-            return [
-                *args,
-                "-vf",
-                "vpp_qsv=w=trunc(iw/2)*2:h=trunc(ih/2)*2:format=nv12",
-                "-codec:v",
-                self.codec_name,
-                "-low_power",
-                "1",
-                "-rc_mode",
-                "cbr",
-                "-b:v",
-                "3M",
-                "-maxrate",
-                "3M",
-                "-bufsize",
-                "6M",
-            ]
-
-        return super().add_video_args(args, ctx)
-
-
-def get_builder_for_codec(codec: str) -> EncoderBuilder:
-    """
-    Return an EncoderBuilder instance for the given concrete codec name.
-
-    Args:
-        codec (str): The concrete codec name.
-
-    Returns:
-        EncoderBuilder: An instance of the corresponding EncoderBuilder.
-
-    """
-    soft = SoftwareBuilder()
-    return {
-        "libx264": soft,
-        "h264_nvenc": NvencBuilder(),
-        "h264_amf": AmfBuilder(),
-        "h264_qsv": QsvBuilder(),
-        "h264_videotoolbox": AppleVideoToolboxBuilder(),
-        "h264_vaapi": VaapiBuilder(),
-    }.get(codec, soft)
-
-
-def encoder_fallback_chain(codec: str) -> tuple[str, ...]:
-    """
-    Return a fallback chain for the given codec.
-
-    Args:
-        codec (str): The concrete codec name.
-
-    Returns:
-        tuple[str, ...]: A tuple of codec names representing the fallback chain.
-
-    """
-    chains: dict[str, list[str]] = {
-        "h264_qsv": ["h264_vaapi", "libx264"],
-        "h264_vaapi": ["libx264"],
-        "h264_nvenc": ["libx264"],
-        "h264_amf": ["libx264"],
-        "h264_videotoolbox": ["libx264"],
-        "libx264": [],
-    }
-
-    return tuple(chains.get(codec, chains["libx264"]))

app/features/streaming/library/segments.py

@@ -1,287 +0,0 @@
-import asyncio
-import os
-import subprocess
-import sys
-import tempfile
-from pathlib import Path
-from secrets import token_hex
-from typing import TYPE_CHECKING, ClassVar
-
-from aiohttp import web
-
-from app.features.streaming.library.ffprobe import ffprobe
-from app.features.streaming.library.segment_encoders import (
-    detect_qsv_capabilities,
-    encoder_fallback_chain,
-    get_builder_for_codec,
-    has_dri_devices,
-    select_encoder,
-)
-from app.library.config import SUPPORTED_CODECS, Config
-from app.library.log import get_logger
-
-if TYPE_CHECKING:
-    from asyncio.subprocess import Process
-
-    from .ffprobe import FFProbeResult
-    from .segment_encoders import EncoderBuilder
-
-LOG = get_logger()
-
-
-class Segments:
-    """
-    Build and stream MPEG-TS segments using ffmpeg with optional on-the-fly
-    transcoding. The class can auto-detect an available hardware encoder on the
-    first video segment and cache that choice for subsequent segments to avoid
-    repeated probing.
-    """
-
-    # Cache of selected video encoder across all segment instances
-    _cached_vcodec: ClassVar[str | None] = None
-    _cache_initialized: ClassVar[bool] = False
-
-    def __init__(self, download_path: str, index: int, duration: float, vconvert: bool, aconvert: bool):
-        config: Config = Config.get_instance()
-        self.download_path: str = download_path
-        "The path where files are downloaded."
-        self.index = int(index)
-        "The index of the segment."
-        self.duration = float(duration)
-        "The duration of the segment."
-        self.vconvert = bool(vconvert)
-        "Whether to convert video."
-        self.aconvert = bool(aconvert)
-        "Whether to convert audio."
-
-        # Default to configured codec; can be replaced by auto-detection
-        self.vcodec: str = config.streamer_vcodec
-        "The video codec to use."
-        self.acodec: str = config.streamer_acodec
-        "The audio codec to use."
-
-        # sadly due to unforeseen circumstances, we have to convert the video for now.
-        self.vconvert = True
-        "Whether to convert video."
-        self.aconvert = True
-        "Whether to convert audio."
-        self.attempted: set[str] = set()
-        "The set of attempted codecs."
-
-    def _make_stream_input(self, file: Path) -> Path:
-        while True:
-            stream_input = Path(tempfile.gettempdir()).joinpath(f"ytptube_stream.{token_hex(8)}")
-            try:
-                stream_input.symlink_to(file, target_is_directory=False)
-                return stream_input
-            except FileExistsError:
-                continue
-
-    async def build_ffmpeg_args(self, file: Path, s_codec: str, *, stream_input: Path | None = None) -> list[str]:
-        try:
-            ff: FFProbeResult = await ffprobe(file)
-        except UnicodeDecodeError:
-            pass
-        input_path = stream_input or file
-
-        startTime: str = f"{0:.6f}" if self.index == 0 else f"{self.duration * self.index:.6f}"
-
-        ctx = {
-            "is_linux": sys.platform.startswith("linux"),
-            "has_dri": has_dri_devices(),
-            "vaapi_device": Config.get_instance().vaapi_device,
-            "qsv": {"full": False, "lp": False},
-        }
-
-        caps: dict[str, dict[str, bool]] = detect_qsv_capabilities()
-        base_codec: str = s_codec.split("_", maxsplit=1)[0]
-        codec_caps: dict[str, bool] = caps.get(base_codec, {"full": False, "lp": False})
-        ctx["qsv"] = codec_caps
-
-        if self.vconvert and ff and hasattr(ff, "has_video") and ff.has_video():
-            builder: EncoderBuilder = get_builder_for_codec(s_codec)
-        else:
-            builder = None
-            ctx: dict = {}
-
-        # Collect encoder-specific input/global flags that must precede the input
-        input_args: list[str] = []
-        if builder:
-            input_args = builder.input_args(ctx)
-
-        fargs: list[str] = [
-            "-xerror",
-            "-hide_banner",
-            "-loglevel",
-            "error",
-            # input trimming before the input file
-            "-ss",
-            str(startTime),
-            "-t",
-            str(f"{self.duration:.6f}"),
-            "-copyts",
-            # hardware/global input options must come before -i
-            *input_args,
-            "-i",
-            f"file:{input_path}",
-            "-map_metadata",
-            "-1",
-        ]
-
-        v_args: list[str] = []
-        if builder:
-            v_args = builder.add_video_args(["-g", "52", "-map", "0:v:0", "-strict", "-2"], ctx)
-        else:
-            v_args += ["-codec:v", "copy"]
-
-        fargs += v_args
-
-        if ff and ff.has_audio():
-            fargs += ["-map", "0:a:0", "-codec:a", self.acodec if self.aconvert else "copy"]
-
-        fargs += ["-sn", "-muxdelay", "0", "-f", "mpegts", "pipe:1"]
-        return fargs
-
-    async def _run(self, resp: web.StreamResponse, file: Path, args: list[str]) -> tuple[bool, int, bool, str]:
-        proc: Process = await asyncio.create_subprocess_exec(
-            "ffmpeg",
-            *args,
-            stdin=asyncio.subprocess.DEVNULL,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-            creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
-        )
-
-        client_disconnected_local = False
-        wrote_any = False
-        stderr_buf = bytearray()
-
-        async def _drain_stderr() -> None:
-            try:
-                assert proc.stderr is not None
-                while True:
-                    chunk: bytes = await proc.stderr.read(4096)
-                    if not chunk:
-                        break
-                    stderr_buf.extend(chunk)
-            except Exception:
-                # best-effort only
-                pass
-
-        stderr_task: asyncio.Task[None] = asyncio.create_task(_drain_stderr())
-
-        LOG.debug(
-            "Streaming segment %s for '%s'.",
-            self.index,
-            file,
-            extra={"file": str(file), "segment_index": self.index, "ffmpeg_args": args},
-        )
-
-        try:
-            assert proc.stdout is not None
-            while True:
-                chunk: bytes = await proc.stdout.read(1024 * 64)
-                if not chunk:
-                    break
-                wrote_any = True
-                try:
-                    await resp.write(chunk)
-                except (asyncio.CancelledError, ConnectionResetError, BrokenPipeError, ConnectionError):
-                    LOG.warning("Client disconnected or connection reset while writing.")
-                    client_disconnected_local = True
-                    break
-        except asyncio.CancelledError:
-            LOG.warning("Client disconnected. Terminating ffmpeg.")
-            client_disconnected_local = True
-            proc.terminate()
-            try:
-                await asyncio.wait_for(proc.wait(), timeout=5)
-            except TimeoutError:
-                LOG.warning(
-                    "Segment stream for '%s' did not stop ffmpeg in time after the client disconnected; killing it.",
-                    file,
-                )
-                proc.kill()
-            raise
-        except ConnectionResetError:
-            LOG.warning("Connection reset by peer. Skipping further writes.")
-            client_disconnected_local = True
-        finally:
-            # Ensure process is stopped when client disconnected, to avoid hangs
-            if client_disconnected_local:
-                proc.terminate()
-                try:
-                    rc: int = await asyncio.wait_for(proc.wait(), timeout=5)
-                except TimeoutError:
-                    LOG.warning(
-                        "Segment stream for '%s' did not stop ffmpeg in time after the client disconnected; killing it.",
-                        file,
-                    )
-                    proc.kill()
-                    try:
-                        rc = await asyncio.wait_for(proc.wait(), timeout=5)
-                    except Exception:
-                        rc = -1
-            else:
-                # Normal termination path with a safety timeout
-                try:
-                    rc = await asyncio.wait_for(proc.wait(), timeout=30)
-                except TimeoutError:
-                    LOG.warning("Segment stream for '%s' did not stop ffmpeg in time; killing the process.", file)
-                    proc.kill()
-                    try:
-                        rc = await asyncio.wait_for(proc.wait(), timeout=5)
-                    except Exception:
-                        rc = -1
-            try:
-                await asyncio.wait_for(stderr_task, timeout=1)
-            except Exception:
-                pass
-
-        return (wrote_any, rc, client_disconnected_local, stderr_buf.decode("utf-8", errors="ignore").strip())
-
-    async def stream(self, file: Path, resp: web.StreamResponse) -> None:
-        codec: str = self.vcodec
-        if Segments._cache_initialized and Segments._cached_vcodec:
-            codec = Segments._cached_vcodec
-        if not codec or codec not in SUPPORTED_CODECS:
-            codec: str = select_encoder(self.vcodec or "")
-
-        LOG.debug("Selected video codec '%s' for segment streaming.", codec, extra={"codec": codec})
-
-        if Segments._cached_vcodec and Segments._cache_initialized:
-            codecs: list[str] = [Segments._cached_vcodec]
-        else:
-            codecs: list[str] = [codec, *list(encoder_fallback_chain(codec))]
-
-        stream_input = self._make_stream_input(file)
-        try:
-            for s_codec in codecs:
-                if s_codec in self.attempted:
-                    continue
-
-                ffmpeg_args: list[str] = await self.build_ffmpeg_args(file, s_codec, stream_input=stream_input)
-                _, rc, client_disconnected, stderr_text = await self._run(resp, file, ffmpeg_args)
-
-                if 0 == rc:
-                    Segments._cached_vcodec = s_codec
-                    Segments._cache_initialized = True
-                    return
-
-                if client_disconnected:
-                    return
-
-                if 0 != rc:
-                    err: str = stderr_text[:500] if stderr_text else "no error output"
-                    LOG.warning(
-                        "Retrying segment %s for '%s' because hardware encoder '%s' failed: %s.",
-                        self.index,
-                        file,
-                        s_codec,
-                        err,
-                        extra={"ffmpeg_args": ffmpeg_args, "returncode": rc, "stderr": err, "codec": s_codec},
-                    )
-                    self.attempted.add(s_codec)
-        finally:
-            if stream_input.is_symlink():
-                stream_input.unlink()

app/features/streaming/library/subtitle.py

@@ -1,155 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any
-
-import anyio
-import pysubs2
-from pysubs2.formats.substation import SubstationFormat
-from pysubs2.time import ms_to_times
-
-from app.library.log import get_logger
-from app.library.Utils import ALLOWED_SUBS_EXTENSIONS, get_file_sidecar
-
-LOG = get_logger()
-
-SOURCE_FORMATS: tuple[str, ...] = ("vtt", "srt", "ass")
-DELIVERY_FORMATS: dict[str, str] = {
-    "vtt": "vtt",
-    "srt": "vtt",
-    "ass": "ass",
-}
-RENDERERS: dict[str, str] = {
-    "vtt": "native",
-    "srt": "native",
-    "ass": "assjs",
-}
-MEDIA_TYPES: dict[str, str] = {
-    "vtt": "text/vtt; charset=UTF-8",
-    "ass": "text/x-ssa; charset=UTF-8",
-}
-TEXT_ENCODINGS: tuple[str, ...] = ("utf-8-sig", "utf-16", "utf-16-le", "utf-16-be", "cp1252")
-
-
-@dataclass(frozen=True, slots=True)
-class SubtitleTrack:
-    file: Path
-    lang: str
-    name: str
-    source_format: str
-    delivery_format: str
-    renderer: str
-
-
-def ms_to_timestamp(ms: int) -> str:
-    ms = max(0, ms)
-    h, m, s, ms = ms_to_times(ms)
-    cs: int = ms // 10
-    return f"{h:01d}:{m:02d}:{s:02d}.{cs:02d}"
-
-
-_substation_format: Any = SubstationFormat
-_substation_format.ms_to_timestamp = ms_to_timestamp
-
-
-class Subtitle:
-    @staticmethod
-    def normalize_format(source_format: str) -> str | None:
-        fmt = source_format.strip().lower().removeprefix(".")
-        if fmt not in SOURCE_FORMATS:
-            return None
-
-        return fmt
-
-    async def read_text(self, file: Path) -> str:
-        async with await anyio.open_file(file, "rb") as f:
-            subtitle_bytes = await f.read()
-
-        return self.decode_bytes(subtitle_bytes)
-
-    @staticmethod
-    def decode_bytes(subtitle_bytes: bytes) -> str:
-        for encoding in TEXT_ENCODINGS:
-            try:
-                return subtitle_bytes.decode(encoding)
-            except UnicodeDecodeError:
-                continue
-
-        return subtitle_bytes.decode("utf-8", errors="replace")
-
-    def media_type(self, file: Path) -> str:
-        fmt = self.normalize_format(file.suffix)
-        if fmt is None:
-            msg = f"File '{file}' subtitle type is not supported."
-            raise Exception(msg)
-
-        return MEDIA_TYPES[DELIVERY_FORMATS[fmt]]
-
-    async def make(self, file: Path) -> str:
-        fmt = self.normalize_format(file.suffix)
-        if fmt is None or file.suffix not in ALLOWED_SUBS_EXTENSIONS:
-            msg: str = f"File '{file}' subtitle type is not supported."
-            raise Exception(msg)
-
-        if fmt == "vtt":
-            return await self.read_text(file)
-
-        subs: pysubs2.SSAFile = pysubs2.load(path=str(file))
-
-        if len(subs.events) < 1:
-            msg = f"No subtitle events were found in '{file}'."
-            raise Exception(msg)
-
-        if len(subs.events) < 2:
-            return subs.to_string("vtt")
-
-        try:
-            if subs.events[0].end == subs.events[len(subs.events) - 1].end:
-                subs.events.pop(0)
-        except Exception:
-            pass
-
-        return subs.to_string("vtt")
-
-    async def make_delivery(self, file: Path) -> tuple[str, str]:
-        fmt = self.normalize_format(file.suffix)
-        if fmt is None or file.suffix not in ALLOWED_SUBS_EXTENSIONS:
-            msg: str = f"File '{file}' subtitle type is not supported."
-            raise Exception(msg)
-
-        if fmt == "ass":
-            return await self.read_text(file), self.media_type(file)
-
-        return await self.make(file), self.media_type(file)
-
-
-def get_subtitle_tracks(file: Path) -> list[SubtitleTrack]:
-    sidecars = get_file_sidecar(file).get("subtitle", [])
-    indexed_tracks: list[tuple[int, SubtitleTrack]] = []
-
-    for index, item in enumerate(sidecars):
-        track_file = item.get("file")
-        if not isinstance(track_file, Path):
-            continue
-
-        fmt = Subtitle.normalize_format(track_file.suffix)
-        if fmt is None:
-            continue
-
-        indexed_tracks.append(
-            (
-                index,
-                SubtitleTrack(
-                    file=track_file,
-                    lang=str(item.get("lang") or "und"),
-                    name=str(item.get("name") or track_file.name),
-                    source_format=fmt,
-                    delivery_format=DELIVERY_FORMATS[fmt],
-                    renderer=RENDERERS[fmt],
-                ),
-            )
-        )
-
-    indexed_tracks.sort(key=lambda item: (SOURCE_FORMATS.index(item[1].source_format), item[0]))
-    return [track for _, track in indexed_tracks]

app/features/streaming/library/thumbnail.py

@@ -1,284 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-import os
-import subprocess
-from pathlib import Path
-
-from app.features.streaming.library.ffprobe import ffprobe
-from app.library.cache import Cache
-from app.library.config import Config
-from app.library.log import get_logger
-from app.library.Utils import FILES_TYPE, get_file_sidecar
-
-LOG = get_logger()
-
-IMAGE_TYPES: tuple[str, ...] = (".jpg", ".jpeg", ".png", ".webp")
-FOLDER_IMAGE_ORDER: tuple[str, ...] = ("thumbnail", "poster", "artwork", "cover", "fanart")
-THUMBNAIL_SAMPLE_FRAMES = 200
-THUMBNAIL_DEFAULT_SEEK_SECONDS = 3.0
-THUMBNAIL_MIN_SEEK_SECONDS = 1.0
-THUMBNAIL_MAX_SEEK_SECONDS = 15.0
-THUMBNAIL_END_MARGIN_SECONDS = 0.1
-THUMBNAIL_MISS_TTL = 3600.0
-
-_LOCK = asyncio.Lock()
-_IN_PROCESS: dict[str, asyncio.Task[Path | None]] = {}
-
-
-_SEM: asyncio.Semaphore | None = None
-_SEM_LIMIT: int | None = None
-
-
-def _get_semaphore() -> asyncio.Semaphore:
-    global _SEM, _SEM_LIMIT  # noqa: PLW0603
-
-    limit: int = max(1, int(Config.get_instance().thumb_concurrency))
-    if _SEM is None or _SEM_LIMIT != limit:
-        _SEM = asyncio.Semaphore(limit)
-        _SEM_LIMIT = limit
-        LOG.info("Configured thumbnail generation to run %s job(s) at a time.", limit, extra={"limit": limit})
-
-    return _SEM
-
-
-def _is_same_stem_image(media_file: Path, image_file: Path) -> bool:
-    if image_file.parent != media_file.parent:
-        return False
-
-    for image_type in FILES_TYPE:
-        if "image" != image_type.get("type"):
-            continue
-
-        rx = image_type.get("rx")
-        if rx and rx.search(image_file.name):
-            break
-    else:
-        return False
-
-    return image_file.stem == media_file.stem
-
-
-def pick_local_thumb(media_file: Path) -> Path | None:
-    sidecar: dict[str, list[dict[str, str]]] = get_file_sidecar(media_file)
-    images: list[dict[str, str]] = sidecar.get("image", [])
-    local_images: list[Path] = [Path(item["file"]) for item in images if isinstance(item.get("file"), Path)]
-
-    for image_file in local_images:
-        if image_file.suffix.lower() in IMAGE_TYPES and _is_same_stem_image(media_file, image_file):
-            return image_file
-
-    by_name: dict[str, Path] = {
-        image_file.stem.lower(): image_file for image_file in local_images if image_file.suffix.lower() in IMAGE_TYPES
-    }
-    for name in FOLDER_IMAGE_ORDER:
-        if image_file := by_name.get(name):
-            return image_file
-
-    return None
-
-
-def _seek_seconds(ff_info) -> float | None:
-    duration: float = 0.0
-    try:
-        duration = float(ff_info.metadata.get("duration") or 0.0)
-    except (TypeError, ValueError):
-        duration = 0.0
-
-    if duration <= 0:
-        return THUMBNAIL_DEFAULT_SEEK_SECONDS
-
-    if duration <= THUMBNAIL_END_MARGIN_SECONDS:
-        return None
-
-    return min(
-        max(duration * 0.10, THUMBNAIL_MIN_SEEK_SECONDS),
-        THUMBNAIL_MAX_SEEK_SECONDS,
-        duration - THUMBNAIL_END_MARGIN_SECONDS,
-    )
-
-
-def _build_ffmpeg_args(media_file: Path, output_file: Path, *, seek_seconds: float | None) -> list[str]:
-    args: list[str] = [
-        "ffmpeg",
-        "-nostdin",
-        "-hide_banner",
-        "-loglevel",
-        "error",
-        "-y",
-    ]
-
-    if seek_seconds is not None and seek_seconds > 0:
-        args.extend(["-ss", f"{seek_seconds:.3f}"])
-
-    args.extend(
-        [
-            "-i",
-            str(media_file),
-            "-vf",
-            f"thumbnail={THUMBNAIL_SAMPLE_FRAMES},scale=1280:-1:force_original_aspect_ratio=decrease",
-            "-frames:v",
-            "1",
-            "-q:v",
-            "3",
-            "-an",
-            str(output_file),
-        ]
-    )
-    return args
-
-
-async def _run_ffmpeg(media_file: Path, output_file: Path) -> Path | None:
-    ff_info = await ffprobe(media_file)
-    if not ff_info.has_video():
-        LOG.debug(
-            "Skipping thumbnail generation for '%s' because no video stream exists.",
-            media_file,
-            extra={"media_file": str(media_file)},
-        )
-        return None
-
-    output_file.parent.mkdir(parents=True, exist_ok=True)
-    temp_file = output_file.with_suffix(".tmp.jpg")
-
-    if temp_file.exists():
-        temp_file.unlink(missing_ok=True)
-
-    seek_seconds: float | None = _seek_seconds(ff_info)
-    attempts: list[float | None] = [seek_seconds]
-    if seek_seconds is not None and seek_seconds > 0:
-        attempts.append(None)
-
-    sem = _get_semaphore()
-    if sem.locked():
-        limit = _SEM_LIMIT or 1
-        LOG.debug(
-            "Waiting for a thumbnail generation slot for '%s' (limit=%s).",
-            media_file,
-            limit,
-            extra={"media_file": str(media_file), "limit": limit},
-        )
-
-    async with sem:
-        last_error: str = "ffmpeg produced an empty thumbnail file"
-        for idx, attempt_seek in enumerate(attempts, start=1):
-            temp_file.unlink(missing_ok=True)
-            args: list[str] = _build_ffmpeg_args(media_file, temp_file, seek_seconds=attempt_seek)
-            LOG.debug(
-                "Generating thumbnail for '%s'. attempt=%s/%s seek=%s",
-                media_file,
-                idx,
-                len(attempts),
-                "none" if attempt_seek is None else f"{attempt_seek:.3f}s",
-                extra={
-                    "media_file": str(media_file),
-                    "attempt": idx,
-                    "attempt_count": len(attempts),
-                    "seek": "none" if attempt_seek is None else f"{attempt_seek:.3f}s",
-                },
-            )
-
-            try:
-                proc = await asyncio.create_subprocess_exec(
-                    *args,
-                    stdout=asyncio.subprocess.PIPE,
-                    stderr=asyncio.subprocess.PIPE,
-                    creationflags=subprocess.CREATE_NO_WINDOW if os.name == "nt" else 0,
-                )
-            except FileNotFoundError as exc:
-                msg = "ffmpeg not found."
-                raise OSError(msg) from exc
-
-            stdout, stderr = await proc.communicate()
-            if 0 == proc.returncode and temp_file.exists() and temp_file.stat().st_size > 0:
-                temp_file.replace(output_file)
-                LOG.info(
-                    "Generated thumbnail '%s' for '%s'.",
-                    output_file,
-                    media_file,
-                    extra={"media_file": str(media_file), "output_file": str(output_file)},
-                )
-                return output_file
-
-            if 0 != proc.returncode:
-                last_error: str = (
-                    f"ffmpeg thumbnail generation failed (rc={proc.returncode}).\n"
-                    f"stdout:\n{stdout.decode('utf-8', errors='replace')}\n"
-                    f"stderr:\n{stderr.decode('utf-8', errors='replace')}"
-                )
-            else:
-                last_error: str = "ffmpeg produced an empty thumbnail file"
-
-            LOG.debug(
-                "Thumbnail generation attempt failed for '%s'. seek=%s",
-                media_file,
-                "none" if attempt_seek is None else f"{attempt_seek:.3f}s",
-                extra={
-                    "media_file": str(media_file),
-                    "seek": "none" if attempt_seek is None else f"{attempt_seek:.3f}s",
-                },
-            )
-
-    temp_file.unlink(missing_ok=True)
-    raise OSError(last_error)
-
-
-async def ensure_thumb(media_file: Path, cache_root: Path, item_id: str | None = None) -> Path | None:
-    config: Config = Config.get_instance()
-    cache: Cache = Cache.get_instance()
-    if bool(config.thumb_generate) is not True:
-        return None
-
-    sidecar: bool = bool(config.thumb_sidecar)
-    if not sidecar and not item_id:
-        msg = "item_id is required when thumbnail sidecar is disabled."
-        raise ValueError(msg)
-
-    thumb_id: str = str(media_file.resolve()) if sidecar else str(item_id)
-    cache_file: Path = media_file.with_name(f"{media_file.name}.jpg") if sidecar else cache_root / f"{item_id}.jpg"
-
-    miss_key: str = f"thumbnail-miss:{thumb_id}"
-
-    if cache_file.exists() and cache_file.stat().st_size > 0:
-        cache.delete(miss_key)
-        return cache_file
-
-    if cache.has(miss_key):
-        return None
-
-    async with _LOCK:
-        if cache_file.exists() and cache_file.stat().st_size > 0:
-            cache.delete(miss_key)
-            return cache_file
-
-        if cache.has(miss_key):
-            return None
-
-        task = _IN_PROCESS.get(thumb_id)
-        if task is not None and not task.done():
-            LOG.debug("Waiting for thumbnail generation for '%s'.", media_file, extra={"media_file": str(media_file)})
-        else:
-            task = asyncio.create_task(_run_ffmpeg(media_file, cache_file), name=f"thumb-{item_id or media_file.stem}")
-            _IN_PROCESS[thumb_id] = task
-            LOG.debug(
-                "Starting thumbnail generation for '%s' -> '%s'.",
-                media_file,
-                cache_file,
-                extra={"media_file": str(media_file), "cache_file": str(cache_file)},
-            )
-
-    try:
-        result: Path | None = await task
-        if result is None:
-            cache.set(miss_key, value=True, ttl=THUMBNAIL_MISS_TTL)
-        else:
-            cache.delete(miss_key)
-        return result
-    except OSError:
-        cache.set(miss_key, value=True, ttl=THUMBNAIL_MISS_TTL)
-        raise
-    finally:
-        async with _LOCK:
-            current = _IN_PROCESS.get(thumb_id)
-            if current is task and task.done():
-                _IN_PROCESS.pop(thumb_id, None)

app/features/streaming/router.py

@@ -1,420 +0,0 @@
-import time
-from datetime import UTC, datetime
-from pathlib import Path
-
-from aiohttp import web
-from aiohttp.web import Request, Response
-from aiohttp.web_response import StreamResponse
-
-from app.features.streaming.library.m3u8 import M3u8
-from app.features.streaming.library.playlist import Playlist
-from app.features.streaming.library.segments import Segments
-from app.features.streaming.library.subtitle import Subtitle, get_subtitle_tracks
-from app.features.streaming.types import StreamingError
-from app.library.config import Config
-from app.library.log import get_logger
-from app.library.router import route
-from app.library.Utils import get_file
-
-LOG = get_logger()
-
-
-@route("GET", "api/player/playlist/{file:.*}.m3u8", "playlist_create")
-async def playlist_create(request: Request, config: Config, app: web.Application) -> Response:
-    """
-    Get the playlist.
-
-    Args:
-        request (Request): The request object.
-        config (Config): The configuration instance.
-        app (web.Application): The aiohttp application instance.
-
-    Returns:
-        Response: The response object.
-
-    """
-    file: str | None = request.match_info.get("file")
-
-    if not file:
-        return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
-
-    base_path: str = config.base_path.rstrip("/")
-
-    try:
-        realFile, status = get_file(download_path=config.download_path, file=file)
-        if web.HTTPFound.status_code == status:
-            return Response(
-                status=web.HTTPFound.status_code,
-                headers={
-                    "Location": str(
-                        app.router["playlist_create"].url_for(
-                            file=str(realFile).replace(config.download_path, "").strip("/")
-                        )
-                    ),
-                },
-            )
-
-        if web.HTTPNotFound.status_code == status:
-            return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
-
-        return web.Response(
-            text=await Playlist(download_path=Path(config.download_path), url=f"{base_path}/").make(file=realFile),
-            headers={
-                "Content-Type": "application/x-mpegURL",
-                "Cache-Control": "no-cache",
-                "Access-Control-Max-Age": "300",
-            },
-            status=web.HTTPOk.status_code,
-        )
-    except StreamingError as e:
-        return web.json_response(data={"error": str(e)}, status=web.HTTPNotFound.status_code)
-
-
-@route("GET", "api/player/m3u8/{mode}/{file:.*}.m3u8", "m3u8_create")
-async def m3u8_create(request: Request, config: Config, app: web.Application) -> Response:
-    """
-    Get the m3u8 file.
-
-    Args:
-        request (Request): The request object.
-        config (Config): The configuration instance.
-        app (web.Application): The aiohttp application instance.
-
-    Returns:
-        Response: The response object.
-
-    """
-    file: str | None = request.match_info.get("file")
-    mode: str | None = request.match_info.get("mode")
-
-    if mode not in ["video", "subtitle"]:
-        return web.json_response(
-            data={"error": "Only video and subtitle modes are supported."}, status=web.HTTPBadRequest.status_code
-        )
-
-    if not file:
-        return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
-
-    duration: float | None = None
-    duration_arg = request.query.get("duration", None)
-
-    if "subtitle" in mode:
-        if not duration_arg:
-            return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
-
-        duration = float(duration_arg)
-
-    base_path: str = config.base_path.rstrip("/")
-
-    try:
-        cls = M3u8(download_path=Path(config.download_path), url=f"{base_path}/")
-
-        realFile, status = get_file(download_path=config.download_path, file=file)
-        if web.HTTPFound.status_code == status:
-            return Response(
-                status=status,
-                headers={
-                    "Location": str(
-                        app.router["m3u8_create"].url_for(
-                            mode=mode, file=str(realFile).replace(config.download_path, "").strip("/")
-                        )
-                    ),
-                },
-            )
-
-        if web.HTTPNotFound.status_code == status:
-            return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
-
-        if "subtitle" in mode:
-            if duration is None:
-                return web.json_response(data={"error": "duration is required."}, status=web.HTTPBadRequest.status_code)
-            text = await cls.make_subtitle(file=realFile, duration=duration)
-        else:
-            text = await cls.make_stream(file=realFile)
-    except StreamingError as e:
-        LOG.exception(
-            "Failed to create %s streaming playlist for '%s': %s.",
-            mode,
-            file,
-            e,
-            extra={"route": "streaming.playlist", "file_path": file, "mode": mode, "exception_type": type(e).__name__},
-        )
-        return web.json_response(data={"error": str(e)}, status=web.HTTPNotFound.status_code)
-
-    return web.Response(
-        text=text,
-        headers={
-            "Content-Type": "application/x-mpegURL",
-            "Cache-Control": "no-cache",
-            "Access-Control-Max-Age": "300",
-        },
-        status=web.HTTPOk.status_code,
-    )
-
-
-@route("GET", r"api/player/segments/{segment:\d+}/{file:.*}.ts", "segments_stream")
-async def segments_stream(request: Request, config: Config, app: web.Application) -> StreamResponse:
-    """
-    Get the segments.
-
-    Args:
-        request (Request): The request object.
-        config (Config): The configuration instance.
-        app (web.Application): The aiohttp application instance.
-
-    Returns:
-        Response: The response object.
-
-    """
-    file: str | None = request.match_info.get("file")
-    segment: str | None = request.match_info.get("segment")
-    sd: str | None = request.query.get("sd")
-    vc: int = int(request.query.get("vc", 0))
-    ac: int = int(request.query.get("ac", 0))
-
-    if not file:
-        return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
-
-    if not segment:
-        return web.json_response(data={"error": "segment id is required."}, status=web.HTTPBadRequest.status_code)
-
-    realFile, status = get_file(download_path=config.download_path, file=file)
-    if web.HTTPFound.status_code == status:
-        return Response(
-            status=status,
-            headers={
-                "Location": str(
-                    app.router["segments_stream"].url_for(
-                        segment=segment,
-                        file=str(realFile).replace(config.download_path, "").strip("/"),
-                    )
-                ),
-            },
-        )
-
-    if web.HTTPNotFound.status_code == status:
-        return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
-
-    mtime = realFile.stat().st_mtime
-
-    if request.if_modified_since and request.if_modified_since.timestamp() == mtime:
-        lastMod = time.strftime("%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple())
-        return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod})
-
-    resp = web.StreamResponse(
-        status=web.HTTPOk.status_code,
-        headers={
-            "Content-Type": "video/mpegts",
-            "X-Accel-Buffering": "no",
-            "Access-Control-Allow-Origin": "*",
-            "Pragma": "public",
-            "Cache-Control": f"public, max-age={time.time() + 31536000}",
-            "Last-Modified": time.strftime(
-                "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple()
-            ),
-            "Expires": time.strftime(
-                "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple()
-            ),
-        },
-    )
-
-    await resp.prepare(request)
-
-    await Segments(
-        download_path=config.download_path,
-        index=int(segment),
-        duration=float(f"{float(sd or M3u8.duration):.6f}"),
-        vconvert=vc == 1,
-        aconvert=ac == 1,
-    ).stream(realFile, resp)
-
-    return resp
-
-
-@route("GET", "api/player/subtitle/{file:.*}.vtt", "subtitles_get")
-async def subtitles_get(request: Request, config: Config, app: web.Application) -> Response:
-    """
-    Get the subtitles.
-
-    Args:
-        request (Request): The request object.
-        config (Config): The configuration instance.
-        app (web.Application): The aiohttp application instance.
-
-    Returns:
-        Response: The response object.
-
-    """
-    file: str | None = request.match_info.get("file")
-
-    if not file:
-        return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
-
-    realFile, status = get_file(download_path=config.download_path, file=file)
-    if web.HTTPFound.status_code == status:
-        return Response(
-            status=status,
-            headers={
-                "Location": str(
-                    app.router["subtitles_get"].url_for(file=str(realFile).replace(config.download_path, "").strip("/"))
-                ),
-            },
-        )
-
-    if web.HTTPNotFound.status_code == status:
-        return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
-
-    mtime = realFile.stat().st_mtime
-
-    if request.if_modified_since and request.if_modified_since.timestamp() == mtime:
-        lastMod = time.strftime("%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple())
-        return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod})
-
-    return web.Response(
-        body=await Subtitle().make(file=realFile),
-        headers={
-            "Content-Type": "text/vtt; charset=UTF-8",
-            "X-Accel-Buffering": "no",
-            "Access-Control-Allow-Origin": "*",
-            "Pragma": "public",
-            "Cache-Control": f"public, max-age={time.time() + 31536000}",
-            "Last-Modified": time.strftime(
-                "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple()
-            ),
-            "Expires": time.strftime(
-                "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple()
-            ),
-        },
-        status=web.HTTPOk.status_code,
-    )
-
-
-@route("GET", "api/player/subtitles/manifest/{file:.*}", "subtitles_manifest_get")
-async def subtitles_manifest_get(request: Request, config: Config, app: web.Application) -> Response:
-    """
-    Get subtitle track metadata for a media file.
-
-    Args:
-        request (Request): The request object.
-        config (Config): The configuration instance.
-        app (web.Application): The aiohttp application instance.
-
-    Returns:
-        Response: The response object.
-
-    """
-    file: str | None = request.match_info.get("file")
-
-    if not file:
-        return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
-
-    realFile, status = get_file(download_path=config.download_path, file=file)
-    if web.HTTPFound.status_code == status:
-        return Response(
-            status=status,
-            headers={
-                "Location": str(
-                    app.router["subtitles_manifest_get"].url_for(
-                        file=str(realFile).replace(config.download_path, "").strip("/")
-                    )
-                ),
-            },
-        )
-
-    if web.HTTPNotFound.status_code == status:
-        return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
-
-    tracks = [
-        {
-            "lang": track.lang,
-            "name": track.name,
-            "source_format": track.source_format,
-            "delivery_format": track.delivery_format,
-            "renderer": track.renderer,
-            "url": str(
-                app.router["subtitles_track_get"].url_for(
-                    source_format=track.source_format,
-                    file=str(track.file).replace(config.download_path, "").strip("/"),
-                )
-            ),
-        }
-        for track in get_subtitle_tracks(realFile)
-    ]
-
-    return web.json_response(data={"subtitles": tracks}, status=web.HTTPOk.status_code)
-
-
-@route("GET", "api/player/subtitles/{source_format}/{file:.*}", "subtitles_track_get")
-async def subtitles_track_get(request: Request, config: Config, app: web.Application) -> Response:
-    """
-    Get a subtitle file using its preferred delivery format.
-
-    Args:
-        request (Request): The request object.
-        config (Config): The configuration instance.
-        app (web.Application): The aiohttp application instance.
-
-    Returns:
-        Response: The response object.
-
-    """
-    file: str | None = request.match_info.get("file")
-    source_format: str | None = request.match_info.get("source_format")
-
-    if not file:
-        return web.json_response(data={"error": "file is required"}, status=web.HTTPBadRequest.status_code)
-
-    fmt = Subtitle.normalize_format(source_format or "")
-    if fmt is None:
-        return web.json_response(
-            data={"error": "Only vtt, srt, and ass subtitle formats are supported."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    realFile, status = get_file(download_path=config.download_path, file=file)
-    if web.HTTPFound.status_code == status:
-        return Response(
-            status=status,
-            headers={
-                "Location": str(
-                    app.router["subtitles_track_get"].url_for(
-                        source_format=fmt,
-                        file=str(realFile).replace(config.download_path, "").strip("/"),
-                    )
-                ),
-            },
-        )
-
-    if web.HTTPNotFound.status_code == status:
-        return web.json_response(data={"error": f"File '{file}' does not exist."}, status=status)
-
-    if Subtitle.normalize_format(realFile.suffix) != fmt:
-        return web.json_response(
-            data={"error": f"Subtitle file '{file}' does not match requested source format '{fmt}'."},
-            status=web.HTTPBadRequest.status_code,
-        )
-
-    mtime = realFile.stat().st_mtime
-
-    if request.if_modified_since and request.if_modified_since.timestamp() == mtime:
-        lastMod = time.strftime("%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple())
-        return web.Response(status=web.HTTPNotModified.status_code, headers={"Last-Modified": lastMod})
-
-    body, content_type = await Subtitle().make_delivery(file=realFile)
-    return web.Response(
-        body=body,
-        headers={
-            "Content-Type": content_type,
-            "X-Accel-Buffering": "no",
-            "Access-Control-Allow-Origin": "*",
-            "Pragma": "public",
-            "Cache-Control": f"public, max-age={time.time() + 31536000}",
-            "Last-Modified": time.strftime(
-                "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(mtime, tz=UTC).timetuple()
-            ),
-            "Expires": time.strftime(
-                "%a, %d %b %Y %H:%M:%S GMT", datetime.fromtimestamp(time.time() + 31536000, tz=UTC).timetuple()
-            ),
-        },
-        status=web.HTTPOk.status_code,
-    )

app/features/streaming/tests/test_ffprobe.py

@@ -1,185 +0,0 @@
-from pathlib import Path
-from unittest.mock import AsyncMock, patch
-
-import pytest
-
-from app.tests.helpers import make_test_temp_dir
-
-
-class TestFFProbe:
-    """Test the ffprobe module functionality."""
-
-    def setup_method(self):
-        """Set up test files."""
-        self.temp_dir = str(make_test_temp_dir("ffprobe"))
-        self.test_file = Path(self.temp_dir) / "test_video.mp4"
-        self.test_file.touch()
-
-    def teardown_method(self):
-        """Clean up test files."""
-        import shutil
-
-        shutil.rmtree(self.temp_dir, ignore_errors=True)
-
-    @pytest.mark.asyncio
-    async def test_ffprobe_with_existing_file(self):
-        """Test ffprobe with an existing file."""
-        from app.features.streaming.library.ffprobe import FFProbeResult, ffprobe
-
-        # Mock subprocess to avoid actual ffprobe execution
-        with patch("asyncio.create_subprocess_exec") as mock_subprocess:
-            # Mock the subprocess result
-            mock_process = AsyncMock()
-            mock_process.wait.return_value = 0
-            mock_process.communicate.return_value = (b'{"format": {"duration": "10.0"}}', b"")
-            mock_subprocess.return_value = mock_process
-
-            with patch("anyio.open_file") as mock_open_file:
-                mock_file = AsyncMock()
-                mock_open_file.return_value.__aenter__.return_value = mock_file
-
-                result = await ffprobe(str(self.test_file))
-                assert isinstance(result, FFProbeResult)
-
-    @pytest.mark.asyncio
-    async def test_ffprobe_with_nonexistent_file(self):
-        """Test ffprobe with a non-existent file."""
-        from app.features.streaming.library.ffprobe import ffprobe
-
-        nonexistent_file = Path(self.temp_dir) / "does_not_exist.mp4"
-
-        with pytest.raises(OSError, match="No such media file"):
-            await ffprobe(str(nonexistent_file))
-
-    @pytest.mark.asyncio
-    async def test_ffprobe_caching_behavior(self):
-        """Test that ffprobe results are cached with enhanced async timed_lru_cache."""
-        from app.features.streaming.library.ffprobe import ffprobe
-
-        assert hasattr(ffprobe, "cache_clear"), (
-            "Test that the function has been decorated with caching - ffprobe should have cache_clear method from timed_lru_cache"
-        )
-        assert hasattr(ffprobe, "cache_info"), "ffprobe should have cache_info method from timed_lru_cache"
-
-        # Clear cache to start fresh
-        ffprobe.cache_clear()
-
-        # Mock subprocess to avoid actual ffprobe execution
-        call_count = 0
-        with patch("asyncio.create_subprocess_exec") as mock_subprocess:
-
-            def create_mock_process(*_args, **_kwargs):
-                nonlocal call_count
-                call_count += 1
-                mock_process = AsyncMock()
-                mock_process.wait.return_value = 0
-                mock_process.communicate.return_value = (b'{"format": {"duration": "10.0"}, "streams": []}', b"")
-                return mock_process
-
-            mock_subprocess.side_effect = create_mock_process
-
-            with patch("anyio.open_file") as mock_open_file:
-                mock_file = AsyncMock()
-                mock_open_file.return_value.__aenter__.return_value = mock_file
-
-                # First call should execute the function
-                result1 = await ffprobe(str(self.test_file))
-                assert isinstance(result1.metadata, dict)
-                first_call_count = call_count
-
-                # Second call with same argument should use cached result
-                result2 = await ffprobe(str(self.test_file))
-                assert isinstance(result2.metadata, dict)
-
-                assert call_count == first_call_count, (
-                    "The subprocess should not be called again for the actual ffprobe execution (it may be called for the -h check, but the main execution should be cached) - Second call should use cached result"
-                )
-
-                assert result1.metadata == result2.metadata, (
-                    "Results should be equivalent (same data, may not be same object due to async nature)"
-                )
-
-    @pytest.mark.asyncio
-    async def test_ffprobe_with_path_object(self):
-        """Test ffprobe with Path object input."""
-        from app.features.streaming.library.ffprobe import ffprobe
-
-        # Mock subprocess to avoid actual ffprobe execution
-        with patch("asyncio.create_subprocess_exec") as mock_subprocess:
-            mock_process = AsyncMock()
-            mock_process.wait.return_value = 0
-            mock_process.communicate.return_value = (b'{"format": {"duration": "10.0"}}', b"")
-            mock_subprocess.return_value = mock_process
-
-            with patch("anyio.open_file") as mock_open_file:
-                mock_file = AsyncMock()
-                mock_open_file.return_value.__aenter__.return_value = mock_file
-
-                # Test with Path object
-                result = await ffprobe(self.test_file)
-                assert result.metadata == {"duration": "10.0"}
-
-    def test_ffprobe_result_properties(self):
-        """Test FFProbeResult object properties."""
-        from app.features.streaming.library.ffprobe import FFStream, FFProbeResult
-
-        result = FFProbeResult()
-
-        assert result.video == [], "Test empty result"
-        assert result.audio == []
-        assert result.subtitle == []
-        assert result.attachment == []
-        assert result.metadata == {}
-
-        # Test adding streams
-        video_stream = FFStream(
-            {"index": 0, "codec_type": "video", "codec_name": "h264", "width": 1920, "height": 1080}
-        )
-
-        audio_stream = FFStream({"index": 1, "codec_type": "audio", "codec_name": "aac", "channels": 2})
-
-        result.video.append(video_stream)
-        result.audio.append(audio_stream)
-
-        assert len(result.video) == 1
-        assert len(result.audio) == 1
-        assert result.video[0].is_video()
-        assert result.audio[0].is_audio()
-
-    def test_stream_object_methods(self):
-        """Test Stream object methods."""
-        from app.features.streaming.library.ffprobe import FFStream
-
-        # Test video stream
-        video_stream = FFStream(
-            {"codec_type": "video", "codec_name": "h264", "width": 1920, "height": 1080, "duration": "10.5"}
-        )
-
-        assert video_stream.is_video()
-        assert not video_stream.is_audio()
-        assert not video_stream.is_subtitle()
-        assert not video_stream.is_attachment()
-
-        # Test audio stream
-        audio_stream = FFStream({"codec_type": "audio", "codec_name": "aac", "channels": 2, "sample_rate": "44100"})
-
-        assert not audio_stream.is_video()
-        assert audio_stream.is_audio()
-        assert not audio_stream.is_subtitle()
-        assert not audio_stream.is_attachment()
-
-        # Test subtitle stream
-        subtitle_stream = FFStream({"codec_type": "subtitle", "codec_name": "subrip"})
-
-        assert not subtitle_stream.is_video()
-        assert not subtitle_stream.is_audio()
-        assert subtitle_stream.is_subtitle()
-        assert not subtitle_stream.is_attachment()
-
-        # Test attachment stream
-        attachment_stream = FFStream({"codec_type": "attachment", "codec_name": "ttf"})
-
-        assert not attachment_stream.is_video()
-        assert not attachment_stream.is_audio()
-        assert not attachment_stream.is_subtitle()
-        assert attachment_stream.is_attachment()

app/features/streaming/tests/test_m3u8.py

@@ -1,146 +0,0 @@
-from pathlib import Path
-from types import SimpleNamespace
-from urllib.parse import quote
-
-import pytest
-
-from app.features.streaming.library.m3u8 import M3u8
-from app.features.streaming.types import StreamingError
-
-
-class _Stream:
-    def __init__(self, v: bool, a: bool, codec: str):
-        self.codec_name = codec
-        self._v = v
-        self._a = a
-
-    def is_video(self) -> bool:
-        return self._v
-
-    def is_audio(self) -> bool:
-        return self._a
-
-
-@pytest.mark.asyncio
-async def test_make_stream_basic_ok_codecs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    base = tmp_path / "dl"
-    base.mkdir()
-    media = base / "dir with space" / "file.mp4"
-    media.parent.mkdir()
-    media.write_text("x")
-
-    async def fake_ffprobe(_file: Path):
-        return SimpleNamespace(
-            metadata={"duration": "12"},
-            streams=lambda: [
-                _Stream(v=True, a=False, codec="h264"),
-                _Stream(v=False, a=True, codec="aac"),
-            ],
-        )
-
-    monkeypatch.setattr("app.features.streaming.library.m3u8.ffprobe", fake_ffprobe)
-
-    m3 = M3u8(download_path=base, url="http://host/")
-    out = await m3.make_stream(media)
-    lines = out.splitlines()
-
-    # Headers
-    assert lines[0] == "#EXTM3U"
-    assert lines[1] == "#EXT-X-VERSION:3"
-    assert lines[2] == "#EXT-X-TARGETDURATION:6"
-    assert lines[3] == "#EXT-X-MEDIA-SEQUENCE:0"
-    assert lines[4] == "#EXT-X-PLAYLIST-TYPE:VOD"
-
-    # Two segments: 0 (no params), 1 (sd present)
-    rel = quote(str(Path("dir with space/file.mp4")))
-    assert lines[5] == "#EXTINF:6.000000,"
-    assert lines[6] == f"http://host/api/player/segments/0/{rel}.ts"
-
-    assert lines[7] == "#EXTINF:6.000000,"
-    assert lines[8].startswith(f"http://host/api/player/segments/1/{rel}.ts?")
-    assert "sd=6.000000" in lines[8]
-
-    assert lines[9] == "#EXT-X-ENDLIST"
-
-
-@pytest.mark.asyncio
-async def test_make_stream_transcode_flags(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    base = tmp_path / "dl"
-    base.mkdir()
-    media = base / "v.mp4"
-    media.write_text("x")
-
-    async def fake_ffprobe(_file: Path):
-        return SimpleNamespace(
-            metadata={"duration": "13"},
-            streams=lambda: [
-                _Stream(v=True, a=False, codec="hevc"),
-                _Stream(v=False, a=True, codec="flac"),
-            ],
-        )
-
-    monkeypatch.setattr("app.features.streaming.library.m3u8.ffprobe", fake_ffprobe)
-
-    m3 = M3u8(download_path=base, url="https://s/")
-    out = await m3.make_stream(media)
-    lines = out.splitlines()
-
-    # 3 segments, last has 1.000000 duration and includes vc/ac + sd
-    rel = quote(str(Path("v.mp4")))
-    # First
-    assert lines[5] == "#EXTINF:6.000000,"
-    assert lines[6].startswith(f"https://s/api/player/segments/0/{rel}.ts?")
-    assert "vc=1" in lines[6]
-    assert "ac=1" in lines[6]
-    # Second
-    assert lines[7] == "#EXTINF:6.000000,"
-    assert lines[8].startswith(f"https://s/api/player/segments/1/{rel}.ts?")
-    assert "vc=1" in lines[8]
-    assert "ac=1" in lines[8]
-    # Third
-    assert lines[9] == "#EXTINF:1.000000,"
-    assert lines[10].startswith(f"https://s/api/player/segments/2/{rel}.ts?")
-    assert "vc=1" in lines[10]
-    assert "ac=1" in lines[10]
-    assert "sd=1.000000" in lines[10]
-    assert lines[11] == "#EXT-X-ENDLIST"
-
-
-@pytest.mark.asyncio
-async def test_make_stream_no_duration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    base = tmp_path / "dl"
-    base.mkdir()
-    media = base / "v.mp4"
-    media.write_text("x")
-
-    async def fake_ffprobe(_file: Path):
-        return SimpleNamespace(metadata={})
-
-    monkeypatch.setattr("app.features.streaming.library.m3u8.ffprobe", fake_ffprobe)
-
-    m3 = M3u8(download_path=base, url="http://s/")
-    with pytest.raises(StreamingError, match="Unable to get"):
-        await m3.make_stream(media)
-
-
-@pytest.mark.asyncio
-async def test_make_subtitle(tmp_path: Path) -> None:
-    base = tmp_path / "dl"
-    base.mkdir()
-    sub = base / "dir" / "cap.srt"
-    sub.parent.mkdir()
-    sub.write_text("x")
-
-    m3 = M3u8(download_path=base, url="http://h/")
-    out = await m3.make_subtitle(sub, duration=42.5)
-    lines = out.splitlines()
-
-    assert lines[0] == "#EXTM3U"
-    assert lines[1] == "#EXT-X-VERSION:3"
-    assert lines[2] == "#EXT-X-TARGETDURATION:6"
-    assert lines[3] == "#EXT-X-MEDIA-SEQUENCE:0"
-    assert lines[4] == "#EXT-X-PLAYLIST-TYPE:VOD"
-    assert lines[5] == "#EXTINF:42.5,"
-    rel = quote(str(Path("dir/cap.srt")))
-    assert lines[6] == f"http://h/api/player/subtitle/{rel}.vtt"
-    assert lines[7] == "#EXT-X-ENDLIST"

app/features/streaming/tests/test_playlist.py

@@ -1,118 +0,0 @@
-from pathlib import Path
-from types import SimpleNamespace
-from urllib.parse import quote
-
-import pytest
-
-from app.features.streaming.library.playlist import Playlist
-from app.features.streaming.types import StreamingError
-
-
-@pytest.mark.asyncio
-async def test_make_playlist_no_subtitles(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    # Setup paths
-    base = tmp_path / "downloads"
-    base.mkdir()
-    # Use a relative subpath with spaces to validate quoting behavior
-    media = base / "My Video.mp4"
-    media.write_text("x")
-
-    # Mock ffprobe to return duration
-    async def fake_ffprobe(_file: Path):
-        return SimpleNamespace(metadata={"duration": "60"})
-
-    monkeypatch.setattr("app.features.streaming.library.playlist.ffprobe", fake_ffprobe)
-    # No sidecar subtitles
-    monkeypatch.setattr("app.features.streaming.library.playlist.get_file_sidecar", lambda _f: {"subtitle": []})
-
-    # Patch module-level quote to be robust for Path objects
-    from urllib.parse import quote as _std_quote
-
-    monkeypatch.setattr("app.features.streaming.library.playlist.quote", lambda v: _std_quote(str(v)))
-
-    pl = Playlist(download_path=base, url="http://localhost/")
-    out = await pl.make(media)
-
-    lines = out.splitlines()
-    assert lines[0] == "#EXTM3U"
-    # No subtitles group when none present
-    assert lines[1] == "#EXT-X-STREAM-INF:PROGRAM-ID=1"
-    # Final line should point to video endpoint with quoted relative path
-    expected_ref = quote(str(Path("My Video.mp4")))
-    assert lines[2] == f"http://localhost/api/player/m3u8/video/{expected_ref}.m3u8"
-
-
-@pytest.mark.asyncio
-async def test_make_playlist_with_subtitles(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    base = tmp_path / "downloads"
-    base.mkdir()
-    media = base / "dir" / "file.mp4"
-    media.parent.mkdir()
-    media.write_text("x")
-
-    # ffprobe returns numeric duration
-    async def fake_ffprobe(_file: Path):
-        return SimpleNamespace(metadata={"duration": "12.5"})
-
-    monkeypatch.setattr("app.features.streaming.library.playlist.ffprobe", fake_ffprobe)
-
-    # Build two subtitle sidecars with names and langs; ensure quoting/relative name used
-    sub1 = media.with_suffix(".en.srt")
-    sub2 = media.with_name("another sub.srt")
-    sidecar = {
-        "subtitle": [
-            {"lang": "en", "file": sub1, "name": "English"},
-            {"lang": "fr", "file": sub2, "name": "French"},
-        ]
-    }
-    monkeypatch.setattr("app.features.streaming.library.playlist.get_file_sidecar", lambda _f: sidecar)
-
-    from urllib.parse import quote as _std_quote
-
-    monkeypatch.setattr("app.features.streaming.library.playlist.quote", lambda v: _std_quote(str(v)))
-
-    pl = Playlist(download_path=base, url="https://server/")
-    out = await pl.make(media)
-
-    lines = out.splitlines()
-    assert lines[0] == "#EXTM3U"
-    # Two EXT-X-MEDIA lines for subtitles
-    assert lines[1].startswith("#EXT-X-MEDIA:TYPE=SUBTITLES,")
-    assert 'NAME="English"' in lines[1]
-    assert 'LANGUAGE="en"' in lines[1]
-    assert "duration=12.5" in lines[1]
-    # URI uses the file name relative to base, replacing the name with sidecar file name and quoted
-    expected_uri1 = quote(str(Path("dir").joinpath(sub1.name)))
-    assert f"/subtitle/{expected_uri1}.m3u8" in lines[1]
-
-    assert lines[2].startswith("#EXT-X-MEDIA:TYPE=SUBTITLES,")
-    assert 'NAME="French"' in lines[2]
-    assert 'LANGUAGE="fr"' in lines[2]
-    expected_uri2 = quote(str(Path("dir").joinpath(sub2.name)))
-    assert f"/subtitle/{expected_uri2}.m3u8" in lines[2]
-
-    # Stream info with SUBTITLES group
-    assert lines[3] == '#EXT-X-STREAM-INF:PROGRAM-ID=1,SUBTITLES="subs"'
-    # Final video URL
-    expected_ref = quote(str(Path("dir/file.mp4")))
-    assert lines[4] == f"https://server/api/player/m3u8/video/{expected_ref}.m3u8"
-
-
-@pytest.mark.asyncio
-async def test_make_playlist_no_duration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    base = tmp_path / "downloads"
-    base.mkdir()
-    media = base / "file.mp4"
-    media.write_text("x")
-
-    # ffprobe missing duration
-    async def fake_ffprobe(_file: Path):
-        return SimpleNamespace(metadata={})
-
-    monkeypatch.setattr("app.features.streaming.library.playlist.ffprobe", fake_ffprobe)
-    monkeypatch.setattr("app.features.streaming.library.playlist.get_file_sidecar", lambda _f: {"subtitle": []})
-
-    pl = Playlist(download_path=base, url="http://localhost/")
-
-    with pytest.raises(StreamingError, match="Unable to get"):
-        await pl.make(media)

app/features/streaming/tests/test_segments.py

@@ -1,422 +0,0 @@
-import asyncio
-import logging
-from pathlib import Path
-from typing import Any
-
-import pytest
-from aiohttp import web
-
-from app.features.streaming.library.segments import Segments
-from app.tests.helpers import get_test_system_temp_root
-
-
-class DummyFF:
-    def __init__(self, v: bool, a: bool) -> None:
-        self._v: bool = v
-        self._a: bool = a
-
-    def has_video(self) -> bool:
-        return self._v
-
-    def has_audio(self) -> bool:
-        return self._a
-
-
-def _ffmpeg_input_path(args: list[str]) -> Path:
-    return Path(args[args.index("-i") + 1].removeprefix("file:"))
-
-
-@pytest.mark.asyncio
-async def test_build_ffmpeg_args_video_and_audio(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    # Create a dummy media file
-    media = tmp_path / "file.mp4"
-    media.write_bytes(b"data")
-
-    # Patch ffprobe to report both video and audio
-    async def fake_ffprobe(_file: Path):
-        return DummyFF(v=True, a=True)
-
-    monkeypatch.setattr("app.features.streaming.library.segments.ffprobe", fake_ffprobe)
-
-    seg = Segments(download_path=str(tmp_path), index=2, duration=5.5, vconvert=False, aconvert=False)
-
-    captured_args: list[list[str]] = []
-
-    class _EmptyProc(_FakeProc):
-        def __init__(self) -> None:
-            super().__init__([b""])
-
-    async def fake_create_subprocess_exec(*args: Any, **_kwargs: Any):
-        captured_args.append(list(args[1:]))
-        return _EmptyProc()
-
-    monkeypatch.setattr("asyncio.create_subprocess_exec", fake_create_subprocess_exec)
-
-    await seg.stream(media, _FakeResp())
-
-    assert captured_args, "ffmpeg was not invoked"
-    args = captured_args[0]
-
-    tmp_file = _ffmpeg_input_path(args)
-
-    # Start time is duration * index with 6 decimals for non-zero index
-    assert "-ss" in args
-    assert args[args.index("-ss") + 1] == f"{5.5 * 2:.6f}"
-    # Duration formatting
-    assert "-t" in args
-    assert args[args.index("-t") + 1] == f"{5.5:.6f}"
-    # Input uses file:<symlink>
-    assert "-i" in args
-    assert tmp_file.parent == get_test_system_temp_root()
-    assert tmp_file.name.startswith("ytptube_stream.")
-    assert not tmp_file.exists()
-    # Includes video and audio mapping and codecs
-    assert "-map" in args
-    assert "0:v:0" in args
-    assert "-codec:v" in args
-    assert "-codec:a" in args
-    # Output format: ensure -f mpegts and pipe:1 present
-    assert args[-1] == "pipe:1"
-    assert "-f" in args
-    assert args[args.index("-f") + 1] == "mpegts"
-
-
-@pytest.mark.asyncio
-async def test_build_ffmpeg_args_audio_only(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    media = tmp_path / "file.mp3"
-    media.write_bytes(b"data")
-
-    async def fake_ffprobe(_file: Path):
-        return DummyFF(v=False, a=True)
-
-    monkeypatch.setattr("app.features.streaming.library.segments.ffprobe", fake_ffprobe)
-
-    seg = Segments(download_path=str(tmp_path), index=0, duration=9.0, vconvert=False, aconvert=False)
-
-    captured_args: list[list[str]] = []
-
-    class _EmptyProc2(_FakeProc):
-        def __init__(self) -> None:
-            super().__init__([b""])
-
-    async def fake_create_subprocess_exec2(*args: Any, **_kwargs: Any):
-        captured_args.append(list(args[1:]))
-        return _EmptyProc2()
-
-    monkeypatch.setattr("asyncio.create_subprocess_exec", fake_create_subprocess_exec2)
-
-    await seg.stream(media, _FakeResp())
-
-    assert captured_args, "ffmpeg was not invoked"
-    args = captured_args[0]
-
-    # Start at 0 for index 0
-    assert "-ss" in args
-    assert args[args.index("-ss") + 1] == f"{0:.6f}"
-    # Should not include video mapping for audio-only file
-    assert "0:v:0" not in args
-    # Should include audio mapping and codec
-    assert "-map" in args
-    assert "0:a:0" in args
-    assert "-codec:a" in args
-
-
-class _FakeStdout:
-    def __init__(self, chunks: list[bytes]) -> None:
-        self._chunks = chunks
-
-    async def read(self, _size: int) -> bytes:
-        if not self._chunks:
-            return b""
-        return self._chunks.pop(0)
-
-
-class _FakeProc:
-    def __init__(self, chunks: list[bytes]) -> None:
-        self.stdout = _FakeStdout(chunks)
-        self.stderr = _FakeStdout([])
-        self.terminated = False
-        self.killed = False
-        self._rc = 0
-
-    async def wait(self) -> int:
-        return self._rc
-
-    def terminate(self) -> None:
-        self.terminated = True
-
-    def kill(self) -> None:
-        self.killed = True
-
-
-class _FakeResp(web.StreamResponse):
-    def __init__(self, fail_with: BaseException | None = None) -> None:
-        self.data: bytearray = bytearray()
-        self.eof = False
-        self._exc = fail_with
-
-    async def write(self, data: bytes | bytearray | memoryview, *_args: Any, **_kwargs: Any) -> None:
-        if self._exc:
-            raise self._exc
-        self.data.extend(data)
-
-    async def write_eof(self, *_args: Any, **_kwargs: Any) -> None:
-        self.eof = True
-
-
-class _FakeProcFail(_FakeProc):
-    def __init__(self, err: bytes = b"") -> None:
-        # no stdout data, immediate failure
-        super().__init__([b""])
-        self.stderr = _FakeStdout([err, b""])
-        self._rc = 1
-
-
-@pytest.mark.asyncio
-async def test_build_ffmpeg_args_no_dri(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    media = tmp_path / "file.mp4"
-    media.write_bytes(b"data")
-
-    async def fake_ffprobe(_file: Path):
-        return DummyFF(v=True, a=True)
-
-    monkeypatch.setattr("app.features.streaming.library.segments.ffprobe", fake_ffprobe)
-    # Simulate no /dev/dri present but GPU encoders otherwise available
-    monkeypatch.setattr("app.features.streaming.library.segment_encoders.has_dri_devices", lambda: False)
-    monkeypatch.setattr(
-        "app.features.streaming.library.segment_encoders.ffmpeg_encoders",
-        lambda: {"h264_nvenc", "h264_qsv", "h264_amf"},
-    )
-
-    # reset encoder cache to ensure clean selection in this test
-    from app.features.streaming.library.segments import Segments as _Seg
-
-    _Seg._cached_vcodec = None
-    _Seg._cache_initialized = False
-
-    seg = Segments(download_path=str(tmp_path), index=0, duration=1.0, vconvert=True, aconvert=True)
-    # Make preferred list try GPUs first
-    seg.vcodec = ""  # empty configured value triggers GPU->software preference
-
-    captured_args: list[list[str]] = []
-
-    class _EmptyProc3(_FakeProc):
-        def __init__(self) -> None:
-            super().__init__([b""])
-
-    async def fake_create_subprocess_exec3(*args: Any, **_kwargs: Any):
-        captured_args.append(list(args[1:]))
-        return _EmptyProc3()
-
-    monkeypatch.setattr("asyncio.create_subprocess_exec", fake_create_subprocess_exec3)
-
-    await seg.stream(media, _FakeResp())
-
-    assert captured_args, "ffmpeg was not invoked"
-    args = captured_args[0]
-    # Expect software encoder selected
-    # Expect that a software attempt occurs eventually; initial selection may be HW
-    assert "-codec:v" in args
-
-
-@pytest.mark.asyncio
-async def test_stream_gpu_fallback(
-    monkeypatch: pytest.MonkeyPatch, tmp_path: Path, caplog: pytest.LogCaptureFixture
-) -> None:
-    async def fake_ffprobe(_file: Path):
-        return DummyFF(v=True, a=True)
-
-    monkeypatch.setattr("app.features.streaming.library.segments.ffprobe", fake_ffprobe)
-    # Allow GPU usage and advertise an NVENC encoder so first pick is GPU
-    monkeypatch.setattr("app.features.streaming.library.segment_encoders.has_dri_devices", lambda: True)
-    monkeypatch.setattr("app.features.streaming.library.segment_encoders.ffmpeg_encoders", lambda: {"h264_nvenc"})
-
-    # First process fails (no data, rc=1), second succeeds and outputs bytes
-    proc_fail = _FakeProcFail(err=b"nvenc failure: encoder not available")
-    proc_ok = _FakeProc([b"gpu-fallback-", b"ok"])  # after fallback we stream this
-
-    calls: list[int] = []
-    captured_args: list[list[str]] = []
-
-    async def fake_create_subprocess_exec(*args: Any, **_kwargs: Any):
-        # args[1:] is the argv for ffmpeg (first element is 'ffmpeg')
-        captured_args.append(list(args[1:]))
-        calls.append(1)
-        return proc_fail if len(calls) == 1 else proc_ok
-
-    monkeypatch.setattr("asyncio.create_subprocess_exec", fake_create_subprocess_exec)
-
-    # reset encoder cache to ensure we try GPU first
-    from app.features.streaming.library.segments import Segments as _Seg
-
-    _Seg._cached_vcodec = None
-    _Seg._cache_initialized = False
-
-    seg = Segments(download_path=str(tmp_path), index=0, duration=1.0, vconvert=True, aconvert=True)
-    # Encourage GPU preference
-    seg.vcodec = ""  # empty -> try GPUs first
-    resp = _FakeResp()
-    with caplog.at_level(logging.WARNING, logger="ytptube"):
-        await seg.stream(tmp_path / "file.mp4", resp)
-
-    # Ensure fallback path streamed data
-    assert len(resp.data) > 0
-    # Ensure we logged the reason for GPU failure (message text may vary)
-    assert any(
-        ("hardware encoder" in r.message.lower()) or ("transcoding has failed" in r.message) for r in caplog.records
-    )
-    assert any("nvenc failure" in r.message for r in caplog.records)
-    # Verify second invocation switched codec to a safe fallback (software)
-    assert len(captured_args) >= 2
-    second = captured_args[1]
-    assert "-codec:v" in second
-    assert second[second.index("-codec:v") + 1] == "libx264"
-
-
-@pytest.mark.asyncio
-async def test_stream_gpu_fallback_switches_codec(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    async def fake_ffprobe(_file: Path):
-        return DummyFF(v=True, a=True)
-
-    monkeypatch.setattr("app.features.streaming.library.segments.ffprobe", fake_ffprobe)
-    # Only QSV advertised so initial build sets QSV
-    # Patch both where it's defined AND where it's imported/used
-    monkeypatch.setattr("app.features.streaming.library.segment_encoders.has_dri_devices", lambda: True)
-    monkeypatch.setattr("app.features.streaming.library.segments.has_dri_devices", lambda: True)
-    monkeypatch.setattr("app.features.streaming.library.segment_encoders.ffmpeg_encoders", lambda: {"h264_qsv"})
-
-    # Fail first, succeed second
-    proc_fail = _FakeProcFail(err=b"qsv failure")
-    proc_ok = _FakeProc([b"ok"])  # after fallback we stream this
-
-    captured_args: list[list[str]] = []
-
-    async def fake_create_subprocess_exec(*args: Any, **_kwargs: Any):
-        captured_args.append(list(args[1:]))
-        return proc_fail if len(captured_args) == 1 else proc_ok
-
-    monkeypatch.setattr("asyncio.create_subprocess_exec", fake_create_subprocess_exec)
-
-    # reset cache
-    from app.features.streaming.library.segments import Segments as _Seg
-
-    _Seg._cached_vcodec = None
-    _Seg._cache_initialized = False
-
-    seg = Segments(download_path=str(tmp_path), index=0, duration=1.0, vconvert=True, aconvert=True)
-    seg.vcodec = "intel"
-    resp = _FakeResp()
-    await seg.stream(tmp_path / "file.mp4", resp)
-
-    # First call had QSV codec and QSV flags
-    first = captured_args[0]
-    assert "-codec:v" in first
-    assert first[first.index("-codec:v") + 1] == "h264_qsv"
-    assert "-init_hw_device" in first
-    assert "qsv=hw:/dev/dri/renderD128" in first
-    assert "-filter_hw_device" in first
-    assert first[first.index("-filter_hw_device") + 1] == "hw"
-    assert "-vf" in first
-    assert "vpp_qsv" in first[first.index("-vf") + 1]
-
-    # Second call (fallback) must switch codec to a safe fallback
-    second = captured_args[1]
-    assert "-codec:v" in second
-    fallback_codec = second[second.index("-codec:v") + 1]
-    assert fallback_codec in {"h264_vaapi", "libx264"}
-    if fallback_codec == "libx264":
-        # No HW flags for software, ensure pix_fmt is set
-        assert "-init_hw_device" not in second
-        assert "-filter_hw_device" not in second
-        if "-vf" in second:
-            vf_val = second[second.index("-vf") + 1]
-            assert "scale_qsv" not in vf_val
-            assert "hwupload" not in vf_val
-        assert "-pix_fmt" in second
-        assert second[second.index("-pix_fmt") + 1] == "yuv420p"
-    else:
-        # VAAPI path: should include vaapi flags
-        assert "-vaapi_device" in second
-        assert "-hwaccel" in second
-        assert "vaapi" in second
-
-
-@pytest.mark.asyncio
-async def test_stream_normal_flow(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    media = tmp_path / "file.mp4"
-    media.write_bytes(b"data")
-
-    async def fake_ffprobe(_file: Path):
-        return DummyFF(v=True, a=True)
-
-    monkeypatch.setattr("app.features.streaming.library.segments.ffprobe", fake_ffprobe)
-
-    # Process that yields two chunks and then EOF
-    proc = _FakeProc([b"abc", b"def", b""])
-
-    captured_args: list[list[str]] = []
-
-    async def fake_create_subprocess_exec(*args: Any, **_kwargs: Any):
-        captured_args.append(list(args[1:]))
-        return proc
-
-    monkeypatch.setattr("asyncio.create_subprocess_exec", fake_create_subprocess_exec)
-
-    seg = Segments(download_path=str(tmp_path), index=0, duration=1.0, vconvert=True, aconvert=True)
-    resp = _FakeResp()
-    await seg.stream(media, resp)
-
-    assert bytes(resp.data) == b"abcdef"
-    assert not _ffmpeg_input_path(captured_args[0]).exists()
-    assert media.exists()
-    # EOF behavior may differ; don't require True
-
-
-@pytest.mark.asyncio
-async def test_stream_client_reset(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    async def fake_ffprobe(_file: Path):
-        return DummyFF(v=True, a=True)
-
-    monkeypatch.setattr("app.features.streaming.library.segments.ffprobe", fake_ffprobe)
-
-    proc = _FakeProc([b"abc", b"def"])  # will attempt to write and fail
-
-    captured_args: list[list[str]] = []
-
-    async def fake_create_subprocess_exec(*args: Any, **_kwargs: Any):
-        captured_args.append(list(args[1:]))
-        return proc
-
-    monkeypatch.setattr("asyncio.create_subprocess_exec", fake_create_subprocess_exec)
-
-    seg = Segments(download_path=str(tmp_path), index=0, duration=1.0, vconvert=True, aconvert=True)
-    resp = _FakeResp(fail_with=ConnectionResetError())
-
-    await seg.stream(tmp_path / "file.mp4", resp)
-
-    # Should not write EOF due to client disconnect
-    assert resp.eof is False
-    assert not _ffmpeg_input_path(captured_args[0]).exists()
-
-
-@pytest.mark.asyncio
-async def test_stream_cancelled(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
-    async def fake_ffprobe(_file: Path):
-        return DummyFF(v=True, a=True)
-
-    monkeypatch.setattr("app.features.streaming.library.segments.ffprobe", fake_ffprobe)
-
-    proc = _FakeProc([b"abc"])  # only one chunk
-
-    async def fake_create_subprocess_exec(*_args: Any, **_kwargs: Any):
-        return proc
-
-    monkeypatch.setattr("asyncio.create_subprocess_exec", fake_create_subprocess_exec)
-
-    seg = Segments(download_path=str(tmp_path), index=0, duration=1.0, vconvert=True, aconvert=True)
-    # Fail with CancelledError on write to hit inner disconnection branch
-    resp = _FakeResp(fail_with=asyncio.CancelledError())
-    await seg.stream(tmp_path / "file.mp4", resp)
-    # Inner branch treats it as client disconnected; no EOF and we terminate ffmpeg
-    assert resp.eof is False
-    assert proc.terminated is True

app/features/streaming/tests/test_subtitle.py

@@ -1,153 +0,0 @@
-from pathlib import Path
-from types import SimpleNamespace
-from unittest.mock import patch
-
-import pytest
-
-from app.features.streaming.library.subtitle import Subtitle, get_subtitle_tracks, ms_to_timestamp
-
-
-class TestMsToTimestamp:
-    def test_ms_to_timestamp_basic(self) -> None:
-        assert ms_to_timestamp(0) == "0:00:00.00"
-        assert ms_to_timestamp(9) == "0:00:00.00"
-        assert ms_to_timestamp(10) == "0:00:00.01"
-        assert ms_to_timestamp(12345) == "0:00:12.34"
-        assert ms_to_timestamp(3600000 + 120000 + 3000) == "1:02:03.00", "1 hour, 2 minutes, 3 seconds"
-        assert ms_to_timestamp(12 * 3600000 + 34 * 60000 + 56 * 1000 + 780) == "12:34:56.78", (
-            "Over 10 hours (SubStation limit is < 10h, our override must exceed)"
-        )
-        assert ms_to_timestamp(37 * 3600000 + 12 * 60000 + 34 * 1000 + 560) == "37:12:34.56", (
-            "Well over 36 hours to ensure no clamping at 9:59:59.99"
-        )
-
-
-@pytest.mark.asyncio
-async def test_make_unsupported_extension(tmp_path: Path) -> None:
-    file = tmp_path / "sub.txt"
-    file.write_text("not a subtitle")
-
-    subtitle = Subtitle()
-    with pytest.raises(Exception, match="subtitle type is not supported"):
-        await subtitle.make(file)
-
-
-@pytest.mark.asyncio
-async def test_make_vtt_reads_file(tmp_path: Path) -> None:
-    vtt = tmp_path / "file.vtt"
-    content = "WEBVTT\n\n00:00:00.00 --> 00:00:01.00\nHello"
-    vtt.write_text(content)
-
-    subtitle = Subtitle()
-    out = await subtitle.make(vtt)
-    assert out == content
-
-
-@pytest.mark.asyncio
-async def test_make_delivery_ass(tmp_path: Path) -> None:
-    ass = tmp_path / "file.ass"
-    content = "[Script Info]\nTitle: Demo\n"
-    ass.write_text(content, encoding="utf-8")
-
-    subtitle = Subtitle()
-    out, media_type = await subtitle.make_delivery(ass)
-    assert out == content
-    assert media_type == "text/x-ssa; charset=UTF-8"
-
-
-def test_tracks_order(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    media = tmp_path / "video.mkv"
-    media.write_text("x", encoding="utf-8")
-    ass_file = tmp_path / "video.ass"
-    ass_file.write_text("ass", encoding="utf-8")
-    vtt_file = tmp_path / "video.vtt"
-    vtt_file.write_text("WEBVTT\n\n", encoding="utf-8")
-    srt_file = tmp_path / "video.en.srt"
-    srt_file.write_text("1\n00:00:00,000 --> 00:00:01,000\nHello\n", encoding="utf-8")
-
-    monkeypatch.setattr(
-        "app.features.streaming.library.subtitle.get_file_sidecar",
-        lambda _file: {
-            "subtitle": [
-                {"file": ass_file, "lang": "en", "name": "ASS"},
-                {"file": srt_file, "lang": "en", "name": "SRT"},
-                {"file": vtt_file, "lang": "en", "name": "VTT"},
-            ]
-        },
-    )
-
-    tracks = get_subtitle_tracks(media)
-
-    assert [track.source_format for track in tracks] == ["vtt", "srt", "ass"]
-    assert [track.delivery_format for track in tracks] == ["vtt", "vtt", "ass"]
-    assert [track.renderer for track in tracks] == ["native", "native", "assjs"]
-
-
-class _DummySubs:
-    def __init__(self, events):
-        self.events = events
-        self.snapshot = None
-
-    def to_string(self, fmt: str) -> str:  # noqa: ARG002
-        # Record end times to verify pop behavior
-        self.snapshot = [e.end for e in self.events]
-        return "OUT"
-
-
-@pytest.mark.asyncio
-async def test_make_no_events_raises(tmp_path: Path) -> None:
-    srt = tmp_path / "sub.srt"
-    srt.write_text("dummy")
-
-    with patch("app.features.streaming.library.subtitle.pysubs2.load") as mock_load:
-        mock_load.return_value = _DummySubs(events=[])
-        subtitle = Subtitle()
-        with pytest.raises(Exception, match="No subtitle events were found"):
-            await subtitle.make(srt)
-
-
-@pytest.mark.asyncio
-async def test_make_single_vtt(tmp_path: Path) -> None:
-    srt = tmp_path / "sub.ass"
-    srt.write_text("dummy")
-
-    single = SimpleNamespace(end=1000)
-    d = _DummySubs(events=[single])
-
-    with patch("app.features.streaming.library.subtitle.pysubs2.load", return_value=d):
-        subtitle = Subtitle()
-        out = await subtitle.make(srt)
-        assert out == "OUT"
-        assert d.snapshot == [1000], "Snapshot should contain the single event"
-
-
-@pytest.mark.asyncio
-async def test_make_two_events_same_end(tmp_path: Path) -> None:
-    srt = tmp_path / "sub.srt"
-    srt.write_text("dummy")
-
-    e1 = SimpleNamespace(end=5000)
-    e2 = SimpleNamespace(end=5000)
-    d = _DummySubs(events=[e1, e2])
-
-    with patch("app.features.streaming.library.subtitle.pysubs2.load", return_value=d):
-        subtitle = Subtitle()
-        out = await subtitle.make(srt)
-        assert out == "OUT"
-        assert d.snapshot == [5000], "Since ends are equal, first should be popped => only last remains"
-
-
-@pytest.mark.asyncio
-async def test_make_two_events_keep_both(tmp_path: Path) -> None:
-    srt = tmp_path / "sub.srt"
-    srt.write_text("dummy")
-
-    e1 = SimpleNamespace(end=5000)
-    e2 = SimpleNamespace(end=6000)
-    d = _DummySubs(events=[e1, e2])
-
-    with patch("app.features.streaming.library.subtitle.pysubs2.load", return_value=d):
-        subtitle = Subtitle()
-        out = await subtitle.make(srt)
-        assert out == "OUT"
-        assert d.snapshot == [5000, 6000], "Both remain since ends differ"

app/features/streaming/tests/test_subtitle_routes.py

@@ -1,132 +0,0 @@
-from pathlib import Path
-from types import SimpleNamespace
-
-import pytest
-from aiohttp import web
-from aiohttp.test_utils import make_mocked_request
-
-from app.features.streaming import router
-from app.library.config import Config
-
-
-class _DummyRoute:
-    def __init__(self, pattern: str) -> None:
-        self.pattern = pattern
-
-    def url_for(self, **params: str) -> str:
-        path = self.pattern
-        for key, value in params.items():
-            path = path.replace(f"{{{key}:.*}}", value)
-            path = path.replace(f"{{{key}}}", value)
-        return path
-
-
-class _DummyRouter(dict[str, _DummyRoute]):
-    def __init__(self) -> None:
-        super().__init__(
-            {
-                "subtitles_manifest_get": _DummyRoute("/api/player/subtitles/manifest/{file:.*}"),
-                "subtitles_track_get": _DummyRoute("/api/player/subtitles/{source_format}/{file:.*}"),
-                "subtitles_get": _DummyRoute("/api/player/subtitle/{file:.*}.vtt"),
-            }
-        )
-
-
-def _make_request(path: str, *, match_info: dict[str, str]) -> web.Request:
-    return make_mocked_request("GET", path, app=SimpleNamespace(router=_DummyRouter()), match_info=match_info)
-
-
-@pytest.mark.asyncio
-async def test_subtitles_manifest_order(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    config = Config.get_instance()
-    config.download_path = str(tmp_path)
-
-    media = tmp_path / "video.mp4"
-    media.write_text("x", encoding="utf-8")
-
-    monkeypatch.setattr(
-        "app.features.streaming.router.get_file",
-        lambda **_kwargs: (media, web.HTTPOk.status_code),
-    )
-    monkeypatch.setattr(
-        "app.features.streaming.router.get_subtitle_tracks",
-        lambda _file: [
-            SimpleNamespace(
-                lang="en",
-                name="English VTT",
-                source_format="vtt",
-                delivery_format="vtt",
-                renderer="native",
-                file=tmp_path / "video.vtt",
-            ),
-            SimpleNamespace(
-                lang="en",
-                name="English ASS",
-                source_format="ass",
-                delivery_format="ass",
-                renderer="assjs",
-                file=tmp_path / "video.ass",
-            ),
-        ],
-    )
-
-    req = _make_request(
-        "/api/player/subtitles/manifest/video.mp4",
-        match_info={"file": "video.mp4"},
-    )
-    response = await router.subtitles_manifest_get(req, config, req.app)
-
-    assert response.status == web.HTTPOk.status_code
-    body = response.text
-    assert '"source_format": "vtt"' in body
-    assert '"renderer": "native"' in body
-    assert '"source_format": "ass"' in body
-    assert '"renderer": "assjs"' in body
-    assert body.index('"source_format": "vtt"') < body.index('"source_format": "ass"')
-
-
-@pytest.mark.asyncio
-async def test_subtitles_track_get_ass(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    config = Config.get_instance()
-    config.download_path = str(tmp_path)
-
-    subtitle = tmp_path / "video.ass"
-    subtitle.write_text("[Script Info]\nTitle: Demo\n", encoding="utf-8")
-
-    monkeypatch.setattr(
-        "app.features.streaming.router.get_file",
-        lambda **_kwargs: (subtitle, web.HTTPOk.status_code),
-    )
-
-    req = _make_request(
-        "/api/player/subtitles/ass/video.ass",
-        match_info={"source_format": "ass", "file": "video.ass"},
-    )
-    response = await router.subtitles_track_get(req, config, req.app)
-
-    assert response.status == web.HTTPOk.status_code
-    assert response.text == "[Script Info]\nTitle: Demo\n"
-    assert response.headers["Content-Type"] == "text/x-ssa; charset=UTF-8"
-
-
-@pytest.mark.asyncio
-async def test_subtitles_track_bad_format(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
-    config = Config.get_instance()
-    config.download_path = str(tmp_path)
-
-    subtitle = tmp_path / "video.ass"
-    subtitle.write_text("[Script Info]\nTitle: Demo\n", encoding="utf-8")
-
-    monkeypatch.setattr(
-        "app.features.streaming.router.get_file",
-        lambda **_kwargs: (subtitle, web.HTTPOk.status_code),
-    )
-
-    req = _make_request(
-        "/api/player/subtitles/vtt/video.ass",
-        match_info={"source_format": "vtt", "file": "video.ass"},
-    )
-    response = await router.subtitles_track_get(req, config, req.app)
-
-    assert response.status == web.HTTPBadRequest.status_code
-    assert b"does not match requested source format" in response.body

app/features/streaming/tests/test_thumbnail.py

@@ -1,358 +0,0 @@
-from __future__ import annotations
-
-import asyncio
-from pathlib import Path
-from typing import Any
-from unittest.mock import AsyncMock
-
-import pytest
-
-from app.tests.helpers import temporary_test_dir
-
-
-@pytest.mark.asyncio
-async def test_pick_same_stem() -> None:
-    from app.features.streaming.library.thumbnail import pick_local_thumb
-
-    with temporary_test_dir("thumb-local") as temp_dir:
-        media = temp_dir / "video.mp4"
-        poster = temp_dir / "poster.jpg"
-        thumb = temp_dir / "video.jpg"
-
-        media.write_text("video")
-        poster.write_text("poster")
-        thumb.write_text("thumb")
-
-        assert pick_local_thumb(media) == thumb
-
-
-@pytest.mark.asyncio
-async def test_singleflight(monkeypatch: pytest.MonkeyPatch) -> None:
-    from app.features.streaming.library import thumbnail
-
-    thumbnail._IN_PROCESS.clear()
-    thumbnail.Cache.get_instance().clear()
-
-    with temporary_test_dir("thumb-singleflight") as temp_dir:
-        media = temp_dir / "video.mp4"
-        cache_root = temp_dir / "cache"
-        media.write_text("video")
-        item_id = "item-1"
-
-        calls = {"count": 0}
-
-        async def fake_run_ffmpeg(_file: Path, output_file: Path) -> Path:
-            calls["count"] += 1
-            await asyncio.sleep(0.01)
-            output_file.parent.mkdir(parents=True, exist_ok=True)
-            output_file.write_text("image")
-            return output_file
-
-        monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
-
-        first, second = await asyncio.gather(
-            thumbnail.ensure_thumb(media, cache_root, item_id=item_id),
-            thumbnail.ensure_thumb(media, cache_root, item_id=item_id),
-        )
-
-        assert first == second
-        assert first is not None
-        assert calls["count"] == 1
-        assert thumbnail._IN_PROCESS == {}
-
-
-@pytest.mark.asyncio
-async def test_cache_hit(monkeypatch: pytest.MonkeyPatch) -> None:
-    from app.features.streaming.library import thumbnail
-
-    thumbnail._IN_PROCESS.clear()
-    thumbnail.Cache.get_instance().clear()
-
-    with temporary_test_dir("thumb-cache") as temp_dir:
-        media = temp_dir / "video.mp4"
-        cache_root = temp_dir / "cache"
-        media.write_text("video")
-        item_id = "item-1"
-
-        cache_file = cache_root / f"{item_id}.jpg"
-        cache_file.parent.mkdir(parents=True, exist_ok=True)
-        cache_file.write_text("image")
-
-        async def fake_run_ffmpeg(_file: Path, _output_file: Path) -> Path:
-            raise AssertionError("ffmpeg should not run when cache exists")
-
-        monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
-
-        result = await thumbnail.ensure_thumb(media, cache_root, item_id=item_id)
-
-        assert result == cache_file
-
-
-@pytest.mark.asyncio
-async def test_disabled(monkeypatch: pytest.MonkeyPatch) -> None:
-    from app.features.streaming.library import thumbnail
-
-    thumbnail._IN_PROCESS.clear()
-    thumbnail.Cache.get_instance().clear()
-
-    with temporary_test_dir("thumb-disabled") as temp_dir:
-        media = temp_dir / "video.mp4"
-        media.write_text("video")
-
-        monkeypatch.setattr(
-            thumbnail.Config,
-            "get_instance",
-            staticmethod(lambda: type("Cfg", (), {"thumb_generate": False, "thumb_sidecar": False})()),
-        )
-
-        result = await thumbnail.ensure_thumb(media, temp_dir / "cache", item_id="item-1")
-
-        assert result is None
-
-
-@pytest.mark.asyncio
-async def test_sidecar_mode(monkeypatch: pytest.MonkeyPatch) -> None:
-    from app.features.streaming.library import thumbnail
-
-    thumbnail._IN_PROCESS.clear()
-    thumbnail._SEM = None
-    thumbnail._SEM_LIMIT = None
-    thumbnail.Cache.get_instance().clear()
-
-    with temporary_test_dir("thumb-sidecar") as temp_dir:
-        media = temp_dir / "video.mp4"
-        media.write_text("video")
-
-        sidecar = media.with_name(f"{media.name}.jpg")
-
-        monkeypatch.setattr(
-            thumbnail.Config,
-            "get_instance",
-            staticmethod(
-                lambda: type(
-                    "Cfg",
-                    (),
-                    {"thumb_generate": True, "thumb_sidecar": True, "thumb_concurrency": 1},
-                )()
-            ),
-        )
-
-        async def fake_run_ffmpeg(_file: Path, out_file: Path) -> Path:
-            out_file.write_text("image")
-            return out_file
-
-        monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
-
-        result = await thumbnail.ensure_thumb(media, temp_dir / "cache")
-
-        assert result == sidecar
-        assert sidecar.exists()
-
-
-@pytest.mark.asyncio
-async def test_miss_cache(monkeypatch: pytest.MonkeyPatch) -> None:
-    from app.features.streaming.library import thumbnail
-
-    thumbnail._IN_PROCESS.clear()
-    thumbnail.Cache.get_instance().clear()
-
-    with temporary_test_dir("thumb-miss") as temp_dir:
-        media = temp_dir / "video.mp4"
-        media.write_text("video")
-        item_id = "item-1"
-
-        monkeypatch.setattr(
-            thumbnail.Config,
-            "get_instance",
-            staticmethod(
-                lambda: type(
-                    "Cfg",
-                    (),
-                    {"thumb_generate": True, "thumb_sidecar": False, "thumb_concurrency": 1},
-                )()
-            ),
-        )
-
-        calls = {"count": 0}
-
-        async def fake_run_ffmpeg(_file: Path, _out_file: Path) -> Path | None:
-            calls["count"] += 1
-            return None
-
-        monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
-
-        first = await thumbnail.ensure_thumb(media, temp_dir / "cache", item_id=item_id)
-        second = await thumbnail.ensure_thumb(media, temp_dir / "cache", item_id=item_id)
-
-        assert first is None
-        assert second is None
-        assert calls["count"] == 1
-
-
-def test_sem_limit(monkeypatch: pytest.MonkeyPatch) -> None:
-    from app.features.streaming.library import thumbnail
-
-    thumbnail._SEM = None
-    thumbnail._SEM_LIMIT = None
-
-    monkeypatch.setattr(
-        thumbnail.Config,
-        "get_instance",
-        staticmethod(
-            lambda: type("Cfg", (), {"thumb_concurrency": 3, "thumb_generate": True, "thumb_sidecar": False})()
-        ),
-    )
-
-    sem = thumbnail._get_semaphore()
-
-    assert sem._value == 3
-    assert thumbnail._SEM_LIMIT == 3
-
-
-def test_seek_bounds() -> None:
-    from app.features.streaming.library.ffprobe import FFProbeResult
-    from app.features.streaming.library.thumbnail import _seek_seconds
-
-    ff_info = FFProbeResult()
-    ff_info.metadata = {"duration": "120.0"}
-    assert _seek_seconds(ff_info) == 12.0
-
-    ff_info.metadata = {"duration": "2.0"}
-    assert _seek_seconds(ff_info) == 1.0
-
-    ff_info.metadata = {"duration": "0.05"}
-    assert _seek_seconds(ff_info) is None
-
-    ff_info.metadata = {}
-    assert _seek_seconds(ff_info) == 3.0
-
-
-@pytest.mark.asyncio
-async def test_cache_name_uses_item_id(monkeypatch: pytest.MonkeyPatch) -> None:
-    from app.features.streaming.library import thumbnail
-
-    thumbnail._IN_PROCESS.clear()
-    thumbnail.Cache.get_instance().clear()
-
-    with temporary_test_dir("thumb-id-name") as temp_dir:
-        media = temp_dir / "video.mp4"
-        cache_root = temp_dir / "cache"
-        media.write_text("video")
-
-        async def fake_run_ffmpeg(_file: Path, output_file: Path) -> Path:
-            output_file.parent.mkdir(parents=True, exist_ok=True)
-            output_file.write_text("image")
-            return output_file
-
-        monkeypatch.setattr(thumbnail, "_run_ffmpeg", fake_run_ffmpeg)
-
-        result = await thumbnail.ensure_thumb(media, cache_root, item_id="item-1")
-
-        assert result == cache_root / "item-1.jpg"
-
-
-@pytest.mark.asyncio
-async def test_retry_no_seek(monkeypatch: pytest.MonkeyPatch) -> None:
-    from app.features.streaming.library.ffprobe import FFProbeResult
-    from app.features.streaming.library import thumbnail
-
-    with temporary_test_dir("thumb-retry") as temp_dir:
-        media = temp_dir / "video.mp4"
-        output = temp_dir / ".jpg"
-        media.write_text("video")
-
-        ff_info: Any = FFProbeResult()
-        ff_info.metadata = {"duration": "120.0"}
-        ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
-
-        monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info))
-
-        calls: list[list[str]] = []
-
-        class DummyProc:
-            def __init__(self, attempt: int, out_path: Path) -> None:
-                self.returncode = 1 if attempt == 1 else 0
-                self._out_path = out_path
-
-            async def communicate(self) -> tuple[bytes, bytes]:
-                if self.returncode == 0:
-                    self._out_path.write_text("image")
-                    return b"", b""
-                return b"", b"seek failed"
-
-        async def fake_create_subprocess_exec(*args, **kwargs):
-            del kwargs
-            calls.append([str(arg) for arg in args])
-            return DummyProc(len(calls), Path(str(args[-1])))
-
-        monkeypatch.setattr(thumbnail.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
-
-        result = await thumbnail._run_ffmpeg(media, output)
-
-        assert result == output
-        assert len(calls) == 2
-        assert "-ss" in calls[0]
-        assert calls[0][calls[0].index("-ss") + 1] == "12.000"
-        assert "-ss" not in calls[1]
-        assert calls[0][calls[0].index("-vf") + 1] == "thumbnail=200,scale=1280:-1:force_original_aspect_ratio=decrease"
-        assert calls[0][-1].endswith(".tmp.jpg")
-
-
-@pytest.mark.asyncio
-async def test_limit_wait(monkeypatch: pytest.MonkeyPatch) -> None:
-    from app.features.streaming.library import thumbnail
-    from app.features.streaming.library.ffprobe import FFProbeResult
-
-    thumbnail._SEM = None
-    thumbnail._SEM_LIMIT = None
-    thumbnail._IN_PROCESS.clear()
-    monkeypatch.setattr(
-        thumbnail.Config,
-        "get_instance",
-        staticmethod(
-            lambda: type("Cfg", (), {"thumb_concurrency": 1, "thumb_generate": True, "thumb_sidecar": False})()
-        ),
-    )
-
-    with temporary_test_dir("thumb-limit") as temp_dir:
-        media1 = temp_dir / "video1.mp4"
-        media2 = temp_dir / "video2.mp4"
-        out1 = temp_dir / "out1.jpg"
-        out2 = temp_dir / "out2.jpg"
-        media1.write_text("video")
-        media2.write_text("video")
-
-        ff_info: Any = FFProbeResult()
-        ff_info.metadata = {"duration": "60.0"}
-        ff_info.video = [type("Video", (), {"codec_type": "video", "codec_name": "h264"})()]
-        monkeypatch.setattr(thumbnail, "ffprobe", AsyncMock(return_value=ff_info))
-
-        active = {"count": 0, "max": 0}
-
-        class DummyProc:
-            def __init__(self, out_path: Path) -> None:
-                self.returncode = 0
-                self._out_path = out_path
-
-            async def communicate(self) -> tuple[bytes, bytes]:
-                active["count"] += 1
-                active["max"] = max(active["max"], active["count"])
-                await asyncio.sleep(0.01)
-                self._out_path.write_text("image")
-                active["count"] -= 1
-                return b"", b""
-
-        async def fake_create_subprocess_exec(*args, **kwargs):
-            del kwargs
-            return DummyProc(Path(str(args[-1])))
-
-        monkeypatch.setattr(thumbnail.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
-
-        first, second = await asyncio.gather(
-            thumbnail._run_ffmpeg(media1, out1),
-            thumbnail._run_ffmpeg(media2, out2),
-        )
-
-        assert first == out1
-        assert second == out2
-        assert active["max"] == 1

app/features/streaming/types.py

@@ -1,6 +0,0 @@
-class StreamingError(Exception):
-    """Raised when an error occurs during streaming."""
-
-
-class FFProbeError(Exception):
-    """Raised when an error occurs during FFProbe processing."""

app/features/tasks/__init__.py

@@ -1 +0,0 @@
-"""Tasks Feature."""

app/features/tasks/definitions/__init__.py

@@ -1 +0,0 @@
-"""Tasks Definitions Feature."""

app/features/tasks/definitions/deps.py

@@ -1,5 +0,0 @@
-from app.features.tasks.definitions.repository import TaskDefinitionsRepository
-
-
-def get_task_definitions_repo() -> TaskDefinitionsRepository:
-    return TaskDefinitionsRepository.get_instance()

app/features/tasks/definitions/handlers/__init__.py

@@ -1 +0,0 @@
-"""Handlers package for task definitions."""

app/features/tasks/definitions/handlers/_base_handler.py

@@ -1,71 +0,0 @@
-from typing import TYPE_CHECKING, Any
-
-from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskResult
-from app.library.config import Config
-from app.library.httpx_client import Globals, build_request_headers, get_async_client, resolve_curl_transport
-
-if TYPE_CHECKING:
-    import httpx
-
-
-class BaseHandler:
-    @staticmethod
-    async def can_handle(task: HandleTask) -> bool:
-        _ = task
-        return False
-
-    @staticmethod
-    async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
-        raise NotImplementedError
-
-    @classmethod
-    async def inspect(cls, task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
-        return await cls.extract(task=task, config=config)
-
-    @staticmethod
-    def parse(url: str) -> Any | None:
-        _ = url
-        return None
-
-    @staticmethod
-    def tests() -> list[tuple[str, bool]]:
-        return []
-
-    @staticmethod
-    async def request(
-        url: str, headers: dict | None = None, ytdlp_opts: dict | None = None, **kwargs
-    ) -> "httpx.Response":
-        """
-        Make an HTTP request.
-
-        Args:
-            url (str): The URL to request.
-            headers (dict | None): Additional headers to include in the request.
-            ytdlp_opts (dict | None): yt-dlp options that may affect the request.
-            **kwargs: Additional arguments to pass to httpx request.
-
-        Returns:
-            httpx.Response: The HTTP response.
-
-        """
-        headers = {} if not isinstance(headers, dict) else headers
-        ytdlp_opts = {} if not isinstance(ytdlp_opts, dict) else ytdlp_opts
-
-        use_curl = resolve_curl_transport()
-        request_headers = build_request_headers(
-            base_headers=headers,
-            user_agent=Globals.get_random_agent(),
-            use_curl=use_curl,
-        )
-
-        proxy = ytdlp_opts.get("proxy", None)
-        client = get_async_client(proxy=proxy, use_curl=use_curl)
-        method = kwargs.pop("method", "GET").upper()
-        timeout = ytdlp_opts.get("timeout", ytdlp_opts.get("socket_timeout", 120))
-        return await client.request(
-            method=method,
-            url=url,
-            headers=request_headers,
-            timeout=timeout,
-            **kwargs,
-        )

app/features/tasks/definitions/handlers/generic.py

@@ -1,925 +0,0 @@
-"""Generic task handler driven by JSON definitions."""
-
-from __future__ import annotations
-
-import asyncio
-import fnmatch
-import hashlib
-import json
-import re
-from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any
-from urllib.parse import urljoin
-
-import httpx
-import jmespath
-from parsel import Selector
-
-from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
-from app.features.tasks.definitions.schemas import (
-    ExtractionRule,
-    TaskDefinition,
-)
-from app.features.ytdlp.extractor import fetch_info
-from app.features.ytdlp.utils import get_archive_id
-from app.library.cache import Cache
-from app.library.config import Config
-from app.library.httpx_client import Globals, build_request_headers, get_async_client, resolve_curl_transport
-from app.library.log import get_logger
-
-from ._base_handler import BaseHandler
-
-if TYPE_CHECKING:
-    from pathlib import Path
-
-    from parsel.selector import SelectorList
-
-LOG = get_logger()
-CACHE: Cache = Cache()
-
-
-class GenericTaskHandler(BaseHandler):
-    """Handler that scrapes arbitrary web pages based on JSON task definitions."""
-
-    _definitions: list[TaskDefinition] = []
-    """Cached loaded task definitions."""
-
-    _sources_mtime: dict[Path, float] = {}
-    """Modification times of source files to detect changes."""
-
-    @classmethod
-    async def refresh_definitions(cls, force: bool = False) -> list[TaskDefinition]:
-        """
-        Refresh the cached task definitions if source files have changed.
-
-        Args:
-            force (bool): If True, force reload even if no changes detected.
-
-        """
-        if cls._definitions and not force:
-            return cls._definitions
-
-        try:
-            from app.features.tasks.definitions.repository import TaskDefinitionsRepository
-            from app.features.tasks.definitions.utils import model_to_schema
-
-            repo = TaskDefinitionsRepository.get_instance()
-            models = await repo.all()
-
-            cls._definitions = [model_to_schema(model) for model in models]
-            return cls._definitions
-        except Exception as exc:
-            LOG.exception(
-                "Failed to load generic task definitions.",
-                extra={"error": str(exc), "exception_type": type(exc).__name__},
-            )
-            return []
-
-    @classmethod
-    async def _find_definition(cls, url: str) -> TaskDefinition | None:
-        """
-        Find a task definition that matches the given URL.
-
-        Args:
-            url (str): The URL to match.
-
-        Returns:
-            (TaskDefinition|None): A matching TaskDefinition if found, None otherwise.
-
-        """
-        await cls.refresh_definitions()
-
-        for definition in cls._definitions:
-            if not definition.enabled:
-                continue
-
-            try:
-                for matcher in definition.match_url:
-                    pattern_str: str | None = None
-
-                    if matcher.startswith("/") and matcher.endswith("/") and len(matcher) > 2:
-                        pattern_str = matcher[1:-1]
-                    else:
-                        pattern_str = fnmatch.translate(matcher)
-
-                    if pattern_str and re.match(pattern_str, url):
-                        return definition
-            except Exception as exc:
-                LOG.exception(
-                    "Failed to match a generic task definition.",
-                    extra={
-                        "definition": definition.name,
-                        "url": url,
-                        "error": str(exc),
-                        "exception_type": type(exc).__name__,
-                    },
-                )
-
-        return None
-
-    @staticmethod
-    async def can_handle(task: HandleTask) -> bool:
-        """
-        Determine if this handler can process the given task.
-
-        Args:
-            task (Task): The task to check.
-
-        Returns:
-            (bool): True if the handler can process the task, False otherwise.
-
-        """
-        definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
-        if definition:
-            LOG.debug(
-                "Task '%s' matched a generic task definition.",
-                task.name,
-                extra={"task_name": task.name, "url": task.url, "definition": definition.name},
-            )
-            return True
-
-        return False
-
-    @staticmethod
-    async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
-        _ = config
-        definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
-        if not definition:
-            return TaskFailure(message="No generic task definition matched the provided URL.")
-
-        ytdlp_opts: dict[str, Any] = task.get_ytdlp_opts().get_all()
-        target_url: str = definition.definition.request.url or task.url
-
-        LOG.debug(
-            "Fetching content for task '%s'.",
-            task.name,
-            extra={
-                "task_name": task.name,
-                "definition": definition.name,
-                "url": target_url,
-                "engine": definition.definition.engine.type,
-            },
-        )
-
-        try:
-            body_text, json_data = await GenericTaskHandler._fetch_content(
-                url=target_url, definition=definition, ytdlp_opts=ytdlp_opts
-            )
-        except httpx.HTTPError as exc:
-            return TaskFailure(message="Failed to fetch target URL.", error=str(exc))
-        except Exception as exc:
-            LOG.exception(
-                "Failed to fetch content for task '%s'.",
-                task.name,
-                extra={
-                    "task_name": task.name,
-                    "definition": definition.name,
-                    "url": target_url,
-                    "exception_type": type(exc).__name__,
-                },
-            )
-            return TaskFailure(message="Failed to fetch target URL.", error=str(exc))
-
-        if "json" == definition.definition.response.type and json_data is None:
-            return TaskFailure(message="Expected JSON response but decoding failed.")
-
-        if "json" != definition.definition.response.type and not body_text:
-            return TaskFailure(message="Received empty response body.")
-
-        raw_items: list[dict[str, str]] = GenericTaskHandler._parse_items(
-            definition=definition, html=body_text or "", base_url=target_url, json_data=json_data
-        )
-
-        task_items: list[TaskItem] = []
-
-        def _generic_id(url):
-            import os
-            from urllib import parse
-
-            return parse.unquote(os.path.splitext(url.rstrip("/").split("/")[-1])[0])
-
-        for entry in raw_items:
-            if not isinstance(entry, dict):
-                continue
-
-            if not (url := entry.get("link") or entry.get("url")):
-                continue
-
-            id_dict: dict[str, str | None] = get_archive_id(url=url)
-            archive_id: str | None = id_dict.get("archive_id")
-            if not archive_id:
-                cache_key: str = hashlib.sha256(f"{task.name}-{url}".encode()).hexdigest()
-                if CACHE.has(cache_key):
-                    archive_id = CACHE.get(cache_key)
-                    if not archive_id:
-                        continue
-                else:
-                    LOG.warning(
-                        "Task '%s' could not generate a static archive ID. Fetching it with yt-dlp.",
-                        task.name,
-                        extra={"definition": definition.name, "task_name": task.name, "url": url},
-                    )
-
-                    (info, _) = await fetch_info(
-                        config=task.get_ytdlp_opts().get_all(),
-                        url=url,
-                        no_archive=True,
-                        no_log=True,
-                        budget_sleep=True,
-                    )
-
-                    if not info:
-                        LOG.error(
-                            "Task '%s' failed to extract info to generate an archive ID. Skipping item.",
-                            task.name,
-                            extra={"definition": definition.name, "task_name": task.name, "url": url},
-                        )
-                        CACHE.set(cache_key, None)
-                        continue
-
-                    if not info.get("id") or not info.get("extractor_key"):
-                        LOG.error(
-                            "Task '%s' returned incomplete info while generating an archive ID. Skipping item.",
-                            task.name,
-                            extra={"definition": definition.name, "task_name": task.name, "url": url},
-                        )
-                        CACHE.set(cache_key, None)
-                        continue
-
-                    archive_id = f"{str(info.get('extractor_key', '')).lower()} {info.get('id')}"
-                    CACHE.set(cache_key, archive_id)
-
-            metadata: dict[str, str] = {
-                k: v for k, v in entry.items() if k not in {"link", "url", "title", "published", "archive_id"}
-            }
-
-            task_items.append(
-                TaskItem(
-                    url=url,
-                    title=entry.get("title"),
-                    archive_id=archive_id,
-                    metadata={"published": entry.get("published"), **metadata},
-                )
-            )
-
-        return TaskResult(
-            items=task_items,
-            metadata={
-                "definition": definition.name,
-                "response_format": definition.definition.response.type,
-            },
-        )
-
-    @staticmethod
-    async def _fetch_content(
-        url: str,
-        definition: TaskDefinition,
-        ytdlp_opts: dict[str, Any],
-    ) -> tuple[str | None, Any | None]:
-        """
-        Fetch the content of the given URL using the specified engine.
-
-        Args:
-            url (str): The URL to fetch.
-            definition (TaskDefinitionRuntimeSchema): The task definition specifying the engine and request details.
-            ytdlp_opts (dict[str, Any]): yt-dlp options that may influence fetching
-
-        Returns:
-            (str|None): The fetched HTML content if successful, None otherwise.
-
-        """
-        if "selenium" == definition.definition.engine.type:
-            return await GenericTaskHandler._fetch_with_selenium(url=url, definition=definition)
-
-        return await GenericTaskHandler._fetch_with_httpx(url=url, definition=definition, ytdlp_opts=ytdlp_opts)
-
-    @staticmethod
-    async def _fetch_with_httpx(
-        url: str,
-        definition: TaskDefinition,
-        ytdlp_opts: dict[str, Any],
-    ) -> tuple[str | None, Any | None]:
-        """
-        Fetch the content using httpx.
-
-        Args:
-            url (str): The URL to fetch.
-            definition (TaskDefinitionRuntimeSchema): The task definition specifying the request details.
-            ytdlp_opts (dict[str, Any]): yt-dlp options that may influence fetching
-
-        Returns:
-            (str|None): The fetched HTML content if successful, None otherwise.
-
-        """
-        headers: dict[str, str] = {**definition.definition.request.headers}
-        use_curl = resolve_curl_transport()
-        request_headers = build_request_headers(
-            base_headers=headers,
-            user_agent=Globals.get_random_agent(),
-            use_curl=use_curl,
-        )
-
-        timeout_value: float | Any = definition.definition.request.timeout or ytdlp_opts.get("socket_timeout", 120)
-
-        client = get_async_client(proxy=ytdlp_opts.get("proxy"), use_curl=use_curl)
-        response: httpx.Response = await client.request(
-            method=definition.definition.request.method.upper(),
-            url=url,
-            params=definition.definition.request.params or None,
-            data=definition.definition.request.data,
-            json=definition.definition.request.json_data,
-            timeout=timeout_value,
-            headers=request_headers,
-        )
-        response.raise_for_status()
-
-        if "json" == definition.definition.response.type:
-            try:
-                json_data: dict[str, Any] = response.json()
-            except Exception as exc:
-                LOG.exception(
-                    "Task definition '%s' returned invalid JSON for '%s'.",
-                    definition.name,
-                    url,
-                    extra={
-                        "definition": definition.name,
-                        "url": url,
-                        "error": str(exc),
-                        "exception_type": type(exc).__name__,
-                    },
-                )
-                return response.text, None
-
-            return response.text, json_data
-
-        return response.text, None
-
-    @staticmethod
-    async def _fetch_with_selenium(
-        url: str,
-        definition: TaskDefinition,
-    ) -> tuple[str | None, Any | None]:
-        """
-        Fetch the content using a Selenium WebDriver.
-
-        Args:
-            url (str): The URL to fetch.
-            definition (TaskDefinitionRuntimeSchema): The task definition specifying the engine options.
-
-        Returns:
-            (str|None): The fetched HTML content if successful, None otherwise.
-
-        """
-        try:
-            from selenium import webdriver
-            from selenium.webdriver.chrome.options import Options as ChromeOptions
-            from selenium.webdriver.common.by import By
-            from selenium.webdriver.support import expected_conditions as EC
-            from selenium.webdriver.support.ui import WebDriverWait
-        except ImportError as exc:
-            LOG.exception(
-                "Task definition '%s' requested Selenium, but Selenium is not installed.",
-                definition.name,
-                extra={"definition": definition.name, "error": str(exc), "exception_type": type(exc).__name__},
-            )
-            return (None, None)
-
-        options_map: dict[str, Any] = definition.definition.engine.options
-        command_executor_value = options_map.get("url")
-        command_executor: str = (
-            str(command_executor_value)
-            if isinstance(command_executor_value, str) and command_executor_value
-            else "http://localhost:4444/wd/hub"
-        )
-        browser: str = str(options_map.get("browser", "chrome")).lower()
-
-        if "chrome" != browser:
-            LOG.error(
-                "Task definition '%s' requested unsupported Selenium browser '%s'.",
-                definition.name,
-                browser,
-                extra={"definition": definition.name, "browser": browser},
-            )
-            return (None, None)
-
-        arguments: list[str] | str = options_map.get("arguments", ["--headless", "--disable-gpu"])
-        if isinstance(arguments, str):
-            arguments = [arguments]
-
-        wait_for: Mapping | None = (
-            options_map.get("wait_for") if isinstance(options_map.get("wait_for"), Mapping) else None
-        )
-        wait_timeout = float(options_map.get("wait_timeout") or 15)
-        page_load_timeout = float(options_map.get("page_load_timeout") or 60)
-
-        def load_page() -> str | None:
-            chrome_options = ChromeOptions()
-            for arg in arguments:
-                chrome_options.add_argument(str(arg))
-
-            driver = webdriver.Remote(command_executor=command_executor, options=chrome_options)
-            try:
-                driver.set_page_load_timeout(page_load_timeout)
-                driver.get(url)
-
-                if wait_for:
-                    wait_type: str = str(wait_for.get("type", "css")).lower()
-                    expression: str | None = wait_for.get("expression") or wait_for.get("value")
-                    if isinstance(expression, str) and expression:
-                        locator = None
-                        locator = (By.XPATH, expression) if "xpath" == wait_type else (By.CSS_SELECTOR, expression)
-                        WebDriverWait(driver, wait_timeout).until(EC.presence_of_element_located(locator))
-
-                return driver.page_source
-            finally:
-                driver.quit()
-
-        html: str | None = await asyncio.to_thread(load_page)
-        return html, None
-
-    @staticmethod
-    def _parse_items(
-        definition: TaskDefinition,
-        html: str,
-        base_url: str,
-        json_data: Any | None = None,
-    ) -> list[dict[str, str]]:
-        """
-        Parse the HTML content and extract items based on the definition.
-
-        Args:
-            definition (TaskDefinitionRuntimeSchema): The task definition specifying the parsers.
-            html (str): The HTML content to parse.
-            base_url (str): The base URL to resolve relative links.
-            json_data (Any|None): The JSON data to parse if applicable.
-
-        Returns:
-            (list[dict[str, str]]): A list of extracted items as dictionaries.
-
-        """
-        if "json" == definition.definition.response.type:
-            return GenericTaskHandler._parse_json_items(definition, json_data, base_url)
-
-        selector = Selector(text=html)
-
-        if definition.definition.parse.get("items"):
-            return GenericTaskHandler._parse_with_container(
-                definition=definition,
-                selector=selector,
-                html=html,
-                base_url=base_url,
-            )
-
-        extracted: dict[str, list[str]] = {}
-
-        for _field, rule_data in definition.definition.parse.field_items():
-            if not isinstance(rule_data, dict):
-                continue
-            rule = ExtractionRule.model_validate(rule_data)
-            values: list[str] = GenericTaskHandler._execute_rule(field=_field, selector=selector, html=html, rule=rule)
-            extracted[_field] = values
-
-        link_values: list[str] = extracted.get("link", [])
-        if not link_values:
-            LOG.debug(
-                "Definition '%s' produced no link values.", definition.name, extra={"definition": definition.name}
-            )
-            return []
-
-        total_items: int = len(link_values)
-        items: list[dict[str, str]] = []
-
-        for index in range(total_items):
-            entry: dict[str, str] = {}
-            link_value: str = link_values[index]
-            if not link_value:
-                continue
-
-            entry["link"] = urljoin(base_url, link_value)
-
-            for _field, values in extracted.items():
-                if "link" == _field:
-                    continue
-
-                value: str | None = values[index] if index < len(values) else None
-                if value is None:
-                    continue
-
-                entry[_field] = value
-
-            items.append(entry)
-
-        return items
-
-    @staticmethod
-    def _parse_json_items(
-        definition: TaskDefinition,
-        json_data: Any | None,
-        base_url: str,
-    ) -> list[dict[str, str]]:
-        if json_data is None:
-            LOG.debug(
-                "Definition '%s' expects JSON but no data was parsed.",
-                definition.name,
-                extra={"definition": definition.name},
-            )
-            return []
-
-        if definition.definition.parse.get("items"):
-            return GenericTaskHandler._parse_json_with_container(definition, json_data, base_url)
-
-        items: list[dict[str, str]] = []
-        entry: dict[str, str] = {}
-
-        for _field, rule_data in definition.definition.parse.field_items():
-            if not isinstance(rule_data, dict):
-                continue
-            rule = ExtractionRule.model_validate(rule_data)
-            values: list[str] = GenericTaskHandler._execute_json_rule(_field, json_data, rule)
-            if values:
-                if "link" == _field:
-                    entry["link"] = urljoin(base_url, values[0])
-                else:
-                    entry[_field] = values[0]
-
-        if "link" in entry:
-            items.append(entry)
-
-        return items
-
-    @staticmethod
-    def _parse_with_container(
-        definition: TaskDefinition,
-        selector: Selector,
-        html: str,
-        base_url: str,
-    ) -> list[dict[str, str]]:
-        container: dict[str, Any] | None = definition.definition.parse.get("items")
-        if not container:
-            return []
-
-        container_type = container.get("type", "css")
-        container_selector = container.get("selector") or container.get("expression") or ""
-        if not container_selector:
-            LOG.error(
-                "Task definition '%s' is missing an item container selector.",
-                definition.name,
-                extra={"definition": definition.name},
-            )
-            return []
-        container_fields = container.get("fields", {})
-
-        selection: SelectorList[Selector] = (
-            selector.css(container_selector) if "css" == container_type else selector.xpath(container_selector)
-        )
-
-        items: list[dict[str, str]] = []
-
-        for node in selection:
-            node_html: Any | str = node.get() or html
-            entry: dict[str, str] = {}
-
-            for _field, rule_data in container_fields.items():
-                rule = ExtractionRule.model_validate(rule_data)
-                values: list[str] = GenericTaskHandler._execute_rule(
-                    field=_field,
-                    selector=node,
-                    html=node_html,
-                    rule=rule,
-                )
-
-                value: str | None = values[0] if values else None
-                if value is None:
-                    continue
-
-                if "link" == _field:
-                    entry["link"] = urljoin(base_url, value)
-                else:
-                    entry[_field] = value
-
-            if "link" not in entry:
-                continue
-
-            items.append(entry)
-
-        return items
-
-    @staticmethod
-    def _parse_json_with_container(
-        definition: TaskDefinition,
-        json_data: Any,
-        base_url: str,
-    ) -> list[dict[str, str]]:
-        container: dict[str, Any] | None = definition.definition.parse.get("items")
-        if not container:
-            return []
-
-        container_type = container.get("type", "css")
-        container_selector = container.get("selector") or container.get("expression", "")
-        container_fields = container.get("fields", {})
-
-        if "jsonpath" != container_type:
-            LOG.error(
-                "JSON response requires container selector type 'jsonpath'. Definition '%s'.",
-                definition.name,
-                extra={"definition": definition.name, "container_type": container_type},
-            )
-            return []
-
-        nodes: Any = GenericTaskHandler._json_search(json_data, container_selector)
-        if nodes is None:
-            return []
-
-        if not isinstance(nodes, list):
-            nodes = [nodes]
-
-        items: list[dict[str, str]] = []
-
-        for node in nodes:
-            entry: dict[str, str] = {}
-
-            for _field, rule_data in container_fields.items():
-                rule = ExtractionRule.model_validate(rule_data)
-                values: list[str] = GenericTaskHandler._execute_json_rule(_field, node, rule)
-                if not values:
-                    continue
-
-                if "link" == _field:
-                    entry["link"] = urljoin(base_url, values[0])
-                else:
-                    entry[_field] = values[0]
-
-            if "link" not in entry:
-                continue
-
-            items.append(entry)
-
-        return items
-
-    @staticmethod
-    def _execute_json_rule(field: str, data: Any, rule: ExtractionRule) -> list[str]:
-        values: list[str] = []
-
-        if "jsonpath" == rule.type:
-            result: Any = GenericTaskHandler._json_search(data, rule.expression)
-            candidates: list | list[Any] = result if isinstance(result, list) else [result]
-            for candidate in candidates:
-                if candidate is None:
-                    continue
-
-                text: str = GenericTaskHandler._coerce_to_string(candidate)
-                processed: str | None = GenericTaskHandler._apply_post_filter(text, rule)
-                if processed is not None:
-                    values.append(processed)
-
-            return values
-
-        if "regex" == rule.type:
-            target: str = GenericTaskHandler._coerce_to_string(data)
-            try:
-                pattern: re.Pattern[str] = re.compile(rule.expression, re.MULTILINE | re.DOTALL)
-            except re.error as exc:
-                LOG.exception(
-                    "Invalid regex expression '%s'.",
-                    rule.expression,
-                    extra={
-                        "field": field,
-                        "expression": rule.expression,
-                        "error": str(exc),
-                        "exception_type": type(exc).__name__,
-                    },
-                )
-                return values
-
-            for match in pattern.finditer(target):
-                raw: str | None = GenericTaskHandler._regex_value(match=match, attribute=rule.attribute)
-                processed = GenericTaskHandler._apply_post_filter(raw, rule)
-                if processed is not None:
-                    values.append(processed)
-
-            return values
-
-        LOG.error(
-            "Unsupported extraction type '%s' for JSON data in field '%s'.",
-            rule.type,
-            field,
-            extra={"field": field, "rule_type": rule.type},
-        )
-        return values
-
-    @staticmethod
-    def _json_search(data: Any, expression: str) -> Any:
-        try:
-            return jmespath.search(expression, data)
-        except Exception as exc:
-            LOG.exception(
-                "JSONPath search failed for expression '%s'.",
-                expression,
-                extra={"expression": expression, "error": str(exc), "exception_type": type(exc).__name__},
-            )
-            return None
-
-    @staticmethod
-    def _coerce_to_string(value: Any) -> str:
-        if isinstance(value, str):
-            return value
-        if isinstance(value, (int, float, bool)) or value is None:
-            return "" if value is None else str(value)
-        try:
-            return json.dumps(value, ensure_ascii=False)
-        except Exception:
-            return str(value)
-
-    @staticmethod
-    def _execute_rule(field: str, selector: Selector, html: str, rule: ExtractionRule) -> list[str]:
-        """
-        Execute a single extraction rule and return the list of extracted values.
-
-        Args:
-            field (str): The name of the field being extracted.
-            selector (Selector): The parsel Selector for the HTML content.
-            html (str): The raw HTML content.
-            rule (ExtractionRuleSchema): The extraction rule to execute.
-
-        Returns:
-            (list[str]): A list of extracted values.
-
-        """
-        values: list[str] = []
-
-        if "regex" == rule.type:
-            try:
-                pattern: re.Pattern[str] = re.compile(rule.expression, re.MULTILINE | re.DOTALL)
-            except re.error as exc:
-                LOG.exception(
-                    "Invalid regex expression '%s'.",
-                    rule.expression,
-                    extra={
-                        "field": field,
-                        "expression": rule.expression,
-                        "error": str(exc),
-                        "exception_type": type(exc).__name__,
-                    },
-                )
-                return values
-
-            for match in pattern.finditer(html):
-                raw: str | None = GenericTaskHandler._regex_value(match=match, attribute=rule.attribute)
-                processed: str | None = GenericTaskHandler._apply_post_filter(raw, rule)
-                if processed is not None:
-                    values.append(processed)
-
-            return values
-
-        if "jsonpath" == rule.type:
-            LOG.error("Field '%s' uses 'jsonpath' on a non-JSON response.", field, extra={"field": field})
-            return values
-
-        selection: SelectorList[Selector] = (
-            selector.css(rule.expression) if "css" == rule.type else selector.xpath(rule.expression)
-        )
-
-        for sel in selection:
-            raw = GenericTaskHandler._selector_value(field, sel, rule.attribute)
-            processed = GenericTaskHandler._apply_post_filter(raw, rule)
-            if processed is not None:
-                values.append(processed)
-
-        return values
-
-    @staticmethod
-    def _regex_value(match: re.Match[str], attribute: str | None) -> str | None:
-        """
-        Extract a value from a regex match based on the attribute.
-
-        Args:
-            match (re.Match[str]): The regex match object.
-            attribute (str|None): Optional group name or index to extract.
-
-        Returns:
-            (str|None): The extracted value if found, None otherwise.
-
-        """
-        if attribute:
-            try:
-                return match.group(attribute)
-            except (IndexError, KeyError):
-                LOG.debug(
-                    "Regex group '%s' not found in pattern '%s'.",
-                    attribute,
-                    match.re.pattern,
-                    extra={"attribute": attribute, "pattern": match.re.pattern},
-                )
-                return None
-
-        if match.groupdict():
-            return next((value for value in match.groupdict().values() if value), None)
-
-        if match.groups():
-            return match.group(1)
-
-        return match.group(0)
-
-    @staticmethod
-    def _selector_value(field: str, sel: Selector, attribute: str | None) -> str | None:
-        """
-        Extract a value from a parsel Selector based on the attribute.
-
-        Args:
-            field (str): The name of the field being extracted.
-            sel (Selector): The parsel Selector object.
-            attribute (str|None): Optional attribute to extract (e.g. 'href', 'src', 'text', etc.).
-
-        Returns:
-            (str|None): The extracted value if found, None otherwise.
-
-        """
-        attr: str | None = attribute.lower() if isinstance(attribute, str) else None
-
-        if attr in {"text", "inner_text"}:
-            return sel.xpath("normalize-space()").get()
-
-        if attr in {"html", "outer_html"}:
-            value: Any = sel.get()
-            return value if value is not None else None
-
-        if attr and attr not in {"html", "outer_html", "text", "inner_text"}:
-            try:
-                attributes: dict[str, str] | None = sel.attrib
-            except AttributeError:
-                attributes = None
-
-            if attributes and attr in attributes:
-                return attributes.get(attr)
-
-            attr_value: str | None = sel.xpath(f"@{attr}").get()
-            if attr_value is not None:
-                return attr_value
-
-        if attr is None and "link" == field.lower():
-            href = None
-            try:
-                attributes: dict[str, str] | None = sel.attrib
-            except AttributeError:
-                attributes = None
-
-            if attributes and "href" in attributes:
-                href: str | None = attributes.get("href")
-            if not href:
-                href: str | None = sel.xpath("@href").get()
-            if href:
-                return href
-
-        if attr is None:
-            text_value: str | None = sel.xpath("normalize-space()").get()
-            if text_value:
-                return text_value
-
-        value = sel.get()
-        return value if value is not None else None
-
-    @staticmethod
-    def _apply_post_filter(value: str | None, rule: ExtractionRule) -> str | None:
-        """
-        Apply the post-filter to the extracted value if defined.
-
-        Args:
-            value (str|None): The extracted value to filter.
-            rule (ExtractionRuleSchema): The extraction rule containing the post-filter.
-
-        Returns:
-            (str|None): The filtered value if applicable, None otherwise.
-
-        """
-        if value is None:
-            return None
-
-        cleaned: str = value.strip()
-        if rule.post_filter:
-            # Apply post-filter inline (removed helper method)
-            try:
-                pattern = re.compile(rule.post_filter.filter)
-                match = pattern.search(cleaned)
-                if not match:
-                    return None
-
-                if rule.post_filter.value:
-                    try:
-                        return match.group(rule.post_filter.value)
-                    except (IndexError, KeyError):
-                        return None
-
-                if match.groupdict():
-                    # Prefer first named group when available
-                    for group_value in match.groupdict().values():
-                        if group_value is not None:
-                            return group_value
-
-                if match.groups():
-                    return match.group(1)
-
-                return match.group(0)
-            except re.error:
-                return None
-
-        return cleaned or None

app/features/tasks/definitions/handlers/rss.py

@@ -1,313 +0,0 @@
-import hashlib
-import re
-from typing import TYPE_CHECKING, Any
-from xml.etree.ElementTree import Element
-
-import httpx
-
-from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
-from app.features.ytdlp.extractor import fetch_info
-from app.features.ytdlp.utils import get_archive_id
-from app.library.cache import Cache
-from app.library.config import Config
-from app.library.log import get_logger
-
-from ._base_handler import BaseHandler
-
-if TYPE_CHECKING:
-    from xml.etree.ElementTree import Element
-
-LOG = get_logger()
-CACHE: Cache = Cache()
-
-
-class RssGenericHandler(BaseHandler):
-    FEED_PATTERN: re.Pattern[str] = re.compile(
-        r"\.(rss|atom)(\?.*)?$|handler=rss",
-        re.IGNORECASE,
-    )
-
-    @staticmethod
-    async def can_handle(task: HandleTask) -> bool:
-        LOG.debug(
-            "Checking if task '%s' uses a parsable RSS feed.",
-            task.name,
-            extra={"task_name": task.name, "url": task.url},
-        )
-        return RssGenericHandler.parse(task.url) is not None
-
-    @staticmethod
-    async def _get(
-        task: HandleTask,
-        params: dict,
-        parsed: dict[str, str],
-    ) -> tuple[str, list[dict[str, str]], int]:
-        """
-        Fetch the feed and return raw entries.
-
-        Args:
-            task (Task): The task containing the feed URL.
-            params (dict): The ytdlp options.
-            parsed (dict): The parsed URL components (contains 'url' key).
-
-        Returns:
-            tuple[str, list[dict[str, str]], int]: The feed URL, list of entry dictionaries, and entry count.
-
-        """
-        from defusedxml.ElementTree import fromstring
-
-        feed_url: str = parsed["url"]
-        LOG.debug(
-            "Fetching RSS/Atom feed for task '%s'.",
-            task.name,
-            extra={"task_name": task.name, "feed_url": feed_url},
-        )
-
-        response = await RssGenericHandler.request(url=feed_url, ytdlp_opts=params)
-        response.raise_for_status()
-
-        root: Element = fromstring(response.text)
-
-        # Define namespaces for different feed formats
-        ns: dict[str, str] = {
-            "atom": "http://www.w3.org/2005/Atom",
-            "rss": "http://www.rssboard.org/specification",
-            "content": "http://purl.org/rss/1.0/modules/content/",
-            "media": "http://search.yahoo.com/mrss/",
-        }
-
-        items: list[dict[str, str]] = []
-        real_count = 0
-
-        # Try to parse as Atom feed first
-        entries = root.findall("atom:entry", ns)
-        if entries:
-            LOG.debug(
-                "'%s': Detected Atom feed format with %s entries",
-                task.name,
-                len(entries),
-                extra={"task_name": task.name, "feed_url": feed_url, "entry_count": len(entries)},
-            )
-            for entry in entries:
-                link_elem: Element | None = entry.find("atom:link[@rel='alternate']", ns)
-                if link_elem is None:
-                    link_elem = entry.find("atom:link", ns)
-
-                url: str = ""
-                if link_elem is not None and link_elem.get("href"):
-                    url = link_elem.get("href", "")
-
-                if not url:
-                    LOG.warning(
-                        "'%s': Atom entry missing URL. Skipping.",
-                        task.name,
-                        extra={"task_name": task.name, "feed_url": feed_url},
-                    )
-                    continue
-
-                title_elem: Element | None = entry.find("atom:title", ns)
-                title: str = title_elem.text if title_elem is not None and title_elem.text else ""
-
-                pub_elem: Element | None = entry.find("atom:published", ns)
-                published: str = pub_elem.text if pub_elem is not None and pub_elem.text else ""
-
-                real_count += 1
-                items.append({"url": url, "title": title, "published": published})
-        else:
-            # Try to parse as RSS feed
-            rss_items = root.findall(".//item")
-            LOG.debug(
-                "'%s': Detected RSS feed format with %s items",
-                task.name,
-                len(rss_items),
-                extra={"task_name": task.name, "feed_url": feed_url, "entry_count": len(rss_items)},
-            )
-
-            for item in rss_items:
-                # Try different link element names (link, url, media:content)
-                url: str = ""
-
-                link_elem = item.find("link")
-                if link_elem is not None and link_elem.text:
-                    url = link_elem.text
-                else:
-                    # Try media:content
-                    media_elem = item.find("media:content", ns)
-                    if media_elem is not None and media_elem.get("url"):
-                        url = media_elem.get("url", "")
-                    else:
-                        # Try enclosure
-                        enclosure_elem = item.find("enclosure")
-                        if enclosure_elem is not None and enclosure_elem.get("url"):
-                            url = enclosure_elem.get("url", "")
-
-                if not url:
-                    LOG.warning(
-                        "'%s': RSS item missing URL. Skipping.",
-                        task.name,
-                        extra={"task_name": task.name, "feed_url": feed_url},
-                    )
-                    continue
-
-                title_elem = item.find("title")
-                title: str = title_elem.text if title_elem is not None and title_elem.text else ""
-
-                pub_elem = item.find("pubDate")
-                published: str = pub_elem.text if pub_elem is not None and pub_elem.text else ""
-
-                real_count += 1
-                items.append({"url": url, "title": title, "published": published})
-
-        return feed_url, items, real_count
-
-    @staticmethod
-    async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
-        _ = config
-        """
-        Extract items from an RSS/Atom feed.
-
-        Args:
-            task (Task): The task containing the feed URL.
-            config (Config | None): Optional handler configuration.
-
-        Returns:
-            TaskResult | TaskFailure: Extraction result with parsed items or failure information.
-
-        """
-        parsed: dict[str, str] | None = RssGenericHandler.parse(task.url)
-        if not parsed:
-            return TaskFailure(message="Unrecognized RSS/Atom feed URL.")
-
-        params: dict = task.get_ytdlp_opts().get_all()
-
-        try:
-            feed_url, items, real_count = await RssGenericHandler._get(task, params, parsed)
-        except httpx.HTTPError as exc:
-            return TaskFailure(message="Failed to fetch RSS/Atom feed.", error=str(exc))
-        except Exception as exc:
-            LOG.exception(
-                "Failed to fetch RSS/Atom feed for task '%s'.",
-                task.name,
-                extra={
-                    "task_id": task.id,
-                    "task_name": task.name,
-                    "url": task.url,
-                    "exception_type": type(exc).__name__,
-                },
-            )
-            return TaskFailure(message="Failed to fetch RSS/Atom feed.", error=str(exc))
-
-        task_items: list[TaskItem] = []
-
-        for entry in items:
-            if not (url := entry.get("url")):
-                continue
-
-            # Try to get static archive ID first
-            id_dict: dict[str, str | None] = get_archive_id(url=url)
-            archive_id: str | None = id_dict.get("archive_id")
-
-            # If static archive_id fails, try to fetch it via yt-dlp (like generic.py)
-            if not archive_id:
-                cache_key: str = hashlib.sha256(f"{task.name}-{url}".encode()).hexdigest()
-
-                if CACHE.has(cache_key):
-                    archive_id = CACHE.get(cache_key)
-                    if not archive_id:
-                        LOG.debug(
-                            "Task '%s' has a cached archive ID lookup failure. Skipping item.",
-                            task.name,
-                            extra={"task_name": task.name, "url": url},
-                        )
-                        continue
-                else:
-                    LOG.warning(
-                        "Task '%s' could not generate a static archive ID. Fetching it with yt-dlp.",
-                        task.name,
-                        extra={"task_name": task.name, "url": url},
-                    )
-
-                    (info, _) = await fetch_info(
-                        config=params,
-                        url=url,
-                        no_archive=True,
-                        no_log=True,
-                        budget_sleep=True,
-                    )
-
-                    if not info:
-                        LOG.error(
-                            "Task '%s' failed to extract info to generate an archive ID. Skipping item.",
-                            task.name,
-                            extra={"task_name": task.name, "url": url},
-                        )
-                        CACHE.set(cache_key, None)
-                        continue
-
-                    if not info.get("id") or not info.get("extractor_key"):
-                        LOG.error(
-                            "Task '%s' returned incomplete info while generating an archive ID. Skipping item.",
-                            task.name,
-                            extra={"task_name": task.name, "url": url},
-                        )
-                        CACHE.set(cache_key, None)
-                        continue
-
-                    archive_id = f"{str(info.get('extractor_key', '')).lower()} {info.get('id')}"
-                    CACHE.set(cache_key, archive_id)
-
-            metadata: dict[str, Any] = {k: v for k, v in entry.items() if k not in {"url", "title", "published"}}
-
-            task_items.append(
-                TaskItem(
-                    url=url,
-                    title=entry.get("title"),
-                    archive_id=archive_id,
-                    metadata={"published": entry.get("published"), **metadata},
-                )
-            )
-
-        return TaskResult(
-            items=task_items,
-            metadata={"feed_url": feed_url, "entry_count": real_count},
-        )
-
-    @staticmethod
-    def parse(url: str) -> dict[str, str] | None:
-        """
-        Parse URL for valid RSS/Atom feed.
-
-        Args:
-            url (str): The URL to parse.
-
-        Returns:
-            dict[str, str] | None: A dictionary with 'url' key if valid RSS/Atom feed, None otherwise.
-
-        """
-        if not isinstance(url, str) or not url:
-            return None
-
-        return {"url": url} if RssGenericHandler.FEED_PATTERN.search(url) else None
-
-    @staticmethod
-    def tests() -> list[tuple[str, bool]]:
-        """
-        Test cases for the URL parser.
-
-        Returns:
-            list[tuple[str, bool]]: A list of tuples containing the URL and expected result.
-
-        """
-        return [
-            ("https://www.example.com/test.rss", True),
-            ("https://www.example.com/test.atom", True),
-            ("https://www.example.com/test.atom#handler=rss", True),
-            ("https://www.example.com/test.atom?handler=rss", True),
-            ("https://www.example.com/feed.rss?version=2.0", True),
-            ("https://www.example.com/test.xml", False),
-            ("https://www.example.com/channel/UC_x5XG1OV2P6uZZ5FSM9Ttw", False),
-            ("https://www.example.com/playlist?list=PLBCF2DAC6FFB574DE", False),
-            ("https://www.example.com/user/SomeUser", False),
-            ("https://example.com/feed.ATOM", True),
-            ("https://example.com/feed.RSS", True),
-        ]

app/features/tasks/definitions/handlers/tver.py

@@ -1,252 +0,0 @@
-import re
-
-import httpx
-
-from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
-from app.features.ytdlp.utils import get_archive_id
-from app.library.config import Config
-from app.library.log import get_logger
-
-from ._base_handler import BaseHandler
-
-LOG = get_logger()
-
-
-class TverHandler(BaseHandler):
-    SERIES_API: str = "https://platform-api.tver.jp/service/api/v1/callSeriesEpisodes/{id}"
-    SESSION_API: str = "https://platform-api.tver.jp/v2/api/platform_users/browser/create"
-    HEADERS: dict[str, str] = {
-        "x-tver-platform-type": "web",
-        "Origin": "https://tver.jp",
-        "Referer": "https://tver.jp/",
-    }
-
-    RX: re.Pattern[str] = re.compile(r"^https?:\/\/(?:www\.|m\.)?tver\.jp\/series\/(?P<id>sr[a-z0-9_]+)$")
-
-    @staticmethod
-    async def can_handle(task: HandleTask) -> bool:
-        LOG.debug(
-            "Checking if task '%s' uses a parsable Tver series URL.",
-            task.name,
-            extra={"task_name": task.name, "url": task.url},
-        )
-        return TverHandler.parse(task.url) is not None
-
-    @staticmethod
-    async def _get_session_tokens() -> dict[str, str] | None:
-        """
-        Create a session and retrieve platform_uid and platform_token.
-
-        Returns:
-            dict[str, str] | None: Dictionary with platform_uid and platform_token, or None on failure.
-
-        """
-        try:
-            response = await TverHandler.request(
-                url=TverHandler.SESSION_API,
-                headers=TverHandler.HEADERS,
-                method="POST",
-                data={"device_type": "pc"},
-            )
-            response.raise_for_status()
-
-            data = response.json()
-            platform_uid = data.get("result", {}).get("platform_uid")
-            platform_token = data.get("result", {}).get("platform_token")
-
-            if not platform_uid or not platform_token:
-                LOG.warning("Failed to extract platform_uid and platform_token from session response.")
-                return None
-
-            return {"platform_uid": platform_uid, "platform_token": platform_token}
-        except Exception as exc:
-            LOG.warning(
-                "Failed to create Tver session.",
-                extra={"error": str(exc), "exception_type": type(exc).__name__},
-                exc_info=True,
-            )
-            return None
-
-    @staticmethod
-    async def _collect_feed(
-        task: HandleTask,
-        params: dict,
-        series_id: str,
-    ) -> tuple[str, list[dict[str, str]], bool]:
-        """
-        Fetch episodes from tver series API.
-
-        Args:
-            task (Task): The task containing the tver series URL.
-            params (dict): The ytdlp options.
-            series_id (str): The tver series ID.
-
-        Returns:
-            tuple[str, list[dict[str, str]], bool]: The feed URL, list of episodes, and whether items were found.
-
-        """
-        tokens = await TverHandler._get_session_tokens()
-        if not tokens:
-            msg = "Could not obtain tver session tokens"
-            raise RuntimeError(msg)
-
-        feed_url = TverHandler.SERIES_API.format(id=series_id)
-
-        LOG.debug(
-            "Fetching Tver episodes for task '%s'.",
-            task.name,
-            extra={"task_name": task.name, "series_id": series_id, "feed_url": feed_url},
-        )
-
-        response = await TverHandler.request(
-            url=feed_url,
-            headers=TverHandler.HEADERS,
-            ytdlp_opts=params,
-            params={
-                "platform_uid": tokens["platform_uid"],
-                "platform_token": tokens["platform_token"],
-            },
-        )
-        response.raise_for_status()
-
-        data = response.json()
-
-        items: list[dict[str, str]] = []
-        has_items = False
-
-        # Parse episodes from result.contents[0].contents
-        try:
-            contents = data.get("result", {}).get("contents", [])
-            if not contents:
-                LOG.warning(
-                    "No contents found in Tver series response for '%s'.",
-                    task.name,
-                    extra={"task_name": task.name, "series_id": series_id, "feed_url": feed_url},
-                )
-                return feed_url, items, has_items
-
-            season_block = contents[0] if contents else {}
-            episodes = season_block.get("contents", [])
-
-            for episode_data in episodes:
-                if "episode" != episode_data.get("type"):
-                    continue
-
-                content = episode_data.get("content", {})
-                episode_id = content.pop("id", None)
-                if not episode_id:
-                    LOG.warning(
-                        "Episode missing ID in '%s' feed. Skipping.",
-                        task.name,
-                        extra={"task_name": task.name, "series_id": series_id, "feed_url": feed_url},
-                    )
-                    continue
-
-                url = f"https://tver.jp/episodes/{episode_id}"
-
-                title = content.pop("title", "")
-
-                id_dict = get_archive_id(url)
-                archive_id = id_dict.get("archive_id")
-                if not archive_id:
-                    LOG.warning(
-                        "Task '%s' could not compute an archive ID for an episode. Skipping item.",
-                        task.name,
-                        extra={"task_name": task.name, "series_id": series_id, "episode_id": episode_id, "url": url},
-                    )
-                    continue
-
-                has_items = True
-                items.append(
-                    {"id": episode_id, "url": url, "title": title, "archive_id": archive_id, "metadata": content}
-                )
-
-        except Exception as exc:
-            LOG.warning(
-                "Failed to parse Tver episodes for task '%s'.",
-                task.name,
-                extra={
-                    "task_id": task.id,
-                    "task_name": task.name,
-                    "series_id": series_id,
-                    "error": str(exc),
-                    "exception_type": type(exc).__name__,
-                },
-                exc_info=True,
-            )
-
-        return feed_url, items, has_items
-
-    @staticmethod
-    async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure:
-        _ = config
-        series_id: str | None = TverHandler.parse(task.url)
-        if not series_id:
-            return TaskFailure(message="Unrecognized Tver series URL.")
-
-        params: dict = task.get_ytdlp_opts().get_all()
-
-        try:
-            feed_url, items, has_items = await TverHandler._collect_feed(task, params, series_id)
-        except httpx.HTTPError as exc:
-            return TaskFailure(message="Failed to fetch Tver feed.", error=str(exc))
-        except Exception as exc:
-            LOG.exception(
-                "Failed to fetch Tver feed for task '%s'.",
-                task.name,
-                extra={
-                    "task_id": task.id,
-                    "task_name": task.name,
-                    "url": task.url,
-                    "series_id": series_id,
-                    "exception_type": type(exc).__name__,
-                },
-            )
-            return TaskFailure(message="Failed to fetch Tver feed.", error=str(exc))
-
-        task_items: list[TaskItem] = []
-
-        for entry in items:
-            if not (url := entry.get("url")):
-                continue
-
-            archive_id: str | None = entry.get("archive_id")
-            metadata = entry.get("metadata", {})
-            if not isinstance(metadata, dict):
-                metadata = {}
-            task_items.append(TaskItem(url=url, title=entry.get("title"), archive_id=archive_id, metadata=metadata))
-
-        return TaskResult(items=task_items, metadata={"feed_url": feed_url, "has_entries": has_items})
-
-    @staticmethod
-    def parse(url: str) -> str | None:
-        """
-        Parse URL to extract series ID.
-
-        Args:
-            url (str): The url to check.
-
-        Returns:
-            str | None: The parsed ID if successful, None otherwise.
-
-        """
-        match: re.Match[str] | None = TverHandler.RX.match(url)
-        return match.group("id") if match else None
-
-    @staticmethod
-    def tests() -> list[tuple[str, bool]]:
-        """
-        Test cases for the URL parser.
-
-        Returns:
-            list[tuple[str, bool]]: A list of tuples containing the URL and expected result.
-
-        """
-        return [
-            ("https://tver.jp/series/sr1jg44pbb", True),
-            ("https://www.tver.jp/series/sr1jg44pbb", True),
-            ("https://m.tver.jp/series/sr_test_id", True),
-            ("http://tver.jp/series/sr123abc456", True),
-            ("https://tver.jp/videos/sr123abc456", False),
-            ("https://youtube.com/watch?v=123", False),
-        ]