[BUGFIX] Fix "File name too long" error when a title is used as a folder name (#1479)

Some presets put the video title into a folder name (not just the file name). The code only shortened the file name, never the folder name. So a title with multi-byte characters (like bold/fancy Unicode, which take 4 bytes each) could make the folder name go over the OS 255-byte limit and crash with OSError: [Errno 36] File name too long.

This fix shortens every folder in the path too, not just the file name, while keeping the file extension. output_directory is left alone so real folders aren't touched.

Added a test with a title made of 4-byte bold Unicode characters.

Relates to #1092, #1189

-- Thanks @Nojyto for the contribution!

.github/workflows/ci-windows.yaml

@@ -19,7 +19,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
     - name: Run unit tests with coverage
       run: |
@@ -29,9 +29,9 @@ jobs:
         move "ffmpeg-master-latest-win64-gpl\bin\ffprobe.exe" "ffprobe.exe"
         
         python -m pip install -e .[test]
-        python -m pytest tests/unit
+        python -m pytest --reruns 3 --reruns-delay 5 tests/unit
 
-  test-soundcloud:
+  test-integration:
     runs-on: windows-latest
     permissions:
       contents: read
@@ -42,9 +42,9 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
-    - name: Run e2e soundcloud tests with coverage
+    - name: Run integration tests with coverage
       run: |
         curl.exe -L -o ffmpeg.zip https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip
         tar -xf ffmpeg.zip
@@ -52,9 +52,10 @@ jobs:
         move "ffmpeg-master-latest-win64-gpl\bin\ffprobe.exe" "ffprobe.exe"
         
         python -m pip install -e .[test]
-        python -m pytest tests/e2e/soundcloud
+        python -m pytest --reruns 3 --reruns-delay 5 tests/integration --ignore tests/integration/prebuilt_presets
 
-  test-bandcamp:
+
+  test-integration-prebuilt-presets:
     runs-on: windows-latest
     permissions:
       contents: read
@@ -65,9 +66,9 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
-    - name: Run e2e soundcloud tests with coverage
+    - name: Run prebuilt preset integration tests with coverage
       run: |
         curl.exe -L -o ffmpeg.zip https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip
         tar -xf ffmpeg.zip
@@ -75,10 +76,9 @@ jobs:
         move "ffmpeg-master-latest-win64-gpl\bin\ffprobe.exe" "ffprobe.exe"
         
         python -m pip install -e .[test]
-        python -m pytest tests/e2e/bandcamp
+        python -m pytest --reruns 3 --reruns-delay 5 tests/integration/prebuilt_presets
 
-
+  test-e2e:
-  test-youtube:
     runs-on: windows-latest
     permissions:
       contents: read
@@ -89,9 +89,9 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
-    - name: Run e2e youtube tests with coverage
+    - name: Run e2e tests with coverage
       run: |
         curl.exe -L -o ffmpeg.zip https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip
         tar -xf ffmpeg.zip
@@ -99,27 +99,4 @@ jobs:
         move "ffmpeg-master-latest-win64-gpl\bin\ffprobe.exe" "ffprobe.exe"
         
         python -m pip install -e .[test]
-        python -m pytest tests/e2e/youtube
+        python -m pytest tests/e2e
-
-  test-plugins:
-    runs-on: windows-latest
-    permissions:
-      contents: read
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-
-    - name: Run e2e plugin tests with coverage
-      run: |
-        curl.exe -L -o ffmpeg.zip https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip
-        tar -xf ffmpeg.zip
-        move "ffmpeg-master-latest-win64-gpl\bin\ffmpeg.exe" "ffmpeg.exe"
-        move "ffmpeg-master-latest-win64-gpl\bin\ffprobe.exe" "ffprobe.exe"
-        
-        python -m pip install -e .[test]
-        python -m pytest tests/e2e/plugins

.github/workflows/ci.yaml

@@ -9,7 +9,7 @@ on:
       - master
 jobs:
   test-lint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     permissions:
       contents: read
 
@@ -19,7 +19,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
     - name: Run linters
       run: |
@@ -27,7 +27,7 @@ jobs:
         make check_lint
 
   test-unit:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     permissions:
       contents: read
 
@@ -37,14 +37,14 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
     - name: Run unit tests with coverage
       run: |
         pip install -e .[test]
         sudo apt-get update
         sudo apt-get install -y ffmpeg
-        coverage run -m pytest tests/unit && coverage xml -o /opt/coverage/unit/coverage.xml
+        coverage run -m pytest --reruns 3 --reruns-delay 5 tests/unit && coverage xml -o /opt/coverage/unit/coverage.xml
 
     - name: Save coverage
       uses: actions/cache@v3
@@ -52,8 +52,8 @@ jobs:
         path: /opt/coverage/unit
         key: ${{github.sha}}-coverage-unit
 
-  test-soundcloud:
+  test-integration:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     permissions:
       contents: read
 
@@ -63,43 +63,23 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
-    - name: Run e2e soundcloud tests with coverage
+    - name: Run integration tests with coverage
       run: |
         pip install -e .[test]
         sudo apt-get update
         sudo apt-get install -y ffmpeg
-        coverage run -m pytest tests/e2e/soundcloud && coverage xml -o /opt/coverage/soundcloud/coverage.xml
+        coverage run -m pytest --reruns 3 --reruns-delay 5 tests/integration --ignore tests/integration/prebuilt_presets && coverage xml -o /opt/coverage/integration/coverage.xml
-
-  test-bandcamp:
-    runs-on: ubuntu-22.04
-    permissions:
-      contents: read
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Set up Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-
-    - name: Run e2e soundcloud tests with coverage
-      run: |
-        pip install -e .[test]
-        sudo apt-get update
-        sudo apt-get install -y ffmpeg
-        coverage run -m pytest tests/e2e/bandcamp && coverage xml -o /opt/coverage/bandcamp/coverage.xml
 
     - name: Save coverage
       uses: actions/cache@v3
       with:
-        path: /opt/coverage/bandcamp
+        path: /opt/coverage/integration
-        key: ${{github.sha}}-coverage-bandcamp
+        key: ${{github.sha}}-coverage-integration
 
-  test-youtube:
+  test-integration-prebuilt-presets:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     permissions:
       contents: read
 
@@ -109,23 +89,23 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
-    - name: Run e2e youtube tests with coverage
+    - name: Run prebuilt preset integration tests with coverage
       run: |
         pip install -e .[test]
         sudo apt-get update
         sudo apt-get install -y ffmpeg
-        coverage run -m pytest tests/e2e/youtube && coverage xml -o /opt/coverage/youtube/coverage.xml
+        coverage run -m pytest --reruns 3 --reruns-delay 5 tests/integration/prebuilt_presets && coverage xml -o /opt/coverage/integration-prebuilt-presets/coverage.xml
 
     - name: Save coverage
       uses: actions/cache@v3
       with:
-        path: /opt/coverage/youtube
+        path: /opt/coverage/integration-prebuilt-presets
-        key: ${{github.sha}}-coverage-youtube
+        key: ${{github.sha}}-coverage-integration-prebuilt-presets
 
-  test-plugins:
+  test-e2e:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     permissions:
       contents: read
 
@@ -135,29 +115,22 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
-        python-version: "3.10"
+        python-version: "3.12"
 
-    - name: Run e2e plugin tests with coverage
+    - name: Run e2e tests with coverage
       run: |
         pip install -e .[test]
         sudo apt-get update
         sudo apt-get install -y ffmpeg
-        coverage run -m pytest tests/e2e/plugins && coverage xml -o /opt/coverage/plugins/coverage.xml
+        coverage run -m pytest tests/e2e && coverage xml -o /opt/coverage/e2e/coverage.xml
-
-    - name: Save coverage
-      uses: actions/cache@v3
-      with:
-        path: /opt/coverage/plugins
-        key: ${{github.sha}}-coverage-plugins
 
   codecov-upload:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     needs: [
       test-unit,
-      test-soundcloud,
+      test-integration,
-      test-bandcamp,
+      test-integration-prebuilt-presets,
-      test-youtube,
+      test-e2e
-      test-plugins
     ]
     permissions:
       contents: read
@@ -169,32 +142,26 @@ jobs:
         path: /opt/coverage/unit
         key: ${{github.sha}}-coverage-unit
 
-    - name: Restore soundcloud test coverage
+    - name: Restore integration test coverage
       uses: actions/cache@v3
       with:
-        path: /opt/coverage/soundcloud
+        path: /opt/coverage/integration
-        key: ${{github.sha}}-coverage-soundcloud
+        key: ${{github.sha}}-coverage-integration
 
-    - name: Restore bandcamp test coverage
+    - name: Restore integration prebuilt presets test coverage
       uses: actions/cache@v3
       with:
-        path: /opt/coverage/bandcamp
+        path: /opt/coverage/integration-prebuilt-presets
-        key: ${{github.sha}}-coverage-bandcamp
+        key: ${{github.sha}}-coverage-integration-prebuilt-presets
 
-    - name: Restore youtube test coverage
+    - name: Restore e2e test coverage
       uses: actions/cache@v3
       with:
-        path: /opt/coverage/youtube
+        path: /opt/coverage/e2e
-        key: ${{github.sha}}-coverage-youtube
+        key: ${{github.sha}}-coverage-e2e
-
-    - name: Restore plugins test coverage
-      uses: actions/cache@v3
-      with:
-        path: /opt/coverage/plugins
-        key: ${{github.sha}}-coverage-plugins
 
     - name: Upload code coverage to codecov.io
       uses: codecov/codecov-action@v3
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
-        files: /opt/coverage/unit/coverage.xml,/opt/coverage/soundcloud/coverage.xml,/opt/coverage/bandcamp/coverage.xml,/opt/coverage/youtube/coverage.xml,/opt/coverage/plugins/coverage.xml
+        files: /opt/coverage/unit/coverage.xml,/opt/coverage/integration/coverage.xml,/opt/coverage/integration-prebuilt-presets/coverage.xml,/opt/coverage/e2e/coverage.xml

.github/workflows/package-gui.yaml

@@ -0,0 +1,292 @@
+name: ytld-sub Docker GUI Build
+
+on:
+  push:
+    # Publish `master` as Docker `latest` image.
+    branches:
+      - master
+
+    # Publish `v1.2.3` tags as releases.
+    tags:
+      - v*
+
+  # Run tests for any PRs.
+  pull_request:
+
+env:
+  IMAGE_NAME: ytdl-sub-gui
+
+jobs:
+  # Push image to GitHub Packages.
+  # See also https://docs.docker.com/docker-hub/builds/
+  version:
+    name: version
+    runs-on: ubuntu-latest
+    outputs:
+      pypi_version: ${{ steps.set_outputs.outputs.pypi_version }}
+      local_version: ${{ steps.set_outputs.outputs.local_version }}
+      init_contents: ${{ steps.set_outputs.outputs.init_contents }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: master
+
+      - name: Set date and commit hash variables
+        run: |
+          echo "DATE=$(date +'%Y.%m.%d')" >> $GITHUB_ENV
+          echo "COMMIT_HASH=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+      - name: Count number of commits on master on day, minus 1 to account for post-push
+        run: |
+          echo "DATE_COMMIT_COUNT=$(($(git rev-list --count master --since='${{ env.DATE }} 00:00:00')-1))" >> $GITHUB_ENV
+      - name: Set pypi and local version values
+        run: |
+          echo "LOCAL_VERSION=${{ env.DATE }}+${{ env.COMMIT_HASH }}" >> $GITHUB_ENV
+          if [ ${{ env.DATE_COMMIT_COUNT }} -le "0" ]
+          then
+            echo "PYPI_VERSION=${{ env.DATE }}" >> $GITHUB_ENV
+          else
+            echo "PYPI_VERSION=${{ env.DATE }}.post${{ env.DATE_COMMIT_COUNT }}" >> $GITHUB_ENV
+          fi
+
+      - name: Test versions
+        run: |
+          echo "${{ env.PYPI_VERSION }}"
+          echo "${{ env.LOCAL_VERSION }}"
+
+      - id: set_outputs
+        run: |
+          echo "pypi_version=${{ env.PYPI_VERSION }}" >> "$GITHUB_OUTPUT"
+          echo "local_version=${{ env.LOCAL_VERSION }}" >> "$GITHUB_OUTPUT"
+          echo 'init_contents=__pypi_version__ = "${{ env.PYPI_VERSION }}";__local_version__ = "${{ env.LOCAL_VERSION }}"' >> "$GITHUB_OUTPUT"
+
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ "3.12" ]
+
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Build Wheel
+        run: |
+          make docker_stage
+
+      - name: Save Python build cache
+        uses: actions/cache@v3
+        with:
+          path: docker/root
+          key: ${{github.sha}}
+
+
+  # Build ARM64 container, only on master branch to save time testing
+  package-arm64:
+    runs-on: ubuntu-latest
+    needs: [
+      build
+    ]
+
+    permissions:
+      contents: read
+
+    if: ${{
+      github.ref == 'refs/heads/master'
+      && !contains(github.event.head_commit.message, '[DEV]')
+      && !contains(github.event.head_commit.message, '[DOCS]')
+      }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Restore Python build cache
+        uses: actions/cache@v3
+        with:
+          path: docker/root
+          key: ${{github.sha}}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: linux/arm64
+
+      - name: Docker Setup Buildx
+        uses: docker/setup-buildx-action@v2.0.0
+
+      - name: Build Docker Image
+        run: |
+          docker buildx build \
+          --platform=linux/arm64 \
+          --cache-to=type=local,dest=/tmp/build-cache/arm64 \
+          --tag $IMAGE_NAME \
+          --label "runnumber=${GITHUB_RUN_ID}" \
+          --file docker/Dockerfile.gui \
+          docker/
+
+      - name: Save ARM64 build cache
+        uses: actions/cache@v3
+        with:
+          path: /tmp/build-cache/arm64
+          key: ${{github.sha}}-arm64
+
+
+  # Build AMD64 container
+  package-amd64:
+    runs-on: ubuntu-latest
+    needs: [
+      build
+    ]
+
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Restore Python build cache
+        uses: actions/cache@v3
+        with:
+          path: docker/root
+          key: ${{github.sha}}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: linux/amd64
+
+      - name: Docker Setup Buildx
+        uses: docker/setup-buildx-action@v2.0.0
+
+      - name: Build Docker Image
+        run: |
+          docker buildx build \
+          --platform=linux/amd64 \
+          --cache-to=type=local,dest=/tmp/build-cache/amd64 \
+          --tag $IMAGE_NAME \
+          --label "runnumber=${GITHUB_RUN_ID}" \
+          --file docker/Dockerfile.gui \
+          docker/
+
+      - name: Save AMD64 build cache
+        uses: actions/cache@v3
+        with:
+          path: /tmp/build-cache/amd64
+          key: ${{github.sha}}-amd64
+
+
+  # On master branch, build the docker manifest file from the cached
+  # docker builds and push to the registry
+  deploy:
+    runs-on: ubuntu-latest
+    needs: [
+      version,
+      build,
+      package-arm64,
+      package-amd64
+    ]
+
+    permissions:
+      packages: write
+      contents: read
+
+    if: ${{
+      github.ref == 'refs/heads/master'
+      && !contains(github.event.head_commit.message, '[DEV]')
+      && !contains(github.event.head_commit.message, '[DOCS]')
+      }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Restore Python build cache
+        uses: actions/cache@v3
+        with:
+          path: docker/root
+          key: ${{github.sha}}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: linux/amd64,linux/arm64
+
+      - name: Docker Setup Buildx
+        uses: docker/setup-buildx-action@v2.0.0
+
+      - name: login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Restore ARM64 build cache
+        uses: actions/cache@v3
+        with:
+          path: /tmp/build-cache/arm64
+          key: ${{github.sha}}-arm64
+
+      - name: Restore AMD64 build cache
+        uses: actions/cache@v3
+        with:
+          path: /tmp/build-cache/amd64
+          key: ${{github.sha}}-amd64
+
+      - name: Format image_id
+        id: formatted-image_id
+        run:  |
+            IMAGE_ID=ghcr.io/${{ github.repository_owner }}/${IMAGE_NAME}
+
+            # Change all uppercase to lowercase
+            IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+            echo IMAGE_ID=${IMAGE_ID}
+
+            echo ::set-output name=IMAGE_ID::${IMAGE_ID}
+
+      - name: Get the version
+        id: formatted_version
+        run: |
+            # Strip git ref prefix from version
+            VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+            # Strip "v" prefix from tag name
+            [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+            # Use Docker `latest` tag convention
+            [ "$VERSION" == "master" ] && VERSION=latest
+            echo VERSION=${VERSION}
+
+            echo ::set-output name=VERSION::${VERSION}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ steps.formatted-image_id.outputs.IMAGE_ID }}
+          labels: |
+            runnumber=${{ github.run_number }}
+            maintainer=${{ github.repository_owner }}
+            org.opencontainers.image.authors=${{ github.repository_owner }}
+            org.opencontainers.image.vendor=${{ github.repository_owner }}
+            org.opencontainers.image.documentation=https://ytdl-sub.readthedocs.io/
+          tags: |
+            type=raw,value=${{ steps.formatted_version.outputs.VERSION }}
+            type=raw,value=${{ needs.version.outputs.pypi_version }}
+
+      - name: Build Docker Image and push to registry
+        uses: docker/build-push-action@v6
+        with:
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          context: "docker/"
+          file: "docker/Dockerfile.gui"
+          cache-from: |
+            type=local,src=/tmp/build-cache/amd64
+            type=local,src=/tmp/build-cache/arm64

.github/workflows/package-ubuntu.yaml

@@ -19,11 +19,54 @@ env:
 jobs:
   # Push image to GitHub Packages.
   # See also https://docs.docker.com/docker-hub/builds/
+  version:
+    name: version
+    runs-on: ubuntu-latest
+    outputs:
+      pypi_version: ${{ steps.set_outputs.outputs.pypi_version }}
+      local_version: ${{ steps.set_outputs.outputs.local_version }}
+      init_contents: ${{ steps.set_outputs.outputs.init_contents }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: master
+
+      - name: Set date and commit hash variables
+        run: |
+          echo "DATE=$(date +'%Y.%m.%d')" >> $GITHUB_ENV
+          echo "COMMIT_HASH=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+      - name: Count number of commits on master on day, minus 1 to account for post-push
+        run: |
+          echo "DATE_COMMIT_COUNT=$(($(git rev-list --count master --since='${{ env.DATE }} 00:00:00')-1))" >> $GITHUB_ENV
+      - name: Set pypi and local version values
+        run: |
+          echo "LOCAL_VERSION=${{ env.DATE }}+${{ env.COMMIT_HASH }}" >> $GITHUB_ENV
+          if [ ${{ env.DATE_COMMIT_COUNT }} -le "0" ]
+          then
+            echo "PYPI_VERSION=${{ env.DATE }}" >> $GITHUB_ENV
+          else
+            echo "PYPI_VERSION=${{ env.DATE }}.post${{ env.DATE_COMMIT_COUNT }}" >> $GITHUB_ENV
+          fi
+
+      - name: Test versions
+        run: |
+          echo "${{ env.PYPI_VERSION }}"
+          echo "${{ env.LOCAL_VERSION }}"
+
+      - id: set_outputs
+        run: |
+          echo "pypi_version=${{ env.PYPI_VERSION }}" >> "$GITHUB_OUTPUT"
+          echo "local_version=${{ env.LOCAL_VERSION }}" >> "$GITHUB_OUTPUT"
+          echo 'init_contents=__pypi_version__ = "${{ env.PYPI_VERSION }}";__local_version__ = "${{ env.LOCAL_VERSION }}"' >> "$GITHUB_OUTPUT"
+
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.10" ]
+        python-version: [ "3.12" ]
 
     permissions:
       contents: read
@@ -49,7 +92,7 @@ jobs:
 
   # Build ARM64 container, only on master branch to save time testing
   package-arm64:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     needs: [
       build
     ]
@@ -57,7 +100,11 @@ jobs:
     permissions:
       contents: read
 
-    if: ${{ github.ref == 'refs/heads/master' && !contains(github.event.head_commit.message, '[DEV]') }}
+    if: ${{
+      github.ref == 'refs/heads/master'
+      && !contains(github.event.head_commit.message, '[DEV]')
+      && !contains(github.event.head_commit.message, '[DOCS]')
+      }}
     steps:
       - uses: actions/checkout@v3
 
@@ -94,7 +141,7 @@ jobs:
 
   # Build AMD64 container
   package-amd64:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     needs: [
       build
     ]
@@ -139,8 +186,9 @@ jobs:
   # On master branch, build the docker manifest file from the cached
   # docker builds and push to the registry
   deploy:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     needs: [
+      version,
       build,
       package-arm64,
       package-amd64
@@ -150,7 +198,11 @@ jobs:
       packages: write
       contents: read
 
-    if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') }}
+    if: ${{
+      github.ref == 'refs/heads/master'
+      && !contains(github.event.head_commit.message, '[DEV]')
+      && !contains(github.event.head_commit.message, '[DOCS]')
+      }}
     steps:
       - uses: actions/checkout@v3
 
@@ -211,13 +263,30 @@ jobs:
 
             echo ::set-output name=VERSION::${VERSION}
 
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ steps.formatted-image_id.outputs.IMAGE_ID }}
+          labels: |
+            runnumber=${{ github.run_number }}
+            maintainer=${{ github.repository_owner }}
+            org.opencontainers.image.authors=${{ github.repository_owner }}
+            org.opencontainers.image.vendor=${{ github.repository_owner }}
+            org.opencontainers.image.documentation=https://ytdl-sub.readthedocs.io/
+          tags: |
+            type=raw,value=ubuntu-${{ steps.formatted_version.outputs.VERSION }}
+            type=raw,value=ubuntu-${{ needs.version.outputs.pypi_version }}
+
       - name: Build Docker Image and push to registry
-        run: | 
+        uses: docker/build-push-action@v6
-          docker buildx build --push \
+        with:
-          --platform=linux/amd64,linux/arm64 \
+          platforms: linux/amd64,linux/arm64
-          --cache-from=type=local,src=/tmp/build-cache/amd64 \
+          push: true
-          --cache-from=type=local,src=/tmp/build-cache/arm64 \
+          tags: ${{ steps.meta.outputs.tags }}
-          --tag ${{ steps.formatted-image_id.outputs.IMAGE_ID }}:ubuntu-${{ steps.formatted_version.outputs.VERSION }} \
+          labels: ${{ steps.meta.outputs.labels }}
-          --label "runnumber=ubuntu-${GITHUB_RUN_ID}" \
+          context: "docker/"
-          --file docker/Dockerfile.ubuntu \
+          file: "docker/Dockerfile.ubuntu"
-          docker/
+          cache-from: |
+            type=local,src=/tmp/build-cache/amd64
+            type=local,src=/tmp/build-cache/arm64

.github/workflows/package.yaml

@@ -19,11 +19,54 @@ env:
 jobs:
   # Push image to GitHub Packages.
   # See also https://docs.docker.com/docker-hub/builds/
+  version:
+    name: version
+    runs-on: ubuntu-latest
+    outputs:
+      pypi_version: ${{ steps.set_outputs.outputs.pypi_version }}
+      local_version: ${{ steps.set_outputs.outputs.local_version }}
+      init_contents: ${{ steps.set_outputs.outputs.init_contents }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          ref: master
+
+      - name: Set date and commit hash variables
+        run: |
+          echo "DATE=$(date +'%Y.%m.%d')" >> $GITHUB_ENV
+          echo "COMMIT_HASH=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+
+      - name: Count number of commits on master on day, minus 1 to account for post-push
+        run: |
+          echo "DATE_COMMIT_COUNT=$(($(git rev-list --count master --since='${{ env.DATE }} 00:00:00')-1))" >> $GITHUB_ENV
+      - name: Set pypi and local version values
+        run: |
+          echo "LOCAL_VERSION=${{ env.DATE }}+${{ env.COMMIT_HASH }}" >> $GITHUB_ENV
+          if [ ${{ env.DATE_COMMIT_COUNT }} -le "0" ]
+          then
+            echo "PYPI_VERSION=${{ env.DATE }}" >> $GITHUB_ENV
+          else
+            echo "PYPI_VERSION=${{ env.DATE }}.post${{ env.DATE_COMMIT_COUNT }}" >> $GITHUB_ENV
+          fi
+
+      - name: Test versions
+        run: |
+          echo "${{ env.PYPI_VERSION }}"
+          echo "${{ env.LOCAL_VERSION }}"
+
+      - id: set_outputs
+        run: |
+          echo "pypi_version=${{ env.PYPI_VERSION }}" >> "$GITHUB_OUTPUT"
+          echo "local_version=${{ env.LOCAL_VERSION }}" >> "$GITHUB_OUTPUT"
+          echo 'init_contents=__pypi_version__ = "${{ env.PYPI_VERSION }}";__local_version__ = "${{ env.LOCAL_VERSION }}"' >> "$GITHUB_OUTPUT"  
+
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.10" ]
+        python-version: [ "3.12" ]
 
     permissions:
       contents: read
@@ -49,7 +92,7 @@ jobs:
 
   # Build ARM64 container, only on master branch to save time testing
   package-arm64:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     needs: [
       build
     ]
@@ -93,7 +136,7 @@ jobs:
 
   # Build AMD64 container
   package-amd64:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     needs: [
       build
     ]
@@ -137,8 +180,9 @@ jobs:
   # On master branch, build the docker manifest file from the cached
   # docker builds and push to the registry
   deploy:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     needs: [
+      version,
       build,
       package-arm64,
       package-amd64
@@ -148,7 +192,11 @@ jobs:
       packages: write
       contents: read
 
-    if: ${{ github.ref == 'refs/heads/master' && !contains(github.event.head_commit.message, '[DEV]') }}
+    if: ${{
+      github.ref == 'refs/heads/master'
+      && !contains(github.event.head_commit.message, '[DEV]')
+      && !contains(github.event.head_commit.message, '[DOCS]')
+      }}
     steps:
       - uses: actions/checkout@v3
 
@@ -209,12 +257,30 @@ jobs:
 
             echo ::set-output name=VERSION::${VERSION}
 
-      - name: Build Docker Image and push to registry
+
-        run: | 
+      - name: Docker meta
-          docker buildx build --push \
+        id: meta
-          --platform=linux/amd64,linux/arm64 \
+        uses: docker/metadata-action@v5
-          --cache-from=type=local,src=/tmp/build-cache/amd64 \
+        with:
-          --cache-from=type=local,src=/tmp/build-cache/arm64 \
+          images: ${{ steps.formatted-image_id.outputs.IMAGE_ID }}
-          --tag ${{ steps.formatted-image_id.outputs.IMAGE_ID }}:${{ steps.formatted_version.outputs.VERSION }} \
+          labels: |
-          --label "runnumber=${GITHUB_RUN_ID}" \
+            runnumber=${{ github.run_number }}
-          docker/
+            maintainer=${{ github.repository_owner }}
+            org.opencontainers.image.authors=${{ github.repository_owner }}
+            org.opencontainers.image.vendor=${{ github.repository_owner }}
+            org.opencontainers.image.documentation=https://ytdl-sub.readthedocs.io/
+          tags: |
+            type=raw,value=${{ steps.formatted_version.outputs.VERSION }}
+            type=raw,value=${{ needs.version.outputs.pypi_version }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          context: "docker/"
+          cache-from: |
+            type=local,src=/tmp/build-cache/amd64
+            type=local,src=/tmp/build-cache/arm64

.github/workflows/release.yaml

@@ -61,9 +61,16 @@ jobs:
     strategy:
       matrix:
         arch: [ "aarch64", "x86_64" ]
-    runs-on: ubuntu-latest
+        include:
+          - arch: "aarch64"
+            runner: "ubuntu-24.04-arm"
+            container: "quay.io/pypa/manylinux_2_28_aarch64"
+          - arch: "x86_64"
+            runner: "ubuntu-latest"
+            container: "quay.io/pypa/manylinux_2_28_x86_64"
+    runs-on: ${{ matrix.runner }}
     container:
-      image: quay.io/pypa/manylinux_2_28_x86_64
+      image: ${{ matrix.container }}
     steps:
       - uses: actions/checkout@v3
       - name: Write version to init file
@@ -76,17 +83,17 @@ jobs:
           dnf install -y epel-release tar wget make gcc openssl-devel bzip2-devel libffi-devel zlib-devel
       - name: Install Python
         run: |
-          wget https://www.python.org/ftp/python/3.10.10/Python-3.10.10.tar.xz
+          wget https://www.python.org/ftp/python/3.12.9/Python-3.12.9.tar.xz
-          tar -xf Python-3.10.10.tar.xz
+          tar -xf Python-3.12.9.tar.xz
-          cd Python-3.10.10 && ./configure --with-ensurepip=install --prefix=/usr/local --enable-shared LDFLAGS="-Wl,-rpath /usr/local/lib"
+          cd Python-3.12.9 && ./configure --with-ensurepip=install --prefix=/usr/local --enable-shared LDFLAGS="-Wl,-rpath /usr/local/lib"
           make -j 8
           make altinstall
-          python3.10 --version
+          python3.12 --version
-          python3.10 -m ensurepip --upgrade
+          python3.12 -m ensurepip --upgrade
       - name: Build Package
         run: |
-          python3.10 -m pip install -e .
+          python3.12 -m pip install -e .
-          python3.10 -m pip install pyinstaller
+          python3.12 -m pip install pyinstaller
           # Build executable
           pyinstaller ytdl-sub.spec
           mkdir -p /opt/builds
@@ -95,7 +102,7 @@ jobs:
           mv dist/ytdl-sub /opt/builds/ytdl-sub_${{ matrix.arch }}
 
       - name: Upload build
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ytdl-sub_${{ matrix.arch }}
           path: /opt/builds/ytdl-sub_${{ matrix.arch }}
@@ -106,13 +113,13 @@ jobs:
     name: build-windows
     needs:
       - version
-    runs-on: windows-2019
+    runs-on: windows-latest
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.12"
       - name: Write version to init file
         run: |
           echo '${{ needs.version.outputs.init_contents }}'> src/ytdl_sub/__init__.py
@@ -124,7 +131,7 @@ jobs:
           .\dist\ytdl-sub.exe -h
 
       - name: Upload build
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ytdl-sub_exe
           path: .\dist\ytdl-sub.exe
@@ -145,19 +152,19 @@ jobs:
           echo '${{ needs.version.outputs.init_contents }}' > src/ytdl_sub/__init__.py
 
       - name: Restore exe build
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ytdl-sub_exe
           path: /opt/builds
 
       - name: Restore aarch64 build
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ytdl-sub_aarch64
           path: /opt/builds
 
       - name: Restore x86_64 build
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: ytdl-sub_x86_64
           path: /opt/builds
@@ -171,7 +178,11 @@ jobs:
           ls -lh /opt/builds
 
       - name: Create Release
-        if: ${{ github.ref == 'refs/heads/master' && !contains(github.event.head_commit.message, '[DEV]') }}
+        if: ${{
+          github.ref == 'refs/heads/master'
+          && !contains(github.event.head_commit.message, '[DEV]')
+          && !contains(github.event.head_commit.message, '[DOCS]')
+          }}
         id: create_release
         uses: softprops/action-gh-release@v1
         with:
@@ -189,13 +200,13 @@ jobs:
     name: pypi-publish
     needs:
       - version
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.12'
       - name: Write version to init file
         run: |
           echo '${{ needs.version.outputs.init_contents }}' > src/ytdl_sub/__init__.py
@@ -207,7 +218,11 @@ jobs:
           python3 -m build
 
       - name: Publish distribution 📦 to PyPI
-        if: ${{ github.ref == 'refs/heads/master' && !contains(github.event.head_commit.message, '[DEV]') }}
+        if: ${{
+          github.ref == 'refs/heads/master'
+          && !contains(github.event.head_commit.message, '[DEV]')
+          && !contains(github.event.head_commit.message, '[DOCS]')
+          }}
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}

.gitignore

@@ -141,8 +141,16 @@ dmypy.json
 docker/*.whl
 docker/root/*.whl
 docker/root/defaults/examples
+docker/testing/volumes
 
 .local/
 
+.ytdl-sub-working-directory
+.ytdl-sub-lock
+
 ffmpeg.exe
 ffprobe.exe
+
+tools/docgen/out
+
+prof/

.readthedocs.yaml

@@ -1,15 +1,17 @@
 version: 2
 
 build:
-  os: ubuntu-20.04
+  os: "ubuntu-22.04"
   tools:
     python: "3.10"
 
 sphinx:
-  configuration: docs/conf.py
+  configuration: docs/source/conf.py
+  fail_on_warning: true
 
 python:
   install:
+    - requirements: docs/source/requirements.txt
     - method: pip
       path: .
       extra_requirements:

Makefile

@@ -1,7 +1,20 @@
+# Defensive settings for make:
+#     https://tech.davis-hansson.com/p/make/
+SHELL:=bash
+.ONESHELL:
+.SHELLFLAGS:=-eu -o pipefail -c
+.SILENT:
+.DELETE_ON_ERROR:
+MAKEFLAGS+=--warn-undefined-variables
+MAKEFLAGS+=--no-builtin-rules
+export PS1?=$$
+# Prefix echoed recipe commands with the recipe line number for debugging:
+export PS4?=:$$LINENO+
+
 # Get version related variables
-export DATE=$(shell date +'%Y.%m.%d')
+export DATE:=$(shell date +'%Y.%m.%d')
-export DATE_COMMIT_COUNT=$(shell git rev-list --count HEAD --since="$(DATE) 00:00:00")
+export DATE_COMMIT_COUNT:=$(shell git rev-list --count HEAD --since="$(DATE) 00:00:00")
-export COMMIT_HASH=$(shell git rev-parse --short HEAD)
+export COMMIT_HASH:=$(shell git rev-parse --short HEAD)
 
 # Set Local version to YYYY.MM.DD-<hash>
 export LOCAL_VERSION="$(DATE)+$(COMMIT_HASH)"
@@ -13,18 +26,26 @@ else
 	export PYPI_VERSION="$(DATE).post$(DATE_COMMIT_COUNT)"
 endif
 
+# Finished with `$(shell)`, echo recipe commands going forward
+.SHELLFLAGS+= -x
+
+
+### Top-level targets:
+
+.PHONY: all
+all: check_lint docs docker docker_ubuntu docker_gui
+
 lint:
-	@-isort .
+	python3 -m ruff format .
-	@-black .
+	python3 -m ruff check --fix .
-	@-pylint src/
+	python3 -m pylint src
-	@-pydocstyle src/*
 check_lint:
-	isort . --check-only --diff  \
+	ruff format --check .  \
-		&& black . --check  \
+        && ruff check .  \
-		&& pylint src/  \
+		&& pylint src/
-		&& pydocstyle src/*
 wheel: clean
-	$(shell echo "__pypi_version__ = \"$(PYPI_VERSION)\"\n__local_version__ = \"$(LOCAL_VERSION)\"" > src/ytdl_sub/__init__.py)
+	$(shell echo "__pypi_version__ = \"$(PYPI_VERSION)\"" > src/ytdl_sub/__init__.py)
+	$(shell echo "__local_version__ = \"$(LOCAL_VERSION)\"" >> src/ytdl_sub/__init__.py)
 	cat src/ytdl_sub/__init__.py
 	pip3 install build
 	python3 -m build
@@ -34,19 +55,23 @@ docker_stage: wheel
 docker: docker_stage
 	sudo docker build --progress=plain --no-cache -t ytdl-sub:local docker/
 docker_ubuntu: docker_stage
-	sudo docker build --progress=plain --no-cache -t ytdl-sub:local_ubuntu -f docker/Dockerfile.ubuntu docker/
+	sudo docker build --progress=plain --no-cache -t ytdl-sub-ubuntu:local -f docker/Dockerfile.ubuntu docker/
+docker_gui: docker_stage
+	sudo docker build --progress=plain --no-cache -t ytdl-sub-gui:local -f docker/Dockerfile.gui docker/
 executable: clean
 	pyinstaller ytdl-sub.spec
 	mv dist/ytdl-sub dist/ytdl-sub${EXEC_SUFFIX}
 docs:
-	sphinx-build -a -b html docs docs/_html
+	REGENERATE_DOCS=1 pytest tests/unit/docgen/test_docgen.py
+	sphinx-build --write-all --fail-on-warning --nitpicky -b html \
+	    "./docs/source/" "./docs/build/"
 clean:
 	rm -rf \
 		.pytest_cache/ \
 		build/ \
 		dist/ \
 		src/ytdl_sub.egg-info/ \
-		docs/_html/ \
+		docs/build/ \
 		.coverage \
 		docker/root/*.whl \
 		docker/root/defaults/examples \

README.md

@@ -33,221 +33,187 @@ maximum flexibility while maintaining simplicity.
 #### Jellyfin
 ![jelly_mv](https://user-images.githubusercontent.com/10107080/182677256-43aeb029-0c3f-4648-9fd2-352b9666b262.PNG)
 
-### SoundCloud Albums and Singles
+### SoundCloud Discography
-#### MusicBee (any file or tag-based music players)
+#### Writes proper music-tags via beets API
 ![sc_mb](https://user-images.githubusercontent.com/10107080/182685415-06adf477-3dd3-475d-bbcd-53b0152b9f0a.PNG)
 
 ### Bandcamp Discography
-#### Navidrome (any file or tag-based music servers)
 ![bc_nav](https://user-images.githubusercontent.com/10107080/212503861-1d8748e6-6f6d-4043-b543-84226cd1f662.png)
 
-
 ## How it Works
-`ytdl-sub` uses YAML configs to define a layout for how you want media to look
+`ytdl-sub` uses YAML files to define subscriptions. Each subscription imports _presets_ that
-after it is downloaded. See our
+define how to handle and output media files. `ytdl-sub` comes packaged with many _prebuilt presets_
-[walk-through guide](https://github.com/jmbannon/ytdl-sub/wiki)
+that do the work of config-building, so you can start downloading immediately.
-on how to get started. Ready-to-use
-[example configurations](https://github.com/jmbannon/ytdl-sub/tree/master/examples)
-can be found here alongside our
-[readthedocs](https://ytdl-sub.readthedocs.io/en/latest/config.html#)
-for detailed information on config fields.
-
-
-### Config
-The `config.yaml` defines how our downloads will look. For this example, let us
-download YouTube channels and generate metadata to look like TV shows using
-ytdl-sub's prebuilt presets. No additional plugins or programs are needed for
-Kodi, Jellyfin, Plex, or Emby to recognize your downloads. This can also be
-used to download any yt-dlp supported URL, including YouTube playlists, Bitchute channels, etc.
-
 
 ```yaml
-# Set the working directory which will be used to stage downloads
+# subscriptions.yaml:
-# before placing them in your desired output directory.
+# Everything in here can be downloaded using the command:
-configuration:
+#   ytdl-sub sub subscriptions.yaml
-  working_directory: '.ytdl-sub-downloads'
 
-# Presets are where you create 'sub-configs' that can can be
+# __preset__ is a place to define global overrides for all subscriptions
-# merged together to dictate what is downloaded, how to format it,
+__preset__:
-# and what metadata to generate.
+  overrides:
-presets:
+    # Root folder of all ytdl-sub TV Shows
+    tv_show_directory: "/tv_shows"
     
-  # Let us create a preset called `only_recent_videos` that will
+    # Root folder of all ytdl-sub Music
-  # only download recent videos in the last 2 months.
+    music_directory: "/music"
-  only_recent_videos:
     
-    # Use the `date_range` plugin to specify ytdl-sub to only
+    # Root folder of all ytdl-sub Music Videos
-    # download videos after today MINUS {download_range}, which
+    music_video_directory: "/music_videos"
-    # is an override variable that we can alter per channel.
-    date_range:
-      after: "today-{download_range}"
     
-    # Any yt-dlp argument can be passed via ytdl-sub. Let us set
+    # For 'Only Recent' preset, only keep vids within this range and limit
-    # yt-dlp's `break_on_reject` to True to stop downloading after
+    only_recent_date_range: "2months"
-    # any video is rejected. Videos will be rejected if they are
+    only_recent_max_files: 30
-    # uploaded after our {download_range}.
-    ytdl_options:
-      break_on_reject: True
     
-    # Deletes any videos uploaded after {download_range}.
+  # Pass any arg directly to yt-dlp's Python API
-    output_options:
+  ytdl_options:
-      keep_files_after: "today-{download_range}"
+    cookiefile: "/config/ytdl-sub-configs/cookie.txt"
 
-    # Set the override variable {download_range} to 2months.
+###################################################################
-    # This will serve as our default value. We can override
+# TV Show Presets. Can replace Plex with Plex/Jellyfin/Emby/Kodi
-    # this per channel or in a child preset.
-    overrides:
-      download_range: "2months"
 
-  ####################################################################
+Plex TV Show by Date:
 
-  # Now let us create a preset that downloads videos and formats
+  # Sets genre tag to "Documentaries"
-  # as TV shows.
+  = Documentaries:
-  tv_show:
+    "NOVA PBS": "https://www.youtube.com/@novapbs"
+    "National Geographic": "https://www.youtube.com/@NatGeo"
+    "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
 
-    # Presets can inherit all attributes from other presets. Our
+  # Sets genre tag to "Kids", "TV-Y" for content rating
-    # `tv_show` preset will inherit these presets built into ytdl-sub.
+  = Kids | = TV-Y:
-    preset:
+    "Jake Trains": "https://www.youtube.com/@JakeTrains"
-      # Let us specify all the TV show by date presets to support all
+    "Kids Toys Play": "https://www.youtube.com/@KidsToysPlayChannel"
-      # players. You only need to specify one, but this ensures
-      # compatibility with all players.
-      - "kodi_tv_show_by_date"
-      - "jellyfin_tv_show_by_date"
-      - "plex_tv_show_by_date"
-      # Now we choose a preset that defines how our seasons and
-      # episode numbers look.
-      - "season_by_year__episode_by_month_day"
 
-    # Set override variables that will be applicable to all downloads
+  = Music:
-    # in main presets.
+    # TV show subscriptions can support multiple urls and store in the same TV Show
-    overrides:
+    "Rick Beato":
-      tv_show_directory: "/tv_shows"  # Replace with desired directory
+      - "https://www.youtube.com/@RickBeato"
+      - "https://www.youtube.com/@rickbeato240"
 
+  # Set genre tag to "News", use `Only Recent` preset to only store videos uploaded recently
+  = News | Only Recent:
+    "BBC News": "https://www.youtube.com/@BBCNews"
+
+Plex TV Show Collection:
+  = Music:
+    # Prefix with ~ to set specific override variables
+    "~Beyond the Guitar":
+      s01_name: "Videos"
+      s01_url: "https://www.youtube.com/c/BeyondTheGuitar"
+      s02_name: "Covers"
+      s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W"
+
+###################################################################
+# Music Presets.
+
+YouTube Releases:
+  = Jazz:  # Sets genre tag to "Jazz"
+    "Thelonious Monk": "https://www.youtube.com/@theloniousmonk3870/releases"
+
+YouTube Full Albums:
+  = Lofi:
+    "Game Chops": "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i"
+
+SoundCloud Discography:
+  = Chill Hop:
+    "UKNOWY": "https://soundcloud.com/uknowymunich"
+  = Synthwave:
+    "Lazerdiscs Records": "https://soundcloud.com/lazerdiscsrecords"
+    "Earmake": "https://soundcloud.com/earmake"
+
+Bandcamp:
+  = Lofi:
+    "Emily Hopkins": "https://emilyharpist.bandcamp.com/"
+
+###################################################################
+# Music Video Presets. Can replace Plex with Plex/Jellyfin/Kodi
+"Plex Music Videos":
+  = Pop:  # Sets genre tag to "Pop"
+    "Rick Astley": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc"
+    "Michael Jackson": "https://www.youtube.com/playlist?list=OLAK5uy_mnY03zP6abNWH929q2XhGzWD_2uKJ_n8E"
 ```
 
-### Subscriptions
+All of this can be downloaded and ready to import to your favorite player 
-The `subscriptions.yaml` file is where we define content to download using
+using the command
-presets in the `config.yaml`. Each subscription can overwrite any field used
+```commandline
-in a preset.
-```yaml
-# The name of our subscription. Let us create one to download
-# ALL of Rick A's videos
-rick_a:
-  # Inherit our `tv_show` preset we made above
-  preset:
-    - "tv_show"
-  
-  # Set override variables to set the channel URL and the
-  # name we want to give the TV show.
-  overrides:
-    tv_show_name: "Rick A"
-    url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
-
-# Let us make another subscription that will only download Rick A's
-# video's in the last 2 weeks.
-rick_a_recent:
-  # Inherit our `tv_show` AND `only_recent_videos` preset
-  # Bottom-most presets take precedence.
-  preset:
-    - "tv_show"
-    - "only_recent_videos"
-  
-  # Set override variables for this subscription. Modify the
-  # `download_range` to only download and keep 2 weeks' worth
-  # of videos.
-  overrides:
-    tv_show_name: "Rick A"
-    url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
-    download_range: "2weeks"
-
-```
-The download can now be performed using:
-```shell
 ytdl-sub sub subscriptions.yaml
 ```
-To preview what your output files before doing any downloads, you can dry run using:
+See our
-```shell
+[example subscriptions](https://github.com/jmbannon/ytdl-sub/tree/master/examples)
-ytdl-sub --dry-run sub subscriptions.yaml
+for more detailed examples and use-cases.
-```
-
-### One-time Download
-There are things we will only want to download once and never again. Anything
-you can define in a subscription can be defined using CLI arguments. This
-example is equivalent to the subscription example above:
-```shell
-ytdl-sub dl \
-    --preset "tv_show" \
-    --overrides.tv_show_name "Rick A" \
-    --overrides.url "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
-```
-
-#### Download Aliases
-In the `config.yaml`, we can define aliases to make `dl` commands shorter.
-```yaml
-configuration:
-  dl_aliases:
-    tv: "--preset tv_show"
-    name: "--overrides.tv_show_name"
-    url: "--overrides.url"
-```
-The above command can now be shortened to
-```shell
-ytdl-sub dl \
-    --tv \
-    --name "Rick A" \
-    --url "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
-```
 
 ### Output
-After `ytdl-sub` runs, the end result will download and format the channel
+After `ytdl-sub` runs, the end result will download and format the files into something ready 
-files into something ready to be consumed by your favorite media player or
+to be consumed by your favorite media player/server.
-server.
 ```
-/path/to/tv_shows/Rick Aß
+tv_shows/
-  /Season 2021
+  Jake Trains/
-    s2021.e0317 - Pattys Day Video-thumb.jpg
+    Season 2021/
-    s2021.e0317 - Pattys Day Video.mp4
+      s2021.e031701 - Pattys Day Video-thumb.jpg
-    s2021.e0317 - Pattys Day Video.nfo
+      s2021.e031701 - Pattys Day Video.mp4
-  /Season 2022
+      s2021.e031701 - Pattys Day Video.nfo
-    s2022.e1225 - Merry Christmas-thumb.jpg
+      s2021.e031702 - Second Pattys Day Video-thumb.jpg
-    s2022.e1225 - Merry Christmas.mp4
+      s2021.e031702 - Second Pattys Day Video.mp4
-    s2022.e1225 - Merry Christmas.nfo
+      s2021.e031702 - Second Pattys Day Video.nfo
-  poster.jpg
+    Season 2022/
-  fanart.jpg
+      s2022.e122501 - Merry Christmas-thumb.jpg
-  tvshow.nfo
+      s2022.e122501 - Merry Christmas.mp4
+      s2022.e122501 - Merry Christmas.nfo
+    poster.jpg
+    fanart.jpg
+    tvshow.nfo
+
+music/
+  Artist/
+    [2022] Some Single/
+      01 - Some Single.mp3
+      folder.jpg
+    [2023] Latest Album/
+      01 - Track Title.mp3
+      02 - Another Track.mp3
+      folder.jpg
+
+music_videos/
+  Elton John/
+    Elton John - Rocketman.jpg
+    Elton John - Rocketman.mp4
 ```
 
-### Beyond TV Shows
+## Custom Configs
-The above example made heavy-use of `ytdl-sub` prebuilt presets and hides many
+Any part of this process is modifiable by using custom configs. See our
-features that are offered. `ytdl-sub` strives to support _any_ use case that first requires
+[walk-through guide](https://ytdl-sub.readthedocs.io/en/latest/guides/index.html)
-a download via yt-dlp. Use `ytdl-sub` to download, format, and convert media for your media
+on how to build your first config from scratch. Ready-to-use
-player to recognize downloads as:
+[example configurations](https://github.com/jmbannon/ytdl-sub/tree/master/examples)
-- Movies
+can be found here alongside our
-- TV shows
+[readthedocs](https://ytdl-sub.readthedocs.io/en/latest/index.html)
-  - From a single channel or playlist
+for detailed information on all config fields.
-  - From multiple channels or playlists
-  - From individual videos
-- Extracted audio as podcasts
-- Music videos
-- Music, including:
-  - Individual songs
-  - Albums
-  - Discographies
 
 ## Installation
 `ytdl-sub` can be installed on the following platforms.
 
-- [Docker Compose](https://ytdl-sub.readthedocs.io/en/latest/install.html#docker-compose_)
+- [Docker Compose](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#install-with-docker-compose)
-  - [with CPU passthrough](https://ytdl-sub.readthedocs.io/en/latest/install.html#cpu-passthrough)
+  - [Web-GUI](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#install-with-docker-compose)
-  - [with GPU passthrough](https://ytdl-sub.readthedocs.io/en/latest/install.html#nvidia-gpu-passthrough)
+  - [Headless](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#install-with-docker-compose)
-- [Docker CLI](https://ytdl-sub.readthedocs.io/en/latest/install.html#docker)
+  - [CPU / GPU Passthrough](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#device-passthrough)
-- [Windows](https://ytdl-sub.readthedocs.io/en/latest/install.html#windows)
+- [Docker CLI](https://ytdl-sub.readthedocs.io/en/latest/guides/install/docker.html#docker-cli)
-- [Unraid](https://ytdl-sub.readthedocs.io/en/latest/install.html#unraid)
+- [Windows](https://ytdl-sub.readthedocs.io/en/latest/guides/install/windows.html)
-- [Linux](https://ytdl-sub.readthedocs.io/en/latest/install.html#linux)
+- [Unraid](https://ytdl-sub.readthedocs.io/en/latest/guides/install/unraid.html)
-- [Linux ARM](https://ytdl-sub.readthedocs.io/en/latest/install.html#linux-arm)
+- [Linux](https://ytdl-sub.readthedocs.io/en/latest/guides/install/linux.html)
-- [PIP](https://ytdl-sub.readthedocs.io/en/latest/install.html#pip)
+- [Linux ARM](https://ytdl-sub.readthedocs.io/en/latest/guides/install/linux.html)
-- [Local Install](https://ytdl-sub.readthedocs.io/en/latest/install.html#local-install)
+- [PIP](https://ytdl-sub.readthedocs.io/en/latest/guides/install/agnostic.html#pip-install)
-- [Local Docker Build](https://ytdl-sub.readthedocs.io/en/latest/install.html#local-docker-build)
+- [Local Install](https://ytdl-sub.readthedocs.io/en/latest/guides/install/agnostic.html#local-install)
+- [Local Docker Build](https://ytdl-sub.readthedocs.io/en/latest/guides/install/agnostic.html#local-docker-build)
+
+### Docker Installation
+Docker installs can be either headless or use the Web-GUI image, which comprises
+[LSIO's](https://www.linuxserver.io/)
+[code-server](https://hub.docker.com/r/linuxserver/code-server)
+Docker image with `ytdl-sub` preinstalled. This is the recommended way to use ``ytdl-sub``.
+
+![image](https://github.com/jmbannon/ytdl-sub/assets/10107080/c2aac8a1-5443-4345-b438-be4b17187c80)
+
 
 ## Contributing
 There are many ways to contribute, even without coding. Please take a look in
@@ -258,5 +224,5 @@ pick up a bug.
 We are pretty active in our
 [Discord channel](https://discord.gg/v8j9RAHb4k)
 if you have any questions. Also see our
-[FAQ](https://github.com/jmbannon/ytdl-sub/wiki/FAQ)
+[FAQ](https://ytdl-sub.readthedocs.io/en/latest/faq/index.html)
 for commonly asked questions.

docker/Dockerfile

@@ -3,18 +3,25 @@ FROM ghcr.io/linuxserver/baseimage-alpine:edge
 ###############################################################################
 # YTDL-SUB INSTALL
 
-# Needed for phantomjs
+# For phantomjs
 ENV OPENSSL_CONF="/etc/ssl"
+# For downloading thumbnails
+ENV SSL_CERT_DIR="/etc/ssl/certs/"
+# Working directory used at both build and run times:
+ENV DEFAULT_WORKSPACE="/config"
 
 COPY root/ /
-RUN mkdir -p /config && \
+RUN mkdir -pv "${DEFAULT_WORKSPACE}" && \
     apk update --no-cache && \
     apk upgrade --no-cache && \
     apk add --no-cache --repository=http://dl-3.alpinelinux.org/alpine/edge/main/ \
       vim \
       g++ \
       nano \
+      unzip \
       make \
+      deno \
+      libffi-dev \
       "python3>=3.10" \
       py3-pip \
       fontconfig \
@@ -25,34 +32,44 @@ RUN mkdir -p /config && \
       "aria2>=1.36.0" && \
     ffmpeg -version && \
     aria2c --version && \
+    deno --version && \
     # Install phantomjs if using x86_64, ensure it is properly installed
     if [[ $(uname -m) == "x86_64" ]]; then \
+      echo "installing phantomjs" && \
       apk add --no-cache gcompat && \
-      cd /usr/share && \
+      tar -xjvf /defaults/phantomjs-2.1.1-linux-x86_64.tar.bz2 && \
-      curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 | tar xj && \
+      mv phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/share/phantomjs && \
-      mv /usr/share/phantomjs-2.1.1-linux-x86_64/bin/phantomjs phantomjs && \
+      rm -rf phantomjs-2.1.1-linux-x86_64 && \
-      rm -rf /usr/share/phantomjs-2.1.1-linux-x86_64 && \
+      rm /defaults/phantomjs-2.1.1-linux-x86_64.tar.bz2 && \
       ln -s /usr/share/phantomjs /usr/bin/phantomjs && \
       echo "Phantom JS version:" && \
       phantomjs --version && \
       cd -; \
     fi && \
-    echo "hi" && \
+    # Configure pip globally
-    # Install ytdl-sub, ensure it is installed properly
+    echo -e "[global]\nbreak-system-packages = true\nroot-user-action = ignore\nno-cache-dir = true" > /etc/pip.conf && \
-    python3 -m pip install --break-system-packages --no-cache-dir ytdl_sub-*.whl && \
+    # Install ytdl-sub and yt-dlp dependencies, ensure they are installed properly
+    python3 -m pip install ytdl_sub-*.whl curl-cffi yt-dlp-ejs && \
       ytdl-sub -h && \
     # Delete unneeded packages after install
     rm ytdl_sub-*.whl && \
     apk del \
       g++ \
       make \
-      py3-pip \
+      libffi-dev && \
-      py3-setuptools
+    python3 -m pip --help
+
 
 ###############################################################################
 # CONTAINER CONFIGS
 
 ENV EDITOR="nano" \
-HOME="/config"
+HOME="${DEFAULT_WORKSPACE}" \
+DOCKER_MODS=linuxserver/mods:universal-stdout-logs|linuxserver/mods:universal-cron \
+CRON_SCRIPT="${DEFAULT_WORKSPACE}/cron" \
+CRON_WRAPPER_SCRIPT="${DEFAULT_WORKSPACE}/.cron_wrapper" \
+LOGS_TO_STDOUT="${DEFAULT_WORKSPACE}/.cron.log" \
+LSIO_FIRST_PARTY=false
 
-VOLUME /config
+VOLUME "${DEFAULT_WORKSPACE}"
+WORKDIR "${DEFAULT_WORKSPACE}"

docker/Dockerfile.gui

@@ -0,0 +1,97 @@
+FROM lscr.io/linuxserver/code-server:4.98.2
+
+# For phantomjs
+ENV OPENSSL_CONF="/etc/ssl"
+# For downloading thumbnails
+ENV SSL_CERT_DIR="/etc/ssl/certs/"
+# Working directory used at both build and run times:
+ENV DEFAULT_WORKSPACE="/config/ytdl-sub-configs"
+
+###############################################################################
+# YTDL-SUB INSTALL
+
+SHELL ["/bin/bash", "-c"]
+COPY root/ /
+RUN mkdir -p /config && \
+    apt-get -y update && \
+    apt-get -y upgrade && \
+    apt-get install --no-install-recommends -y \
+      software-properties-common && \
+    apt-get -y update && \
+    apt-get -y upgrade && \
+    apt-get install --no-install-recommends -y \
+      vim \
+      g++ \
+      nano \
+      unzip \
+      make \
+      python3-pip \
+      fontconfig \
+      xz-utils \
+      bzip2 \
+      aria2 \
+      python3-venv && \
+    if [[ $(uname -m) == "x86_64" ]]; then \
+      curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz && \
+      tar -xf ffmpeg.tar.gz && \
+      chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffmpeg && \
+      chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffprobe && \
+      mv ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/bin/ffmpeg && \
+      mv ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/bin/ffprobe && \
+      rm ffmpeg.tar.gz && \
+      rm -rf ffmpeg-master-latest-linux64-gpl/ ; \
+    else \
+      curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz && \
+      tar -xf ffmpeg.tar.gz && \
+      chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg && \
+      chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe && \
+      mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg /usr/bin/ffmpeg && \
+      mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe /usr/bin/ffprobe && \
+      rm ffmpeg.tar.gz && \
+      rm -rf ffmpeg-master-latest-linuxarm64-gpl/ ; \
+    fi && \
+    # Ensure ffmpeg is installed
+    ffmpeg -version && \
+    # Install phantomjs if using x86_64, ensure it is properly installed
+    if [[ $(uname -m) == "x86_64" ]]; then \
+      echo "installing phantomjs" && \
+      tar -xjvf /defaults/phantomjs-2.1.1-linux-x86_64.tar.bz2 && \
+      mv phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/bin/phantomjs && \
+      rm -rf phantomjs-2.1.1-linux-x86_64 && \
+      rm /defaults/phantomjs-2.1.1-linux-x86_64.tar.bz2 && \
+      echo "Phantom JS version:" && \
+      phantomjs --version ; \
+    fi && \
+    # Install Deno, required for YouTube downloads
+    curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh -s -- -y --no-modify-path && \
+    deno --help && \
+    # Configure pip globally
+    echo -e "[global]\nbreak-system-packages = true\nroot-user-action = ignore\nno-cache-dir = true" > /etc/pip.conf && \
+    # Install ytdl-sub and yt-dlp dependencies, ensure they are installed properly
+    python3 -m pip install ytdl_sub-*.whl curl-cffi yt-dlp-ejs && \
+      ytdl-sub -h && \
+    # Delete unneeded packages after install
+    rm ytdl_sub-*.whl && \
+    apt-get remove -y \
+      g++ \
+      make \
+      xz-utils \
+      bzip2 && \
+    apt-get autoremove -y && \
+    apt-get purge -y --auto-remove && \
+    rm -rf /var/lib/apt/lists/* && \
+    python3 -m pip --help
+
+###############################################################################
+# CONTAINER CONFIGS
+
+ENV EDITOR="nano" \
+HOME="/config" \
+DOCKER_MODS=linuxserver/mods:universal-stdout-logs|linuxserver/mods:universal-cron \
+CRON_SCRIPT="${DEFAULT_WORKSPACE}/cron" \
+CRON_WRAPPER_SCRIPT="/config/.cron_wrapper" \
+LOGS_TO_STDOUT=/config/.cron.log \
+LSIO_FIRST_PARTY=false
+
+VOLUME /config
+WORKDIR "${DEFAULT_WORKSPACE}"

docker/Dockerfile.headless

@@ -0,0 +1 @@
+Dockerfile

docker/Dockerfile.ubuntu

@@ -1,17 +1,21 @@
-FROM ghcr.io/linuxserver/baseimage-ubuntu:jammy
+FROM ghcr.io/linuxserver/baseimage-ubuntu:noble
 
 # https://askubuntu.com/questions/972516/debian-frontend-environment-variable
 ARG DEBIAN_FRONTEND=noninteractive
 
-# Needed for phantomjs
+# For phantomjs
-ENV OPENSSL_CONF=/etc/ssl
+ENV OPENSSL_CONF="/etc/ssl"
+# For downloading thumbnails
+ENV SSL_CERT_DIR="/etc/ssl/certs/"
+# Working directory used at both build and run times:
+ENV DEFAULT_WORKSPACE="/config"
 
 ###############################################################################
 # YTDL-SUB INSTALL
 
 SHELL ["/bin/bash", "-c"]
 COPY root/ /
-RUN mkdir -p /config && \
+RUN mkdir -pv "${DEFAULT_WORKSPACE}" && \
     apt-get -y update && \
     apt-get -y upgrade && \
     apt-get install --no-install-recommends -y \
@@ -22,8 +26,8 @@ RUN mkdir -p /config && \
       vim \
       g++ \
       nano \
+      unzip \
       make \
-      python3.10-dev \
       python3-pip \
       fontconfig \
       xz-utils \
@@ -53,16 +57,21 @@ RUN mkdir -p /config && \
     ffmpeg -version && \
     # Install phantomjs if using x86_64, ensure it is properly installed
     if [[ $(uname -m) == "x86_64" ]]; then \
-      curl -L -o phantomjs.tar.bz2 https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 && \
+      echo "installing phantomjs" && \
-      tar -xvf phantomjs.tar.bz2 && \
+      tar -xjvf /defaults/phantomjs-2.1.1-linux-x86_64.tar.bz2 && \
       mv phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/bin/phantomjs && \
-      rm -rf phantomjs-2.1.1-linux-x86_64/ && \
+      rm -rf phantomjs-2.1.1-linux-x86_64 && \
-      rm phantomjs.tar.bz2 && \
+      rm /defaults/phantomjs-2.1.1-linux-x86_64.tar.bz2 && \
       echo "Phantom JS version:" && \
       phantomjs --version ; \
     fi && \
-    # Install ytdl-sub, ensure it is installed properly
+    # Install Deno, required for YouTube downloads
-    pip install --no-cache-dir ytdl_sub-*.whl && \
+    curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh -s -- -y --no-modify-path && \
+    deno --help && \
+    # Configure pip globally
+    echo -e "[global]\nbreak-system-packages = true\nroot-user-action = ignore\nno-cache-dir = true" > /etc/pip.conf && \
+    # Install ytdl-sub and yt-dlp dependencies, ensure they are installed properly
+    python3 -m pip install ytdl_sub-*.whl curl-cffi yt-dlp-ejs && \
       ytdl-sub -h && \
     # Delete unneeded packages after install
     rm ytdl_sub-*.whl && \
@@ -70,17 +79,22 @@ RUN mkdir -p /config && \
       g++ \
       make \
       xz-utils \
-      bzip2 \
+      bzip2 && \
-      python3.10-dev \
-      python3-venv && \
     apt-get autoremove -y && \
     apt-get purge -y --auto-remove && \
-    rm -rf /var/lib/apt/lists/*
+    rm -rf /var/lib/apt/lists/* && \
+    python3 -m pip --help
 
 ###############################################################################
 # CONTAINER CONFIGS
 
 ENV EDITOR="nano" \
-HOME="/config"
+HOME="${DEFAULT_WORKSPACE}" \
+DOCKER_MODS=linuxserver/mods:universal-stdout-logs|linuxserver/mods:universal-cron \
+CRON_SCRIPT="${DEFAULT_WORKSPACE}/cron" \
+CRON_WRAPPER_SCRIPT="${DEFAULT_WORKSPACE}/.cron_wrapper" \
+LOGS_TO_STDOUT="${DEFAULT_WORKSPACE}/.cron.log" \
+LSIO_FIRST_PARTY=false
 
-VOLUME /config
+VOLUME "${DEFAULT_WORKSPACE}"
+WORKDIR "${DEFAULT_WORKSPACE}"

docker/root/custom-cont-init.d/defaults

@@ -0,0 +1,86 @@
+#!/usr/bin/with-contenv bash
+
+echo "Starting ytdl-sub..."
+
+# copy config
+[[ ! -e "$DEFAULT_WORKSPACE/config.yaml" ]] && \
+    mkdir -p "$DEFAULT_WORKSPACE" && \
+    cp /defaults/config.yaml "$DEFAULT_WORKSPACE/config.yaml"
+[[ ! -e "$DEFAULT_WORKSPACE/subscriptions.yaml" ]] && \
+    mkdir -p "$DEFAULT_WORKSPACE" && \
+    cp /defaults/subscriptions.yaml "$DEFAULT_WORKSPACE/subscriptions.yaml"
+[[ ! -d "$DEFAULT_WORKSPACE/examples" ]] && \
+    mkdir -p "$DEFAULT_WORKSPACE/examples" && \
+    cp -r /defaults/examples/* "$DEFAULT_WORKSPACE/examples"
+
+[[ ! -e "/config/.bashrc" ]] && \
+  echo "alias ls='ls --color=auto'" > /config/.bashrc && \
+  echo "cd ." >> /config/.bashrc
+
+# always create empty cron log file on start
+echo "" > "$LOGS_TO_STDOUT"
+
+# permissions
+chown -R ${PUID:-abc}:${PGID:-abc} \
+    /config
+
+# update command reference:
+# https://github.com/yt-dlp/yt-dlp/wiki/Installation#with-pip
+if [ "$UPDATE_YT_DLP_ON_START" == "stable" ] ; then
+    echo "UPDATE_YT_DLP_ON_START is set to stable, attempting to update to a new stable version of yt-dlp if it exists."
+    python3 -m pip install -U "yt-dlp[default]" 
+elif [ "$UPDATE_YT_DLP_ON_START" == "nightly" ] ; then
+    echo "UPDATE_YT_DLP_ON_START is set to nightly, attempting to update to the latest nightly version of yt-dlp."
+    python3 -m pip install -U --pre "yt-dlp[default]"
+elif [ "$UPDATE_YT_DLP_ON_START" == "master" ] ; then
+    echo "UPDATE_YT_DLP_ON_START is set to master, pulling yt-dlp's latest commit for install."
+    python3 -m pip install -U pip hatchling wheel
+    python3 -m pip install --force-reinstall "yt-dlp[default] @ https://github.com/yt-dlp/yt-dlp/archive/master.tar.gz"
+else
+    echo "UPDATE_YT_DLP_ON_START is not set, using packaged version."
+fi
+
+# set up cron
+if [ "$CRON_SCHEDULE" != "" ] ; then
+    [[ ! -e "$CRON_SCRIPT" ]] && \
+    cp /defaults/cron "$CRON_SCRIPT"
+
+    # create cron script wrapper
+    echo '#!/bin/bash' > "$CRON_WRAPPER_SCRIPT"
+    # Echo commands for easier user debugging:
+    echo "set -x" >> "$CRON_WRAPPER_SCRIPT"
+    echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" >> "$CRON_WRAPPER_SCRIPT"
+    echo "cd \"$DEFAULT_WORKSPACE\"" >> "$CRON_WRAPPER_SCRIPT"
+    echo ". \"$CRON_SCRIPT\" >> \"$LOGS_TO_STDOUT\" 2>&1" >> "$CRON_WRAPPER_SCRIPT"
+    chmod +x "$CRON_WRAPPER_SCRIPT"
+    chown abc:abc "$CRON_WRAPPER_SCRIPT"
+
+    # Set the crontab file to the schedule, cleanly
+    CRON_SCHEDULE_CLEAN="${CRON_SCHEDULE//\"/}"
+    CRON_SCHEDULE_CLEAN="${CRON_SCHEDULE_CLEAN//\'/}"
+
+    echo "# min   hour    day     month   weekday command" > /config/crontabs/abc
+    echo "$CRON_SCHEDULE_CLEAN   $CRON_WRAPPER_SCRIPT" >> /config/crontabs/abc
+
+    chmod +x "$CRON_SCRIPT"
+    chown abc:abc "$CRON_SCRIPT"
+
+    crontab -u abc /config/crontabs/abc
+
+    CRON_SUCCESS=$?
+    if [ $CRON_SUCCESS -eq 0 ] ; then
+      echo "Cron enabled with schedule $CRON_SCHEDULE_CLEAN"
+
+      if [ "$CRON_RUN_ON_START" = true ] ; then
+        echo "Running cron script on start in the background"
+        # ensure it runs as abc to respect puid/guid with delay for tail to start
+        su -s "/bin/bash" -c "sleep 5 && . '$CRON_WRAPPER_SCRIPT'" abc > /dev/null 2>&1 &
+      fi
+    else
+      echo "Error in CRON_SCHEDULE definition, disabling cron."
+      exit 1
+    fi
+else
+  echo "CRON_SCHEDULE not specified, leaving crontabs as-is. Current configuration in /config/crontabs/abc"
+  cat /config/crontabs/abc
+fi

docker/root/defaults/config.yaml

@@ -1,23 +1,22 @@
 # Bare-bones config. Here are some useful links to get started:
-#   Walk-through Guide:  https://github.com/jmbannon/ytdl-sub/wiki/1.-Introduction
+#   Walk-through Guide:  https://ytdl-sub.readthedocs.io/en/latest/guides/index.html
 #   Config Examples:     https://github.com/jmbannon/ytdl-sub/tree/master/examples
-#   Prebuilt Presets:    https://ytdl-sub.readthedocs.io/en/latest/presets.html
+#   Prebuilt Presets:    https://ytdl-sub.readthedocs.io/en/latest/prebuilt_presets/index.html
-#   Config Docs:         https://ytdl-sub.readthedocs.io/en/latest/config.html
+#   Config Reference:    https://ytdl-sub.readthedocs.io/en/latest/config_reference/index.html
 #
-# Examples are included in the /config/examples/ directory.
+# The subscriptions in `subscriptions.yaml` uses prebuilt presets which do not require
-# Any config and subscription can be ran using:
+# any additions to this config. They can be downloaded using the command:
-#   ytdl-sub --config /path/to/config.yaml sub /path/to/subscriptions.yaml
+#
+#   ytdl-sub --config config.yaml sub subscriptions.yaml
 #
 # Or dry-ran with:
-#   ytdl-sub --dry-run --config /path/to/config.yaml sub /path/to/subscriptions.yaml
+#
+#   ytdl-sub --dry-run --config config.yaml sub subscriptions.yaml
+#
+# See the documentation above on how to build your own custom presets.
 #
 configuration:
-  working_directory: '/tmp/ytdl-sub-downloads'
+  # Avoid unnecessarily long large file renames, set this to a path on the same
-
+  # filesystem as the destination for downloaded files in the `overrides: /
-presets:
+  # *_directory:` paths:
-
+  working_directory: ".ytdl-sub-working-directory"
-  video:
-    output_options:
-      output_directory: "/tmp/ytdl-sub-output"
-      file_name: "{uid}.{ext}"
-      thumbnail_name: "{uid}.{thumbnail_ext}"

docker/root/defaults/cron

@@ -0,0 +1,15 @@
+# Place your ytdl-sub command(s) here.
+#
+# This script is executed in the same directory as this file which also contains the
+# default `./config.yaml` and `./subscriptions.yaml`, so you don't need to use the
+# `--config` CLI option or pass a `SUBPATH` to the `$ ytdl-sub sub` sub-command.
+#
+# Test your configuration and subscriptions carefully before automating downloads to
+# prevent triggering throttles or bans:
+#
+# https://ytdl-sub.readthedocs.io/en/latest/guides/getting_started/downloading.html
+#
+# Once you've tested your configuration and you're ready to download entries unattended,
+# remove the next line and un-comment the following line:
+echo "WARNING: Read /config/ytdl-sub-configs/cron and modify to automate downloads."
+# ytdl-sub sub

docker/root/defaults/subscriptions.yaml

@@ -1,17 +1,88 @@
-# Bare-bones config. Here are some useful links to get started:
+# subscriptions.yaml:
-#   Walk-through Guide:  https://github.com/jmbannon/ytdl-sub/wiki/1.-Introduction
+# Everything in here can be downloaded using the command:
-#   Config Examples:     https://github.com/jmbannon/ytdl-sub/tree/master/examples
+#   ytdl-sub sub subscriptions.yaml
-#   Prebuilt Presets:    https://ytdl-sub.readthedocs.io/en/latest/presets.html
-#   Config Docs:         https://ytdl-sub.readthedocs.io/en/latest/config.html
-#
-# Examples are included in the /config/examples/ directory.
-# Any config and subscription can be ran using:
-#   ytdl-sub --config /path/to/config.yaml sub /path/to/subscriptions.yaml
-#
-# Or dry-ran with:
-#   ytdl-sub --dry-run --config /path/to/config.yaml sub /path/to/subscriptions.yaml
-#
-rammstein_music_videos:
-  preset: "video"
-  download: "https://youtube.com/playlist?list=PLVTLbc6i-h_iuhdwUfuPDLFLXG2QQnz-x"
 
+# __preset__ is a place to define global overrides for all subscriptions
+__preset__:
+  overrides:
+    # Root folder of all ytdl-sub TV Shows
+    tv_show_directory: "/tv_shows"
+
+    # Root folder of all ytdl-sub Music
+    music_directory: "/music"
+
+    # Root folder of all ytdl-sub Music Videos
+    music_video_directory: "/music_videos"
+
+    # For 'Only Recent' preset, only keep vids within this range and limit
+    # only_recent_date_range: "2months"
+    # only_recent_max_files: 30
+
+  # Pass any arg directly to yt-dlp's Python API
+  # ytdl_options:
+  #   cookiefile: "/config/ytdl-sub-configs/cookie.txt"
+
+###################################################################
+# Subscriptions nested under this will use the
+# `Plex TV Show by Date` preset.
+#
+# Can choose between:
+#   - Plex TV Show by Date:
+#   - Jellyfin TV Show by Date:
+#   - Kodi TV Show by Date:
+Plex TV Show by Date:
+
+  # Sets genre tag to "Documentaries"
+  = Documentaries:
+    "NOVA PBS": "https://www.youtube.com/@novapbs"
+    # "National Geographic": "https://www.youtube.com/@NatGeo"
+    # "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
+
+  # Sets genre tag to "Kids", "TV-Y" for content rating
+  # = Kids | = TV-Y:
+  #   "Jake Trains": "https://www.youtube.com/@JakeTrains"
+  #   "Kids Toys Play": "https://www.youtube.com/@KidsToysPlayChannel"
+
+  # = Music:
+  #   # TV show subscriptions can support multiple urls and store in the same TV Show
+  #   "Rick Beato":
+  #     - "https://www.youtube.com/@RickBeato"
+  #     - "https://www.youtube.com/@rickbeato240"
+
+  # Set genre tag to "News", use `Only Recent` preset to only store videos uploaded recently
+  # = News | Only Recent:
+  #   "BBC News": "https://www.youtube.com/@BBCNews"
+
+###################################################################
+# Subscriptions nested under these will use the various prebuilt
+# music presets
+
+# YouTube Releases:
+#   = Jazz:  # Sets genre tag to "Jazz"
+#     "Thelonious Monk": "https://www.youtube.com/@theloniousmonk3870/releases"
+
+# YouTube Full Albums:
+#   = Lofi:
+#     "Game Chops": "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i"
+
+# SoundCloud Discography:
+#   = Chill Hop:
+#     "UKNOWY": "https://soundcloud.com/uknowymunich"
+#   = Synthwave:
+#     "Lazerdiscs Records": "https://soundcloud.com/lazerdiscsrecords"
+#     "Earmake": "https://soundcloud.com/earmake"
+
+# Bandcamp:
+#   = Lofi:
+#     "Emily Hopkins": "https://emilyharpist.bandcamp.com/"
+
+###################################################################
+# Can choose between:
+#   - Plex Music Videos:
+#   - Jellyfin Music Videos:
+#   - Kodi Music Videos:
+
+# "Plex Music Videos":
+#   = Pop:  # Sets genre tag to "Pop"
+#     "Rick Astley": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc"
+#     "Michael Jackson": "https://www.youtube.com/playlist?list=OLAK5uy_mnY03zP6abNWH929q2XhGzWD_2uKJ_n8E"

docker/root/etc/cont-init.d/30-config

@@ -1,13 +0,0 @@
-#!/usr/bin/with-contenv bash
-
-# copy config
-[[ ! -e /config/config.yaml ]] && \
-    cp /defaults/config.yaml /config/config.yaml
-[[ ! -e /config/subscriptions.yaml ]] && \
-    cp /defaults/subscriptions.yaml /config/subscriptions.yaml
-[[ ! -d /config/examples ]] && \
-    cp -R /defaults/examples /config/
-
-# permissions
-chown -R ${PUID:-abc}:${PGID:-abc} \
-    /config

docker/root/etc/s6-overlay/s6-rc.d/init-adduser/branding

@@ -0,0 +1,15 @@
+――――――――――――――――――――――――――――――――――――
+██╗   ██╗████████╗██████╗ ██╗
+╚██╗ ██╔╝╚══██╔══╝██╔══██╗██║
+ ╚████╔╝    ██║   ██║  ██║██║
+  ╚██╔╝     ██║   ██║  ██║██║
+   ██║      ██║   ██████╔╝███████╗
+   ╚═╝      ╚═╝   ╚═════╝ ╚══════╝
+
+    ███████╗██╗   ██╗██████╗
+    ██╔════╝██║   ██║██╔══██╗
+    ███████╗██║   ██║██████╔╝
+    ╚════██║██║   ██║██╔══██╗
+    ███████║╚██████╔╝██████╔╝
+    ╚══════╝ ╚═════╝ ╚═════╝
+――――――――――――――――――――――――――――――――――――

docker/testing/Makefile

@@ -0,0 +1,46 @@
+# Local building and testing of the Docker image variants.
+
+# Defensive settings for make:
+#     https://tech.davis-hansson.com/p/make/
+SHELL:=bash
+.ONESHELL:
+.SHELLFLAGS:=-eu -o pipefail -c
+.SILENT:
+.DELETE_ON_ERROR:
+MAKEFLAGS+=--warn-undefined-variables
+MAKEFLAGS+=--no-builtin-rules
+export PS1?=$$
+# Prefix echoed recipe commands with the recipe line number for debugging:
+export PS4?=:$$LINENO+
+
+VARIANTS=headless gui ubuntu
+ROOT_PREREQS:=$(shell find ../root -type f)
+
+# Finished with `$(shell)`, echo recipe commands going forward
+.SHELLFLAGS+= -x
+
+
+### Top-level targets:
+
+.PHONY: all
+all: build
+
+.PHONY: build
+build: $(VARIANTS:%=./build/ytdl-sub-%.log)
+
+.PHONY: run
+run: build $(VARIANTS:%=./volumes/ytdl-sub-%/)
+	docker compose up
+
+
+### Real targets:
+
+# Re-build the local images when changes require it.
+./build/ytdl-sub-%.log: ../Dockerfile.% $(ROOT_PREREQS)
+	mkdir -pv "$(dir $(@))"
+	docker compose build "$(@:build/ytdl-sub-%.log=ytdl-sub-%)" |&
+	    tee -a "$(@)"
+
+# Ensure volumes are owned by the developer's normal user:
+./volumes/ytdl-sub-%/:
+	mkdir -pv "$(@)"

docker/testing/docker-compose.yml

@@ -0,0 +1,50 @@
+services:
+  ytdl-sub-gui:
+    build:
+      context: "../"
+      dockerfile: "./Dockerfile.gui"
+    image: "ytdl-sub-gui:local"
+    container_name: "ytdl-sub-gui"
+    environment:
+      PUID: "1000"
+      PGID: "1000"
+      TZ: "America/Los_Angeles"
+      CRON_SCHEDULE: '*/1 * * * *'
+      CRON_RUN_ON_START: "true"
+      UPDATE_YT_DLP_ON_START: "stable"
+    volumes:
+      - "./volumes/ytdl-sub-gui/:/config/"
+    ports:
+      - "8443:8443"
+    restart: "unless-stopped"
+  ytdl-sub-headless:
+    build:
+      context: "../"
+    image: "ytdl-sub:local"
+    container_name: "ytdl-sub-headless"
+    environment:
+      PUID: "1000"
+      PGID: "1000"
+      TZ: "America/Los_Angeles"
+      CRON_SCHEDULE: '*/1 * * * *'
+      CRON_RUN_ON_START: "true"
+      UPDATE_YT_DLP_ON_START: "stable"
+    volumes:
+      - "./volumes/ytdl-sub-headless/:/config/"
+    restart: "unless-stopped"
+  ytdl-sub-ubuntu:
+    build:
+      context: "../"
+      dockerfile: "./Dockerfile.ubuntu"
+    image: "ytdl-sub-ubuntu:local"
+    container_name: "ytdl-sub-ubuntu"
+    environment:
+      PUID: "1000"
+      PGID: "1000"
+      TZ: "America/Los_Angeles"
+      CRON_SCHEDULE: '*/1 * * * *'
+      CRON_RUN_ON_START: "true"
+      UPDATE_YT_DLP_ON_START: "stable"
+    volumes:
+      - "./volumes/ytdl-sub-ubuntu/:/config/"
+    restart: "unless-stopped"

docs/Makefile

@@ -5,8 +5,8 @@
 # from the environment for the first two.
 SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = .
+SOURCEDIR     = source
-BUILDDIR      = _build
+BUILDDIR      = build
 
 # Put it first so that "make" without argument is like "make help".
 help:

docs/conf.py

@@ -1,67 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# This file only contains a selection of the most common options. For a full
-# list see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Path setup --------------------------------------------------------------
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-import os
-import sys
-
-sys.path.insert(0, os.path.abspath("../src"))
-
-
-# -- Project information -----------------------------------------------------
-
-project = "ytdl-sub"
-copyright = "2022, Jesse Bannon"
-author = "Jesse Bannon"
-
-
-# -- General configuration ---------------------------------------------------
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = [
-    "sphinx.ext.autodoc",
-    "sphinx.ext.napoleon",
-]
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ["_templates"]
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
-
-
-# -- Options for HTML output -------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
-html_theme = "sphinx_rtd_theme"
-
-html_theme_options = {"navigation_depth": 10}
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-# html_static_path = ["_static"]
-
-# Do not show full module path in api docs
-add_module_names = False
-python_use_unqualified_type_names = False
-napoleon_numpy_docstrings = True
-napoleon_use_rtype = False
-
-# -- Options for autodocs -------------------------------------------------
-
-autodoc_default_options = {"autodoc_typehints_format": "short"}

docs/config.rst

@@ -1,374 +0,0 @@
-Config
-======
-ytdl-sub is configured using a ``config.yaml`` file.
-
-.. _config_yaml:
-
-config.yaml
------------
-
-The ``config.yaml`` is made up of two sections:
-
-.. code-block:: yaml
-
-   configuration:
-   presets:
-
-You can jump to any section and subsection of the config using the navigation
-section to the left.
-
-Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``.
-If you wish to represent paths like Windows, you will need to ``C:\\double\\bashslash\\paths``
-in order to escape the backslash character.
-
-configuration
-^^^^^^^^^^^^^
-The ``configuration`` section contains app-wide configs applied to all presets
-and subscriptions.
-
-.. autoclass:: ytdl_sub.config.config_validator.ConfigOptions()
-  :members:
-  :member-order: bysource
-  :exclude-members: persist_logs, experimental
-
-persist_logs
-""""""""""""
-Within ``configuration``, define whether logs from subscription downloads
-should be persisted.
-
-.. code-block:: yaml
-
-   configuration:
-     persist_logs:
-       logs_directory: "/path/to/log/directory"
-
-Log files are stored as
-``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``.
-
-.. autoclass:: ytdl_sub.config.config_validator.PersistLogsValidator()
-  :members:
-  :member-order: bysource
-
-presets
-^^^^^^^
-``presets`` define a `formula` for how to format downloaded media and metadata.
-
-download_strategy
-"""""""""""""""""
-Download strategies dictate what is getting downloaded from a source. Each
-download strategy has its own set of parameters.
-
-.. _url:
-
-url
-'''
-.. autoclass:: ytdl_sub.downloaders.url.url.UrlDownloadOptions()
-  :members: url, playlist_thumbnails, source_thumbnails, download_reverse
-  :member-order: bysource
-
-multi_url
-'''''''''
-.. autoclass:: ytdl_sub.downloaders.url.multi_url.MultiUrlDownloadOptions()
-  :members: urls, variables
-
--------------------------------------------------------------------------------
-
-output_options
-""""""""""""""
-
-.. autoclass:: ytdl_sub.config.preset_options.OutputOptions()
-  :members:
-  :member-order: bysource
-  :exclude-members: get_upload_date_range_to_keep, partial_validate
-
--------------------------------------------------------------------------------
-
-.. _ytdl_options:
-
-ytdl_options
-""""""""""""
-.. autoclass:: ytdl_sub.config.preset_options.YTDLOptions()
-
--------------------------------------------------------------------------------
-
-.. _overrides:
-
-overrides
-"""""""""
-.. autoclass:: ytdl_sub.config.preset_options.Overrides()
-
-.. _parent preset:
-
-preset
-""""""
-Presets support inheritance by defining a parent preset:
-
-.. code-block:: yaml
-
-   presets:
-     custom_preset:
-       ...
-     parent_preset:
-       ...
-     child_preset:
-       preset: "parent_preset"
-
-In the example above, ``child_preset`` inherits all fields defined in ``parent_preset``.
-It is advantageous to use parent presets where possible to reduce duplicate yaml
-definitions.
-
-Presets also support inheritance from multiple presets:
-
-.. code-block:: yaml
-
-     child_preset:
-       preset:
-         - "custom_preset"
-         - "parent_preset"
-
-In this example, ``child_preset`` will inherit all fields from ``custom_preset``
-and ``parent_preset`` in that order. The bottom-most preset has the highest
-priority.
-
-If you are only inheriting from one preset, the syntax ``preset: "parent_preset"`` is
-valid YAML. Inheriting from multiple presets require use of a list.
-
--------------------------------------------------------------------------------
-
-
-Plugins
-"""""""
-Plugins are used to perform any type of post-processing to the already downloaded files.
-
-audio_extract
-'''''''''''''
-.. autoclass:: ytdl_sub.plugins.audio_extract.AudioExtractOptions()
-  :members:
-  :member-order: bysource
-  :exclude-members: partial_validate
-
--------------------------------------------------------------------------------
-
-chapters
-''''''''
-.. autoclass:: ytdl_sub.plugins.chapters.ChaptersOptions()
-  :members:
-  :member-order: bysource
-  :exclude-members: partial_validate
-
--------------------------------------------------------------------------------
-
-date_range
-''''''''''
-.. autoclass:: ytdl_sub.plugins.date_range.DateRangeOptions()
-  :members:
-  :member-order: bysource
-  :exclude-members: partial_validate
-
--------------------------------------------------------------------------------
-
-embed_thumbnail
-''''''''''''''''
-
-.. autoclass:: ytdl_sub.plugins.embed_thumbnail.EmbedThumbnailOptions()
-
--------------------------------------------------------------------------------
-
-file_convert
-''''''''''''
-.. autoclass:: ytdl_sub.plugins.file_convert.FileConvertOptions()
-  :members:
-  :member-order: bysource
-  :exclude-members: partial_validate
-
--------------------------------------------------------------------------------
-
-match_filters
-'''''''''''''
-.. autoclass:: ytdl_sub.plugins.match_filters.MatchFiltersOptions()
-  :members:
-  :exclude-members: partial_validate
-
--------------------------------------------------------------------------------
-
-music_tags
-''''''''''
-.. autoclass:: ytdl_sub.plugins.music_tags.MusicTagsOptions()
-
--------------------------------------------------------------------------------
-
-nfo_tags
-''''''''
-.. autoclass:: ytdl_sub.plugins.nfo_tags.NfoTagsOptions()
-  :members: nfo_name, nfo_root, tags, kodi_safe
-  :member-order: bysource
-  :exclude-members: partial_validate
-
--------------------------------------------------------------------------------
-
-output_directory_nfo_tags
-'''''''''''''''''''''''''
-.. autoclass:: ytdl_sub.plugins.output_directory_nfo_tags.OutputDirectoryNfoTagsOptions()
-  :members: nfo_name, nfo_root, tags, kodi_safe
-  :member-order: bysource
-  :exclude-members: partial_validate
-
--------------------------------------------------------------------------------
-
-regex
-'''''
-.. autoclass:: ytdl_sub.plugins.regex.RegexOptions()
-  :members: skip_if_match_fails
-
-.. autoclass:: ytdl_sub.plugins.regex.VariableRegex()
-  :members: match, capture_group_names, capture_group_defaults, exclude
-  :member-order: bysource
-  :exclude-members: partial_validate
-
--------------------------------------------------------------------------------
-
-split_by_chapters
-'''''''''''''''''
-.. autoclass:: ytdl_sub.plugins.split_by_chapters.SplitByChaptersOptions()
-  :members: when_no_chapters
-  :member-order: bysource
-  :exclude-members: partial_validate
-
--------------------------------------------------------------------------------
-
-subtitles
-'''''''''
-.. autoclass:: ytdl_sub.plugins.subtitles.SubtitleOptions()
-  :members: subtitles_name, subtitles_type, embed_subtitles, languages, allow_auto_generated_subtitles
-  :member-order: bysource
-  :exclude-members: partial_validate
-
--------------------------------------------------------------------------------
-
-video_tags
-''''''''''
-.. autoclass:: ytdl_sub.plugins.video_tags.VideoTagsOptions()
-
--------------------------------------------------------------------------------
-
-.. _subscription_yaml:
-
-subscription.yaml
------------------
-The ``subscription.yaml`` file is where we use our `presets`_ in the `config.yaml`_
-to define a `subscription`: something we want to recurrently download such as a specific
-channel or playlist.
-
-The only difference between a ``subscription`` and ``preset`` is that the subscription
-must have all required fields and ``{variables}`` defined so it can perform a download.
-
-Below is an example that downloads a YouTube playlist:
-
-.. code-block:: yaml
-  :caption: config.yaml
-
-   presets:
-     playlist_preset_ex:
-       download:
-         download_strategy: "url"
-         url: "{url}"
-       output_options:
-         output_directory: "{output_directory}/{playlist_name}"
-         file_name: "{playlist_name}.{title}.{ext}"
-       overrides:
-         output_directory: "/path/to/ytdl-sub-videos"
-
-.. code-block:: yaml
-  :caption: subscription.yaml
-
-   my_subscription_name:
-     preset: "playlist_preset_ex"
-     overrides:
-       playlist_name: "diy-playlist"
-       url: "https://youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
-
-Our preset ``playlist_preset_ex`` defines three
-custom variables: ``{output_directory}``, ``{playlist_name}``, and ``{url}``. The subscription sets
-the `parent preset`_ to ``playlist_preset_ex``, and must define the variables ``{playlist_name}``
-and ``{url}`` since the preset did not.
-
-File Preset
-^^^^^^^^^^^
-You can apply a preset to all subscriptions in the ``subscription.yaml`` file
-by using the file-wide ``__preset__``:
-
-.. code-block:: yaml
-  :caption: subscription.yaml
-
-   __preset__:
-     preset: "playlist_preset_ex"
-
-   my_subscription_name:
-     overrides:
-       url: "https://youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
-       playlist_name: "diy-playlist"
-
-This ``subscription.yaml`` is equivalent to the one above it because all
-subscriptions automatically set ``__preset__`` as a `parent preset`_.
-
-File Subscription Value
-^^^^^^^^^^^^^^^^^^^^^^^
-With a clever config and use of ``__preset__``, your subscriptions can typically boil
-down to a name and url. You can set ``__value__`` to the name of an override variable,
-and use the override variable ``subscription_name`` to achieve one-liner subscriptions.
-Using the example above, we can do:
-
-.. code-block:: yaml
-  :caption: subscription.yaml
-
-   __preset__:
-     preset: "playlist_preset_ex"
-     overrides:
-       playlist_name: "{subscription_name}"
-
-   __value__: "url"
-
-   # single-line subscription
-   "diy-playlist": "https://youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
-
-``"diy-playlist"`` gets assigned to the ``playlist_name`` override variable by setting
-it with ``subscription_name`` , and the url gets assigned to ``url`` by setting ``__value__``
-to write values to it.
-
-Traditional subscriptions that can override presets will still work when using ``__value__``.
-
--------------------------------------------------------------------------------
-
-.. _source-variables:
-
-Source Variables
-----------------
-
-.. autoclass:: ytdl_sub.entries.variables.entry_variables.EntryVariables
-   :members:
-   :inherited-members:
-   :undoc-members:
-
-Override Variables
-------------------
-
-.. autoclass:: ytdl_sub.config.preset_options.OverridesVariables()
-   :members:
-
--------------------------------------------------------------------------------
-
-Config Types
-------------
-The `config.yaml`_ uses various types for its configurable fields. Below is a definition for each type.
-
-.. autoclass:: ytdl_sub.validators.string_formatter_validators.StringFormatterValidator()
-
-.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesStringFormatterValidator()
-
-.. autoclass:: ytdl_sub.validators.file_path_validators.StringFormatterFileNameValidator()
-
-.. autoclass:: ytdl_sub.validators.string_datetime.StringDatetimeValidator()
-
-.. autoclass:: ytdl_sub.validators.string_formatter_validators.DictFormatterValidator()
-
-.. autoclass:: ytdl_sub.validators.string_formatter_validators.OverridesDictFormatterValidator()

docs/deprecation_notices.rst

@@ -1,51 +0,0 @@
-Deprecation Notices
-===================
-
-July 2023
----------
-
-music_tags
-^^^^^^^^^^
-
-Music tags are getting simplified. ``tags`` will now reside directly under music_tags, and
-``embed_thumbnail`` is getting moved to its own plugin (supports video files as well). Convert from:
-
-.. code-block:: yaml
-
-   my_example_preset:
-     music_tags:
-       embed_thumbnail: True
-       tags:
-         artist: "Elvis Presley"
-
-To the following:
-
-.. code-block:: yaml
-
-   my_example_preset:
-     embed_thumbnail: True
-     music_tags:
-       artist: "Elvis Presley"
-
-The old format will be removed in October 2023.
-
-video_tags
-^^^^^^^^^^
-
-Video tags are getting simplified as well. ``tags`` will now reside directly under video_tags.
-Convert from:
-
-.. code-block:: yaml
-
-   my_example_preset:
-     video_tags:
-       tags:
-         title: "Elvis Presley Documentary"
-
-To the following:
-
-.. code-block:: yaml
-
-   my_example_preset:
-     video_tags:
-       title: "Elvis Presley Documentary"

docs/getting_started.rst

@@ -1,29 +0,0 @@
-Getting Started
-===============
-
-Walk-through Guide
--------------------
-If you haven't read it yet, it's highly recommended to go through our
-`walk-through guide <https://github.com/jmbannon/ytdl-sub/wiki/1.-Introduction>`_
-to get familiar with how ``ytdl-sub`` works.
-
-Example Configs
----------------
-If you are ready to start downloading, see our
-`examples directory <https://github.com/jmbannon/ytdl-sub/tree/master/examples>`_
-for ready-to-use configs and subscriptions. Read through them carefully before use.
-
-Using Example Configs
-^^^^^^^^^^^^^^^^^^^^^^
-Copy and paste the examples into local yaml files, modify the
-``working_directory`` and ``output_directory`` with your desired paths,
-and perform a dry-run using
-
-.. code-block:: bash
-
-   ytdl-sub \
-     --dry-run \
-     --config path/to/config.yaml \
-     sub path/to/subscriptions.yaml
-
-This will simulate what a download will look like.

docs/index.rst

@@ -1,29 +0,0 @@
-ytdl-sub readthedocs
-====================
-
-Our readthedocs page is dedicated towards ytdl-sub config documentation.
-
-If you are new to ytdl-sub, head over to the
-`GitHub Wiki <https://github.com/jmbannon/ytdl-sub/wiki>`_
-to see our
-`walkthrough <https://github.com/jmbannon/ytdl-sub/wiki/1.-Introduction>`_ and
-`FAQ <https://github.com/jmbannon/ytdl-sub/wiki/FAQ>`_. For full examples of
-ytdl-sub configs, check out the
-`examples directory <https://github.com/jmbannon/ytdl-sub/tree/master/examples>`_.
-
-
-For navigating config docs, use the left-side bar on the
-:ref:`config_yaml` page to find every available ytdl-sub field.
-
-Contents
-========
-
-.. toctree::
-   :maxdepth: 2
-
-   install
-   usage
-   getting_started
-   presets
-   config
-   deprecation_notices

docs/install.rst

@@ -1,209 +0,0 @@
-Install
-=======
-``ytdl-sub`` can be installed on the following platforms.
-
-.. contents::
-    :depth: 2
-
-All installations require a 64-bit CPU. 32-bit is not supported.
-
-Docker Compose
---------------
-The ytdl-sub docker image uses
-`Linux Server's <https://www.linuxserver.io/>`_
-`base alpine image <https://github.com/linuxserver/docker-baseimage-alpine/>`_
-It looks, feels, and operates like other LinuxServer images. This is the
-recommended way to use ytdl-sub.
-
-The docker image is intended to be used as a console. For automating
-``subscriptions.yaml`` downloads to pull new media, see
-`this guide <https://github.com/jmbannon/ytdl-sub/wiki/7.-Automate-Downloading-New-Content-Using-Your-Configs/>`_
-on how set up a cron job in the docker container.
-
-.. code-block:: yaml
-
-   services:
-     ytdl-sub:
-       image: ghcr.io/jmbannon/ytdl-sub:latest
-       container_name: ytdl-sub
-       environment:
-         - PUID=1000
-         - PGID=1000
-         - TZ=America/Los_Angeles
-         - DOCKER_MODS=linuxserver/mods:universal-cron
-       volumes:
-         - <path/to/ytdl-sub/config>:/config
-         - <path/to/tv_shows>:/tv_shows  # optional
-         - <path/to/movies>:/movies  # optional
-         - <path/to/music_videos>:/music_videos  # optional
-         - <path/to/music>:/music  # optional
-       restart: unless-stopped
-
-CPU Passthrough
-^^^^^^^^^^^^^^^^^^^^^^
-For CPU passthrough, you must use the ``ytdl-sub`` Ubuntu version with the following additions:
-
-.. code-block:: yaml
-
-   services:
-     ytdl-sub:
-       image: ghcr.io/jmbannon/ytdl-sub:ubuntu-latest
-       container_name: ytdl-sub
-       environment:
-         - PUID=1000
-         - PGID=1000
-         - TZ=America/Los_Angeles
-         - DOCKER_MODS=linuxserver/mods:universal-cron
-       volumes:
-         - <path/to/ytdl-sub/config>:/config
-         - <path/to/tv_shows>:/tv_shows  # optional
-         - <path/to/movies>:/movies  # optional
-         - <path/to/music_videos>:/music_videos  # optional
-         - <path/to/music>:/music  # optional
-       devices:
-         - /dev/dri:/dev/dri  # CPU passthrough
-       restart: unless-stopped
-
-Nvidia GPU Passthrough
-^^^^^^^^^^^^^^^^^^^^^^
-For GPU passthrough, you must use the ``ytdl-sub`` Ubuntu version with the following additions:
-
-.. code-block:: yaml
-
-   services:
-     ytdl-sub:
-       image: ghcr.io/jmbannon/ytdl-sub:ubuntu-latest
-       container_name: ytdl-sub
-       environment:
-         - PUID=1000
-         - PGID=1000
-         - TZ=America/Los_Angeles
-         - DOCKER_MODS=linuxserver/mods:universal-cron
-         - NVIDIA_DRIVER_CAPABILITIES=all  # Nvidia ENV args
-         - NVIDIA_VISIBLE_DEVICES=all
-       volumes:
-         - <path/to/ytdl-sub/config>:/config
-         - <path/to/tv_shows>:/tv_shows  # optional
-         - <path/to/movies>:/movies  # optional
-         - <path/to/music_videos>:/music_videos  # optional
-         - <path/to/music>:/music  # optional
-       deploy:
-         resources:
-           reservations:
-             devices:
-               - capabilities: [gpu]  # GPU passthrough
-       restart: unless-stopped
-
-Docker
---------------
-.. code-block:: bash
-
-   docker run -d \
-       --name=ytdl-sub \
-       -e PUID=1000 \
-       -e PGID=1000 \
-       -e TZ=America/Los_Angeles \
-       -e DOCKER_MODS=linuxserver/mods:universal-cron \
-       -v <path/to/ytdl-sub/config>:/config \
-       -v <OPTIONAL/path/to/tv_shows>:/tv_shows \
-       -v <OPTIONAL/path/to/movies>:/movies \
-       -v <OPTIONAL/path/to/music_videos>:/music_videos \
-       -v <OPTIONAL/path/to/music>:/music \
-       --restart unless-stopped \
-       ghcr.io/jmbannon/ytdl-sub:latest
-
-Windows
---------------
-From powershell, run:
-
-.. code-block:: powershell
-
-   # Download ffmpeg/ffprobe dependencies from yt-dlp
-   curl.exe -L -o ffmpeg.zip https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip
-   tar -xf ffmpeg.zip
-   move "ffmpeg-master-latest-win64-gpl\bin\ffmpeg.exe" "ffmpeg.exe"
-   move "ffmpeg-master-latest-win64-gpl\bin\ffprobe.exe" "ffprobe.exe"
-
-   # Download ytdl-sub
-   curl.exe -L -o ytdl-sub.exe https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub.exe
-   ytdl-sub.exe -h
-
-Unraid
---------------
-See the
-`community app <https://unraid.net/community/apps?q=ytdl-sub#r>`_
-``ytdl-sub``. Uses Docker under the hood.
-
-Linux
---------------
-Requires ffmpeg as a dependency. Can typically be installed with any Linux package manager.
-
-.. code-block:: bash
-
-   curl -L -o ytdl-sub https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub
-   chmod +x ytdl-sub
-   ytdl-sub -h
-
-You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to date:
-
-.. code-block:: bash
-
-   curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz
-   tar -xf ffmpeg.tar.gz
-   chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffmpeg
-   chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffprobe
-
-   # May need sudo / root permissions to perform
-   mv ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/bin/ffmpeg
-   mv ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/bin/ffprobe
-
-Linux ARM
---------------
-Requires ffmpeg as a dependency. Can typically be installed with any Linux package manager.
-
-.. code-block:: bash
-
-   curl -L -o ytdl-sub https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub_aarch64
-   chmod +x ytdl-sub
-   ytdl-sub -h
-
-You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to date:
-
-.. code-block:: bash
-
-   curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz
-   tar -xf ffmpeg.tar.gz
-   chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg
-   chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe
-
-   # May need sudo / root permissions to perform
-   mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg /usr/bin/ffmpeg
-   mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe /usr/bin/ffprobe
-
-
-PIP
---------------
-You can install our
-`PyPI package <https://pypi.org/project/ytdl-sub/>`_.
-Both ffmpeg and Python 3.10 or greater are required.
-
-.. code-block:: bash
-
-   python3 -m pip install -U ytdl-sub
-
-Local Install
---------------
-With a Python 3.10 virtual environment, you can clone and install the repo using
-
-.. code-block:: bash
-
-   git clone https://github.com/jmbannon/ytdl-sub.git
-   cd ytdl-sub
-
-   pip install -e .
-
-Local Docker Build
--------------------
-Run ``make docker`` in the root directory of this repo to build the image. This
-will build the python wheel and install it in the Dockerfile.
-

docs/make.bat

@@ -7,8 +7,8 @@ REM Command file for Sphinx documentation
 if "%SPHINXBUILD%" == "" (
 	set SPHINXBUILD=sphinx-build
 )
-set SOURCEDIR=.
+set SOURCEDIR=source
-set BUILDDIR=_build
+set BUILDDIR=build
 
 %SPHINXBUILD% >NUL 2>NUL
 if errorlevel 9009 (

docs/presets.rst

@@ -1,183 +0,0 @@
-Presets
-=======
-``ytdl-sub`` offers a number of built-in presets using best practices for formatting
-media in various players. For advanced users, you can find the prebuilt preset
-definitions
-`here <https://github.com/jmbannon/ytdl-sub/tree/master/src/ytdl_sub/prebuilt_presets>`_.
-
-TV Shows
---------
-
-There are two main methods for downloading and formatting videos as a TV show.
-
-TV Show by Date
-^^^^^^^^^^^^^^^
-
-TV Show by Date will organize something like a YouTube channel or playlist
-into a tv show, where seasons and episodes are organized using upload date.
-
-Player Presets
-""""""""""""""
-
-* ``kodi_tv_show_by_date``
-* ``jellyfin_tv_show_by_date``
-* ``plex_tv_show_by_date``
-
-Episode Formatting Presets
-""""""""""""""""""""""""""
-
-* ``season_by_year__episode_by_month_day``
-* ``season_by_year_month__episode_by_day``
-* ``season_by_year__episode_by_month_day_reversed``
-   * Episode numbers are reversed, meaning more recent episodes appear at the
-     top of a season by having a lower value.
-* ``season_by_year__episode_by_download_index``
-   * Episodes are numbered by the download order. NOTE that this fetched using
-     the length of the download archive. Do not use if you intend to remove
-     old videos.
-
-Usage
-"""""
-
-A preset/subscription requires specifying a player and episode formatting preset
-and overriding the following variables:
-
-.. code-block:: yaml
-
-   rick_a_tv_show_by_date:
-     preset:
-       - "jellyfin_tv_show_by_date"
-       - "season_by_year__episode_by_month_day"
-     overrides:
-       # required
-       tv_show_name: "Rick A"
-       tv_show_directory: "/path/to/youtube_shows"
-       url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
-       # can be modified from their default value
-       # tv_show_genre: "ytdl-sub"
-       # tv_show_content_rating: "TV-14"
-       # episode_title: "{upload_date_standardized} - {title}"
-       # episode_description: "{webpage_url}"
-
-In addition, you can add additional URLs to create a single TV by using the override variables
-``url2``, ``url3``, ..., ``url20``:
-
-.. code-block:: yaml
-
-     overrides:
-       tv_show_name: "Rick A"
-       tv_show_directory: "/path/to/youtube_shows"
-       url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
-       url2: "https://www.youtube.com/@just.rick_6"
-
-
-TV Show Collection
-^^^^^^^^^^^^^^^^^^
-
-TV Show Collections are made up from multiple URLs, where each URL is a season.
-If a video belongs to multiple URLs (i.e. a channel and a channel's playlist),
-it will resolve to the bottom-most season.
-
-Two main use cases of a collection are:
-   1. Organize a YouTube channel TV show where Season 1 contains any video
-      not in a 'season playlist', Season 2 for 'Playlist A', Season 3 for
-      'Playlist B', etc.
-   2. Organize one or more YouTube channels/playlists, where each season
-      represents a separate channel/playlist.
-
-Player Presets
-""""""""""""""
-
-* ``kodi_tv_show_collection``
-* ``jellyfin_tv_show_collection``
-* ``plex_tv_show_collection``
-
-Episode Formatting Presets
-""""""""""""""""""""""""""
-
-* ``season_by_collection__episode_by_year_month_day``
-* ``season_by_collection__episode_by_year_month_day_reversed``
-* ``season_by_collection__episode_by_playlist_index``
-   * Only use playlist_index episode formatting for playlists that
-     will be fully downloaded once and never again. Otherwise,
-     indices can change.
-* ``season_by_collection__episode_by_playlist_index_reversed``
-
-Season Presets
-""""""""""""""
-
-* ``collection_season_1``
-* ``collection_season_2``
-* ``collection_season_3``
-* ``collection_season_4``
-* ``...``
-* ``collection_season_40``
-
-Example
-"""""""
-
-A preset/subscription requires specifying a player, episode formatting, and
-one or more season presets, with the following override variables:
-
-.. code-block:: yaml
-
-   rick_a_tv_show_collection:
-     preset:
-       - "jellyfin_tv_show_collection"
-       - "season_by_collection__episode_by_year_month_day_reversed"
-       - "collection_season_1"
-       - "collection_season_2"
-     overrides:
-       # required
-       tv_show_name: "Rick A"
-       tv_show_directory: "/path/to/youtube_shows"
-       collection_season_1_url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
-       collection_season_1_name: "All Videos"
-       collection_season_2_url: "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc"
-       collection_season_2_name: "Official Music Videos"
-       # can be modified from their default value
-       # tv_show_genre: "ytdl-sub"
-       # episode_title: "{upload_date_standardized} - {title}"
-       # episode_description: "{webpage_url}"
-
-Common
-------
-
-Common presets are applicable to any config.
-
-Best Video Quality
-^^^^^^^^^^^^^^^^^^
-
-Add the following preset to download the best available video and audio quality, and remux
-it into an MP4 container:
-
-* ``best_video_quality``
-
-
-Max 1080p
-^^^^^^^^^^^^^^^^^^
-
-Add the following preset to download the best available audio and video quality, with the video not greater than 1080p, and remux it into an MP4 container:
-
-* ``max_1080p``
-
-Chunk Initial Download
-^^^^^^^^^^^^^^^^^^^^^^
-
-If you are archiving a large channel, ``ytdl-sub`` will try pulling each video's metadata from
-newest to oldest before starting any downloads. It is a long process and not ideal. A better method
-is to chunk the process by using the following preset:
-
-* ``chunk_initial_download``
-
-It will download videos starting from the oldest one, and only download 20 at a time. You can
-change this number by setting:
-
-.. code-block:: yaml
-
-   ytdl_options:
-     max_downloads: 30  # Desired number to download per invocation
-
-Once the entire channel is downloaded, remove this preset. Then it will pull metadata from newest to
-oldest again, and stop pulling additional metadata once it reaches a video that has already been
-downloaded.

docs/source/conf.py

@@ -0,0 +1,70 @@
+# Configuration file for the Sphinx documentation builder.
+
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = "ytdl-sub"
+copyright = "2026, Jesse Bannon"
+author = "Jesse Bannon"
+release = ""
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+    "sphinx.ext.autosectionlabel",
+    "sphinx.ext.extlinks",
+    "sphinx_copybutton",
+    "sphinx_design",
+]
+
+templates_path = ["_templates"]
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = "sphinx_book_theme"
+
+html_theme_options = {
+    "icon_links": [
+        {
+            "name": "GitHub",
+            "url": "https://github.com/jmbannon/ytdl-sub",
+            "icon": "fa-brands fa-square-github",
+            "type": "fontawesome",
+        },
+        {
+            "name": "Discord",
+            "url": "https://discord.gg/v8j9RAHb4k",
+            "icon": "https://img.shields.io/discord/994270357957648404?logo=Discord",
+            "type": "url",
+        },
+    ],
+    "announcement": "",
+    "navigation_depth": 10,
+    "show_toc_level": 10,
+}
+
+html_static_path = ["_static"]
+html_css_files = ["custom.css"]
+
+
+# Make sure the all autosectionlabel targets are unique
+autosectionlabel_prefix_document = True
+
+suppress_warnings = [
+    "autosectionlabel.*",
+]
+
+extlinks = {
+    "yt-dlp": ("https://github.com/yt-dlp/yt-dlp/%s", "yt-dlp%s"),
+    "unraid": ("https://unraid.net/%s", "unraid%s"),
+    "lsio": ("https://www.linuxserver.io/%s", "lsio%s"),
+    "lsio-gh": ("https://github.com/linuxserver/%s", "%s image"),
+    "ytdl-sub-gh": ("https://github.com/jmbannon/ytdl-sub/%s", "src %s"),
+}

docs/source/config_reference/config_yaml.rst

@@ -0,0 +1,134 @@
+..
+  WARNING: This RST file is generated from docstrings in:
+    The respective function docstrings within ytdl_sub/config/config_validator.py
+  In order to make a change to this file, edit the respective docstring
+  and run `make docs`. This will automatically sync the Python RST-based
+  docstrings into this file. If the docstrings and RST file are out of sync,
+  it will fail TestDocGen tests in GitHub CI.
+
+Configuration File
+==================
+ytdl-sub is configured using a ``config.yaml`` file.
+
+The ``config.yaml`` is made up of two sections:
+
+.. code-block:: yaml
+
+  configuration:
+  presets:
+
+
+Note for Windows users, paths can be represented with ``C:/forward/slashes/like/linux``.
+If you prefer to use a Windows backslash, note that it must have
+``C:\\double\\bashslash\\paths`` in order to escape the backslash character. This is due
+to it being a YAML escape character.
+
+.. code-block:: yaml
+
+  configuration:
+    dl_aliases:
+      mv: "--preset music_video"
+      u: "--download.url"
+
+    experimental:
+      enable_update_with_info_json: True
+
+    ffmpeg_path: "/usr/bin/ffmpeg"
+    ffprobe_path: "/usr/bin/ffprobe"
+
+    file_name_max_bytes: 255
+    lock_directory: "/tmp"
+
+    persist_logs:
+      keep_successful_logs: True
+      logs_directory: "/var/log/ytdl-sub-logs"
+
+    umask: "022"
+    working_directory: ".ytdl-sub-working-directory"
+
+dl_aliases
+----------
+.. _dl_aliases:
+
+Alias definitions to shorten :ref:`dl arguments <usage:Download Options>`. For example,
+
+.. code-block:: yaml
+
+   configuration:
+     dl_aliases:
+       mv: "--preset music_video"
+       u: "--download.url"
+
+Simplifies
+
+.. code-block:: bash
+
+   ytdl-sub dl --preset "Jellyfin Music Videos" --download.url "youtube.com/watch?v=a1b2c3"
+
+to
+
+.. code-block:: bash
+
+   ytdl-sub dl --mv --u "youtube.com/watch?v=a1b2c3"
+
+experimental
+------------
+Experimental flags reside under the ``experimental`` key.
+
+``enable_update_with_info_json``
+
+Enables modifying subscription files using info.json files using the argument
+``--update-with-info-json``. This feature is still being tested and has the ability to
+destroy files. Ensure you have a full backup before usage. You have been warned!
+
+ffmpeg_path
+-----------
+Path to ffmpeg executable. Defaults to ``/usr/bin/ffmpeg`` for Linux,
+``./ffmpeg.exe`` in the same directory as ytdl-sub for Windows.
+
+ffprobe_path
+------------
+Path to ffprobe executable. Defaults to ``/usr/bin/ffprobe`` for Linux,
+``./ffprobe.exe`` in the same directory as ytdl-sub for Windows.
+
+file_name_max_bytes
+-------------------
+Max file name size in bytes. Most OS's typically default to 255 bytes.
+
+lock_directory
+--------------
+The directory to temporarily store file locks, which prevents multiple instances
+of ``ytdl-sub`` from running. Note that file locks do not work on
+network-mounted directories. Ensure that this directory resides on the host
+machine. Defaults to ``/tmp``.
+
+persist_logs
+------------
+By default, no logs are persisted. Specifying this key will enable persisted logs. The following
+options are available.
+
+``keep_successful_logs``
+
+Defaults to ``True``. When this key is ``False``, only write log files for failed
+subscriptions.
+
+``logs_directory``
+
+Required field. Write log files to this directory with names like
+``YYYY-mm-dd-HHMMSS.subscription_name.(success|error).log``.
+
+umask
+-----
+Umask in octal format to apply to every created file. Defaults to ``022``.
+
+working_directory
+-----------------
+The directory to temporarily store downloaded files before moving them into their final
+directory. Defaults to ``.ytdl-sub-working-directory``, created in the same directory
+that ytdl-sub is invoked from.
+
+Presets
+=======
+Custom presets are defined in this section. Refer to the
+:ref:`Getting Started Guide<guides/getting_started/first_config:Basic Configuration>`
+on how to configure.

docs/source/config_reference/index.rst

@@ -0,0 +1,51 @@
+=========
+Reference
+=========
+
+This section contains direct references to the code of ``ytdl-sub`` and information on
+how it functions.
+
+
+Terminology
+-----------
+
+Must-know terminology:
+
+- ``subscription``: URL(s) that you want to download with specific metadata
+  requirements.
+- ``preset``: A media profile comprised of YAML configuration that can specify anything
+  from metadata layout, media quality, or any feature of ytdl-sub, to apply to
+  subscriptions. A preset can inherit other presets.
+- ``prebuilt preset``: Presets that are included in ytdl-sub. These do most of the work
+  defining plugins, overrides, etc in order to make downloads ready for player
+  consumption.
+- ``override``: Verb describing the act of overriding something in a preset. For
+  example, the TV Show presets practically expect you to *override* the URL variable to
+  tell ytdl-sub where to download from.
+- ``override variables``: User-defined variables that are intended to *override*
+  something.
+- ``subscription file``: The file to specify all of your subscriptions and some override
+  variables.
+
+Intermediate terminology:
+
+- ``plugin``: Modular logic to apply to a subscription. To use a plugin, it must be
+  defined in a preset.
+- ``config file``: An optional file where you can define custom presets and other
+  advanced configuration.
+- ``yt-dlp``: The underlying application that handles downloading for ytdl-sub.
+
+Advanced terminology:
+
+- ``entry variables``: Variables that derive from a downloaded yt-dlp entry (media).
+- ``static variables``: Variables that do not have a dependency to entry variables.
+- ``scripting``: Syntax that allows the use of entry variables, static variables, and
+  functions in override variables.
+
+
+.. toctree:: 
+    config_yaml
+    subscription_yaml
+    plugins
+    scripting/index
+    prebuilt_presets/index

docs/source/config_reference/prebuilt_presets/common.rst

@@ -0,0 +1,33 @@
+=======================
+Common
+=======================
+
+.. highlight:: yaml
+
+
+Filter Keywords
+---------------
+
+.. literalinclude::
+   /../../src/ytdl_sub/prebuilt_presets/helpers/filter_keywords.yaml
+
+
+Filter Duration
+---------------
+
+.. literalinclude::
+   /../../src/ytdl_sub/prebuilt_presets/helpers/filter_duration.yaml
+
+
+Media Quality
+-------------
+
+.. literalinclude::
+   /../../src/ytdl_sub/prebuilt_presets/helpers/media_quality.yaml
+
+
+Only Recent Videos
+------------------
+
+.. literalinclude::
+   /../../src/ytdl_sub/prebuilt_presets/helpers/download_deletion_options.yaml

docs/source/config_reference/prebuilt_presets/index.rst

@@ -0,0 +1,11 @@
+=========================
+Prebuilt Preset Reference
+=========================
+
+This section contains the code for the prebuilt presets. If you just want to understand
+how to use the presets, check :doc:`this section instead</prebuilt_presets/index>`.
+
+.. toctree::
+  common
+  tv_show
+  music

docs/source/config_reference/prebuilt_presets/music.rst

@@ -0,0 +1,10 @@
+=====
+Music
+=====
+
+All audio music based presets inherit from ``_music_base``.
+
+.. highlight:: yaml
+
+.. literalinclude::
+   /../../src/ytdl_sub/prebuilt_presets/music/singles.yaml

docs/source/config_reference/prebuilt_presets/tv_show.rst

@@ -0,0 +1,10 @@
+========================
+TV Show
+========================
+
+All TV show based presets inherit from ``_episode_base``.
+
+.. highlight:: yaml
+
+.. literalinclude::
+   /../../src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml

docs/source/config_reference/scripting/entry_variables.rst

@@ -0,0 +1,688 @@
+..
+  WARNING: This RST file is generated from docstrings in:
+    src/ytdl_sub/entries/script/variable_definitions.py
+  In order to make a change to this file, edit the respective docstring
+  and run `make docs`. This will automatically sync the Python RST-based
+  docstrings into this file. If the docstrings and RST file are out of sync,
+  it will fail TestDocGen tests in GitHub CI.
+
+Entry Variables
+===============
+
+Entry Variables
+---------------
+
+channel
+~~~~~~~
+:type: ``String``
+:description:
+  The channel name if it exists, otherwise returns the uploader.
+
+channel_id
+~~~~~~~~~~
+:type: ``String``
+:description:
+  The channel id if it exists, otherwise returns the entry uploader ID.
+
+chapters
+~~~~~~~~
+:type: ``Array``
+:description:
+  Chapters if they exist
+
+comments
+~~~~~~~~
+:type: ``Array``
+:description:
+  Comments if they are requested
+
+creator
+~~~~~~~
+:type: ``String``
+:description:
+  The creator name if it exists, otherwise returns the channel.
+
+description
+~~~~~~~~~~~
+:type: ``String``
+:description:
+  The description if it exists. Otherwise, returns an emtpy string.
+
+duration
+~~~~~~~~
+:type: ``Integer``
+:description:
+  The duration of the entry in seconds if it exists. Defaults to zero otherwise.
+
+epoch
+~~~~~
+:type: ``Integer``
+:description:
+  The unix epoch of when the metadata was scraped by yt-dlp.
+
+epoch_date
+~~~~~~~~~~
+:type: ``String``
+:description:
+  The epoch's date, in YYYYMMDD format.
+
+epoch_hour
+~~~~~~~~~~
+:type: ``String``
+:description:
+  The epoch's hour
+
+ext
+~~~
+:type: ``String``
+:description:
+  The downloaded entry's file extension
+
+extractor
+~~~~~~~~~
+:type: ``String``
+:description:
+  The yt-dlp extractor name
+
+extractor_key
+~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The yt-dlp extractor key
+
+height
+~~~~~~
+:type: ``Integer``
+:description:
+  Height in pixels of the video. If this value is unavailable (i.e. audio download), it
+  will default to 0.
+
+ie_key
+~~~~~~
+:type: ``String``
+:description:
+  The ie_key, used in legacy yt-dlp things as the 'info-extractor key'.
+  If it does not exist, return ``extractor_key``
+
+info_json_ext
+~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The "info.json" extension
+
+requested_subtitles
+~~~~~~~~~~~~~~~~~~~
+:type: ``Map``
+:description:
+  Subtitles if they are requested and exist
+
+sponsorblock_chapters
+~~~~~~~~~~~~~~~~~~~~~
+:type: ``Array``
+:description:
+  Sponsorblock Chapters if they are requested and exist
+
+thumbnail_ext
+~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The download entry's thumbnail extension. Will always return 'jpg'. Until there is a
+  need to support other image types, we always convert to jpg.
+
+title
+~~~~~
+:type: ``String``
+:description:
+  The title of the entry. If a title does not exist, returns its unique ID.
+
+title_sanitized_plex
+~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The sanitized title with additional sanitizing for Plex. It replaces numbers with
+  fixed-width numbers so Plex does not recognize them as season or episode numbers.
+
+uid
+~~~
+:type: ``String``
+:description:
+  The entry's unique ID
+
+uid_sanitized_plex
+~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The sanitized uid with additional sanitizing for Plex. Replaces numbers with
+  fixed-width numbers so Plex does not recognize them as season or episode numbers.
+
+uploader
+~~~~~~~~
+:type: ``String``
+:description:
+  The uploader if it exists, otherwise return the uploader ID.
+
+uploader_id
+~~~~~~~~~~~
+:type: ``String``
+:description:
+  The uploader id if it exists, otherwise return the unique ID.
+
+uploader_url
+~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The uploader url if it exists, otherwise returns the webpage_url.
+
+webpage_url
+~~~~~~~~~~~
+:type: ``String``
+:description:
+  The url to the webpage.
+
+width
+~~~~~
+:type: ``Integer``
+:description:
+  Width in pixels of the video. If this value is unavailable (i.e. audio download), it
+  will default to 0.
+
+----------------------------------------------------------------------------------------------------
+
+Metadata Variables
+------------------
+
+entry_metadata
+~~~~~~~~~~~~~~
+:type: ``Map``
+:description:
+  The entry's info.json
+
+playlist_metadata
+~~~~~~~~~~~~~~~~~
+:type: ``Map``
+:description:
+  Metadata from the playlist (i.e. the parent metadata, like playlist -> entry)
+
+sibling_metadata
+~~~~~~~~~~~~~~~~
+:type: ``Array``
+:description:
+  Metadata from any sibling entries that reside in the same playlist as this entry.
+
+source_metadata
+~~~~~~~~~~~~~~~
+:type: ``Map``
+:description:
+  Metadata from the source
+  (i.e. the grandparent metadata, like channel -> playlist -> entry)
+
+----------------------------------------------------------------------------------------------------
+
+Playlist Variables
+------------------
+
+playlist_count
+~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  Playlist count if it exists, otherwise returns ``1``.
+
+  Note that for channels/playlists, any change (i.e. adding or removing a video) will make
+  this value change. Use with caution.
+
+playlist_description
+~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The playlist description if it exists, otherwise returns the entry's description.
+
+playlist_index
+~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  Playlist index if it exists, otherwise returns ``1``.
+
+  Note that for channels/playlists, any change (i.e. adding or removing a video) will make
+  this value change. Use with caution.
+
+playlist_index_padded
+~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  playlist_index padded two digits
+
+playlist_index_padded6
+~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  playlist_index padded six digits.
+
+playlist_index_reversed
+~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  Playlist index reversed via ``playlist_count - playlist_index + 1``
+
+playlist_index_reversed_padded
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  playlist_index_reversed padded two digits
+
+playlist_index_reversed_padded6
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  playlist_index_reversed padded six digits.
+
+playlist_max_upload_date
+~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  Max upload_date for all entries in this entry's playlist if it exists, otherwise returns
+  ``upload_date``
+
+playlist_max_upload_year
+~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  Max upload_year for all entries in this entry's playlist if it exists, otherwise returns
+  ``upload_year``
+
+playlist_max_upload_year_truncated
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The max playlist truncated upload year for all entries in this entry's playlist if it
+  exists, otherwise returns ``upload_year_truncated``.
+
+playlist_title
+~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  Name of its parent playlist/channel if it exists, otherwise returns its title.
+
+playlist_uid
+~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The playlist unique ID if it exists, otherwise return the entry unique ID.
+
+playlist_uploader
+~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The playlist uploader if it exists, otherwise return the entry uploader.
+
+playlist_uploader_id
+~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The playlist uploader id if it exists, otherwise returns the entry uploader ID.
+
+playlist_uploader_url
+~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The playlist uploader url if it exists, otherwise returns the playlist webpage_url.
+
+playlist_webpage_url
+~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The playlist webpage url if it exists. Otherwise, returns the entry webpage url.
+
+----------------------------------------------------------------------------------------------------
+
+Release Date Variables
+----------------------
+
+release_date
+~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The entry’s release date, in YYYYMMDD format. If not present, return the upload date.
+
+release_date_standardized
+~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The uploaded date formatted as YYYY-MM-DD
+
+release_day
+~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The upload day as an integer (no padding).
+
+release_day_of_year
+~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The day of the year, i.e. February 1st returns ``32``
+
+release_day_of_year_padded
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The upload day of year, but padded i.e. February 1st returns "032"
+
+release_day_of_year_reversed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The upload day, but reversed using ``{total_days_in_year} + 1 - {release_day}``,
+  i.e. February 2nd would have release_day_of_year_reversed of ``365 + 1 - 32`` = ``334``
+
+release_day_of_year_reversed_padded
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The reversed upload day of year, but padded i.e. December 31st returns "001"
+
+release_day_padded
+~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The entry's upload day padded to two digits, i.e. the fifth returns "05"
+
+release_day_reversed
+~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The upload day, but reversed using ``{total_days_in_month} + 1 - {release_day}``,
+  i.e. August 8th would have release_day_reversed of ``31 + 1 - 8`` = ``24``
+
+release_day_reversed_padded
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The reversed upload day, but padded. i.e. August 30th returns "02".
+
+release_month
+~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The upload month as an integer (no padding).
+
+release_month_padded
+~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The entry's upload month padded to two digits, i.e. March returns "03"
+
+release_month_reversed
+~~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The upload month, but reversed using ``13 - {release_month}``, i.e. March returns ``10``
+
+release_month_reversed_padded
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The reversed upload month, but padded. i.e. November returns "02"
+
+release_year
+~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The entry's upload year
+
+release_year_truncated
+~~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The last two digits of the upload year, i.e. 22 in 2022
+
+release_year_truncated_reversed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The upload year truncated, but reversed using ``100 - {release_year_truncated}``, i.e.
+  2022 returns ``100 - 22`` = ``78``
+
+----------------------------------------------------------------------------------------------------
+
+Source Variables
+----------------
+
+source_count
+~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The source count if it exists, otherwise returns ``1``.
+
+source_description
+~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The source description if it exists, otherwise returns the playlist description.
+
+source_index
+~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  Source index if it exists, otherwise returns ``1``.
+
+  It is recommended to not use this unless you know the source will never add new content
+  (it is easy for this value to change).
+
+source_index_padded
+~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The source index, padded two digits.
+
+source_title
+~~~~~~~~~~~~
+:type: ``String``
+:description:
+  Name of the source (i.e. channel with multiple playlists) if it exists, otherwise
+  returns its playlist_title.
+
+source_uid
+~~~~~~~~~~
+:type: ``String``
+:description:
+  The source unique id if it exists, otherwise returns the playlist unique ID.
+
+source_uploader
+~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The source uploader if it exists, otherwise return the playlist_uploader
+
+source_uploader_id
+~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The source uploader id if it exists, otherwise returns the playlist_uploader_id
+
+source_uploader_url
+~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The source uploader url if it exists, otherwise returns the source webpage_url.
+
+source_webpage_url
+~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The source webpage url if it exists, otherwise returns the playlist webpage url.
+
+----------------------------------------------------------------------------------------------------
+
+Upload Date Variables
+---------------------
+
+upload_date
+~~~~~~~~~~~
+:type: ``String``
+:description:
+  The entry’s uploaded date, in YYYYMMDD format. If not present, return today’s date.
+
+upload_date_standardized
+~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The uploaded date formatted as YYYY-MM-DD
+
+upload_day
+~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The upload day as an integer (no padding).
+
+upload_day_of_year
+~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The day of the year, i.e. February 1st returns ``32``
+
+upload_day_of_year_padded
+~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The upload day of year, but padded i.e. February 1st returns "032"
+
+upload_day_of_year_reversed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The upload day, but reversed using ``{total_days_in_year} + 1 - {upload_day}``,
+  i.e. February 2nd would have upload_day_of_year_reversed of ``365 + 1 - 32`` = ``334``
+
+upload_day_of_year_reversed_padded
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The reversed upload day of year, but padded i.e. December 31st returns "001"
+
+upload_day_padded
+~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The entry's upload day padded to two digits, i.e. the fifth returns "05"
+
+upload_day_reversed
+~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The upload day, but reversed using ``{total_days_in_month} + 1 - {upload_day}``,
+  i.e. August 8th would have upload_day_reversed of ``31 + 1 - 8`` = ``24``
+
+upload_day_reversed_padded
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The reversed upload day, but padded. i.e. August 30th returns "02".
+
+upload_month
+~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The upload month as an integer (no padding).
+
+upload_month_padded
+~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The entry's upload month padded to two digits, i.e. March returns "03"
+
+upload_month_reversed
+~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The upload month, but reversed using ``13 - {upload_month}``, i.e. March returns ``10``
+
+upload_month_reversed_padded
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The reversed upload month, but padded. i.e. November returns "02"
+
+upload_year
+~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The entry's upload year
+
+upload_year_truncated
+~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The last two digits of the upload year, i.e. 22 in 2022
+
+upload_year_truncated_reversed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The upload year truncated, but reversed using ``100 - {upload_year_truncated}``, i.e.
+  2022 returns ``100 - 22`` = ``78``
+
+----------------------------------------------------------------------------------------------------
+
+Ytdl-Sub Variables
+------------------
+
+download_index
+~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The i'th entry downloaded. NOTE that this is fetched dynamically from the download
+  archive.
+
+download_index_padded6
+~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The download_index padded six digits
+
+upload_date_index
+~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The i'th entry downloaded with this upload date.
+
+upload_date_index_padded
+~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The upload_date_index padded two digits
+
+upload_date_index_reversed
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  100 - upload_date_index
+
+upload_date_index_reversed_padded
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The upload_date_index padded two digits
+
+ytdl_sub_input_url
+~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The input URL used in ytdl-sub to create this entry.
+
+ytdl_sub_input_url_count
+~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The total number of input URLs as defined in the subscription.
+
+ytdl_sub_input_url_index
+~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``Integer``
+:description:
+  The index of the input URL as defined in the subscription, top-most being the 0th index.
+
+ytdl_sub_keep_files_date_eval
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:type: ``String``
+:description:
+  The standardized date variable supplied in ``output_options.keep_files_date_eval``.

docs/source/config_reference/scripting/index.rst

@@ -0,0 +1,210 @@
+=========
+Scripting
+=========
+
+``ytdl-sub`` fields (file-names, tags, etc) are defined using variables and scripts. The
+links below contain reference documentation for each built-in variable and scripting
+function.
+
+.. toctree::
+  :maxdepth: 1
+
+  entry_variables
+  static_variables
+  scripting_functions
+  scripting_types
+
+
+How it Works
+------------
+
+Fields in the config that support ``formatters`` mean they support scripting, and will
+*format* the field using its defined script.
+
+In its most basic form, a script is a string comprised of variables and/or functions.
+
+Static String
+~~~~~~~~~~~~~
+
+The following example sets ``ytdl-sub``'s output directory. It is
+considered *static* because it does not depend on anything from an entry.
+
+.. code-block:: yaml
+
+   output_options:
+     output_directory: "/path/to/tv_shows/Custom YTDL-SUB TV Show"
+
+Static Variables
+~~~~~~~~~~~~~~~~
+
+``ytdl-sub`` offers a few built-in static variables, including ``subscription_name``.
+We can use this instead of hard-coding it above:
+
+.. code-block:: yaml
+
+   output_options:
+     output_directory: "/path/to/tv_shows/{subscription_name}"
+
+The syntax for variable usage is curly-braces with the variable name within it. Assuming
+our subscription is actually named "Custom YTDL-SUB TV Show", then ``ytdl-sub`` will
+actually write to that directory.
+
+Entry Variables
+~~~~~~~~~~~~~~~
+
+For context, an *entry* is a video or audio file downloaded from ``yt-dlp``.  *Entry
+variables* are variables that are derived from an entry's ``info.json`` file. This file
+comes from ``yt-dlp`` and contains every piece of metadata that it scraped.
+
+These variables are not considered static since they change per entry download. There
+are a few fields in ``ytdl-sub`` (i.e. ``output_directory``) that must be static. For
+others, we are free to use values that derive from an entry.
+
+Suppose we want to customize the name of an entry's output file and thumbnail to include
+its title in its name. We can do that using entry variables:
+
+.. code-block:: yaml
+
+   output_options:
+     output_directory: "/path/to/tv_shows/{subscription_name}"
+     file_name: "{title}.{ext}"
+     thumbnail_name: "{title}.{thumbnail_ext}"
+
+Creating Custom Variables
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Suppose we want to include the date in our file names. This means we'd need to update
+both the ``file_name`` and ``thumbnail_name`` fields to include it.
+
+Instead, we can create a custom *override variable*. This is ``ytdl-sub``'s method for
+creating and overriding custom variables.
+
+These are created in the ``overrides`` section. Let's take our above example and create
+a ``custom_file_name`` variable to use for the entry file and thumbnail fields:
+
+.. code-block:: yaml
+
+   output_options:
+     output_directory: "/path/to/tv_shows/{subscription_name}"
+     file_name: "{custom_file_name}.{ext}"
+     thumbnail_name: "{custom_file_name}.{thumbnail_ext}"
+
+   overrides:
+     custom_file_name: "{upload_date_standardized} {title}"
+
+Sanitizing Variables
+~~~~~~~~~~~~~~~~~~~~
+
+For experienced ``yt-dlp`` scrapers, you may be thinking:
+
+- What if the title has characters that do not play nice with my operating system?
+
+``ytdl-sub`` is able to *sanitize* any variable, meaning it replaces any problematic
+characters with safe alternatives that can be used in file names. We can ensure our file
+names and directories are safe by using:
+
+.. code-block:: yaml
+
+   output_options:
+     output_directory: "/path/to/tv_shows/{subscription_name_sanitized}"
+     file_name: "{custom_file_name}.{ext}"
+     thumbnail_name: "{custom_file_name}.{thumbnail_ext}"
+
+   overrides:
+     custom_file_name: "{upload_date_standardized} {title_sanitized}"
+
+Simply add a ``_sanitized`` suffix to any variable name to make it sanitized.
+
+.. note::
+
+   Make sure you do not sanitize custom variables that intentionally create directories,
+   (i.e. sanitizing ``/path/to/tv_shows/``) otherwise they will... be sanitized and not
+   resolve to directories!
+
+
+Using Scripting Functions
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Let's suppose you are an avid command-line user, and like all of your file names to be
+``snake_cased_with_no_spaces``. We can use the `replace
+<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#replace>`_
+*scripting function* to create and use a snake-cased title.
+
+.. code-block:: yaml
+
+   output_options:
+     output_directory: "/path/to/tv_shows/{subscription_name_sanitized}"
+     file_name: "{custom_file_name}.{ext}"
+     thumbnail_name: "{custom_file_name}.{thumbnail_ext}"
+
+   overrides:
+     snake_cased_title: >-
+       {
+         %replace( title, ' ', '_' )
+       }
+     custom_file_name: "{upload_date_standardized}_{snake_cased_title_sanitized}"
+
+Scripting functions are similar to variables - they must be used within curly-braces.
+It is good practice to use ``>-`` when defining variables that use functions. It is
+YAML's way of saying:
+
+- Allow a string to be multi-lined, and do not include newlines before or after it.
+
+See for yourself `here
+<https://yaml-online-parser.appspot.com/?yaml=output_options%3A%0A%20%20output_directory%3A%20%22%7Bsubscription_name_sanitized%7D%22%0A%20%20file_name%3A%20%22%7Bcustom_file_name%7D.%7Bext%7D%22%0A%20%20thumbnail_name%3A%20%22%7Bcustom_file_name%7D.%7Bthumbnail_ext%7D%22%0A%0Aoverrides%3A%0A%20%20snake_cased_title%3A%20%3E-%0A%20%20%20%20%7B%0A%20%20%20%20%20%20%25replace%28%20title%2C%20%27%20%27%2C%20%27_%27%20%29%0A%20%20%20%20%7D%0A%20%20custom_file_name%3A%20%22%7Bupload_date_standardized%7D%20%7Bsnake_cased_title_sanitized%7D%22&type=json>`_.
+Any whitespace within curly-braces is okay since it will be parsed out. This is needed
+to make scripting function usage readable.
+
+.. important::
+
+   It is important to use ``>-`` over other YAML new-line directives like ``>`` because
+   they add newlines before or after curly-braces, and will be included in your
+   variable's output string.
+
+
+Advanced Scripting
+------------------
+
+Accessing ``info.json`` Fields
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The entirety of an entry's ``info.json`` file resides in the `Map
+<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_types.html#map>`_
+variable `entry_metadata
+<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/entry_variables.html#entry-metadata>`_.
+
+Any field can be accessed by using the `map_get
+<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#map-get>`_
+function like so:
+
+.. code-block:: yaml
+  :caption:
+     Fetches the 'artist' value from the .info.json, returns null if it does not exist.
+
+   artist: >-
+     { %map_get( entry_metadata, "artist", null ) }
+
+Creating Custom Functions
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Custom functions can be created in the overrides section using the following syntax:
+
+.. code-block:: yaml
+
+   overrides:
+     "%get_entry_metadata_field": >-
+       { %map_get( entry_metadata, $0, null ) }
+
+Custom function definitions must have ``%`` as a prefix to the function name, be
+surrounded by quotes to make YAML parsing happy, and can support arguments using ``$0``,
+``$1``, ... to indicate their first argument, second argument, etc.
+
+Using our new custom function, we can simply the ``artist`` variable definition above to:
+
+.. code-block:: yaml
+
+   overrides:
+     "%get_entry_metadata_field": >-
+       { %map_get( entry_metadata, $0, null ) }
+     artist: >-
+       { get_entry_metadata_field("artist") }

docs/source/config_reference/scripting/scripting_functions.rst

@@ -0,0 +1,889 @@
+..
+  WARNING: This RST file is generated from docstrings in:
+    The respective function files under src/ytdl_sub/script/functions/
+  In order to make a change to this file, edit the respective docstring
+  and run `make docs`. This will automatically sync the Python RST-based
+  docstrings into this file. If the docstrings and RST file are out of sync,
+  it will fail TestDocGen tests in GitHub CI.
+
+Scripting Functions
+===================
+
+Array Functions
+---------------
+
+array
+~~~~~
+:spec: ``array(maybe_array: AnyArgument) -> Array``
+
+:description:
+  Tries to cast an unknown variable type to an Array.
+
+array_apply
+~~~~~~~~~~~
+:spec: ``array_apply(array: Array, lambda_function: Lambda) -> Array``
+
+:description:
+  Apply a lambda function on every element in the Array.
+:usage:
+
+.. code-block:: python
+
+   {
+     %array_apply( [1, 2, 3] , %string )
+   }
+
+   # ["1", "2", "3"]
+
+array_apply_fixed
+~~~~~~~~~~~~~~~~~
+:spec: ``array_apply_fixed(array: Array, fixed_argument: AnyArgument, lambda2_function: LambdaTwo, reverse_args: Optional[Boolean]) -> Array``
+
+:description:
+  Apply a lambda function on every element in the Array, with ``fixed_argument``
+  passed as a second argument to every invocation.
+
+array_at
+~~~~~~~~
+:spec: ``array_at(array: Array, idx: Integer, default: Optional[AnyArgument]) -> AnyArgument``
+
+:description:
+  Return the element in the Array at index ``idx``. If ``idx`` exceeds the array length,
+  either return ``default`` if provided or throw an error.
+
+array_contains
+~~~~~~~~~~~~~~
+:spec: ``array_contains(array: Array, value: AnyArgument) -> Boolean``
+
+:description:
+  Return True if the value exists in the Array. False otherwise.
+
+array_enumerate
+~~~~~~~~~~~~~~~
+:spec: ``array_enumerate(array: Array, lambda_function: LambdaTwo) -> Array``
+
+:description:
+  Apply a lambda function on every element in the Array, where each arg
+  passed to the lambda function is ``idx, element`` as two separate args.
+
+array_extend
+~~~~~~~~~~~~
+:spec: ``array_extend(arrays: Array, ...) -> Array``
+
+:description:
+  Combine multiple Arrays into a single Array.
+
+array_first
+~~~~~~~~~~~
+:spec: ``array_first(array: Array, fallback: AnyArgument) -> AnyArgument``
+
+:description:
+  Returns the first element whose boolean conversion is True. Returns fallback
+  if all elements evaluate to False.
+
+array_flatten
+~~~~~~~~~~~~~
+:spec: ``array_flatten(array: Array) -> Array``
+
+:description:
+  Flatten any nested Arrays into a single-dimensional Array.
+
+array_index
+~~~~~~~~~~~
+:spec: ``array_index(array: Array, value: AnyArgument) -> Integer``
+
+:description:
+  Return the index of the value within the Array if it exists. If it does not, it will
+  throw an error.
+
+array_overlay
+~~~~~~~~~~~~~
+:spec: ``array_overlay(array: Array, overlap: Array, only_missing: Optional[Boolean]) -> Array``
+
+:description:
+  Overlaps ``overlap`` onto ``array``. Can optionally only overlay missing indices.
+
+array_product
+~~~~~~~~~~~~~
+:spec: ``array_product(arrays: Array, ...) -> Array``
+
+:description:
+  Returns the Cartesian product of elements from different arrays
+
+array_reduce
+~~~~~~~~~~~~
+:spec: ``array_reduce(array: Array, lambda_reduce_function: LambdaReduce) -> AnyArgument``
+
+:description:
+  Apply a reduce function on pairs of elements in the Array, until one element remains.
+  Executes using the left-most and reduces in the right direction.
+
+array_reverse
+~~~~~~~~~~~~~
+:spec: ``array_reverse(array: Array) -> Array``
+
+:description:
+  Reverse an Array.
+
+array_size
+~~~~~~~~~~
+:spec: ``array_size(array: Array) -> Integer``
+
+:description:
+  Returns the size of an Array.
+
+array_slice
+~~~~~~~~~~~
+:spec: ``array_slice(array: Array, start: Integer, end: Optional[Integer]) -> Array``
+
+:description:
+  Returns the slice of the Array.
+
+----------------------------------------------------------------------------------------------------
+
+Boolean Functions
+-----------------
+
+and
+~~~
+:spec: ``and(values: AnyArgument, ...) -> Boolean``
+
+:description:
+  ``and`` operator. Returns True if all values evaluate to True. False otherwise.
+
+bool
+~~~~
+:spec: ``bool(value: AnyArgument) -> Boolean``
+
+:description:
+  Cast any type to a Boolean.
+
+eq
+~~
+:spec: ``eq(left: AnyArgument, right: AnyArgument) -> Boolean``
+
+:description:
+  ``==`` operator. Returns True if left == right. False otherwise.
+
+gt
+~~
+:spec: ``gt(left: AnyArgument, right: AnyArgument) -> Boolean``
+
+:description:
+  ``>`` operator. Returns True if left > right. False otherwise.
+
+gte
+~~~
+:spec: ``gte(left: AnyArgument, right: AnyArgument) -> Boolean``
+
+:description:
+  ``>=`` operator. Returns True if left >= right. False otherwise.
+
+is_array
+~~~~~~~~
+:spec: ``is_array(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is a Map. False otherwise.
+
+is_bool
+~~~~~~~
+:spec: ``is_bool(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is a Float. False otherwise.
+
+is_float
+~~~~~~~~
+:spec: ``is_float(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is a Float. False otherwise.
+
+is_int
+~~~~~~
+:spec: ``is_int(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is an Integer. False otherwise.
+
+is_map
+~~~~~~
+:spec: ``is_map(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is a Map. False otherwise.
+
+is_null
+~~~~~~~
+:spec: ``is_null(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is null (i.e. an empty string). False otherwise.
+
+is_numeric
+~~~~~~~~~~
+:spec: ``is_numeric(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is either an Integer or Float. False otherwise.
+
+is_string
+~~~~~~~~~
+:spec: ``is_string(value: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if a value is a String. False otherwise.
+
+lt
+~~
+:spec: ``lt(left: AnyArgument, right: AnyArgument) -> Boolean``
+
+:description:
+  ``<`` operator. Returns True if left < right. False otherwise.
+
+lte
+~~~
+:spec: ``lte(left: AnyArgument, right: AnyArgument) -> Boolean``
+
+:description:
+  ``<=`` operator. Returns True if left <= right. False otherwise.
+
+ne
+~~
+:spec: ``ne(left: AnyArgument, right: AnyArgument) -> Boolean``
+
+:description:
+  ``!=`` operator. Returns True if left != right. False otherwise.
+
+not
+~~~
+:spec: ``not(value: Boolean) -> Boolean``
+
+:description:
+  ``not`` operator. Returns the opposite of value.
+
+or
+~~
+:spec: ``or(values: AnyArgument, ...) -> Boolean``
+
+:description:
+  ``or`` operator. Returns True if any value evaluates to True. False otherwise.
+
+xor
+~~~
+:spec: ``xor(values: AnyArgument, ...) -> Boolean``
+
+:description:
+  ``^`` operator. Returns True if exactly one value is set to True. False otherwise.
+
+----------------------------------------------------------------------------------------------------
+
+Conditional Functions
+---------------------
+
+elif
+~~~~
+:spec: ``elif(if_elif_else: AnyArgument, ...) -> AnyArgument``
+
+:description:
+  Conditional ``if`` statement that is capable of doing else-ifs (``elif``) via
+  adjacent arguments. It is expected for there to be an odd number of arguments >= 3 to
+  supply at least one conditional and an else.
+:usage:
+
+  .. code-block:: python
+
+     %elif(
+        condition1,
+        return1,
+        condition2,
+        return2,
+        ...
+        else_return
+     )
+
+if
+~~
+:spec: ``if(condition: Boolean, true: ReturnableArgumentA, false: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
+
+:description:
+  Conditional ``if`` statement that returns the ``true`` or ``false`` parameter
+  depending on the ``condition`` value.
+
+if_passthrough
+~~~~~~~~~~~~~~
+:spec: ``if_passthrough(maybe_true_arg: ReturnableArgumentA, else_arg: ReturnableArgumentB) -> Union[ReturnableArgumentA, ReturnableArgumentB]``
+
+:description:
+  Conditional ``if`` statement that returns the ``maybe_true_arg`` if it evaluates to True,
+  otherwise returns ``else_arg``.
+
+----------------------------------------------------------------------------------------------------
+
+Date Functions
+--------------
+
+datetime_strftime
+~~~~~~~~~~~~~~~~~
+:spec: ``datetime_strftime(posix_timestamp: Integer, date_format: String) -> String``
+
+:description:
+  Converts a posix timestamp to a date using strftime formatting.
+
+----------------------------------------------------------------------------------------------------
+
+Error Functions
+---------------
+
+assert
+~~~~~~
+:spec: ``assert(value: ReturnableArgument, assert_message: String) -> ReturnableArgument``
+
+:description:
+  Explicitly throw an error with the provided assert message if ``value`` evaluates to
+  False. If it evaluates to True, it will return ``value``.
+
+assert_eq
+~~~~~~~~~
+:spec: ``assert_eq(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument``
+
+:description:
+  Explicitly throw an error with the provided assert message if ``value`` does not equal
+  ``equals``. If they do equal, then return ``value``.
+
+assert_ne
+~~~~~~~~~
+:spec: ``assert_ne(value: ReturnableArgument, equals: AnyArgument, assert_message: String) -> ReturnableArgument``
+
+:description:
+  Explicitly throw an error with the provided assert message if ``value`` equals
+  ``equals``. If they do equal, then return ``value``.
+
+assert_then
+~~~~~~~~~~~
+:spec: ``assert_then(value: AnyArgument, ret: ReturnableArgument, assert_message: String) -> ReturnableArgument``
+
+:description:
+  Explicitly throw an error with the provided assert message if ``value`` evaluates to
+  False. If it evaluates to True, it will return ``ret``.
+
+throw
+~~~~~
+:spec: ``throw(error_message: String) -> AnyArgument``
+
+:description:
+  Explicitly throw an error with the provided error message.
+
+----------------------------------------------------------------------------------------------------
+
+Json Functions
+--------------
+
+from_json
+~~~~~~~~~
+:spec: ``from_json(argument: String) -> AnyArgument``
+
+:description:
+  Converts a JSON string into an actual type.
+
+----------------------------------------------------------------------------------------------------
+
+Map Functions
+-------------
+
+map
+~~~
+:spec: ``map(maybe_mapping: AnyArgument) -> Map``
+
+:description:
+  Tries to cast an unknown variable type to a Map.
+
+map_apply
+~~~~~~~~~
+:spec: ``map_apply(mapping: Map, lambda_function: LambdaTwo) -> Array``
+
+:description:
+  Apply a lambda function on the Map, where each arg
+  passed to the lambda function is ``key, value`` as two separate args.
+
+map_contains
+~~~~~~~~~~~~
+:spec: ``map_contains(mapping: Map, key: AnyArgument) -> Boolean``
+
+:description:
+  Returns True if the key is in the Map. False otherwise.
+
+map_enumerate
+~~~~~~~~~~~~~
+:spec: ``map_enumerate(mapping: Map, lambda_function: LambdaThree) -> Array``
+
+:description:
+  Apply a lambda function on the Map, where each arg
+  passed to the lambda function is ``idx, key, value`` as three separate args.
+
+map_extend
+~~~~~~~~~~
+:spec: ``map_extend(maps: Map, ...) -> Map``
+
+:description:
+  Return maps combined in the order from left-to-right. Duplicate keys will use the
+  right-most map's value.
+
+map_get
+~~~~~~~
+:spec: ``map_get(mapping: Map, key: AnyArgument, default: Optional[AnyArgument]) -> AnyArgument``
+
+:description:
+  Return ``key``'s value within the Map. If ``key`` does not exist, and ``default`` is
+  provided, it will return ``default``. Otherwise, will error.
+
+map_get_non_empty
+~~~~~~~~~~~~~~~~~
+:spec: ``map_get_non_empty(mapping: Map, key: AnyArgument, default: AnyArgument) -> AnyArgument``
+
+:description:
+  Return ``key``'s value within the Map. If ``key`` does not exist or is an empty string,
+  return ``default``. Otherwise, will error.
+
+map_size
+~~~~~~~~
+:spec: ``map_size(mapping: Map) -> Integer``
+
+:description:
+  Returns the size of a Map.
+
+----------------------------------------------------------------------------------------------------
+
+Numeric Functions
+-----------------
+
+add
+~~~
+:spec: ``add(values: Numeric, ...) -> Numeric``
+
+:description:
+  ``+`` operator. Returns the sum of all values.
+
+div
+~~~
+:spec: ``div(left: Numeric, right: Numeric) -> Numeric``
+
+:description:
+  ``/`` operator. Returns ``left / right``.
+
+float
+~~~~~
+:spec: ``float(value: AnyArgument) -> Float``
+
+:description:
+  Cast to Float.
+
+int
+~~~
+:spec: ``int(value: AnyArgument) -> Integer``
+
+:description:
+  Cast to Integer.
+
+max
+~~~
+:spec: ``max(values: Numeric, ...) -> Numeric``
+
+:description:
+  Returns max of all values.
+
+min
+~~~
+:spec: ``min(values: Numeric, ...) -> Numeric``
+
+:description:
+  Returns min of all values.
+
+mod
+~~~
+:spec: ``mod(left: Numeric, right: Numeric) -> Numeric``
+
+:description:
+  ``%`` operator. Returns ``left % right``.
+
+mul
+~~~
+:spec: ``mul(values: Numeric, ...) -> Numeric``
+
+:description:
+  ``*`` operator. Returns the product of all values.
+
+pow
+~~~
+:spec: ``pow(base: Numeric, exponent: Numeric) -> Numeric``
+
+:description:
+  ``**`` operator. Returns the exponential of the base and exponent value.
+
+range
+~~~~~
+:spec: ``range(end: Integer, start: Optional[Integer], step: Optional[Integer]) -> Array``
+
+:description:
+  Returns the desired range of Integers in the form of an Array.
+
+sub
+~~~
+:spec: ``sub(values: Numeric, ...) -> Numeric``
+
+:description:
+  ``-`` operator. Subtracts all values from left to right.
+
+----------------------------------------------------------------------------------------------------
+
+Print Functions
+---------------
+
+print
+~~~~~
+:spec: ``print(message: AnyArgument, passthrough: ReturnableArgument, level: Optional[Integer]) -> ReturnableArgument``
+
+:description:
+  Log the ``message`` and return ``passthrough``. Optionally can pass level,
+  where < 0 is debug, 0 is info, 1 is warning, > 1 is error. (default ``0``)
+
+print_if_false
+~~~~~~~~~~~~~~
+:spec: ``print_if_false(message: AnyArgument, passthrough: ReturnableArgument, level: Optional[Integer]) -> ReturnableArgument``
+
+:description:
+  Log the ``message`` if ``passthrough`` evaluates to ``false``. Return
+  ``passthrough``. Optionally can pass level, where < 0 is debug, 0 is info, 1
+  is warning, > 1 is error. (default ``0``)
+
+print_if_true
+~~~~~~~~~~~~~
+:spec: ``print_if_true(message: AnyArgument, passthrough: ReturnableArgument, level: Optional[Integer]) -> ReturnableArgument``
+
+:description:
+  Log the ``message`` if ``passthrough`` evaluates to ``true``. Return
+  ``passthrough``. Optionally can pass level, where < 0 is debug, 0 is info, 1
+  is warning, > 1 is error. (default ``0``)
+
+----------------------------------------------------------------------------------------------------
+
+Regex Functions
+---------------
+
+regex_capture_groups
+~~~~~~~~~~~~~~~~~~~~
+:spec: ``regex_capture_groups(regex: String) -> Integer``
+
+:description:
+  Returns number of capture groups in regex
+
+regex_capture_many
+~~~~~~~~~~~~~~~~~~
+:spec: ``regex_capture_many(string: String, regex_array: Array, default: Optional[Array]) -> Array``
+
+:description:
+  Returns the input string and first regex's capture groups that match to the string
+  in an array. If a default is not provided, then all number of regex capture groups
+  must be equal across all regex strings. In addition, an error will be thrown if
+  no matches are found.
+
+  If the default is provided, then the number of capture groups must be less than
+  or equal to the length of the default value array. Any element not captured
+  will return the respective default value.
+:usage:
+
+.. code-block:: python
+
+   {
+     %regex_capture_many(
+       "2020-02-27",
+       [
+         "No (.*) matches here",
+         "([0-9]+)-([0-9]+)-27"
+       ],
+       [ "01", "01" ]
+     )
+   }
+
+   # ["2020-02-27", "2020", "02"]
+
+regex_capture_many_required
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:spec: ``regex_capture_many_required(string: String, regex_array: Array) -> Array``
+
+:description:
+  Deprecated. Use %regex_capture_many instead.
+
+regex_capture_many_with_defaults
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:spec: ``regex_capture_many_with_defaults(string: String, regex_array: Array, default: Optional[Array]) -> Array``
+
+:description:
+  Deprecated. Use %regex_capture_many instead.
+
+regex_fullmatch
+~~~~~~~~~~~~~~~
+:spec: ``regex_fullmatch(regex: String, string: String) -> Array``
+
+:description:
+  Checks for entire string to be a match. If a match exists, returns
+  the string as the first element of the Array. If there are capture groups, returns each
+  group as a subsequent element in the Array.
+
+regex_match
+~~~~~~~~~~~
+:spec: ``regex_match(regex: String, string: String) -> Array``
+
+:description:
+  Checks for a match only at the beginning of the string. If a match exists, returns
+  the string as the first element of the Array. If there are capture groups, returns each
+  group as a subsequent element in the Array.
+
+regex_search
+~~~~~~~~~~~~
+:spec: ``regex_search(regex: String, string: String) -> Array``
+
+:description:
+  Checks for a match anywhere in the string. If a match exists, returns
+  the string as the first element of the Array. If there are capture groups, returns each
+  group as a subsequent element in the Array.
+
+regex_search_any
+~~~~~~~~~~~~~~~~
+:spec: ``regex_search_any(string: String, regex_array: Array) -> Boolean``
+
+:description:
+  Returns True if any regex pattern in the regex array matches the string. False otherwise.
+
+regex_sub
+~~~~~~~~~
+:spec: ``regex_sub(regex: String, replacement: String, string: String) -> String``
+
+:description:
+  Returns the string obtained by replacing the leftmost non-overlapping occurrences of the
+  pattern in string by the replacement string. The replacement string can reference the
+  match groups via backslash escapes. Callables as replacement argument are not supported.
+
+----------------------------------------------------------------------------------------------------
+
+String Functions
+----------------
+
+capitalize
+~~~~~~~~~~
+:spec: ``capitalize(string: String) -> String``
+
+:description:
+  Capitalize the first character in the string.
+
+concat
+~~~~~~
+:spec: ``concat(values: AnyArgument, ...) -> String``
+
+:description:
+  Concatenate multiple Strings into a single String.
+
+contains
+~~~~~~~~
+:spec: ``contains(string: String, contains: String) -> Boolean``
+
+:description:
+  Returns True if ``contains`` is in ``string``. False otherwise.
+
+contains_all
+~~~~~~~~~~~~
+:spec: ``contains_all(string: String, contains_array: Array) -> Boolean``
+
+:description:
+    Returns true if all elements in ``contains_array`` are in ``string``. False otherwise.
+
+contains_any
+~~~~~~~~~~~~
+:spec: ``contains_any(string: String, contains_array: Array) -> Boolean``
+
+:description:
+    Returns true if any element in ``contains_array`` is in ``string``. False otherwise.
+
+join
+~~~~
+:spec: ``join(separator: String, array: Array) -> String``
+
+:description:
+    Join all elements in the array together as a string, and insert the
+    separator between them.
+
+:usage:
+
+.. code-block:: python
+
+   {
+     %join( ", ", ["item1", "item2"] )
+   }
+
+   # "item1, item2"
+
+lower
+~~~~~
+:spec: ``lower(string: String) -> String``
+
+:description:
+  Lower-case the entire String.
+
+pad
+~~~
+:spec: ``pad(string: String, length: Integer, char: String) -> String``
+
+:description:
+  Pads the string to the given length
+
+pad_zero
+~~~~~~~~
+:spec: ``pad_zero(numeric: Numeric, length: Integer) -> String``
+
+:description:
+  Pads a numeric with zeros to the given length
+
+replace
+~~~~~~~
+:spec: ``replace(string: String, old: String, new: String, count: Optional[Integer]) -> String``
+
+:description:
+  Replace the ``old`` part of the String with the ``new``. Optionally only replace it
+  ``count`` number of times.
+
+slice
+~~~~~
+:spec: ``slice(string: String, start: Integer, end: Optional[Integer]) -> String``
+
+:description:
+  Returns the slice of the Array.
+
+split
+~~~~~
+:spec: ``split(string: String, sep: String, max_split: Optional[Integer]) -> Array``
+
+:description:
+  Splits the input string into multiple strings.
+
+string
+~~~~~~
+:spec: ``string(value: AnyArgument) -> String``
+
+:description:
+  Cast to String.
+
+strip
+~~~~~
+:spec: ``strip(string: String) -> String``
+
+:description:
+    Strip a string of all its whitespace at the beginning and end.
+
+:usage:
+
+.. code-block:: python
+
+   {
+     %trim(" delete the outer! ")
+   }
+
+   # "delete the outer!"
+
+titlecase
+~~~~~~~~~
+:spec: ``titlecase(string: String) -> String``
+
+:description:
+  Capitalize each word in the string.
+
+unescape
+~~~~~~~~
+:spec: ``unescape(string: String) -> String``
+
+:description:
+  Unescape symbols like newlines or tabs to their true form.
+
+:usage:
+
+.. code-block:: python
+
+   {
+     %unescape( "Hello\nWorld" )
+   }
+
+   # Hello
+   # World
+
+upper
+~~~~~
+:spec: ``upper(string: String) -> String``
+
+:description:
+  Upper-case the entire String.
+
+----------------------------------------------------------------------------------------------------
+
+Ytdl-Sub Functions
+------------------
+
+legacy_bracket_safety
+~~~~~~~~~~~~~~~~~~~~~
+:spec: ``legacy_bracket_safety(value: ReturnableArgument) -> ReturnableArgument``
+
+ytdl-sub used to replace brackets ('{', '}') with unicode brackets ('｛', '｝') to not
+interfere with its legacy variable scripting system. This function replicates that
+behavior.
+
+sanitize
+~~~~~~~~
+:spec: ``sanitize(value: AnyArgument, ...) -> String``
+
+Sanitize a string using yt-dlp's ``sanitize_filename`` method to ensure it's safe to use
+for file/directory names on any OS.
+
+sanitize_plex_episode
+~~~~~~~~~~~~~~~~~~~~~
+:spec: ``sanitize_plex_episode(string: String) -> String``
+
+Sanitize a string using ``sanitize`` and replace numerics with their respective fixed-width
+numbers. This is used to have Plex avoid scraping numbers like ``4x4`` as the
+season and/or episode.
+
+to_date_metadata
+~~~~~~~~~~~~~~~~
+:spec: ``to_date_metadata(yyyymmdd: String) -> Map``
+
+Takes a date in the form of YYYYMMDD and returns a Map containing:
+
+- date (String, YYYYMMDD)
+- date_standardized (String, YYYY-MM-DD)
+- year (Integer)
+- month (Integer)
+- day (Integer)
+- year_truncated (Integer, YY from YY[YY])
+- month_padded (String)
+- day_padded (String)
+- year_truncated_reversed (Integer, 100 - year_truncated)
+- month_reversed (Integer, 13 - month)
+- month_reversed_padded (String)
+- day_reversed (Integer, total_days_in_month + 1 - day)
+- day_reversed_padded (String)
+- day_of_year (Integer)
+- day_of_year_padded (String, padded 3)
+- day_of_year_reversed (Integer, total_days_in_year + 1 - day_of_year)
+- day_of_year_reversed_padded (String, padded 3)
+
+to_native_filepath
+~~~~~~~~~~~~~~~~~~
+:spec: ``to_native_filepath(filepath: String) -> String``
+
+Convert any unix-based path separators ('/') with the OS's native
+separator. In addition, expand ~ to absolute directories.
+
+truncate_filepath_if_too_long
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+:spec: ``truncate_filepath_if_too_long(filepath: String) -> String``
+
+If a file-path is too long for the OS, this function will truncate it while preserving
+the extension.

docs/source/config_reference/scripting/scripting_types.rst

@@ -0,0 +1,408 @@
+===============
+Scripting Types
+===============
+
+
+Types
+-----
+
+String
+~~~~~~
+
+Strings are a series of characters surrounded by quotes.
+
+.. code-block:: yaml
+
+   string_variable: "This is a String variable"
+
+.. note::
+
+   For non-String types, they must be defined as parameters to scripting functions. This
+   is because anything in a variable definition that is not within curly-braces gets
+   evaluated as a String.
+
+We can define Strings within curly-braces by setting them as parameters to a function:
+
+.. tab-set::
+
+  .. tab-item:: Multi-Line Single Quote
+
+    .. code-block:: yaml
+
+      string_variable: >-
+        {
+          %string('This is a String variable')
+        }
+
+  .. tab-item:: Multi-Line Double Quote
+
+    .. code-block:: yaml
+
+      string_variable: >-
+        {
+          %string("This is a String variable")
+        }
+
+There are a few ways to make variables that use curly braces more compact, including:
+
+.. tab-set::
+
+  .. tab-item:: New-Line Single Quote
+
+    .. code-block:: yaml
+
+      string_variable: >-
+        { %string('This is a String variable') }
+
+  .. tab-item:: New-Line Double Quote
+
+    .. code-block:: yaml
+
+      string_variable: >-
+        { %string("This is a String variable") }
+
+  .. tab-item:: Same-Line
+
+    .. code-block:: yaml
+
+      string_variable: "{ %string('This is a String variable') }"
+
+In the case that you want to define a string variable that contains both single and
+double quotes, triple-quotes can be used to avoid *closing* the String.
+
+.. tab-set::
+
+  .. tab-item:: Triple-Single Quote
+
+    .. code-block:: yaml
+
+      string_variable: >-
+        {
+          %string('''This has both " and ' in it.''')
+        }
+
+  .. tab-item:: Triple-Double Quote
+
+    .. code-block:: yaml
+
+      string_variable: >-
+        {
+          %string("""This has both " and ' in it.""")
+        }
+
+If you want a plain string that contains literal curly braces, you can escape them like
+so:
+
+.. code-block:: yaml
+
+   string_variable: "This contains \\{ literal curly braces \\}"
+
+Integer
+~~~~~~~
+
+Integers are whole numbers with no decimal.
+
+.. tab-set::
+
+  .. tab-item:: Multi-Line
+
+    .. code-block:: yaml
+
+       int_variable: >-
+         {
+           %int(2022)
+         }
+
+  .. tab-item:: New-Line
+
+    .. code-block:: yaml
+
+       int_variable: >-
+         { %int(2022) }
+
+
+  .. tab-item:: Same-Line
+
+    .. code-block:: yaml
+
+       int_variable: "{ %int(2022) }"
+
+Float
+~~~~~
+
+Floats are floating-point decimals numbers.
+
+.. tab-set::
+
+  .. tab-item:: Multi-Line
+
+    .. code-block:: yaml
+
+       float_variable: >-
+         {
+           %float(3.14)
+         }
+
+  .. tab-item:: New-Line
+
+    .. code-block:: yaml
+
+       float_variable: >-
+         { %float(3.14) }
+
+  .. tab-item:: Same-Line
+
+    .. code-block:: yaml
+
+       float_variable: "{ %float(3.14) }"
+
+Boolean
+~~~~~~~
+
+A type is considered boolean if it spells out ``True`` or ``False``, case-insensitive.
+
+.. tab-set::
+
+  .. tab-item:: Multi-Line
+
+    .. code-block:: yaml
+
+       bool_variable: >-
+         {
+           %bool(True)
+         }
+
+  .. tab-item:: New-Line
+
+    .. code-block:: yaml
+
+       bool_variable: >-
+         { %bool(True) }
+
+  .. tab-item:: Same-Line
+
+    .. code-block:: yaml
+
+       bool_variable: "{ %bool(FALSE) }"
+
+Array
+~~~~~
+
+An Array contains multiple types of any kind, including nested Arrays and Maps.  Arrays
+are defined using brackets (``[ ]``), and are accessed using zero-based indexing.
+
+.. tab-set::
+
+  .. tab-item:: Multi-Line
+
+    .. code-block:: yaml
+
+       array_variable: >-
+         {
+           [
+             "element with index 0",
+             1,
+             2.0,
+             [ "Nested Array 3" ]
+           ]
+         }
+       element_0: >-
+         {
+           %array_at(array_variable, 0)
+         }
+
+  .. tab-item:: New-Line
+
+    .. code-block:: yaml
+
+       array_variable: >-
+         { ["element with index 0", 1, 2.0, ["Nested Array 3"]] }
+       element_0: >-
+         { %array_at(array_variable, 0) }
+
+  .. tab-item:: Same-Line
+
+    .. code-block:: yaml
+
+       array_variable: "{ ['element with index 0', 1, 2.0, ['Nested Array 3' ]] }"
+       element_0: "{ %array_at(array_variable, 0) }"
+
+Map
+~~~
+
+A Map is a key-value store, containing mappings between keys and values.  Maps are
+defined using curly-braces (``{ }``), and are accessed using their keys.
+
+.. tab-set::
+
+  .. tab-item:: Multi-Line
+
+    .. code-block:: yaml
+
+       map_variable: >-
+         {
+           {
+             "string_key": "string_value",
+             1: "int_key",
+             "list_value": [ "elem0", 1, 2.0 ]
+           }
+         }
+       string_value: >-
+         {
+           %map_get(map_variable, "string_key")
+         }
+
+  .. tab-item:: New-Line
+
+    .. code-block:: yaml
+
+       map_variable: >-
+         { {"string_key": "string_value", 1: "int_key", "list_value": ["elem0", 1, 2.0]} }
+       string_value: >-
+         { %map_get(map_variable, "string_key") }
+
+  .. tab-item:: Same-Line
+
+    .. code-block:: yaml
+
+       map_variable: "{ {'string_key': 'string_value', 1: 'int_key', 'list_value': [ 'elem0', 1, 2.0 ]} }"
+       string_value: "{ %map_get(map_variable, 'string_key') }"
+
+Null
+~~~~
+
+Null is represented by an empty String, and can be conveyed by spelling out ``null``,
+case-insensitive.
+
+.. tab-set::
+
+  .. tab-item:: Literal
+
+    .. code-block:: yaml
+
+       null_variable: ""
+
+  .. tab-item:: New-Line
+
+    .. code-block:: yaml
+
+      null_variable: >-
+        { %string(null) }
+
+  .. tab-item:: Same-Line
+
+    .. code-block:: yaml
+
+      null_variable: "{ %string(null) }"
+
+
+Function Type-Hints
+-------------------
+
+AnyArgument
+~~~~~~~~~~~
+
+AnyArgument means any of the above Types are valid as input or output to a scripting
+function.
+
+.. note::
+
+   Strict typing is enforced. For functions that return ``AnyArgument`` need to be casted before
+   passing into functions that expect a particular type.
+
+Numeric
+~~~~~~~
+
+Numeric refers to either an Integer or Float.
+
+Optional
+~~~~~~~~
+
+Optional means a particular scripting function argument can be either provided or not
+included.  For example, the function `map_get
+<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#map-get>`_
+has an optional default value. Both of these usages are valid:
+
+.. tab-set::
+
+  .. tab-item:: Map Get
+
+    .. code-block:: yaml
+
+       will_throw_key_does_not_exist_error: "{ %map_get( {}, 'key' ) }"
+
+  .. tab-item:: Map Get with Optional Default Value
+
+    .. code-block:: yaml
+
+      will_return_default: "{ %map_get( {}, 'key', 'default value' ) }"
+
+Lambda
+~~~~~~
+
+Lambda parameters are a reference to a function, and will call that lambda function on
+the input. In this example,
+
+.. code-block:: yaml
+
+   lambda_array_numeric_to_string: >-
+     {
+       %array_apply( [ 1, 2, 3, 4], %string )
+     }
+
+We apply ``%string`` as a lambda function to `array_apply
+<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-apply>`_,
+which is called on every element in the input array. The output becomes ``["1", "2",
+"3", "4"]``.
+
+This example has one input-argument being passed into the lambda. For other lambda-based
+functions like `array_enumerate
+<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-enumerate>`_,
+it expects the lambda function to have two input arguments. These are denoted using
+``LambdaTwo``, ``LambdaThree``, etc within the function spec.
+
+LambdaReduce
+~~~~~~~~~~~~
+
+LambdaReduce parameters are a reference to a function that will perform a *reduce* - an
+operation that reduces an Array to a single value by calling the LambdaReduce function
+repeatedly on two elements in the Array until it is reduced to a single value.
+
+In this example,
+
+.. code-block:: yaml
+
+   lambda_reduce_sum: >-
+     {
+       %array_reduce( [ 1, 2, 3, 4], %add )
+     }
+
+We call `array_reduce
+<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#array-reduce>`_
+on the input array, using `add
+<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#add>`_
+as the LambdaReduce function. This will reduce the Array to a single value by internally
+calling
+
+- *reduce-call 1*: ``%add(1, 2) = 3`` (first two elements)
+- *reduce-call 2*: ``%add(3, 3) = 6`` (output from first two and third element)
+- *reduce-call 3*: ``%add(6, 4) = 10`` (output from first three elements and fourth element)
+
+And evaluate to ``10``.
+
+ReturnableArguments
+~~~~~~~~~~~~~~~~~~~
+
+Returnable arguments are used in conditional functions like `if
+<https://ytdl-sub.readthedocs.io/en/latest/config_reference/scripting/scripting_functions.html#if>`_,
+which implies the argument passed into the function is the function's output. For
+example,
+
+.. code-block:: yaml
+
+   conditional_function: >-
+     {
+       %if( True, "Return this if True", "Return this if False" )
+     }
+
+is going to return ``"Return this if True"`` since the condition parameter is ``True``.

docs/source/config_reference/scripting/static_variables.rst

@@ -0,0 +1,103 @@
+..
+  WARNING: This RST file is generated from docstrings in:
+    src/ytdl_sub/entries/variables/override_variables.py
+  In order to make a change to this file, edit the respective docstring
+  and run `make docs`. This will automatically sync the Python RST-based
+  docstrings into this file. If the docstrings and RST file are out of sync,
+  it will fail TestDocGen tests in GitHub CI.
+
+Static Variables
+================
+
+Subscription Variables
+----------------------
+
+subscription_array
+~~~~~~~~~~~~~~~~~~
+For subscriptions in the form of
+
+.. code-block:: yaml
+
+   "Subscription Name":
+     - "https://url1.com/..."
+     - "https://url2.com/..."
+
+Store all values into an array named ``subscription_array``.
+
+subscription_has_download_archive
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Returns True if the subscription has any entries recorded in a download archive. False
+otherwise.
+
+subscription_indent_i
+~~~~~~~~~~~~~~~~~~~~~
+For subscriptions where the ancestor keys contain the ``= ...`` prefix, the
+variables ``subscription_indent_1``, ``subscription_indent_2``, and so on get
+set to each subsequent value. For example, given the following subscriptions
+file snippet:
+
+.. code-block:: yaml
+
+   Preset 1 | = Indent Value 1 | Preset 2:
+     Preset 3 | = Indent Value 2 | Preset 4:
+       "Subscription Name": "https://..."
+
+The ``{subscription_indent_1}`` variable will be ``Indent Value 1`` and
+``{subscription_indent_2}`` will be ``Indent Value 2``. The most common use of
+these variables is to :doc:`set the genre and rating for subscriptions from the
+YAML keys <../prebuilt_presets/tv_show>`.
+
+subscription_map
+~~~~~~~~~~~~~~~~
+For subscriptions in the form of
+
+.. code-block:: yaml
+
+   + Subscription Name:
+     Music Videos:
+       - "https://url1.com/..."
+     Concerts:
+       - "https://url2.com/..."
+
+Stores all the contents under the subscription name into the override variable
+``subscription_map`` as a Map value. The above example is stored as:
+
+.. code-block:: python
+
+   {
+     "Music Videos": [
+       "https://url1.com/..."
+     ],
+     "Concerts: [
+       "https://url2.com/..."
+     ]
+   }
+
+subscription_name
+~~~~~~~~~~~~~~~~~
+Name of the subscription. For subscriptions types that use a prefix (``~``, ``+``),
+the prefix and all whitespace afterwards is stripped from the subscription name.
+
+subscription_value
+~~~~~~~~~~~~~~~~~~
+For subscriptions in the form of
+
+.. code-block:: yaml
+
+   "Subscription Name": "https://..."
+
+``subscription_value`` gets set to ``https://...``.
+
+subscription_value_i
+~~~~~~~~~~~~~~~~~~~~
+For subscriptions in the form of
+
+.. code-block:: yaml
+
+   "Subscription Name":
+     - "https://url1.com/..."
+     - "https://url2.com/..."
+
+``subscription_value_1`` and ``subscription_value_2`` get set to ``https://url1.com/...``
+and ``https://url2.com/...``. Note that ``subscription_value_1`` also gets set to
+``subscription_value``.

docs/source/config_reference/subscription_yaml.rst

@@ -0,0 +1,128 @@
+==================
+Subscription File
+==================
+
+A subscription file is designed to both define and organize many things to download in
+condensed YAML.
+
+.. hint::
+
+  Read the :ref:`getting started guide <guides/getting_started/index:Getting Started>`
+  first before reviewing this section.
+
+
+File Preset
+-----------
+
+Many examples show ``__preset__`` at the top. This is known as the *subscription file
+preset*.  It is where a single :ref:`preset <guides/getting_started/first_config:Custom
+Preset Definition>` can be defined that gets applied to each subscription within the
+file.
+
+This is a good place to apply file-wide variables such as ``tv_show_directory`` or
+supply a cookies file path.
+
+.. code-block:: yaml
+
+  __preset__:
+    # Variables that override defaults from `overrides:` for presets in YAML keys:
+    overrides:
+      tv_show_directory: "/tv_shows"
+
+    # Directly set plugin options:
+    ytdl_options:
+      cookiefile: "/config/ytdl-sub-configs/cookie.txt"
+
+Layout
+------
+
+A subscription file is comprised of YAML keys and values. Keys can be either
+
+- a preset
+- an override value
+- a subscription name
+
+Take the following example:
+
+.. code-block:: yaml
+
+  Jellyfin TV Show by Date:
+    = News:
+      "Breaking News": "https://www.youtube.com/@SomeBreakingNews"
+      "BBC News": "https://www.youtube.com/@BBCNews"
+
+All three types of keys are used for the following:
+
+- ``Jellyfin TV Show by Date`` - a prebuilt preset
+- ``= News`` - an override value for genre
+- ``Breaking News``, ``BBC News`` - The subscription names
+
+The lowest level, most indented keys should always be the subscription name.  It is good
+practice to put subscription names in quotes to differentiate between preset names and
+subscription names.
+
+Values should always be the subscription itself. The simplest form is just the
+URL. Further sections will show more exotic examples that go beyond a single URL.
+
+
+Inheritance
+-----------
+
+A subscription inherits every key above it. In the above example, both ``Breaking News``
+and ``BBC News`` inherits the ``Jellyfin TV Show by Date`` preset and the ``= News``
+override value.
+
+.. note::
+
+  There are no limits or boundaries on how one structures their presets. This
+  flexibility is intended for subscription authors to organize their downloads as they
+  see fit.
+
+
+Multi Keys
+----------
+
+Subscription keys support pipe syntax, or ``|``, which allows multiple keys to be
+defined on a single line. The following is equivalent to the above example:
+
+.. code-block:: yaml
+
+  Jellyfin TV Show by Date | = News:
+    "Breaking News": "https://www.youtube.com/@SomeBreakingNews"
+    "BBC News": "https://www.youtube.com/@BBCNews"
+
+
+Override Mode
+-------------
+
+Often times, it is convenient to set multiple override values for a single
+subscription. We can put a preset in *override mode* by using tilda syntax, or ``~``.
+
+Suppose we want to apply the :ref:`Only Recent <prebuilt_presets/helpers:Only Recent>`
+preset to the above examples. But for ``BBC News`` specifically, we want to set the date
+range to be different than the default ``2months`` value to ``2weeks``.
+
+We can change it as follows:
+
+.. code-block:: yaml
+
+  Jellyfin TV Show by Date
+    = News | Only Recent:
+      "Breaking News": "https://www.youtube.com/@SomeBreakingNews"
+      "~BBC News":
+        url: "https://www.youtube.com/@BBCNews"
+        only_recent_date_range: "2weeks"
+
+.. important::
+
+  When using override mode, we need to set the ``url`` variable since we are no longer
+  using the simplified *subscription_value*. For more info on how this works, read about
+  :ref:`subscription variables <config_reference/scripting/static_variables:Subscription
+  Variables>`.
+
+
+Map Mode
+--------
+
+Map mode is for highly advanced presets that benefit from a more complex subscription
+definition. TODO: Show music video example here.

docs/source/debugging.rst

@@ -0,0 +1,78 @@
+Debugging
+=========
+
+Run with ``--log-level debug`` to show all log messages, often too much information for
+normal operation but useful when investigating a specific problem.
+
+:ref:`ytdl-sub builds on yt-dlp <introduction:motivation>`, which is in itself a complex
+tool. It performs an intricate and fragile task, web scraping, which in turn :ref:`is
+subject to the whims of external services <guides/getting_started/index:minimize the
+work to only what's necessary>` outside its control. Finally, because :ref:`ytdl-sub is
+a lower-level tool <guides/getting_started/index:prerequisite knowledge>`, many users,
+if not most, will have problems getting their configuration working and it can be
+difficult to determine when the root cause is their configuration, just a limit imposed
+by the services, or, least likely, a bug in one of the tools involved.
+
+To expedite resolution and conserve the limited resources of both yourself and
+volunteers, do as much investigation yourself as you can:
+
+#. Start by assuming the issue is your configuration:
+
+   Review :doc:`the guides <./guides/index>` to confirm your understanding. Increase
+   output using the ``--log-level`` CLI option and read the output carefully for hints
+   and clues. Use those clues to `search the docs`_. Read :doc:`the reference docs
+   <./config_reference/index>` of the involved ``ytdl-sub`` components.
+
+#. Try to determine if the issue is happening in ``yt-dlp`` or ``ytdl-sub``:
+
+   The user's configuration tells ``ytdl-sub`` how to run ``yt-dlp``. ``yt-dlp`` handles
+   all the web scraping and downloading. ``ytdl-sub`` then assembles the files and metadata
+   produced by ``yt-dlp`` and places them in your library.
+
+   If the issue is happening while scraping or downloading from the external service,
+   then it's happening in the running of ``yt-dlp``. Look for output showing failed
+   downloads, ``403`` errors, or signs of throttles. That doesn't mean it's a bug in
+   ``yt-dlp``, it could be in how your configuration tells ``ytdl-sub`` to run
+   ``yt-dlp`` or limits imposed by the service that are constantly changing, but you may
+   be able to find answers from other ``yt-dlp`` users running into similar issues.
+
+   See `the yt-dlp known issues`_ and `search their issues`_ for clues and hints. Read
+   the comments for more understanding, workarounds, and maybe even fixes. If you still
+   don't understand the cause after reading everything you can find there, try to find
+   help in `the yt-dlp Discord`_.
+
+#. If the issue is happening in ``ytdl-sub``, reach out for help:
+
+   Once you've done everything you can to get your configuration working and you've
+   determined that the issue isn't happening in ``yt-dlp``, look for answers in
+   ``ytdl-sub``:
+
+   #. Cut your configuration and subscriptions down to the minimum that reproduces the
+      issue.
+
+   #. Run with the ``--log-level debug`` CLI option and copy the full output.
+
+   #. `Search the ytdl-sub issues`_ using clues and hints from the output.
+
+   #. `Open a support post in Discord`_ with those details and all other relevant
+      details.
+
+   #. If someone from the Discord discussion directs you to, then `open a new issue`_
+      with those same details.
+
+
+.. _`the yt-dlp known issues`:
+   https://github.com/yt-dlp/yt-dlp/wiki/FAQ#known-issues
+.. _`search their issues`:
+   https://github.com/yt-dlp/yt-dlp/issues
+.. _`the yt-dlp Discord`:
+   https://discord.gg/H5MNcFW63r
+
+.. _`search the docs`:
+   https://ytdl-sub.readthedocs.io/en/latest/search.html
+.. _`search the ytdl-sub issues`:
+   https://github.com/jmbannon/ytdl-sub/issues
+.. _`open a support post in Discord`:
+   https://discord.com/channels/994270357957648404/1084886228266127460
+.. _`open a new issue`:
+   https://github.com/jmbannon/ytdl-sub/issues/new

docs/source/deprecation_notices.rst

@@ -0,0 +1,113 @@
+Deprecation Notices
+===================
+
+Dec 2025
+--------
+
+Override variables names can no longer be plugin names, to avoid the common pitfall of
+defining a plugin underneath ``overrides``.
+
+In the past, there was usage of a ``date_range`` override variable in a few example configs
+that complimented the ``Only Recent`` preset. This overrride variable usage needs to be
+replaced with ``only_recent_date_range``.
+
+Sep 2024
+--------
+
+regex plugin
+~~~~~~~~~~~~
+
+Regex plugin has been removed in favor of scripting. The function
+:ref:`config_reference/scripting/scripting_functions:regex_capture_many` has been
+created to replicate the plugin's behavior. See the following converted example:
+
+.. code-block:: yaml
+  :caption: regex plugin
+
+    regex:
+      from:
+        title:
+          match:
+            - ".*? - (.*)"  # Captures 'Some - Song' from 'Emily Hopkins - Some - Song'
+          capture_group_names:
+            - "captured_track_title"
+          capture_group_defaults:
+            - "{title}"
+    overrides:
+      track_title: "{captured_track_title}"
+
+.. code-block:: yaml
+  :caption: scripting
+
+    overrides:
+      # Captures 'Some - Song' from 'Emily Hopkins - Some - Song'
+      captured_track_title: >-
+        {
+          %regex_capture_many(
+            title,
+            [ ".*? - (.*)" ],
+            [ title ]
+          )
+        }
+      track_title: "{%array_at(captured_track_title, 1)}"
+
+
+Oct 2023
+--------
+
+subscription preset and value
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The use of ``__value__`` will go away in Dec 2023 in favor of the method found in
+:ref:`config_reference/subscription_yaml:Subscription File`. ``__preset__`` will still
+be supported for the time being.
+
+July 2023
+---------
+
+music_tags
+~~~~~~~~~~
+
+Music tags are getting simplified. ``tags`` will now reside directly under music_tags,
+and ``embed_thumbnail`` is getting moved to its own plugin (supports video files as
+well). Convert from:
+
+.. code-block:: yaml
+
+  my_example_preset:
+    music_tags:
+      embed_thumbnail: True
+      tags:
+        artist: "Elvis Presley"
+
+To the following:
+
+.. code-block:: yaml
+
+  my_example_preset:
+    embed_thumbnail: True
+    music_tags:
+      artist: "Elvis Presley"
+
+The old format will be removed in October 2023.
+
+video_tags
+~~~~~~~~~~
+
+Video tags are getting simplified as well. ``tags`` will now reside directly under
+video_tags.  Convert from:
+
+.. code-block:: yaml
+
+  my_example_preset:
+    video_tags:
+      tags:
+        title: "Elvis Presley Documentary"
+
+To the following:
+
+.. code-block:: yaml
+
+  my_example_preset:
+    video_tags:
+      title: "Elvis Presley Documentary"

docs/source/faq/index.rst

@@ -0,0 +1,294 @@
+===
+FAQ
+===
+
+Since ytdl-sub is relatively new to the public, there has not been many question asked
+yet. We will update this as more questions get asked.
+
+.. contents:: Frequently Asked Questions
+  :depth: 3
+
+
+How do I...
+-----------
+
+...remove the date in the video title?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :ref:`config_reference/prebuilt_presets/tv_show:TV Show` presets by default include
+the upload date in the ``episode_title`` override variable. This variable is used to set
+the title in things like the video metadata, NFO file, etc, which is subsequently read
+by media players. This can be overwritten as you see fit by redefining it:
+
+.. code-block:: yaml
+
+   overrides:
+     episode_title: "{title}"  # Only sets the video title
+
+...download age-restricted YouTube videos?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+See `yt-dl's recommended way
+<https://github.com/ytdl-org/youtube-dl#how-do-i-pass-cookies-to-youtube-dl>`_ to
+download your YouTube cookie, then add it to your :ref:`ytdl options
+<config_reference/plugins:ytdl_options>` section of your config:
+
+.. code-block:: yaml
+
+  ytdl_options:
+    cookiefile: "/path/to/cookies/file.txt"
+
+...automate my downloads?
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:doc:`This page </guides/getting_started/automating_downloads>` shows how to set up
+``ytdl-sub`` to run automatically on various platforms.
+
+...download large channels?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+See the prebuilt preset :doc:`chunk_initial_download </prebuilt_presets/helpers>`.
+
+...filter to include or exclude based on certain keywords?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+See the prebuilt preset :doc:`Filter Keywords </prebuilt_presets/helpers>`.
+
+...prevent creation of NFO file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Creation of NFO files is done by the NFO tags plugin. It, as any other plugin, can be
+disabled:
+
+.. code-block:: yaml
+
+  nfo_tags:
+    enabled: False
+
+...prevent download of images
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :ref:`config_reference/prebuilt_presets/tv_show:TV Show` presets by default
+downloads images corresponding to show and each episode.  This can be prevented by
+overriding following variables:
+
+.. code-block:: yaml
+
+  overrides:
+    tv_show_fanart_file_name: ""  # to stop creation of fanart.jpg in subscription
+    tv_show_poster_file_name: ""  # to stop creation of poster.jpg in subscription
+    thumbnail_name: ""            # to stop creation of episode thumbnails
+
+...use only part of the media's title
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ytdl-sub offers a range of functions that can be used to parse a subset of a title for
+use in your media player. Consider the example:
+
+* I want to remove "NOVA PBS - " from the title ``NOVA PBS - Hidden Cities All Around
+  Us``.
+
+There are several solutions using ytdl-sub's scripting capabilities to override
+``episode_title`` by manipulating the original media's ``title``.
+
+.. code-block:: yaml
+   :caption: Replace exclusion with empty string
+
+   "~Nova PBS":
+     url: "https://www.youtube.com/@novapbs"
+     episode_title: >-
+       {
+         %replace( title, "NOVA PBS - ", "" )
+       }
+
+.. code-block:: yaml
+   :caption: Split once using delimiter, grab last value in the split array.
+
+   "~Nova PBS":
+     url: "https://www.youtube.com/@novapbs"
+     episode_title: >-
+       {
+         %array_at( %split(title, " - ", 1), -1 )
+       }
+
+.. code-block:: yaml
+   :caption:
+      Regex capture. Supports multiple capture strings and default values if captures
+      are unsuccessful.
+
+   "~Nova PBS":
+     url: "https://www.youtube.com/@novapbs"
+     captured_episode_title: >-
+       {
+         %regex_capture_many(
+           title,
+           [ "NOVA PBS - (.*)" ],
+           [ title ]
+         )
+       }
+     episode_title: >-
+        { %array_at( captured_episode_title, 1 ) }
+
+There is no single solution to this problem - it will vary case-by-case. See our full
+suite of :ref:`scripting functions
+<config_reference/scripting/scripting_functions:Scripting Functions>` to create your own
+clever scraping mechanisms.
+
+...force ytdl-sub to re-download a file
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes users may wish to replace a file already in the archive, for example, if the
+current file is a lower resolution than desired, missing subtitles, corrupt, etc..
+
+``ytdl-sub`` decides what files have already been downloaded by entries in :ref:`the
+download archive file <config_reference/plugins:output_options>`,
+``./.ytdl-sub-...-download-archive.json``, at the top of the subscription/series/show
+:ref:`output directory <config_reference/plugins:output_options>` in the appropriate
+``overrides: / ..._directory:`` library path, *and* the presence of the corresponding
+downloaded files under the same path. To force ``ytdl-sub`` to re-download an entry both
+need to be removed:
+
+- Move aside the downloaded files:
+
+  Rename or move the downloaded files, including the associated files with the same
+  base/stem name, such as ``./*.nfo``, ``./*.info-json``, etc..
+
+- Ensure ``ytdl-sub`` is not running and won't run, such as by cron:
+
+  ``ytdl-sub`` loads the ``./.ytdl-sub-...-download-archive.json`` file early, keeps it
+  in memory, and writes it back out late. If it's running or starts running while you're
+  modifying that file, then your changes will be overwritten when it exits.
+
+- Remove the ``./.ytdl-sub-...-download-archive.json`` JSON array item:
+
+  Search for the stem name, the basename without any extension or suffix, common to all
+  the downloaded files in this file and delete that whole entry, from the YouTube ID
+  string to the closing curly braces. Be ware of JSON traling commas.
+
+- Run ``$ ytdl-sub sub`` again with the appropriate CLI plugin options:
+
+  In normal operation, :ref:`yt-dlp minimizes requests and the files considered for
+  download <guides/getting_started/index:minimize the work to only what's
+  necessary>`. To re-download, those options must be disabled or modified. Disable
+  :ref:`the 'break_on_existing' option <config_reference/plugins:ytdl_options>`, set
+  :ref:`the 'date_range:' plugin <config_reference/plugins:date_range>`, and :ref:`limit
+  the subscriptions <guides/getting_started/downloading:preview>` to
+  download only the files that you've renamed in the steps above.
+
+  Set the appropriate dates, :ref:`including a sufficient margin
+  <config_reference/plugins:date_range>`, and subscription name to include only the
+  files you've renamed, and re-run. For example, if you've renamed all the files from
+  2024 in the ``NOVA PBS`` subscription:
+
+    .. code-block:: shell
+
+       ytdl-sub --match="NOVA PBS" sub -o "\
+       --ytdl_options.break_on_existing False \
+       --date_range.after 20240101 \
+       --date_range.before 20250101 \
+       "
+
+...download a file missing from the archive
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The root causes are unknown, but sometimes even after successful, complete runs, some
+files will be missing from the archive. To attempt to download those missing files,
+use `the same CLI options as re-downloading a file`_
+
+.. _`the same CLI options as re-downloading a file`:
+   `...force ytdl-sub to re-download a file`_
+
+...get support?
+~~~~~~~~~~~~~~~
+
+See :doc:`the debugging documentation <../debugging>`.
+
+...reach out to contribute?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you would like to contribute, we're happy to accept any help, including from
+non-coders! To find out how you can help this project, you can:
+
+- `Join our Discord <https://discord.gg/v8j9RAHb4k>`_ and leave a comment in
+  #development with where you think you can assist or what skills you would like to
+  contribute.
+
+- If you just want to fix one thing, you're welcome to :ytdl-sub-gh:`submit a pull
+  request <compare>` with information on what issue you're resolving and it will be
+  reviewed as soon as possible.
+
+
+There is a bug where...
+-----------------------
+
+...ytdl-sub is not downloading
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+...ytdl-sub is downloading at 360p or other lower quality
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+...ytdl-sub downloads 2-4 videos and then fails
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+These are often just limits imposed by the external services that are not bugs. There
+may be little that can be done about them, but see :ref:`the '_throttle_protection'
+preset <prebuilt_presets/helpers:_throttle_protection>` for more information.
+
+...date_range is not downloading older videos after I changed the range
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Your preset most likely has ``break_on_existing`` set to True, which will stop
+downloading additional metadata/videos if the video exists in your download archive. Set
+the following in your config to skip downloading videos that exist instead of stopping
+altogether.
+
+.. code-block:: yaml
+
+  ytdl_options:
+    break_on_existing: False
+
+After you download your new date_range duration, re-enable ``break_on_existing`` to
+speed up successive downloads.
+
+...it is downloading non-English title and description metadata
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Most likely the video has a non-English language set to its 'native' language. You can
+tell yt-dlp to explicitly download English metadata using.
+
+.. code-block:: yaml
+
+  ytdl_options:
+    extractor_args:
+      youtube:
+        lang:
+          - "en"
+
+...Plex is not showing my TV shows correctly
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+1. Set the following for your ytdl-sub library that has been added to Plex.
+
+.. figure:: ../../images/plex_scanner_agent.png
+  :alt:
+     The Plex library editor, under the advanced settings, showing the required options
+     for Plex to show the TV shows correctly.
+
+- **Scanner:** Plex Series Scanner
+- **Agent:** Personal Media shows
+- **Visibility:** Exclude from home screen and global search
+- **Episode sorting:** Library default
+- **YES** Enable video preview thumbnails
+
+2. Under **Settings** > **Agents**, confirm Plex Personal Media Shows/Movies scanner has
+   **Local Media Assets** enabled.
+
+.. figure:: ../../images/plex_agent_sources.png
+  :alt:
+     The Plex Agents settings page has Local Media Assets enabled for Personal Media
+     Shows and Movies tabs.
+
+...ytdl-sub errors when downloading a 360p video with resolution assert
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:ref:`See how to either ignore this specific video or disable resolution assertion entirely here. <resolution assert handling>`

docs/source/guides/development/index.rst

@@ -0,0 +1,98 @@
+Development and Contributing
+============================
+
+
+Requirements
+------------
+
+- python >= 3.10
+- ffmpeg/ffprobe 4.4.5 (test checksums rely on this version)
+- make
+
+
+Local Install
+-------------
+.. tab-set-code::
+
+    .. code-block:: shell
+
+        pip install -e .[test,lint,docs]
+
+    .. code-block:: zsh
+
+        pip install -e .\[test,lint,docs\]
+
+
+Linter
+------
+
+All source code contributed must be formatted to our linter specification.  Run the
+following to auto-format and check for any issues with your code:
+
+.. code-block:: shell
+
+   make lint
+
+
+Adding Documentation
+--------------------
+
+Docs can be found in ``ytdl-sub/docs/source/``, and are built using the command:
+
+
+.. code-block:: shell
+   :caption:
+      Viewable at http://localhost:63342/ytdl-sub/docs/build/html/index.html once built
+
+   make docs
+
+Some of the documentation is built using doc-strings from the python source code. The
+above command will rebuild those as well.
+
+
+Testing
+-------
+
+Tests are written using pytest. Many of them evaluate checksums of output files to
+ensure no unintended changes are introduced to the way ``ytdl-sub`` produces files. This
+checksum can be inaccurate for end-to-end tests, but are reliable for integration tests.
+
+If integration tests are failing, ensure...
+
+- you're using the correct ffmpeg version
+- you are developing on Linux or Mac (have not tested windows yet)
+- your local ``ytdl-sub`` dependencies are up-to-date
+
+
+Docker
+------
+
+Test changes to the Docker image variants locally:
+
+.. code-block:: shell
+
+   cd ./docker/testing/
+   make -j run
+
+See ``./docker/testing/docker-compose.yml`` for the Compose services for each image
+variant.
+
+
+IDE Setup
+---------
+
+PyCharm is our preferred IDE. The codebase is simple enough to where it's not required,
+but is highly recommended.
+
+TODO: screenshots of configuration
+
+
+Reproducing a Failing Subscription
+----------------------------------
+
+Subscriptions will dump their entire *compiled* yaml at the beginning of exeuction
+:doc:`when using '--log-level debug' <../../debugging>`. This can be copy-pasted into
+the file ``resources/file_fixtures/repro.yaml``.
+
+Running the test ``e2e.test_debug_repro.TestReproduce.test_debug_log_repro`` will fully
+reproduce that subscription in order to debug it.

docs/source/guides/getting_started/automating_downloads.rst

@@ -0,0 +1,132 @@
+Automating
+==========
+
+Automate downloading your subscriptions by running the :ref:`'sub' sub-command
+<usage:subscriptions options>` periodically. There are various tools that can run
+commands on a schedule you may use any of them that work with your installation
+method. Most users use `cron`_ in `Docker containers <docker and unraid_>`_.
+
+
+Docker and Unraid
+-----------------
+
+:doc:`The 'ytdl-sub' Docker container images <../install/docker>` provide optional cron
+support. Enable cron support by setting `a cron schedule`_ in the ``CRON_SCHEDULE``
+environment variable:
+
+.. code-block:: yaml
+  :caption: ./compose.yaml
+  :emphasize-lines: 4
+
+  services:
+    ytdl-sub:
+      environment:
+        CRON_SCHEDULE: "0 */6 * * *"
+	# WARNING: See "Getting Started" -> "Automating" docs regarding throttles/bans:
+	# CRON_RUN_ON_START: false
+
+Then recreate the container to apply the change and start it to generate the default
+``/config/ytdl-sub-configs/cron`` script. Read the comments in that script and edit as
+appropriate.
+
+The container cron wrapper script will write output from the cron job to
+``/config/ytdl-sub-configs/.cron.log``. The default image ``ENTRYPOINT`` will ``$ tail
+...`` that file so you can monitor the cron job in the container's output and thus also
+in the Docker logs.
+
+You may also set the ``CRON_RUN_ON_START`` environment variable to ``true`` to have the
+image run your cron script whenever the container starts in addition to the cron
+schedule.
+
+.. warning::
+
+   Using ``CRON_RUN_ON_START`` may cause your cron script to run too often and may
+   trigger throttles and bans. When enabled, your cron script will run *whenever* the
+   container starts including when the host reboots, when ``# dockerd`` restarts such as
+   when upgrading Docker itself, when a new image is pulled, when something applies
+   Compose changes, etc.. This may result in running ``ytdl-sub`` right before or after
+   the next cron scheduled run.
+
+
+.. _linux-setup:
+
+Linux, Mac OS X, BSD, or other UNIX's
+-------------------------------------
+
+For installations on systems already running ``# crond``, you can also use cron to run
+``ytdl-sub`` periodically. Write a script to run ``ytdl-sub`` in the cron job. Be sure
+the script changes to the same directory as your configuration and uses the full path to
+``ytdl-sub``:
+
+.. code-block:: shell
+   :caption: ~/.local/bin/ytdl-sub-cron
+   :emphasize-lines: 2,3
+
+   #!/bin/bash
+   cd "~/.config/ytdl-sub/"
+   ~/.local/bin/ytdl-sub --dry-run sub -o '--ytdl_options.max_downloads 3' |&
+       tee -a "~/.local/state/ytdl-sub/.cron.log"
+
+Then tell ``# crond`` when to run the script:
+
+.. code-block:: console
+
+   echo "0 */6 * * * ${HOME}/.local/bin/ytdl-sub-cron" | crontab "-"
+
+Remove the ``--dry-run`` and ``-o ...`` CLI options from your cron script when you've
+tested your configuration and you're ready to download entries unattended.
+
+
+.. _windows-setup:
+
+Windows
+-------
+
+For most Windows users, the best way to run commands periodically is `the Task
+Scheduler`_:
+
+.. attention::
+
+   These instructions are untested. Use at your own risk. If you use them, whether they
+   work or not, please let us know how it went in `a support post in Discord`_ or `a new
+   GitHub issue`_.
+
+#. Open the Task Scheduler app.
+
+#. Click ``Create Basic Task`` at the top of the right sidebar.
+
+#. Set all the fields as appropriate until you get to the ``Action``...
+
+#. For the ``Action``, select ``Start a program``...
+
+#. Click ``Browse...`` to the installed ``ytdl-sub.exe`` executable...
+
+#. Add CLI arguments to ``Add arguments (optional):``, for example ``--dry-run sub -o
+   '--ytdl_options.max_downloads 3'``...
+
+#. Set ``Start in (optional):`` to the directory containing your configuration.
+
+#. Finish the rest of the ``Create Basic Task`` wizard.
+
+
+Next Steps
+----------
+
+At this point, ``ytdl-sub`` should run periodically and keep your subscriptions current
+in your media library without your intervention. As your :doc:`subscriptions file
+<./subscriptions>` grows or you discover new use cases, it becomes worth while to
+simplify things by :doc:`defining your own custom presets <./first_config>`.
+
+
+
+.. _`cron`:
+   https://en.wikipedia.org/wiki/Cron
+.. _`a cron schedule`:
+   https://crontab.cronhub.io/
+
+.. _`the Task Scheduler`:
+   https://learn.microsoft.com/en-us/windows/win32/taskschd/task-scheduler-start-page
+.. _`a support post in Discord`:
+   https://discord.com/channels/994270357957648404/1084886228266127460
+.. _`a new GitHub issue`:
+   https://github.com/jmbannon/ytdl-sub/issues/new

docs/source/guides/getting_started/downloading.rst

@@ -0,0 +1,69 @@
+Downloading
+===========
+
+Once you've :doc:`defined your subscriptions <./subscriptions>`, it's time to test your
+configuration and try your first download. As a web scraping tool, :ref:`it's important
+to minimize the requests sent to external services
+<guides/getting_started/index:minimize the work to only what's necessary>` to avoid
+triggering throttling or bans. Further, a full download of even one subscription can
+take significant time. Test each change to your subscriptions carefully and quickly as
+follows.
+
+
+Preview
+-------
+
+Preview what ``ytdl-sub`` would do for this subscription. Run the :ref:`'sub'
+sub-command <usage:subscriptions options>` with CLI options to restrict requests as much
+as possible:
+
+- Pull metadata and *simulate* a download without actually downloading any media files
+  using the ``--dry-run`` CLI option.
+
+- Limit requests by narrowing the run to one subscription by giving a
+  subscription name to the ``--match`` CLI option.
+
+- Stop after just a few downloads to further minimize requests and make testing faster
+  using the ``max_downloads`` setting from ``yt-dlp``.
+
+Change to the directory containing your ``./subscriptions.yaml`` file and run with those
+options:
+
+.. code-block:: console
+
+   cd "/config/ytdl-sub-configs/"
+   ytdl-sub --dry-run --match="NOVA PBS" sub -o '--ytdl_options.max_downloads 3'
+
+Examine the output carefully, investigate anything that doesn't look right and repeat
+this step until everything looks right.
+
+
+Review
+------
+
+Review the results of real downloads. Run it again without the ``--dry-run`` option to
+actually download media and place the files in your library:
+
+.. code-block:: console
+
+   ytdl-sub --match="NOVA PBS" sub -o '--ytdl_options.max_downloads 3'
+
+Examine the output carefully again. Then examine how the resulting downloads work in
+your library. Repeat with a larger value for ``max_downloads`` and examine the output
+and downloads again.
+
+
+Next Steps
+----------
+
+Once you're `previewed <preview_>`_ and `reviewed <review_>`_ successful downloads of
+each of your subscriptions, you're ready to run a full download of all your
+subscriptions. Run the sub-command without the CLI options you used to limit what
+``ytdl-sub`` does while testing:
+
+.. code-block:: console
+
+   ytdl-sub sub
+
+If you're ready to let ``ytdl-sub`` run unattended, it's time to :doc:`automate
+downloads <./automating_downloads>`.

docs/source/guides/getting_started/first_config.rst

@@ -0,0 +1,263 @@
+Basic Configuration
+===================
+
+A configuration file serves two purposes:
+
+1. Set application-level functionality that is not specifiable in a subscription file.
+
+  .. note::
+
+   ytdl-sub does not require a configuration file. However,
+   certain application settings may be desirable for tweak, such as setting
+   ``working_directory`` to make ytdl-sub perform the initial download
+   to an SSD drive.
+
+2. Create custom presets.
+
+  .. note::
+
+    In the prior Initial Subscription examples, we leveraged the prebuilt preset
+    ``Jellyfin TV Show by Date``. This preset is entirely built using the same
+    YAML configuration system offered to users by using a configuration file.
+
+The following section attempts to demystify and explain how to...
+
+- Set an application setting
+- Know whether or not custom presets are actually needed
+- How to create a custom preset
+- How to use a custom preset on subscriptions
+
+-------------
+
+how this works, and show-case how
+
+.. code-block:: yaml
+  :linenos:
+
+  configuration:
+    working_directory: ".ytdl-sub-working-directory"
+
+  presets:
+    TV Show:
+      preset:
+        - "Jellyfin TV Show by Date"
+        - "Max 1080p"
+
+      embed_thumbnail: True
+
+      throttle_protection:
+         sleep_per_download_s:
+           min: 2.2
+           max: 10.8
+         sleep_per_subscription_s:
+           min: 9.0
+           max: 14.1
+         max_downloads_per_subscription:
+           min: 10
+           max: 36
+
+      overrides:
+        tv_show_directory: "/tv_shows"
+
+    TV Show Only Recent:
+      preset:
+        - "TV Show"
+        - "Only Recent"
+
+
+Configuration Section
+---------------------
+
+The :ref:`configuration <config_reference/config_yaml:Configuration File>` section sets
+options for ytdl-sub execution. Most users should set the path where ``ytdl-sub``
+temporarily stores downloaded data before assembling it and moving it into your
+library. To avoid unnecessarily long large file renames, use a path on the same
+filesystem as your library in the ``overrides: / *_directory:`` paths:
+
+
+.. code-block:: yaml
+  :lineno-start: 1
+
+  configuration:
+    working_directory: ".ytdl-sub-working-directory"
+
+
+Preset Section
+--------------
+
+Underneath ``presets``, we define two custom presets with the names ``TV Show`` and ``TV
+Show Only Recent``.
+
+.. code-block:: yaml
+
+  presets:
+    TV Show:
+      ...
+    TV Show Only Recent:
+      ...
+
+The indentation example above shows how to define multiple presets.
+
+
+Custom Preset Definition
+------------------------
+
+Before we break down the above ``TV Show`` preset, lets first outline a preset layout:
+
+.. code-block:: yaml
+
+    Preset Name:
+      preset:
+        ...
+
+      plugin(s):
+        ...
+
+      overrides:
+        ...
+
+Presets can contain three important things:
+
+1. ``preset`` section, which can inherit :ref:`prebuilt presets
+   <config_reference/prebuilt_presets/index:Prebuilt Preset Reference>` or other presets
+   defined in your config.
+2. :ref:`Plugin definitions <config_reference/plugins:Plugins>`
+3. :ref:`overrides <config_reference/plugins:overrides>`, which can override inherited
+   preset variables
+
+Presets do not have to define all of these, as we'll see in the ``TV Show Only Recent``
+preset.
+
+Inheriting Presets
+~~~~~~~~~~~~~~~~~~
+
+.. code-block:: yaml
+  :lineno-start: 5
+
+    TV Show:
+      preset:
+        - "Jellyfin TV Show by Date"
+        - "Max 1080p"
+
+The following snippet shows that the ``TV Show`` preset will inherit all properties of
+the prebuilt presets ``Jellyfin TV Show by Date`` and ``Max 1080p`` in that order.
+
+Order matters for preset inheritance. Bottom-most presets will override ones above them.
+
+It is highly advisable to use :ref:`prebuilt presets
+<config_reference/prebuilt_presets/index:Prebuilt Preset Reference>` as a starting point
+for custom preset building, as they do the work of preset building to ensure things show
+as expected in their respective media players. Read on to see how to override prebuilt
+preset specifics such as title.
+
+Defining Plugins
+~~~~~~~~~~~~~~~~
+
+.. code-block:: yaml
+  :lineno-start: 10
+
+      embed_thumbnail: True
+
+      throttle_protection:
+         sleep_per_download_s:
+           min: 2.2
+           max: 10.8
+         sleep_per_subscription_s:
+           min: 9.0
+           max: 14.1
+         max_downloads_per_subscription:
+           min: 10
+           max: 36
+
+Our ``TV Show`` sets two plugins, :ref:`throttle_protection
+<config_reference/plugins:throttle_protection>` and :ref:`embed_thumbnail
+<config_reference/plugins:embed_thumbnail>`. Each plugin's documentation shows the
+respective fields that they support.
+
+If an inherited preset defines the same plugin, the custom preset will use
+'merge-and-append' strategy to combine their definitions. What this means is:
+
+1. If the field is a map (i.e. has sub-params like ``sleep_per_download_s`` above) or
+   array, it will try to merge them
+2. If both the inherited preset and custom preset set the same exact field and value
+   (i.e. ``embed_thumbnail``) the custom preset will overwrite it
+
+Setting Override Variables
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: yaml
+  :lineno-start: 23
+
+      overrides:
+        tv_show_directory: "/ytdl_sub_tv_shows"
+
+All override variables reside underneath the :ref:`overrides
+<config_reference/plugins:overrides>` section.
+
+It is important to remember that individual subscriptions can override specific override
+variables.  When defining variables in a preset, it is best practice to define them with
+the intention that
+
+1. All subscriptions will use its value them
+2. Use them as placeholders to perform other logic, then have subscriptions or child
+   presets define their specific value
+
+For simplicity, we'll focus on (1) for now. The above snippet sets the
+``tv_show_directory`` variable to a file path. This variable name is specific to the
+prebuilt TV show presets.
+
+See the :ref:`prebuilt preset reference
+<config_reference/prebuilt_presets/index:Prebuilt Preset Reference>` to see all
+available variables that are overridable.
+
+
+Using Custom Presets in Subscriptions
+--------------------------------------
+
+Subscription files can use custom presets just like any other prebuilt preset.  Below
+shows a complete subscription file using the above two custom presets.
+
+.. code-block:: yaml
+
+  TV Show:
+    = Documentaries:
+      "NOVA PBS": "https://www.youtube.com/@novapbs"
+
+    = Kids | = TV-Y:
+      "Jake Trains": "https://www.youtube.com/@JakeTrains"
+
+  TV Show Only Recent:
+    = News:
+      "BBC News": "https://www.youtube.com/@BBCNews"
+
+Notice how we do not need to define ``tv_show_directory`` in the ``__preset__`` section
+like in prior examples. This is because our custom presets do the work of defining it.
+
+
+Reference Custom Config in the CLI
+----------------------------------
+
+Be sure to tell ytdl-sub to use your config by using the argument ``--config
+/path/to/config.yaml``.
+
+If you run ytdl-sub in the same directory, and the config file is named ``config.yaml``,
+it will use it by default.
+
+Visualizing a subscription in Preset form
+-----------------------------------------
+
+Subscription file syntax is designed to minimize boiler-plate when authoring new subscriptions.
+You can unpack any subscription using the ``inspect`` sub-command to see its boiler-plate *preset format*.
+
+.. code-block:: bash
+
+   ytdl-sub inspect --match "BBC News" /path/to/subscriptions.yaml
+
+This can be utilized for numerous purposes including:
+
+* Ensuring your custom preset is getting applied correctly.
+* Figuring out which variables set things like file names, metadata, etc.
+* Understanding how subscription syntax translates to preset representation.
+
+The default ``--level`` of inspect will fill in defined variables. Using ``--level original`` will
+present the subscription's raw layout with no fill.

docs/source/guides/getting_started/index.rst

@@ -0,0 +1,157 @@
+Getting Started
+===============
+
+
+Prerequisite Knowledge
+----------------------
+
+Using ``ytdl-sub`` requires some technical knowledge. You must be able to:
+
+- do `basic CLI shell navigation`_
+- read and write `YAML text files`_
+
+If you plan on using a :ref:`Docker headless image variant
+<guides/install/docker:headless image>` of ``ytdl-sub``, you can:
+
+- use ``$ nano /config/...`` to edit configuration files inside the container
+- or bind mount ``/config/`` as a Docker volume and use the editor of your choice from
+  the host
+
+Soon, it's time to start configuring ``ytdl-sub``. We provide a :doc:`./quick_start`
+with rigid, rote instructions on how to get a minimal configuration up and running, but
+if that serves all your needs, then you're probably better off with :ref:`one of the
+more user-friendly yt-dlp wrappers available <introduction:motivation>`. As a lower
+level tool with no GUI, most ``ytdl-sub`` users will need to understand at least some of
+how ``ytdl-sub`` works, how it "thinks". So before you start configuring ``ytdl-sub``,
+`read on <architecture>`_ to learn how ``ytdl-sub`` works.
+
+.. _`basic CLI shell navigation`:
+   https://developer.mozilla.org/en-US/docs/Learn_web_development/Getting_started/Environment_setup/Command_line
+.. _`YAML text files`: http://thomasloven.com/blog/2018/08/YAML-For-Nonprogrammers/
+
+
+
+Architecture
+------------
+
+For most users, ``ytdl-sub`` works as follows:
+
+Subscriptions use presets
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Run ``$ ytdl-sub sub`` to read :doc:`a subscription file <./subscriptions>` that defines
+what subscriptions to download and place into your media library. Each subscription
+selects which :doc:`presets <../../prebuilt_presets/index>` to apply. Those presets
+configure how each subscription is downloaded and placed in the media library.
+
+Presets configure plugins
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:doc:`A preset <../../prebuilt_presets/index>` is effectively a set of plugin
+configurations. Specifically, a preset consists of:
+
+- base presets that it inherits from and extends
+- plugin configurations
+
+When a preset has multiple base presets and more than one of those base presets
+configures the same keys for a plugin, the later/lower base preset overrides the plugin
+key configurations of earlier/higher base presets. Similarly, when the preset configures
+the same keys for a plugin that one of its base plugins configures, the preset
+configuration overrides the base presets.
+
+Plugins do the work
+~~~~~~~~~~~~~~~~~~~
+
+``ytdl-sub`` applies the plugins that the presets configure when it downloads a
+subscription. :doc:`The plugins <../../config_reference/plugins>` control how to run
+``yt-dlp``, which media in the subscription to download, how to collect and format
+metadata for those media, how to place the resulting files into your media library, and
+more.
+
+Presets and subscriptions accept overrides
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Presets accept override keys and values and the preset uses those overrides to modify
+their plugin configurations. Similarly, individual subscriptions can supply overrides of
+their presets for just that subscription.
+
+Subscriptions are grouped by indentation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Most subscriptions have more in common with each other than not. Thus, defining the
+presets and overrides for each subscription would result in mostly repetition and would
+multiply the burden of management for the user. The more subscriptions the more work.
+
+To avoid this redundant work, and so that the subscription configurations describe the
+intent of the user, subscriptions are nested/indented under parent/ancestor keys that
+define their shared configuration. To support this, ``ytdl-sub`` uses special handling
+of the ancestor YAML keys above each subscription. A subscription is the most
+nested/indented/descendant key that specifies the URLs for that subscription. The
+ancestor keys above that subscription describe the shared presets of that subscription
+and all the other descendant subscriptions under them.
+
+Genres are also more often shared between subscriptions than not. To accommodate that
+reality, the ancestor keys of subscriptions may also use :ref:`the special '= ...'
+prefix to pass specific overrides
+<config_reference/scripting/static_variables:subscription_indent_i>` supported by the
+preset. By convention in the pre-built media type presets, the first ``= ...`` value
+specifies the genre for all descendant subscriptions.
+
+Finally, ancestor keys may use :ref:`the '... | ...' special character
+<config_reference/subscription_yaml:multi keys>` to combine multiple presets and/or
+genres for the descendant subscriptions beneath.
+
+The configuration file extends pre-defined presets
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Users define additional presets in :doc:`their configuration file <./first_config>` that
+they then use in most of their subscriptions. Most user-defined presets extend the
+:doc:`../../prebuilt_presets/index` provided by ``ytdl-sub``.
+
+Minimize the work to only what's necessary
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Throttling and bans are a core problem for any web scraping tool, perhaps even more so
+for ``yt-dlp``, and no good actor *wants* to be an onerous burden on a
+service. Similarly, many web scraping use cases involve very large sets of data that are
+too big to process as a whole for performance. It's important to narrow the amount of
+data considered and minimize requests.
+
+To these ends, most presets tell ``yt-dlp`` not to consider files before the most
+recently downloaded file using :ref:`the 'break_on_existing' option
+<config_reference/plugins:ytdl_options>`. Similarly, and particularly for huge channels
+or playlists, most users should use either :ref:`an 'Only Recent' preset
+<prebuilt_presets/helpers:only recent>` and/or :ref:`the 'Chunk Downloads' preset
+<prebuilt_presets/helpers:chunk downloads>` to restrict the number of downloads
+considered.
+
+
+Caveats
+~~~~~~~
+
+Some of these descriptions are not technically complete. For example, a subscription may
+use no preset at all and will just run ``yt-dlp`` without any customization or post
+processing. The subscriptions file has special support for :ref:`overriding the presets
+of all subscriptions in the file <config_reference/subscription_yaml:file preset>`. The
+configuration file supports :ref:`a few special options
+<config_reference/config_yaml:Configuration File>` that are not about defining presets. See
+:doc:`the reference documentation <../../config_reference/index>` for technically
+complete details, but for almost all of the use cases served by ``ytdl-sub``, the above
+is accurate and representative.
+
+
+Next Steps
+----------
+
+With ``ytdl-sub`` installed and the above understood, the next step is to :doc:`start
+adding subscriptions <./subscriptions>`.
+
+
+.. toctree::
+  :hidden:
+
+  subscriptions
+  downloading
+  automating_downloads
+  first_config
+  quick_start

docs/source/guides/getting_started/quick_start.rst

@@ -0,0 +1,54 @@
+Quick Start
+===========
+
+:ref:`Again <guides/getting_started/index:prerequisite knowledge>`, if the following
+serves all your needs, then you're probably better off with :ref:`one of the more
+user-friendly yt-dlp wrappers available <introduction:motivation>`. If you still want to
+get ``ytdl-sub`` up and running quickly and without understanding, then follow these
+instructions to the letter.
+
+#. Install using :ref:`the official Docker GUI image variant <guides/install/docker:gui
+   image>`.
+
+#. Update the paths for your media library:
+
+   Edit :ref:`the subscriptions file <guides/install/docker:configuration>`. Near the
+   top, under ``__preset__:`` and then ``overrides:``, update the values under the
+   ``*_directory:`` keys with the correct paths for your media library *as they appear
+   inside the container*.
+
+#. Select your media library software:
+
+   Change the ``Plex TV Show by Date:`` *key itself* to the preset for your media
+   library software. See the comment above for the available options.
+
+#. Select the genre:
+
+   Under the library software preset key from the previous step, change the ``=
+   Documentaries`` *key itself* to the genre for this subscription prefixed with ``=
+   ...``. When adding other subscriptions that have the same genre, place them under the
+   same key.
+
+#. Update the subscription name and URL:
+
+   Under the genre key from the previous step, update the ``"NOVA PBS":`` key to the
+   directory name the downloaded files should be placed beneath. This directory will be
+   created under the ``tv_show_directory:`` from step #2. Then update the
+   ``"https://www.youtube.com/@novapbs"`` value to the URL of the channel or playlist
+   for this subscription.
+
+#. :ref:`Preview <guides/getting_started/downloading:preview>` and :ref:`Review
+   <guides/getting_started/downloading:review>` the subscription.
+
+#. Add the rest of your subscriptions:
+
+   Repeat steps #3-6 for each of your subscriptions. Be sure to repeat the preview and
+   review steps for each subscription. In general, move slowly and carefully review
+   everything. It's best to catch issues early :ref:`to avoid repeating downloads and to
+   minimize requests <guides/getting_started/index:minimize the work to only what's
+   necessary>`.
+
+#. Automate downloads:
+
+   :ref:`Set up ytdl-sub to run periodically
+   <guides/getting_started/automating_downloads:docker and unraid>`.

docs/source/guides/getting_started/subscriptions.rst

@@ -0,0 +1,167 @@
+Subscriptions
+=============
+
+Once you understand :ref:`how ytdl-sub works
+<guides/getting_started/index:architecture>`, it's time to start writing your
+:doc:`../../config_reference/subscription_yaml`.
+
+
+Media library paths
+-------------------
+
+Everyone's media library may use different paths so ``ytdl-sub`` can't provide
+defaults. Tell ``ytdl-sub`` where to put your media using :ref:`overrides
+<guides/getting_started/index:presets and subscriptions accept overrides>`:
+
+.. code-block:: yaml
+  :caption: subscriptions.yaml
+  :emphasize-lines: 3-
+
+  __preset__:
+    overrides:
+      tv_show_directory: "/tv_shows"
+      music_directory: "/music"
+      music_video_directory: "/music_videos"
+
+See the reference documentation for details about :ref:`the '__preset__:' special key
+<config_reference/subscription_yaml:file preset>`.
+
+
+Media library software and media types
+--------------------------------------
+
+Different media library software, such as `Jellyfin`_, `Kodi`_, Plex, or Emby, have
+different requirements for where media files are placed, how those files are named, how
+metadata is formatted, and more. Those software also have different requirements for
+different types of media, such as shows/series, music, music videos, etc.. Use
+:doc:`prebuilt presets <../../prebuilt_presets/index>` in :ref:`YAML keys
+<guides/getting_started/index:subscriptions are grouped by indentation>` to tell
+``ytdl-sub`` which media library software and media type to process downloaded files
+for.
+
+The actual subscription is defined in the lowest indentation level YAML keys. The
+example below defines a subscription named ``NOVA PBS`` to archive downloads from the
+entries in the ``https://www.youtube.com/@novapbs`` URL.
+
+.. code-block:: yaml
+  :caption: subscriptions.yaml
+  :emphasize-lines: 1,4
+
+  Jellyfin TV Show by Date:
+    "NOVA PBS": "https://www.youtube.com/@novapbs"
+
+  Bandcamp:
+    "Emily Hopkins": "https://emilyharpist.bandcamp.com/"
+
+.. _`Jellyfin`:
+   https://jellyfin.org/
+.. _`Kodi`:
+   https://kodi.tv/
+
+
+Which entries
+-------------
+
+The :doc:`helper presets <../../prebuilt_presets/helpers>` also provide support for
+controlling which entries are downloaded and archived. These presets are intended to be
+combined with the library software and media type presets.
+
+Combine presets using :ref:`the '.. | ...' special character
+<guides/getting_started/index:subscriptions are grouped by indentation>` in the YAML
+keys:
+
+.. code-block:: yaml
+  :caption: subscriptions.yaml
+  :emphasize-lines: 2,6
+
+  # Only download entries whose upload date is within the past 2 months:
+  Kodi TV Show by Date | Only Recent:
+    "NOVA PBS": "https://www.youtube.com/@novapbs"
+
+  # Only download 20 entries per run:
+  Soundcloud Discography | Chunk Downloads:
+    "UKNOWY": "https://soundcloud.com/uknowymunich"
+
+
+What format, quality, or resolution
+-----------------------------------
+
+The :doc:`media quality presets <../../prebuilt_presets/media_quality>` provide support
+for controlling which ``yt-dlp`` media "format" to download, such as ``1080p`` video
+resolution or ``320k`` audio bitrate.
+
+Users may also group and combine presets :ref:`using the YAML hierarchy
+<guides/getting_started/index:subscriptions are grouped by indentation>`. Subscriptions
+merge all the presets from their ancestor YAML keys. The hierarchy indentation depth may
+be as deep as needed to group your subscriptions for easy maintenance:
+
+.. code-block:: yaml
+  :caption: subscriptions.yaml
+  :emphasize-lines: 3,7,12
+
+  Jellyfin TV Show by Date | Only Recent:
+    # Download the highest resolution available:
+    Max Video Quality:
+      "NOVA PBS": "https://www.youtube.com/@novapbs"
+      "National Geographic": "https://www.youtube.com/@NatGeo"
+    # Download the highest resolution available that is 720p or less:
+    Max 720p:
+      "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
+
+  Soundcloud Discography | Chunk Downloads:
+    # Only download audio using the Opus codec, not MP3 or other codecs:
+    Max Opus Quality:
+      "UKNOWY": "https://soundcloud.com/uknowymunich"
+
+
+Genre and rating metadata
+-------------------------
+
+Presets may also support using arbitrary values from :ref:`YAML keys prefixed with '=
+...' <guides/getting_started/index:subscriptions are grouped by indentation>`. The ``=
+...`` prefix may be used at any indentation depth and may also be combined with presets
+and other ``= ...`` values using the ``... | ...`` special character to best group your
+subscriptions.
+
+:ref:`By convention <config_reference/scripting/static_variables:subscription_indent_i>`
+in the built-in library software and media type presets, the first ``= ...`` value
+specifies the genre for all descendant subscriptions. For the ``TV Show ...`` presets,
+the second ``= ...`` value specifies the rating for all descendant subscriptions:
+
+.. code-block:: yaml
+  :caption: subscriptions.yaml
+  :emphasize-lines: 1,3
+
+  = Kids:
+
+    Jellyfin TV Show by Date | = TV-Y:
+      "Jake Trains": "https://www.youtube.com/@JakeTrains"
+      "Kids Toys Play": "https://www.youtube.com/@KidsToysPlayChannel"
+
+    Soundcloud Discography:
+      "Foo Kids Band": "https://soundcloud.com/foo-kids-band"
+
+
+Override variables for one subscription
+---------------------------------------
+
+Most variable overrides aren't actually specific to just one subscription and should be
+set in :doc:`your own custom presets <./first_config>`. But use :ref:`the override mode
+'~...' prefix <config_reference/subscription_yaml:override mode>` when an override is
+specific to only one subscription and will never be shared with another:
+
+.. code-block:: yaml
+  :caption: subscriptions.yaml
+  :emphasize-lines: 2-
+
+  Jellyfin TV Show by Date:
+    "~NOVA PBS":
+      url: "https://www.youtube.com/@novapbs"
+      tv_show_directory: "/media/Unique/Series/Path"
+
+
+Next Steps
+----------
+
+Once you've defined your subscriptions, it's time to :doc:`test your configuration and
+try your first download <./downloading>`.

docs/source/guides/index.rst

@@ -0,0 +1,7 @@
+Guides
+======
+
+.. toctree::
+  install/index
+  getting_started/index
+  development/index

docs/source/guides/install/agnostic.rst

@@ -0,0 +1,49 @@
+====================
+Environment Agnostic
+====================
+
+The PIP install method is not recommended; use of this method may cause unintended
+requirement conflicts if you have other locally installed apps that depend on ffmpeg.
+
+
+PIP Install
+--------------
+
+You can install our `PyPI package <https://pypi.org/project/ytdl-sub/>`_.  Both ffmpeg
+and Python 3.10 or greater are required.
+
+.. code-block:: bash
+
+  python3 -m pip install -U ytdl-sub
+
+Install for Development
+=======================
+
+These environment-agnostic methods of installing ``ytdl-sub`` are meant for local
+development of ``ytdl-sub``. If you want to contribute your changes, please read
+:doc:`/guides/development/index`.
+
+Local Install
+--------------
+
+With a Python 3.10 virtual environment, you can clone and install the repo.
+
+.. code-block:: bash
+
+  git clone https://github.com/jmbannon/ytdl-sub.git
+  cd ytdl-sub
+
+  pip install -e .
+
+Local Docker Build
+-------------------
+
+Run ``make docker`` in the root directory of this repo to build the image. This will
+build the python wheel and install it in the Dockerfile.
+
+.. code-block:: bash
+
+  git clone https://github.com/jmbannon/ytdl-sub.git
+  cd ytdl-sub
+
+  make docker

docs/source/guides/install/docker.rst

@@ -0,0 +1,146 @@
+======
+Docker
+======
+
+The ``ytdl-sub`` Docker images use :lsio:`LSIO-based images <\ >` and install ytdl-sub
+on top. There are two flavors or variants to choose from. For a more user-friendly
+experience editing the `configuration`_, we recommend the `GUI image`_
+variant. :ref:`Docker Compose <guides/install/docker:install with docker compose>` is
+the recommended way of managing a ``ytdl-sub`` docker container.  See :ref:`Automating
+Downloads <guides/getting_started/automating_downloads:docker and unraid>` for how to
+automate running ``ytdl-sub`` in a container running either variant.
+
+
+GUI Image
+---------
+
+The GUI image is based on LSIO's :lsio-gh:`docker-code-server` to provide you full
+management of ``ytdl-sub``, such as file editing and terminal access, all within your
+browser using the VS Code web UI. See its documentation regarding environment variables
+and other details. Once running, open `the web UI`_ to edit the `configuration`_ and run
+``ytdl-sub``.
+
+.. _`the web UI`: http://localhost:8443
+
+
+Headless Image
+--------------
+
+The headless image is based on LSIO's :lsio-gh:`docker-baseimage-alpine`. Once running,
+the default command just starts services including cron for :ref:`Automating Downloads
+<guides/getting_started/automating_downloads:docker and unraid>` but otherwise doesn't
+run ``ytdl-sub``. You may run arbitrary ``ytdl-sub`` commands using the
+``--rm --user="${PUID}:${PGID}" --entrypoint="ytdl-sub"`` options to either ``$ docker
+run`` or ``$ docker compose run``. Overriding the image's ``ENTRYPOINT`` is important so
+that cron doesn't run ``ytdl-sub`` while you're running it manually.
+
+For example::
+
+  $ docker compose run --rm --user="${PUID}:${PGID}" --entrypoint="ytdl-sub" ytdl-sub sub
+
+.. note::
+
+   In `the recommended GUI image <gui image_>`_, the ``DEFAULT_WORKSPACE`` directory is
+   ``/config/ytdl-sub-configs/`` which is used throughout the documentation and
+   examples. In the headless images, that directory is just ``/config/``, so substitute
+   that path if using a headless image.
+
+
+Install with Docker Compose
+---------------------------
+
+Docker Compose provides a declarative way to configure and orchestrate containers which
+makes them easier to manage and re-use. Create a ``compose.yaml`` file in your project
+directory such as:
+
+.. code-block:: yaml
+   :caption: compose.yaml
+
+   services:
+     ytdl-sub:
+       # The GUI image variant:
+       image: ghcr.io/jmbannon/ytdl-sub-gui:latest
+       # Or use the headless image variant:
+       # image: ghcr.io/jmbannon/ytdl-sub:latest
+       # For CPU/GPU passthrough, use the GUI image above or the headless Ubuntu image:
+       # image: ghcr.io/jmbannon/ytdl-sub:ubuntu-latest
+       container_name: ytdl-sub
+       restart: unless-stopped
+       environment:
+         - TZ=America/Los_Angeles
+         # Set these as appropriate so your users can access the downloaded files in
+         # your library:
+         - PUID=1000
+         - PGID=1000
+         # Optionally passthrough your NVidia GPU:
+         # - NVIDIA_DRIVER_CAPABILITIES=all
+         # - NVIDIA_VISIBLE_DEVICES=all
+       volumes:
+         - <path/to/ytdl-sub/config>:/config
+         - <path/to/tv_shows>:/tv_shows  # optional
+         - <path/to/movies>:/movies  # optional
+         - <path/to/music_videos>:/music_videos  # optional
+         - <path/to/music>:/music  # optional
+       # Not necessary for the headless image variant:
+       ports:
+         - 8443:8443
+       # Optionally passthrough the CPU for hardware acceleration:
+       # devices:
+       #   - /dev/dri:/dev/dri
+       # Optionally passthrough the GPU:
+       # deploy:
+       #   resources:
+       #     reservations:
+       #       devices:
+       #         - capabilities: ["gpu"]
+
+
+Docker CLI
+----------
+
+You can run the container on an ad-hoc basis without Docker Compose using the Docker CLI
+instead. It will not restart if stopped for any reason, including rebooting the
+host. The following command is for the gui image:
+
+.. code-block:: bash
+
+  docker run -d \
+    --name=ytdl-sub \
+    -e PUID=1000 \
+    -e PGID=1000 \
+    -e TZ=America/Los_Angeles \
+    -p 8443:8443 \
+    -v <path/to/ytdl-sub/config>:/config \
+    -v <OPTIONAL/path/to/tv_shows>:/tv_shows \
+    -v <OPTIONAL/path/to/movies>:/movies \
+    -v <OPTIONAL/path/to/music_videos>:/music_videos \
+    -v <OPTIONAL/path/to/music>:/music \
+    ghcr.io/jmbannon/ytdl-sub-gui:latest
+
+See `the Docker reference <https://docs.docker.com/engine/reference/run/>`_ for further
+details.
+
+Environment Variables
+---------------------
+``ytdl-sub`` docker images support the following environment variables.
+
+.. csv-table:: Docker Environment Variables
+   :header: "Name", "Supported Values", "Description"
+   :widths: 15, 10, 60
+
+   "``PUID``", "integer", "User ID"
+   "``PGID``", "integer", "Group ID"
+   "``TZ``", "timezone", "Optional. Timezone to use in the logs. For supported values, see this `list <https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List>`_. "
+   "``CRON_SCHEDULE``", "cron schedule `format <https://crontab.guru/#0_*/6_*_*_*>`_", "Optional. Schedule to run the ``cron`` file in ytdl-sub's container. More info :ref:`here <guides/getting_started/automating_downloads:docker and unraid>`."
+   "``CRON_RUN_ON_START``", "true/false", "Optional. Whether to run the cron script on container start."
+   "``UPDATE_YT_DLP_ON_START``", "stable/nightly/master", "Optional. Whether to update yt-dlp to the latest configured version on container start."
+
+For the GUI image, you can set LSIO's underlying code-server `env variables <https://docs.linuxserver.io/images/docker-code-server/#environment-variables-e>`_ as well."
+
+Configuration
+-------------
+
+In these examples, the configuration files will be at
+``<path/to/ytdl-sub/config>/config.yaml`` and
+``<path/to/ytdl-sub/config>/subscriptions.yaml``. Start the container the first time to
+populate those files with default examples.

docs/source/guides/install/index.rst

@@ -0,0 +1,35 @@
+Install by Platform
+===================
+
+``ytdl-sub`` can be installed on the following platforms.
+
+All installations require a 64-bit CPU. 32-bit is not supported.
+
+.. margin:: 
+
+  .. tip::
+    
+    The recommended install method of ``ytdl-sub`` is one of our :doc:`docker containers
+    </guides/install/docker>`.
+
+:doc:`/guides/install/docker`
+
+:doc:`/guides/install/unraid`
+
+:doc:`/guides/install/linux`
+
+:doc:`/guides/install/windows`
+
+:doc:`/guides/install/agnostic`
+
+Once you've completed your installation, please refer to the
+:doc:`../getting_started/index` guide for next steps
+
+.. toctree::
+  :hidden:
+
+  docker
+  linux
+  unraid
+  windows
+  agnostic

docs/source/guides/install/linux.rst

@@ -0,0 +1,52 @@
+=====
+Linux
+=====
+
+``ytdl-sub`` should be installable using any Linux package manager, and requires ffmpeg
+to be installed.
+
+.. tab-set:: 
+
+  .. tab-item:: Linux
+
+    .. code-block:: bash
+
+      curl -L -o ytdl-sub https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub
+      chmod +x ytdl-sub
+      ./ytdl-sub -h
+
+    You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to
+    date:
+
+    .. code-block:: bash
+
+      curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz
+      tar -xf ffmpeg.tar.gz
+      chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffmpeg
+      chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffprobe
+
+      # May need sudo / root permissions to perform
+      mv ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/bin/ffmpeg
+      mv ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/bin/ffprobe
+
+  .. tab-item:: Linux ARM
+
+    .. code-block:: bash
+
+      curl -L -o ytdl-sub https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub_aarch64
+      chmod +x ytdl-sub
+      ./ytdl-sub -h
+
+    You can also install using yt-dlp's ffmpeg builds. This ensures your ffmpeg is up to
+    date:
+
+    .. code-block:: bash
+
+      curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz
+      tar -xf ffmpeg.tar.gz
+      chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg
+      chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe
+
+      # May need sudo / root permissions to perform
+      mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg /usr/bin/ffmpeg
+      mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe /usr/bin/ffprobe

docs/source/guides/install/unraid.rst

@@ -0,0 +1,29 @@
+======
+Unraid
+======
+
+You can install our :unraid:`unraid community apps <community/apps?q=ytdl-sub#r>`
+through the `Unraid Community Apps plugin <https://unraid.net/community/apps>`_.
+
+If you installed the ``ytdl-sub-gui`` app, the code-server will be running at
+http://localhost:8443 (replace ``localhost`` with the IP of the computer running Unraid
+if you aren't trying to access ``ytdl-sub`` on that computer). Open this page in a
+browser to access and interact with ``ytdl-sub``.
+
+If you installed the ``ytdl-sub`` app (headless), open the normal app-specific console
+to access and interact with ``ytdl-sub``. Once open, you must first run ``su abc -s
+/bin/bash`` to change to the non-root user. You can confirm that this command worked by
+running ``whoami`` and verifying that the result is ``abc``.
+
+.. warning::
+
+   If you use the below option to access the ``ytdl-sub`` console, be sure to run ``su
+   abc -s /bin/bash`` first thing. You can confirm that this command worked by running
+   ``whoami`` and verifying that the result is ``abc``. Do **NOT** run ``ytdl-sub`` as
+   the root user! Running as root will set the owner of all modified files to root,
+   which prevents most media managers and players from accessing the files.
+
+   .. figure:: ../../../images/unraid_badconsole.png
+     :alt:
+	The Unraid community app plugin GUI, with an arrow pointing at the "Console"
+	option in the dropdown after selecting ytdl-sub-gui

docs/source/guides/install/windows.rst

@@ -0,0 +1,17 @@
+=======
+Windows
+=======
+
+From powershell, run:
+
+.. code-block:: powershell
+
+  # Download ffmpeg/ffprobe dependencies from yt-dlp
+  curl.exe -L -o ffmpeg.zip https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl.zip
+  tar -xf ffmpeg.zip
+  move "ffmpeg-master-latest-win64-gpl\bin\ffmpeg.exe" "ffmpeg.exe"
+  move "ffmpeg-master-latest-win64-gpl\bin\ffprobe.exe" "ffprobe.exe"
+
+  # Download ytdl-sub
+  curl.exe -L -o ytdl-sub.exe https://github.com/jmbannon/ytdl-sub/releases/latest/download/ytdl-sub.exe
+  ytdl-sub.exe -h

docs/source/index.rst

@@ -0,0 +1,15 @@
+ytdl-sub User Guide
+===================
+
+.. toctree::
+  :maxdepth: 2
+  :titlesonly:
+
+  introduction
+  guides/index
+  prebuilt_presets/index
+  usage
+  config_reference/index
+  debugging
+  faq/index
+  deprecation_notices

docs/source/introduction.rst

@@ -0,0 +1,98 @@
+=================
+What is ytdl-sub?
+=================
+
+.. _yt-dlp: https://github.com/yt-dlp/yt-dlp
+.. _kodi: https://github.com/xbmc/xbmc
+.. _jellyfin: https://github.com/jellyfin/jellyfin
+.. _plex: https://github.com/plexinc/pms-docker
+.. _emby: https://github.com/plexinc/pms-docker
+
+``ytdl-sub`` is a command-line tool that builds on and orchestrates `yt-dlp`_ to
+download media from YouTube and/or other online services. It provides a declarative,
+expressive YAML configuration system that allows you to describe which media to download
+and how it should appear in your media library servers and applications such as
+`Jellyfin`_, `Plex`_, `Emby`_, `Kodi`_, modern music players, etc..
+
+To these ends, ``ytdl-sub``:
+
+- wraps and runs `yt-dlp`_, per your configuration to:
+
+  - download the media, remux and/or optionally transcode it
+
+- prepares additional metadata both embedded and in external files
+
+- renames the resulting files
+
+- places them in your library
+
+.. figure:: https://user-images.githubusercontent.com/10107080/182677243-b4184e51-9780-4094-bd40-ea4ff58555d0.PNG
+  :alt: The Jellyfin web interface, showing the thumbnails of various YouTube shows.
+
+  Youtube channels as TV shows in Jellyfin
+
+.. figure:: https://user-images.githubusercontent.com/10107080/182677256-43aeb029-0c3f-4648-9fd2-352b9666b262.PNG
+  :alt: The Jellyfin web interace, showing the thumbnails of various music videos starring the Red Hot Chili Peppers
+
+  Music videos and concerts in Jellyfin
+
+.. figure:: https://user-images.githubusercontent.com/10107080/182677268-d1bf2ff0-9b9c-4a04-98ec-443a67ada734.png
+  :alt: The Kodi app interface, showing a list of artists available to watch under the "Music videos" heading
+
+  Music videos and concerts in Kodi
+
+.. figure:: https://user-images.githubusercontent.com/10107080/182685415-06adf477-3dd3-475d-bbcd-53b0152b9f0a.PNG
+  :alt: The MusicBee app interface, showing a list of album artists and the thumbnails of all downloaded songs produced by the currently selected artist
+
+  SoundCloud albums and singles in MusicBee
+
+
+Motivation
+----------
+
+`yt-dlp`_ has grown into a well maintained, central repository of the intricate,
+inscrutable, and extensive technical knowledge required to automate downloading media
+from online services. When those services change their APIs or otherwise change
+behavior, `yt-dlp`_ is the central, low-level tool to update. It does a best-in-class
+job at that task, and it does that job more effectively by narrowing focus to just that.
+As much knowledge as it encapsulates and as well as it does that, it still requires a
+great deal of additional knowledge to make its output accessible to end-users. Mostly
+this gap is about extracting and formatting metadata and correctly placing the resulting
+output files in a media library.
+
+A number of tools, applications, and other projects have grown up around that central
+`yt-dlp`_ pillar to fill in those gaps, and this project was one of the early
+entrants. Many are `full-featured services that provide web UIs`_ including some that
+`provide media player web UIs`_. Most of those other projects necessarily narrow their
+scope to provide a more polished and integrated user experience.
+
+Similarly, ``ytdl-sub`` can run automatically to accomplish the same goals, but aims to
+serve users that need lower-level control and/or have use cases not covered by the more
+narrow scope of those other projects. To some degree, this makes this project
+intrinsically less user friendly and requires more technical experience or learning.
+
+Want something that "Just Works", try one of the other projects; we recommend
+`Pinchflat`_ as the next step towards that end. Want to download from more than just
+YouTube? Don't like the other restrictions inherent in the goals of those other
+projects? Have unique use cases? Then dig in, learn, and we hope ``ytdl-sub`` gives you
+enough rope and `a foot-gun`_ to get you there.
+
+.. _`full-featured services that provide web UIs`:
+   https://github.com/kieraneglin/pinchflat
+.. _`provide media player web UIs`:
+   https://www.tubearchivist.com/
+.. _`Pinchflat`: `full-featured services that provide web UIs`_
+.. _`a foot-gun`: https://en.wiktionary.org/wiki/footgun
+
+
+Why download instead of stream?
+-------------------------------
+
+Most of the tools in this `yt-dlp`_ ecosystem serve a similar set of larger, more
+general use cases, and so does ``ytdl-sub``:
+
+- Don't rely on profit-driven corporate persons to keep more obscure content available.
+- Even if they do, don't depend on them to make it possible to use it in different ways.
+- Even when you pay, don't count on them not inserting ads later.
+- Regardless, don't depend on them to curate content for yourself and/or your family.
+- Free yourself and/or your family from what the algorithm would feed them next.

docs/source/prebuilt_presets/helpers.rst

@@ -0,0 +1,237 @@
+
+
+
+==============
+Helper Presets
+==============
+
+.. hint::
+
+   See how to apply helper presets :doc:`here </prebuilt_presets/index>`
+
+
+Only Recent
+-----------
+
+To only download a recent number of videos, apply the ``Only Recent`` preset. Once a
+video's upload date is outside of the range, or you hit max files, older videos will be
+deleted automatically.
+
+.. code-block:: yaml
+
+   __preset__:
+     overrides:
+       # Set to a non-zero value to only keep this many files at once per sub
+       only_recent_max_files: 0
+       only_recent_date_range: "7days"
+
+   Plex TV Show by Date | Only Recent:
+
+     = Documentaries:
+       "NOVA PBS": "https://www.youtube.com/@novapbs"
+
+To prevent deletion of files, use the preset ``Only Recent Archive`` instead.
+
+
+Filter Keywords
+---------------
+
+``Filter Keywords`` can include or exclude media with any of the listed keywords. Both
+keywords and title/description are lower-cased before filtering.
+
+Default behavior for Keyword evaluation is ANY, meaning the filter will succeed if any
+of the keywords are present. This can be set to ANY or ALL using the respective
+``_eval`` variable.
+
+Supports the following override variables:
+
+* ``title_include_keywords``, ``title_include_eval``
+* ``title_exclude_keywords``, ``title_exclude_eval``
+* ``description_include_keywords``, ``title_exclude_eval``
+* ``description_exclude_keywords``, ``title_exclude_eval``
+
+.. tip::
+
+   Use the `~` tilda subscription mode to set a subscription's list override variables.
+   Tilda mode allows override variables to be set directly underneath it.
+
+   .. code-block:: yaml
+
+      Plex TV Show by Date | Filter Keywords:
+
+        = Documentaries:
+          "~NOVA PBS":
+            url: "https://www.youtube.com/@novapbs"
+            title_exclude_keywords:
+              - "preview"
+              - "trailer"
+
+          "~To Catch a Smuggler":
+            url: "https://www.youtube.com/@NatGeo"
+            title_include_keywords:
+              - "To Catch a Smuggler"
+
+        = Sports:
+          "~Maple Leafs Highlights":
+            url: "https://www.youtube.com/@NHL"
+            title_include_eval: "ALL"
+            title_include_keywords:
+              - "maple leafs"
+              - "highlights"
+
+
+Filter Duration
+---------------
+
+``Filter Duration`` can include or exclude media based on its duration.
+
+Supports the following override variables:
+
+* ``filter_duration_min_s``
+* ``filter_duration_max_s``
+
+.. tip::
+
+   Use the `~` tilda subscription mode to set a subscription's list override variables.
+   Tilda mode allows override variables to be set directly underneath it.
+
+   .. code-block:: yaml
+
+      Plex TV Show by Date | Filter Duration:
+
+        = Documentaries:
+          "~NOVA PBS":
+            url: "https://www.youtube.com/@novapbs"
+            filter_duration_min_s: 120  # Only download videos at least 2m long
+
+        = Sports:
+          "~Maple Leafs Highlights":
+            url: "https://www.youtube.com/@NHL"
+            filter_duration_max_s: 180  # Only get highlight videos less than 3m long
+
+
+Chunk Downloads
+---------------
+
+If you are archiving a large channel, ``ytdl-sub`` will try pulling each video's
+metadata from newest to oldest before starting any downloads. It is a long process and
+not ideal. A better method is to chunk the process by using the following preset:
+
+``Chunk Downloads``
+
+It will download videos starting from the oldest one, and only download 20 at a time by
+default. You can change this number by setting the override variable
+``chunk_max_downloads``.
+
+.. code-block:: yaml
+
+   __preset__:
+     overrides:
+       chunk_max_downloads: 20
+
+   Plex TV Show by Date:
+
+     # Chunk these ones
+     = Documentaries | Chunk Downloads:
+       "NOVA PBS": "https://www.youtube.com/@novapbs"
+       "National Geographic": "https://www.youtube.com/@NatGeo"
+
+     # But not these ones
+     = Documentaries:
+       "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
+
+Once the entire channel is downloaded, remove the usage of this preset. It will then
+pull metadata from newest to oldest again, and stop once it reaches a video that has
+already been downloaded.
+
+
+_throttle_protection
+--------------------
+
+.. note::
+
+   This preset is already a base preset of those higher-level presets that require it,
+   so users seldom need to use it directly, for example, unless they're writing presets
+   from scratch.
+
+This preset is primarily a sensible default configuration of :ref:`the
+'throttle_protection' plugin <config_reference/plugins:throttle_protection>` along with
+an override to disable the plugin:
+
+.. code-block:: yaml
+
+   overrides:
+     # Disable throttle protection:
+     enable_throttle_protection: false
+
+In addition to throttling by denying download requests, some services also throttle
+downloads by only allowing downloads of the lowest resolution quality. At the time of
+writing, only YouTube does this by allowing only 360p downloads when throttled. To work
+around this kind of throttling, this preset includes :ref:`an assertion
+<config_reference/scripting/scripting_functions:error functions>` that will stop
+downloading when ``ytdl-sub`` downloads a video at 360p or lower. It supports the
+following overrides:
+
+.. code-block:: yaml
+
+   overrides:
+     # Disable resolution quality throttle protection:
+     enable_resolution_assert: false
+     # Change the resolution below which to assume downloading is throttled:
+     resolution_assert_height_gte: 720
+
+.. _resolution assert handling:
+
+Handling Low Quality Videos
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A side effect from throttle protection's resolution assert is, if the only resolution available is 360p or lower, it will
+error. You can either disable resolution assert entirely (see above), or ignore specific titles in the subscription
+using the ``resolution_assert_ignore_titles`` variable. Add a subset of the title (case-sensitive) as a list entry
+to your subscription, like so:
+
+.. code-block:: yaml
+
+   # use tilda mode to set override variables to the subscription
+   "~My Subscription":
+     url: "https://youtube.com/@channel"
+     resolution_assert_ignore_titles:
+       - "This 360p Video Title"
+
+_url
+----
+
+All prebuilt presets share the same internal ``_multi_url`` preset which comes equipped with
+a few available customizations.
+
+Sibling Metadata
+~~~~~~~~~~~~~~~~
+
+*Sibling* refers to any entry within the same *playlist*. For channel downloads, this would
+imply **every** video that gets downloaded since yt-dlp treats the channel as the *playlist*.
+
+Setting the variable ``include_sibling_metadata`` will include all sibling metadata within
+each individual entry's metadata. This is used specifically for music presets. When downloading
+a playlist as an album for example, it will take the max year amongst all the other sibling's metadata
+to have a consistent album year that can be used in file or directory naming.
+
+Webpage URL
+~~~~~~~~~~~
+
+``ytdl-sub`` performs downloads in two stages.
+
+1. Metadata scrape from the original URL
+2. Individual entry downloads
+
+For step 2, ``ytdl-sub`` will use the ``webpage_url`` variable by default for the input URL to yt-dlp.
+This can be modified in case it's not working as expected by using the variable ``modified_webpage_url``.
+
+Example:
+
+.. code-block:: yaml
+  :caption:
+     Removes yt-dlp smuggle data from the URL
+
+   overrides:
+     modified_webpage_url: >-
+       { %regex_sub("#__youtubedl_smuggle=.*", "", webpage_url) }

docs/source/prebuilt_presets/index.rst

@@ -0,0 +1,37 @@
+================
+Prebuilt Presets
+================
+
+``ytdl-sub`` offers a number of built-in presets using best practices for formatting
+media in various players.
+
+.. hint::
+
+   Apply multiple presets to your subscriptions using pipes. Pipes can define multiple
+   presets and values on the same line to apply to all subscriptions nested below them.
+
+   .. code-block:: yaml
+     :caption:
+	Applies Max Video Quality preset to all TV shows, and Chunk Downloads preset to
+	some
+
+     Plex TV Show by Date | Max Video Quality:
+
+       = Documentaries | Chunk Downloads:
+         "NOVA PBS": "https://www.youtube.com/@novapbs"
+         "National Geographic": "https://www.youtube.com/@NatGeo"
+
+       = Documentaries:
+         "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
+
+For advanced users, you can review the prebuilt preset definitions :doc:`here
+</config_reference/prebuilt_presets/index>`.
+
+.. toctree:: 
+  :titlesonly:
+
+  tv_shows
+  music
+  music_videos
+  media_quality
+  helpers

docs/source/prebuilt_presets/media_quality.rst

@@ -0,0 +1,33 @@
+======================
+Media Quality Presets
+======================
+
+.. hint::
+
+   See how to apply media quality presets :doc:`here </prebuilt_presets/index>`
+
+
+Video
+-----
+
+The following presets set video quality specifications to yt-dlp.
+
+- ``Max Video Quality``
+- ``Max 2160p``
+- ``Max 1440p``
+- ``Max 1080p``
+- ``Max 720p``
+- ``Max 480p``
+
+
+Audio
+-----
+
+The following presets set audio quality specifications to yt-dlp.  These assume you are
+only extracting audio (no video).
+
+- ``Max Audio Quality``, format is determined by the source
+- ``Max MP3 Quality``
+- ``Max Opus Quality``
+- ``MP3 320k``
+- ``MP3 128k``

docs/source/prebuilt_presets/music.rst

@@ -0,0 +1,90 @@
+=============
+Music Presets
+=============
+
+Music downloadable by yt-dlp comes in many flavors. ``ytdl-sub`` offers a suite of
+various presets for handling some of the most popular forms of uploaded music content.
+
+
+YouTube Releases
+----------------
+
+Many artists, especially those auto-uploaded as ``Topics`` in YouTube have a section on
+their channel named "Releases", or "Albums and Singles". The ``YouTube Releases`` preset
+aims to scrape this *playlist of playlists*.
+
+Playlists are recognized as the album, and videos within it are tracks.
+
+.. code-block:: yaml
+
+   YouTube Releases:
+     = Jazz:  # Sets genre tag to "Jazz"
+       "Thelonious Monk": "https://www.youtube.com/@officialtheloniousmonk/releases"
+
+If you are only interested in a subset of albums, you can provide their playlists as
+separate values in the form of an array, like so:
+
+.. code-block:: yaml
+
+   YouTube Releases:
+     = Jazz:
+       "Thelonious Monk":
+         - "https://www.youtube.com/playlist?list=OLAK5uy_lcqINwfzkw73TPnAt6MlpB6V0gM9VzQu8"  # Monk on Monk
+         - "https://www.youtube.com/playlist?list=OLAK5uy_nhuvjuZOO3yLIWCbQzbiWfyzkGapSIuYw"  # Late Night Thelonious Monk
+
+
+YouTube Full Albums
+-------------------
+
+In many cases, albums are uploaded to YouTube as a single video, where each track as
+separated by either chapters or timestamps in a description. The ``YouTube Full Albums``
+preset will take each video and split it by the chapters to form an album.
+
+Videos are recognized as the album, and chapters within it are tracks.
+
+.. code-block:: yaml
+
+   YouTube Full Albums:
+     = Lofi:
+       "Game Chops": "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i"
+
+If you are only interested in a subset of albums, you can provide their video as
+separate values in the form of an array, like so:
+
+.. code-block:: yaml
+
+   YouTube Full Albums:
+     = Lofi:
+       "Game Chops":
+         - "https://www.youtube.com/watch?v=m7vBrD7LMLI"  # Zelda & Sleep Ensemble Collection
+         - "https://www.youtube.com/watch?v=w0XebCwSpKI"  # Study Buddy ~ video game lofi mix
+
+
+Soundcloud Discography
+----------------------
+
+SoundCloud tracks can be uploaded as either a single, part of an album, or a
+collaboration with another artist. At this time, ``SoundCloud Discography`` only scrapes
+singles and albums.  It will attempt to group tracks into albums before falling back to
+single format.
+
+.. code-block:: yaml
+
+   SoundCloud Discography:
+     = Chill Hop:
+       "UKNOWY": "https://soundcloud.com/uknowymunich"
+     = Synthwave:
+       "Lazerdiscs Records": "https://soundcloud.com/lazerdiscsrecords"
+       "Earmake": "https://soundcloud.com/earmake"
+
+
+Bandcamp
+--------
+
+Bandcamp albums and singles can be scraped using the ``Bandcamp`` preset.
+
+.. code-block:: yaml
+
+   Bandcamp:
+     = Lofi:
+       "Emily Hopkins": "https://emilyharpist.bandcamp.com/"

docs/source/prebuilt_presets/music_videos.rst

@@ -0,0 +1,5 @@
+===================
+Music Video Presets
+===================
+
+WIP

docs/source/prebuilt_presets/tv_shows.rst

@@ -0,0 +1,255 @@
+===============
+TV Show Presets
+===============
+
+
+Player-Specific Presets
+-----------------------
+
+``ytdl-sub`` provides player-specific versions of certain presets, which apply settings
+to optimize the downloads for that player.
+
+The following actions are taken based on the indicated player:
+
+Kodi
+~~~~
+
+* Everything that the Jellyfin version does
+* Enables ``kodi_safe`` NFOs, replacing 4-byte unicode characters that break kodi with
+  ``□``
+
+Jellyfin
+~~~~~~~~
+
+* Places any season-specific poster art in the main show folder
+* Generates NFO tags
+
+Emby
+~~~~
+
+* Places any season-specific poster art in the main show folder
+* Generates NFO tags
+
+  * For named seasons, creates a ``season.nfo`` file per season
+
+Plex
+~~~~~~~~
+
+* :ref:`Special sanitization
+  <config_reference/scripting/entry_variables:title_sanitized_plex>` of numbers so Plex
+  doesn't recognize numbers that are part of the title as the episode number
+* Converts all downloaded videos to the mp4 format
+* Places any season-specific poster art into the season folder
+
+----------------------------------------------
+
+
+TV Show by Date
+---------------
+
+TV Show by Date will organize something like a YouTube channel or playlist into a tv
+show, where seasons and episodes are organized using upload date.
+
+Example
+~~~~~~~
+
+Must define ``tv_show_directory``. Available presets:
+
+* ``Kodi TV Show by Date``
+* ``Jellyfin TV Show by Date``
+* ``Emby TV Show by Date``
+* ``Plex TV Show by Date``
+
+.. code-block:: yaml
+
+   __preset__:
+     overrides:
+       tv_show_directory: "/tv_shows"
+
+   Plex TV Show by Date:
+
+     # Sets genre tag to "Documentaries"
+     = Documentaries:
+       "NOVA PBS": "https://www.youtube.com/@novapbs"
+       "National Geographic": "https://www.youtube.com/@NatGeo"
+       "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
+
+     # Sets genre tag to "Kids", "TV-Y" for content rating
+     = Kids | = TV-Y:
+       "Jake Trains": "https://www.youtube.com/@JakeTrains"
+       "Kids Toys Play": "https://www.youtube.com/@KidsToysPlayChannel"
+
+     = Music:
+       # TV show subscriptions can support multiple urls and store in the same TV Show
+       "Rick Beato":
+         - "https://www.youtube.com/@RickBeato"
+         - "https://www.youtube.com/@rickbeato240"
+
+Advanced Usage
+~~~~~~~~~~~~~~
+
+If you prefer a different season/episode organization method, you can set the following
+override variables.
+
+.. code-block:: yaml
+
+   __preset__:
+     overrides:
+       tv_show_directory: "/tv_shows"
+       tv_show_by_date_season_ordering: "upload-year-month"
+       tv_show_by_date_episode_ordering: "upload-day"
+
+Or for a specific preset
+
+.. code-block:: yaml
+
+       "~Kids Toys Play":
+          url: "https://www.youtube.com/@KidsToysPlayChannel"
+          tv_show_by_date_season_ordering: "upload-year-month"
+          tv_show_by_date_episode_ordering: "upload-day"
+
+The following are supported. Be sure the combined season + episode ordering include the
+year, month, day, i.e. upload-year + upload-month-day.
+
+Season Ordering
+"""""""""""""""
+
+``tv_show_by_date_season_ordering`` supports one of the following:
+
+* ``upload-year`` (default)
+* ``upload-year-month``
+* ``release-year``
+* ``release-year-month``
+
+Episode Ordering
+""""""""""""""""
+
+``tv_show_by_date_episode_ordering`` supports one of the following:
+
+* ``upload-month-day`` (default)
+* ``upload-month-day-reversed``
+
+  * Reversed means more recent episodes appear at the top of a season by having a lower
+    value.
+* ``upload-day``
+* ``release-day``
+* ``release-month-day``
+* ``release-month-day-reversed``
+* ``download-index``
+
+  * Episodes are numbered by the download order. **NOTE**: this is fetched using the
+    length of the download archive. Do not use if you intend to remove old videos.
+
+TV Show by Date presets use the following for defaults:
+
+.. code-block:: yaml
+
+   tv_show_by_date_season_ordering: "upload-year"
+   tv_show_by_date_episode_ordering: "upload-month-day"
+
+TV Show Collection
+------------------
+
+TV Show Collections set each URL as its own season. If a video belongs to multiple URLs
+(i.e. a channel and a channel's playlist), the video will only download once and reside
+in the higher-numbered season.
+
+Two main use cases of a collection are:
+   1. Organize a YouTube channel TV show where Season 1 contains any video not in a
+      'season playlist', Season 2 for 'Playlist A', Season 3 for 'Playlist B', etc.
+   2. Organize one or more YouTube channels/playlists, where each season represents a
+      separate channel/playlist.
+
+Today, ytdl-supports up to 40 seasons with 11 URLs per season.
+
+Example
+~~~~~~~
+
+Must define ``tv_show_directory``. Available presets:
+
+* ``Kodi TV Show Collection``
+* ``Jellyfin TV Show Collection``
+* ``Emby TV Show Collection``
+* ``Plex TV Show Collection``
+
+.. code-block:: yaml
+
+   __preset__:
+     overrides:
+       tv_show_directory: "/tv_shows"
+
+   Plex TV Show Collection:
+     = Music:
+       # Prefix with ~ to set specific override variables
+       "~Beyond the Guitar":
+         s01_name: "Videos"
+         s01_url: "https://www.youtube.com/c/BeyondTheGuitar"
+         s02_name: "Covers"
+         s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W"
+
+Other notable features include:
+
+* TV show poster info is pulled from the first URL in s01.
+* Duplicate videos in different URLs (channel /videos vs playlist) will not download twice.
+
+  * The video will attributed to the season with the highest number.
+* Individual seasons support both single and multi URL.
+* s00 is supported for specials.
+
+.. code-block:: yaml
+
+       "~Beyond the Guitar":
+         s00_name: "Specials"
+         s00_url:
+           - "https://www.youtube.com/watch?v=vXzguOdulAI"
+           - "https://www.youtube.com/watch?v=IGwYDvaGAz0"
+         s01_name: "Videos"
+         s01_url:
+           - "https://www.youtube.com/c/BeyondTheGuitar"
+           - "https://www.youtube.com/@BeyondTheGuitarAcademy"
+         s02_name: "Covers"
+         s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W"
+
+Advanced Usage
+~~~~~~~~~~~~~~
+
+If you prefer a different episode organization method, you can set the following
+override variables.
+
+.. code-block:: yaml
+
+   __preset__:
+     overrides:
+       tv_show_directory: "/tv_shows"
+       tv_show_collection_episode_ordering: "release-year-month-day"
+
+Or for a specific preset
+
+.. code-block:: yaml
+
+       "~Beyond the Guitar":
+         tv_show_collection_episode_ordering: "release-year-month-day"
+         s01_name: "Videos"
+         s01_url: "https://www.youtube.com/c/BeyondTheGuitar"
+         s02_name: "Covers"
+         s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W"
+
+
+The following are supported.
+
+Episode Ordering
+""""""""""""""""
+
+``tv_show_collection_episode_ordering`` supports one of the following:
+
+* ``upload-year-month-day`` (default)
+* ``upload-year-month-day-reversed``
+* ``release-year-month-day``
+* ``release-year-month-day-reversed``
+* ``playlist-index``
+
+  * Only use ``playlist-index`` episode formatting for playlists that will be fully
+    downloaded once and never again. Otherwise, indices can change.
+* ``playlist-index-reversed``
+
+TV Show Collection presets use upload-year-month-day as the default.

docs/source/requirements.txt

@@ -0,0 +1,3 @@
+sphinx-book-theme==1.1.0
+sphinx-copybutton==0.5.2
+sphinx-design==0.5.0

docs/source/usage.rst

@@ -0,0 +1,146 @@
+Usage
+=====
+
+.. code-block::
+
+  ytdl-sub [GENERAL OPTIONS] {sub,dl,view} [COMMAND OPTIONS]
+
+For Windows users, it would be ``ytdl-sub.exe``
+
+
+General Options
+---------------
+
+CLI options common to all sub-commands. Must be specified before the sub-command, for
+example ``$ ytdl-sub --dry-run sub ...``:
+
+.. code-block:: text
+
+  -h, --help            show this help message and exit
+  -v, --version         show program's version number and exit
+  -c CONFIGPATH, --config CONFIGPATH
+                        path to the config yaml, uses config.yaml if not provided
+  -d, --dry-run         preview what a download would output, does not perform any video downloads or writes to output directories
+  -l quiet|info|verbose|debug, --log-level quiet|info|verbose|debug
+                        level of logs to print to console, defaults to verbose
+  -t TRANSACTIONPATH, --transaction-log TRANSACTIONPATH
+                        path to store the transaction log output of all files added, modified, deleted
+  -st, --suppress-transaction-log
+                        do not output transaction logs to console or file
+  -nc, --suppress-colors
+                        do not use colors in ytdl-sub output
+  -m MATCH [MATCH ...], --match MATCH [MATCH ...]
+                        match subscription names to one or more substrings, and only run those subscriptions
+
+
+Subscriptions Options
+---------------------
+
+Download all subscriptions specified in each :doc:`subscriptions file
+<./guides/getting_started/subscriptions>`.
+
+.. code-block::
+
+   ytdl-sub [GENERAL OPTIONS] sub [SUBPATH ...]
+
+``SUBPATH`` is one or more paths to subscription files and defaults to
+``./subscriptions.yaml`` if none are given.  It will use the config specified by
+``--config``, or ``./config.yaml``, if not provided.
+
+.. code-block:: text
+  :caption: Additional Options
+
+  -u, --update-with-info-json
+                        update all subscriptions with the current config using info.json files
+  -o DL_OVERRIDE, --dl-override DL_OVERRIDE
+                        override all subscription config values using `dl` syntax, i.e. --dl-override='--ytdl_options.max_downloads 3'
+
+
+Download Options
+----------------
+
+Download a single subscription in the form of CLI arguments instead of from :doc:`a
+subscriptions file <./guides/getting_started/subscriptions>`:
+
+.. code-block::
+
+  ytdl-sub [GENERAL OPTIONS] dl [SUBSCRIPTION ARGUMENTS]
+
+``SUBSCRIPTION ARGUMENTS`` are the same as YAML arguments, but use periods (``.``)
+instead of indents. For example, you can represent this subscription:
+
+.. code-block:: yaml
+
+  rick_a:
+    preset:
+      - "tv_show"
+    overrides:
+      tv_show_name: "Rick A"
+      url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
+
+Using the command:
+
+.. code-block:: bash
+
+  ytdl-sub dl \
+      --preset "tv_show" \
+      --overrides.tv_show_name "Rick A" \
+      --overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
+
+See how to shorten commands using :ref:`download aliases <config_reference/config_yaml:dl_aliases>`.
+
+
+View Options
+------------
+
+Preview the source variables for a given URL. Helpful to create new subscriptions:
+
+.. code-block::
+
+   ytdl-sub view [-sc] [URL]
+
+.. code-block:: text
+  :caption: Additional Options
+
+  -sc, --split-chapters
+                        View source variables after splitting by chapters
+
+CLI to SUB Options
+------------------
+Convert yt-dlp cli arguments to ytdl-sub `ytdl_options` arguments.
+
+.. code-block::
+
+   ytdl-sub cli-to-sub [YT-DLP ARGS]
+
+Inspect
+-------
+Inspect a single subscription's underlying preset representation.
+This can be utilized for numerous purposes including:
+
+* Ensuring your custom preset is getting applied correctly.
+* Figuring out which variables set things like file names, metadata, etc.
+* Understanding how subscription syntax translates to preset representation.
+
+Usage:
+
+.. code-block:: bash
+
+   ytdl-sub inspect --match "Game Chops" --mock 'title=Lets Play' examples/music_subscriptions.yaml
+
+.. code-block:: text
+  :caption: Additional Options
+
+     -l 0,1,2,3, --level 0,1,2,3
+                           level of inspection to perform:
+                               0 - original   present the subscription as-is
+                               1 - fill       fill in defined values
+                               2 - resolve    resolve all possible variables (default)
+                               3 - internal   resolve all variables to their internal representation
+
+     -m MATCH [MATCH ...], --match MATCH [MATCH ...]
+                           match subscription names to one or more substrings, and only run those subscriptions
+     -o DL_OVERRIDE, --dl-override DL_OVERRIDE
+                           override all subscription config values using `dl` syntax, i.e. --dl-override='--ytdl_options.max_downloads 3'
+     -k VAR=VALUE, --mock VAR=VALUE
+                           ability to mock one or more variable values, i.e. --mock 'title=Lets Play'

docs/usage.rst

@@ -1,84 +0,0 @@
-Usage
-=======
-
-.. code-block::
-
-   ytdl-sub [GENERAL OPTIONS] {sub,dl,view} [COMMAND OPTIONS]
-
-For Windows users, it would be ``ytdl-sub.exe``
-
-General Options
----------------
-
-General options must be specified before the command (i.e. ``sub``).
-
-.. code-block:: text
-
-  -h, --help            show this help message and exit
-  -v, --version         show program's version number and exit
-  -c CONFIGPATH, --config CONFIGPATH
-                        path to the config yaml, uses config.yaml if not provided
-  -d, --dry-run         preview what a download would output, does not perform any video downloads or writes to output directories
-  -l quiet|info|verbose|debug, --log-level quiet|info|verbose|debug
-                        level of logs to print to console, defaults to info
-  -t TRANSACTIONPATH, --transaction-log TRANSACTIONPATH
-                        path to store the transaction log output of all files added, modified, deleted
-  -st, --suppress-transaction-log
-                        do not output transaction logs to console or file
-
-Sub Options
------------
-Download all subscriptions specified in each ``SUBPATH``.
-
-.. code-block::
-
-   ytdl-sub [GENERAL OPTIONS] sub [SUBPATH ...]
-
-``SUBPATH`` is one or more paths to subscription files, uses ``subscriptions.yaml`` if not provided.
-It will use the config specified by ``--config``, or ``config.yaml`` if not provided.
-
-Download Options
------------------
-Download a single subscription in the form of CLI arguments.
-
-.. code-block::
-
-   ytdl-sub [GENERAL OPTIONS] dl [SUBSCRIPTION ARGUMENTS]
-
-``SUBSCRIPTION ARGUMENTS`` are exactly the same as YAML arguments, but use periods (``.``) instead
-of indents for specifying YAML from the CLI. For example, you can represent this subscription:
-
-.. code-block:: yaml
-
-   rick_a:
-     preset:
-       - "tv_show"
-     overrides:
-       tv_show_name: "Rick A"
-       url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
-
-Using the command:
-
-.. code-block:: bash
-
-   ytdl-sub dl \
-       --preset "tv_show" \
-       --overrides.tv_show_name "Rick A" \
-       --overrides.url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
-
-See how to shorten commands using
-`download aliases <https://ytdl-sub.readthedocs.io/en/latest/config.html#ytdl_sub.config.config_validator.ConfigOptions.dl_aliases>`_.
-
-View Options
------------------
-.. code-block::
-
-   ytdl-sub view [-sc] [URL]
-
-.. code-block:: text
-
-  -sc, --split-chapters
-                        View source variables after splitting by chapters
-
-
-Preview the source variables for a given URL. Helps when creating new configs.

examples/README.md

@@ -1,7 +1,13 @@
 # Example Configurations
-This directory shows how you can use ytdl-sub for various use cases. These
+This directory shows how use ytdl-sub's built-in presets to start downloading immediately with no
-are the configs I personally use and have incorporated as part of the e2e tests.
+configuration required. Simply run:
 
-Each example has a `config.yaml` and `subscription.yaml`. The config defines
+### Unix
-_how_ you format your media, whereas the subscription defines _what_ you 
+```commandline
-download plus some additional configuring if needed.
+ytdl-sub sub tv_show_subscriptions.yaml
+```
+
+### Windows
+```commandline
+./ytdl-sub.exe sub tv_show_subscriptions.yaml
+```

examples/advanced/tv_show_config.yaml

@@ -0,0 +1,101 @@
+###############################################################################
+# Top-level configurations to apply umask and write log files
+configuration:
+  umask: "002"
+  persist_logs:
+    logs_directory: '/config/logs'
+    keep_successful_logs: False
+
+presets:
+
+  ###############################################################################
+  # Set tv_show_directory here instead of in the subscriptions file
+  tv_show_paths:
+    overrides:
+      tv_show_directory: "/tv_shows"
+
+  ###############################################################################
+  # Filter out any YouTube shorts
+  no_shorts:
+    match_filters:
+      filters:
+        - "original_url!*=/shorts/"
+
+  ###############################################################################
+  # Remove all the following sponsorblock sections
+  sponsorblock:
+    chapters:
+      sponsorblock_categories:
+        - "outro"
+        - "selfpromo"
+        - "preview"
+        - "interaction"
+        - "sponsor"
+        - "music_offtopic"
+        - "intro"
+      remove_sponsorblock_categories: "all"
+      force_key_frames: False
+
+  ###############################################################################
+  # Wait 2 days before downloading in hopes to get more accurate sponsorblock
+  sponsorblock_wait:
+    # Import the sponsorblock preset defined above
+    preset:
+      - "sponsorblock"
+
+    date_range:
+      before: "today-2days"
+
+
+  ###############################################################################
+  # base preset to use on all TV Show-based subscriptions
+  base:
+    preset:
+      - "Kodi TV Show by Date"  # Set intended player
+      - "best_video_quality"    # prebuilt preset to get best quality
+      - "tv_show_paths"
+
+    # Embed chapters into video files
+    chapters:
+      embed_chapters: True
+
+    # Embed English subtitles into video files (supports more)
+    subtitles:
+      embed_subtitles: True
+      languages:
+        - "en"
+      allow_auto_generated_subtitles: True
+
+    # ytdl_options lets you pass any arg into yt-dlp's Python API
+    ytdl_options:
+      # Set the cookie file
+      # cookiefile: "/config/ytdl-sub-configs/youtube_cookies.txt"
+
+      # For YouTube, get English metadata if multiple languages are present
+      extractor_args:
+        youtube:
+          lang:
+            - "en"
+
+  ###############################################################################
+  # Custom preset to archive an entire channel
+  TV Show Full Archive:
+    preset:
+      - "base"
+      - "sponsorblock_wait"  # wait for sponsorblock when full-archiving
+
+  ###############################################################################
+  # Custom preset to only fetch and keep recent videos.
+  # Format the videos in reverse order, so the first video is the most recent
+  # Also include the prebuilt "Only Recent" preset
+  TV Show Only Recent:
+    preset:
+      - "base"
+      - "sponsorblock"
+      - "no_shorts"
+      - "season_by_year__episode_by_month_day_reversed"
+      - "Only Recent"
+
+    overrides:
+      only_recent_date_range: "2months"
+      only_recent_max_files: 30

examples/advanced/tv_show_subscriptions.yaml

@@ -0,0 +1,29 @@
+# Subscriptions using custom presets made in `tv_show_config.yaml`
+
+TV Show Full Archive:
+  = Documentaries | = TV-PG:
+    "NOVA PBS": "https://www.youtube.com/@novapbs"
+    "National Geographic": "https://www.youtube.com/@NatGeo"
+    "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
+
+  = Kids | = TV-Y:
+    "Jake Trains": "https://www.youtube.com/@JakeTrains"
+    "Kids Toys Play": "https://www.youtube.com/@KidsToysPlayChannel"
+
+  = Gardening | = TV-Y:
+    # TV Show presets support multiple URLs as a list
+    "Gardening with Ciscoe":
+      - "https://www.youtube.com/@gardeningwithciscoe4430"
+      - "https://www.youtube.com/playlist?list=PLi8V8UemxeG6lo5if5H5g5EbsteELcb0_"
+      - "https://www.youtube.com/playlist?list=PLsJlQSR-KjmaQqqJ9jq18cF6XXXAR4kyn"
+      - "https://www.youtube.com/watch?v=2vq-vPubS5I"
+
+TV Show Only Recent:
+  = News | = TV-14:
+    # Subscriptions can prefix a tilda to specify override variables
+    # to set only for that subscriptions
+    "~BBC News":
+      url: "https://www.youtube.com/@BBCNews"  # use url2, url3, ... for multi-url in this form
+      only_recent_date_range: "2weeks"
+    "Frontline PBS": "https://www.youtube.com/@frontline"
+    "Whitehouse": "https://www.bitchute.com/channel/zWsYVmCOu4JA/"  # Supports non-YT sites

examples/music_audio_config.yaml

@@ -1,194 +0,0 @@
-# This config builds presets to download and format the following use-cases:
-#   - YouTube
-#     - Songs
-#       - From a single video
-#     - Albums
-#       - From a video, where each song is represented by chapters
-#       - From a playlist where each video is a song on the album
-#     - Discographies
-#       - From a channel or a channel's set of playlists
-#   - Bandcamp
-#     - Track, Album, Artist URL as a Discography
-#   - Soundcloud
-#     - Track, Album, Artist URL as a Discography
-#
-# All files will look like:
-#
-#   music_directory/
-#     Artist/
-#       [2022] Some Single/
-#         01 - Some Single.mp3
-#         folder.jpg
-#       [2023] Latest Album/
-#         01 - Track Title.mp3
-#         02 - Another Track.mp3
-#         folder.jpg
-#
-# The idea is to format files under music_directory as `Artist/Album/Song`. Singles that do not
-# belong to an album will be represented like albums with a single track.
-#
-# Each file will be properly tagged regardless of file extension (suppers flac, ogg, mp3, etc)
-# and should show up nicely in music players that take advantage of tags.
-configuration:
-  working_directory: '.ytdl-sub-downloads'
-  umask: "002"
-
-presets:
-
-  # The `base` preset that represents how all files will be structured after downloading.
-  # Every other preset afterwards will inherit this preset.
-  base:
-    # Store all music under our music_directory (to be set as an override variable).
-    # Store each resulting file with its full track path, and treat thumbnails as the album covers.
-    #
-    # Maintain a download archive. This will produce a hidden file in your music directory
-    # containing an archive of all audio previously downloaded. This makes it so we do not
-    # re-download files we already have.
-    output_options:
-      output_directory: "{music_directory}"
-      file_name: "{track_full_path}"
-      thumbnail_name: "{album_cover_path}"
-      maintain_download_archive: True
-
-    # Set break_on_existing to True. This will stop fetching any more metadata once we
-    # hit a video/audio that we already have downloaded.
-    ytdl_options:
-      break_on_existing: True
-
-    # Extract any audio from files using this codec and quality.
-    audio_extract:
-      codec: "mp3"
-      quality: 320
-
-    # Set these music tags on every resulting audio file.
-    # It is recommended to keep most of this as-is, and use override
-    # variables to set them to be what you want.
-    music_tags:
-      artist: "{track_artist}"
-      artists: "{track_artist}"
-      albumartist: "{track_artist}"
-      albumartists: "{track_artist}"
-      title: "{track_title}"
-      album: "{track_album}"
-      track: "{track_number}"
-      tracktotal: "{track_total}"
-      year: "{track_year}"
-      genre: "{track_genre}"
-
-    # Optionally embed the thumbnail into the track
-    embed_thumbnail: False
-
-    # For every configurable field, make it an override variable,
-    # so we can carefully tune different use-cases by only modifying override variables.
-    overrides:
-      # Track Overrides. By default, set overrides to make each song its own album
-      track_title: "{title}"
-      track_album: "{title}"
-      track_artist: "{channel}"
-      track_number: "1"
-      track_number_padded: "01"
-      track_total: "1"
-      track_year: "{upload_year}"
-      track_genre: "Unset"
-      # Filename Overrides
-      track_file_name: "{track_number_padded} - {track_title_sanitized}.{ext}"
-      album_file_name: "folder.{thumbnail_ext}"
-      # Directory Name Overrides
-      music_directory: "OVERRIDE THIS WITH YOUR MUSIC DIRECTORY"
-      artist_dir: "{track_artist_sanitized}"
-      album_dir: "[{track_year}] {track_album_sanitized}"
-      # Full Filepath Overrides
-      track_full_path: "{artist_dir}/{album_dir}/{track_file_name}"
-      album_cover_path: "{artist_dir}/{album_dir}/{album_file_name}"
-
-####################################################################################################
-
-  # Make the 'single` preset accept a single URL using the override variable 'url'
-  # Each audio file will reside in its own album.
-  single:
-    # Inherit from `base`
-    preset: "base"
-
-    download:
-      url: "{url}"
-
-####################################################################################################
-
-  # Make the 'albums_from_playlists' preset format audio files to reside under albums, where
-  # each album is a playlist. If a file downloaded using this preset is not part of a playlist,
-  # it will default to how it'd look as a `single`.
-  albums_from_playlists:
-    # Inherit from single
-    preset: "single"
-
-    # Override various track properties using playlist variables.
-    overrides:
-      track_album: "{playlist_title}"
-      track_number: "{playlist_index}"
-      track_number_padded: "{playlist_index_padded}"
-      track_total: "{playlist_count}"
-      track_year: "{playlist_max_upload_year}"
-
-####################################################################################################
-
-  # Make the 'albums_from_chapters' preset format audio files to reside under an album, where the
-  # video itself is an album containing all the songs in it represented by chapters.
-  albums_from_chapters:
-    # Inherit from single
-    preset: "single"
-
-    # Embed chapters if present. If no chapters are present, allow parsing comments that contain
-    # timestamps to each song, and use those as chapters.
-    chapters:
-      embed_chapters: True
-      allow_chapters_from_comments: True
-
-    # Split each file by its chapters. If a file does not have chapters, simply 'pass' on it and
-    # process the next audio/video file.
-    split_by_chapters:
-      when_no_chapters: "pass"
-
-    # Override various track properties using chapter variables.
-    overrides:
-      track_title: "{chapter_title}"   # Chapter title is the track title
-      track_album: "{title}"           # Video's title is the album title
-      track_number: "{chapter_index}"
-      track_number_padded: "{chapter_index_padded}"
-      track_total: "{chapter_count}"
-
-####################################################################################################
-
-  # Make the 'soundcloud_discography' preset specially made for ripping SoundCloud artists.
-  # We will use the 'multi_url' approach to download both albums and non-album tracks
-  soundcloud_discography:
-    preset: "base"
-
-    # Download using the multi_url strategy
-    download:
-      # The first URL will be all the artist's tracks.
-      # Treat these as singles - an album with a single track
-      - url: "{sc_artist_url}/tracks"
-        variables:
-          sc_track_album: "{title}"
-          sc_track_number: "1"
-          sc_track_number_padded: "01"
-          sc_track_total: "1"
-          sc_track_year: "{upload_year}"
-      # Set the second URL to the artist's albums. If a track belongs to both
-      # to an album and tracks (in the URL above), it will resolve to this
-      # URL and include the album metadata we set below.
-      - url: "{sc_artist_url}/albums"
-        variables:
-          sc_track_album: "{playlist_title}"
-          sc_track_number: "{playlist_index}"
-          sc_track_number_padded: "{playlist_index_padded}"
-          sc_track_total: "{playlist_count}"
-          sc_track_year: "{playlist_max_upload_year}"
-
-    # Override various track properties using playlist variables.
-    overrides:
-      track_album: "{sc_track_album}"
-      track_number: "{sc_track_number}"
-      track_number_padded: "{sc_track_number_padded}"
-      track_total: "{sc_track_total}"
-      track_year: "{sc_track_year}"

examples/music_audio_subscriptions.yaml

@@ -1,137 +0,0 @@
-# Define any number of subscriptions in a single file, or use multiple
-# files to organize (i.e by use-case, website, genre, etc)
-#
-# Each example show-cases every use case:
-#   - YouTube
-#     - Songs
-#       - From a single video
-#     - Albums
-#       - From a video, where each song is represented by chapters
-#       - From a playlist where each video is a song on the album
-#     - Discographies
-#       - From a channel or a channel's set of playlists
-#   - Bandcamp
-#     - Track, Album, Artist URL as a Discography
-#   - Soundcloud
-#     - Track, Album, Artist URL as a Discography
-#
-####################################################################################################
-# YOUTUBE - PLAYLIST OF ALBUMS
-# See https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i
-#
-# Downloads the GameChops Albums' playlist. Each video in the playlist is an album with
-# chapters representing songs.
-game_chops:
-  preset: "albums_from_chapters"
-
-  # Perform regex to extract title from the chapter's title to not include the track number.
-  # Configuring this hard is a bit overkill, but I'm a perfectionist :-)
-  regex:
-    from:
-      chapter_title:
-        match:
-          - "^\\d+\\.\\.(.*)"  # Captures 'title' from '1..title'
-        capture_group_names:
-          - "captured_track_title"
-        capture_group_defaults:
-          - "{chapter_title}"
-
-  # Explicitly set the URL, track artist, and genre.
-  # Override various track properties using the regex variables we extracted.
-  overrides:
-    url: "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i"
-    track_artist: "GameChops"
-    track_genre: "Lofi"
-    track_title: "{captured_track_title}"
-
-
-####################################################################################################
-# YOUTUBE - MANY ALBUMS FROM MANY ARTISTS UPLOADED BY A CHANNEL
-# See https://www.youtube.com/@heavymetalofeasternbloc
-#
-# Downloads the entire 'Heavy Metal from the Eastern Bloc' channel. Each video they upload
-# is an album from Eastern Europe with chapters representing songs.
-eastern_bloc:
-  # Inherit albums from chapters
-  preset: "albums_from_chapters"
-
-  # Perform regex on many fields (title, description, chapter title) to extract as much
-  # metadata as possible
-  regex:
-    skip_if_match_fails: False  # Error if regex match fails
-    from:
-      title:
-        match:
-          - "^(.*) - (.*) \\|\\|.*"  # Captures artist, album from 'artist - album ||...'
-          - "^(.*) - (.*) \\[.*"  # Captures artist, ablum from 'artist - album [...'
-          - "^(.*) - (.*)"  # Captures artist, ablum from 'artist - album'
-        capture_group_names:
-          - "captured_track_artist"
-          - "captured_track_album"
-      description:
-        match:
-          - "Genre:\\s*(.*)\\s*\n(?:Rec.+|Year):\\s*.*(\\d{4})\\s*\\n"  # Captures genre and recorded year
-        capture_group_names:
-          - "captured_track_genre"
-          - "captured_track_year"
-      chapter_title:
-        match:
-          - "^(?:\\d+\\.\\s*|)(.*)"  # Captures title from '1. title'
-        capture_group_names:
-          - "captured_track_title"
-        capture_group_defaults:
-          - "{chapter_title}"
-
-  # Explicitly set the URL.
-  # Override various track properties using the regex variables we extracted.
-  overrides:
-    url: "https://www.youtube.com/@heavymetalofeasternbloc"
-    track_artist: "{captured_track_artist}"
-    track_album: "{captured_track_album}"
-    track_title: "{captured_track_title}"
-    track_year: "{captured_track_year}"
-    track_genre: "Eastern Bloc {captured_track_genre}"
-
-
-####################################################################################################
-# BANDCAMP - DISCOGRAPHY
-# See https://emilyharpist.bandcamp.com/
-#
-# Downloads Emily's entire bandcamp discography. yt-dlp represents albums as playlist.
-emily_hopkins:
-  preset: "albums_from_playlists"
-  regex:
-    from:
-      title:
-        match:
-          - "^Emily Hopkins - (.*)"  # Captures 'title' from 'Emily Hopkins - title'
-        capture_group_names:
-          - "captured_track_title"
-        capture_group_defaults:
-          - "{title}"
-
-  # Explicitly set the URL, track artist, and genre.
-  # Override various track properties using the regex variables we extracted.
-  overrides:
-    url: "https://emilyharpist.bandcamp.com/"
-    track_artist: "Emily Hopkins"
-    track_genre: "Lofi"
-    track_title: "{captured_track_title}"
-
-
-####################################################################################################
-# SOUNDCLOUD - DISCOGRAPHY
-# See https://soundcloud.com/jessebannon
-#
-# Downloads my acoustic 'album' and various tracks from SoundCloud using the soundcloud
-# discography preset. I'm not very good so keep your expectations low :-)
-jmbannon:
-  # Inherit soundcloud_discography preset
-  preset: "soundcloud_discography"
-
-  # Explicitly set the SoundCloud artist url, track artist, and genre.
-  overrides:
-    sc_artist_url: "sc_artist_url"
-    track_artist: "jmbannon"
-    track_genre: "Acoustic"
-

examples/music_subscriptions.yaml

@@ -0,0 +1,52 @@
+# Files will be stored in the form of:
+#
+#   music_directory/
+#     Artist/
+#       [2022] Some Single/
+#         01 - Some Single.mp3
+#         folder.jpg
+#       [2023] Latest Album/
+#         01 - Track Title.mp3
+#         02 - Another Track.mp3
+#         folder.jpg
+
+# Override variables globally for all subscriptions
+__preset__:
+  overrides:
+    music_directory: "/music"
+
+# Supports downloading YouTube /releases tab. Also works for any playlist (or playlist of playlists)
+# where each video is a single track.
+YouTube Releases:
+  = Jazz:  # Sets genre tag to "Jazz"
+    "Lester Young": "https://www.youtube.com/channel/UCsItMF6_fP754ihIsSRLk5A/playlists"
+    "Thelonious Monk": "https://www.youtube.com/@theloniousmonk3870/releases"
+    "Stan Getz": "https://www.youtube.com/@stangetzofficial/releases"
+    "Art Blakey": "https://www.youtube.com/channel/UCMki-b0zfAQiMQ0nbsrIuBQ/playlists"
+
+# Supports downloading playlists or individual videos where a single video is a full album.
+# ytdl-sub will split the album based on video chapters, and make each chapter a track.
+YouTube Full Albums:
+  = Lofi:
+    "Game Chops": "https://www.youtube.com/playlist?list=PLBsm_SagFMmdWnCnrNtLjA9kzfrRkto4i"
+
+# Supports downloading a SoundCloud's artists albums + singles. Be sure to not include
+# any extension after the SoundCloud artist's name in the URL.
+SoundCloud Discography:
+  = Chill Hop:
+    "UKNOWY": "https://soundcloud.com/uknowymunich"
+  = Electronic:
+    "Italo Brutalo": "https://soundcloud.com/italobrutalo"
+    "SURVIVE": "https://soundcloud.com/s-u-r-v-i-v-e"
+    "French79": "https://soundcloud.com/french79music"
+    "VHS Dreams": "https://soundcloud.com/vhsdreamsofficial"
+  = Synthwave:
+    "Lazerdiscs Records": "https://soundcloud.com/lazerdiscsrecords"
+    "Earmake": "https://soundcloud.com/earmake"
+    "Poly Poly": "https://soundcloud.com/poly_poly"
+
+# Supports downloading a Bandcamp artist
+Bandcamp:
+  = Lofi:
+    "Emily Hopkins": "https://emilyharpist.bandcamp.com/"
+

examples/music_video_subscriptions.yaml

@@ -0,0 +1,45 @@
+# Files will be stored in the form of:
+#
+#   music_videos/
+#     Elton John/
+#       Elton John - Rocketman.jpg
+#       Elton John - Rocketman.mp4
+#    System of a Down/
+#       System of a Down - Chop Suey.jpg
+#       System of a Down - Chop Suey.mp4
+#       ...
+
+# Override variables globally for all subscriptions
+__preset__:
+  overrides:
+    music_video_directory: "/music_videos"
+
+# Choose between Jellyfin/Kodi/Plex Music Videos preset:
+#   - Plex Music Videos:
+#   - Jellyfin Music Videos:
+#   - Kodi Music Videos:
+"Plex Music Videos":
+
+  = Pop:  # Sets genre tag to "Pop"
+    "Rick Astley": "https://www.youtube.com/playlist?list=PLlaN88a7y2_plecYoJxvRFTLHVbIVAOoc"
+    "Michael Jackson": "https://www.youtube.com/playlist?list=OLAK5uy_mnY03zP6abNWH929q2XhGzWD_2uKJ_n8E"
+
+  = Blues:
+    "Eric Clapton": "https://www.youtube.com/playlist?list=PLABGggHhsbEeaRtdzqnxYoEINsJE_4GF4"
+
+  = Rock:
+    # Prefixing with '+' puts the subscription into 'map-mode'.
+    # Music video presets in map-mode support grouping videos into different
+    # categories, which get set on the album field.
+    #
+    # URLs can either be strings, or maps that can overload title, year, date
+    "+ Guns N' Roses":
+      Music Videos:
+        - "https://www.youtube.com/playlist?list=PLOTK54q5K4INNXaHKtmXYr6J7CajWjqeJ"
+      Concerts:
+        - title: "Live at The Ritz - New York City"
+          year: "1988"
+          url: "https://www.youtube.com/watch?v=OldpIhHPsbs"
+        - title: "Live at The Hollywood Bowl"
+          date: "2023-01-11"
+          url: "https://www.youtube.com/watch?v=Z7hutGlvq9I"

examples/music_videos_config.yaml

@@ -1,65 +0,0 @@
-# This example shows how to download and format a music video OR playlist
-# of music videos to display in Kodi as a music video. The format will
-# look like:
-#
-#   /path/to/Music Videos
-#     Elton John/
-#       Elton John - Rocketman-thumb.jpg
-#       Elton John - Rocketman.mp4
-#       Elton John - Rocketman.nfo
-#    System of a Down/
-#       System of a Down - Chop Suey-thumb.jpg
-#       System of a Down - Chop Suey.mp4
-#       System of a Down - Chop Suey.nfo
-#       ...
-#
-configuration:
-  working_directory: '.ytdl-sub-downloads'
-
-presets:
-  music_video:
-    # We will only use a single URL to download music video(s).
-    # Make {url} an override variable to set later.
-    download:
-      - "{url}"
-
-    # For advanced YTDL users only; any YTDL parameter can be set here.
-    # To download age-restricted videos, you will need to set your cookie
-    # file here as a ytdl parameter. For more info, see
-    # https://ytdl-sub.readthedocs.io/en/latest/faq.html#download-age-restricted-youtube-videos
-    ytdl_options:
-      # cookiefile: "path/to/cookie_file.txt
-      break_on_existing: True  # Stop downloads if it already exists
-
-    # For each video downloaded, set the file and thumbnail name here.
-    # We set both with {music_video_name}, which is a variable we define in
-    # the overrides section further below to represent consistent naming format.
-    output_options:
-      output_directory: "{music_video_directory}"
-      file_name: "{music_video_name}.{ext}"
-      thumbnail_name: "{music_video_name}-thumb.jpg"
-      info_json_name: "{music_video_name}.{info_json_ext}"
-      maintain_download_archive: True
-
-    # For each video downloaded, add a music video NFO file for it. Populate it
-    # with tags that Kodi will read and use to display it in the music or music
-    # videos section.
-    nfo_tags:
-      nfo_name: "{music_video_name}.nfo"
-      nfo_root: "musicvideo"
-      tags:
-        artist: "{artist}"
-        title: "{title}"
-        album: "Music Videos"
-        year: "{upload_year}"
-
-    # Overrides is a section where we can define our own variables, and use them in
-    # any other section. We define our music video directory and episode file name
-    # here, which gets reused above for the video, thumbnail, and NFO file.
-    # Recommended to override the artist variable since {channel} is not always
-    # the artist's exact name.
-    overrides:
-      music_video_directory: "path/to/Music Videos"
-      music_video_name: "{artist_sanitized}/{artist_sanitized} - {title_sanitized}"
-      artist: "{channel}"
-

examples/music_videos_subscriptions.yaml

@@ -1,24 +0,0 @@
-###############################################################################
-# DOWNLOAD MUSIC VIDEO PLAYLIST
-john_smith:
-  preset: "music_video"
-  overrides:
-    url: "https://youtube.com/playlist?list=UCsvn_Po0SmunchJYtttWpOxMg"
-    artist: "John Smith and the Instrument Players"
-
-###############################################################################
-# DOWNLOAD SINGLE MUSIC VIDEO VIA CLI
-
-# It is not always ideal to download all of an artist's music videos.
-# Maybe you only like one song of theirs. We can reuse our preset
-# to download a single video instead.
-#
-# Of course, defining yaml configuration to download a single video once
-# and never again seems weird. Instead, we can perform this download via
-# command-line:
-#
-#     ytdl-sub dl                                                        \
-#         --preset "music_video"                                         \
-#         --overrides.url "https://youtube.com/watch?v=QhY6r6oAErg"      \
-#         --overrides.artist "John Smith and the Instrument Players"
-#

examples/tv_show_config.yaml

@@ -1,127 +0,0 @@
-# This config uses prebuilt presets included with ytdl-sub to download and format
-# channels from YouTube or other sites supported by yt-dlp into a TV show for
-# your favorite player. The directory format will look something like
-#
-#   /tv_shows
-#       /Season 2021
-#           s2021.e0317 - Pattys Day Video-thumb.jpg
-#           s2021.e0317 - Pattys Day Video.mp4
-#           s2021.e0317 - Pattys Day Video.nfo
-#       /Season 2022
-#           s2022.e1225 - Merry Christmas-thumb.jpg
-#           s2022.e1225 - Merry Christmas.mp4
-#           s2022.e1225 - Merry Christmas.nfo
-#       poster.jpg
-#       fanart.jpg
-#       tvshow.nfo
-#
-# The idea is to use dates as numerics to represent season and episode numbers.
-configuration:
-  working_directory: '.ytdl-sub-downloads'
-
-presets:
-
-  # Your main TV show preset - all your tv show subscriptions will use this.
-  tv_show:
-    preset:
-      # Choose one of the following player types:
-      # - "kodi_tv_show_by_date"
-      # - "jellyfin_tv_show_by_date"
-      # - "plex_tv_show_by_date"
-
-      - "kodi_tv_show_by_date"  # replace with desired player type
-
-      # Choose one of the following season/episode formats:
-      # - "season_by_year__episode_by_month_day"
-      # - "season_by_year_month__episode_by_day"
-      # - "season_by_year__episode_by_month_day_reversed"
-      # - "season_by_year__episode_by_download_index"
-
-      - "season_by_year__episode_by_month_day"  # replace with desired season/episode format
-
-      # Include any of the presets listed below in your 'main preset' if you want
-      # it applied to every TV show. Or, use them on the individual subscriptions.
-      # - "only_recent_videos"
-      # - "add_subtitles"
-      # - "sponsorblock"
-      # - "include_info_json"
-
-    # To download age-restricted videos, you will need to uncomment and set your cookie
-    # file here as a ytdl parameter. For more info, see
-    # https://ytdl-sub.readthedocs.io/en/latest/faq.html#download-age-restricted-youtube-videos
-    #
-    # ytdl_options:
-    #   cookiefile: "/config/cookie_file.txt"  # replace with actual cookie file path
-
-    overrides:
-      tv_show_directory: "/tv_shows"  # replace with path to tv show directory
-      # Fields in the prebuilt preset that can be changed:
-      #
-      # episode_title: "{upload_date_standardized} - {title}"
-      # episode_plot: "{webpage_url}"  # source variable for the video description is {description}
-
-####################################################################################################
-
-  # Preset to only download and keep recent videos
-  only_recent_videos:
-
-    # Only download videos within the download_range
-    date_range:
-      after: "today-{download_range}"
-
-    # Stops fetching metadata if the video is out of range (saves time + bandwidth)
-    ytdl_options:
-      break_on_reject: True
-
-    # Deletes any videos older than download_range. WARNING: do not use
-    # "season_by_year__episode_by_download_index" if you plan to delete older videos
-    output_options:
-      keep_files_after: "today-{download_range}"
-
-    # Set the duration of download_range, defaults to 2 months
-    overrides:
-      download_range: "2months"
-
-####################################################################################################
-
-  # Preset to download subtitles (either by file or embedded)
-  add_subtitles:
-    subtitles:
-      # Embed subtitles into the video
-      embed_subtitles: True
-      # And/or download them as a file. Uncomment to download as file:
-      # subtitles_name: "{episode_file_path}.{lang}.{subtitles_ext}"
-      # subtitles_type: "srt"
-
-      languages: "en"  # supports list of multiple languages
-      allow_auto_generated_subtitles: True  # allow auto subtitles
-
-####################################################################################################
-
-  # Preset to cut sponsor segments from videos
-  sponsorblock:
-    # If you download using cron, it is wise to add a delay before downloading ad-filled content to
-    # give folks time to submit sponsor segments. Uncomment to wait 2 days before download a video.
-    # date_range:
-    #  before: "today-2days"
-
-    chapters:
-      # Remove all of these sponsorblock categories
-      sponsorblock_categories:
-        - "intro"
-        - "outro"
-        - "selfpromo"
-        - "preview"
-        - "interaction"
-        - "sponsor"
-        - "music_offtopic"
-      remove_sponsorblock_categories: "all"
-      force_key_frames: False
-
-####################################################################################################
-
-  # Preset for the hoarders who want to also save the info.json file
-  include_info_json:
-    output_options:
-      info_json_name: "{episode_file_path}.{info_json_ext}"
-

examples/tv_show_subscriptions.yaml

@@ -1,16 +1,77 @@
-# Define any number subscriptions in a single file, or use multiple
+# This example downloads the entirety of a channel (not limited to YouTube).
-# files to organize (i.e by genre)
+# Files will be stored in the form of:
-
+#
-rick_a:
+#   /tv_shows
-  # Mix and match any number of presets to use. You can either lump all settings into
+#       /Season 2021
-  # a single preset to save some lines, or modularize them and use many.
+#           s2021.e031701 - Pattys Day Video-thumb.jpg
-  preset:
+#           s2021.e031701 - Pattys Day Video.mp4
-    - "tv_show"
+#           s2021.e031701 - Pattys Day Video.nfo
-    - "only_recent_videos"
+#           s2021.e031702 - Second Pattys Day Video-thumb.jpg
+#           s2021.e031702 - Second Pattys Day Video.mp4
+#           s2021.e031702 - Second Pattys Day Video.nfo
+#       /Season 2022
+#           s2022.e122501 - Merry Christmas-thumb.jpg
+#           s2022.e122501 - Merry Christmas.mp4
+#           s2022.e122501 - Merry Christmas.nfo
+#       poster.jpg
+#       fanart.jpg
+#       tvshow.nfo
+#
+# The idea is to use dates as numerics to represent season and episode numbers.
 
+# Overrides to the prebuilt presets
+__preset__:
   overrides:
-    # Required to specify for every subscription
+    tv_show_directory: "/tv_shows"  # Root folder of all ytdl-sub TV Shows
-    tv_show_name: "Rick A"
+
-    url: "https://www.youtube.com/channel/UCuAXFkgsw1L7xaCfnd5JJOw"
+    # For 'Only Recent' preset, only keep vids within this range and limit
-    # Optional
+    only_recent_date_range: "2months"
-    tv_show_genre: "Music"
+    only_recent_max_files: 30
+
+# Choose the player you intend to use by setting the top-level key to be either:
+#   - Plex TV Show by Date:
+#   - Jellyfin TV Show by Date:
+#   - Emby TV Show by Date:
+#   - Kodi TV Show by Date:
+Plex TV Show by Date:
+
+  # Sets genre tag to "Documentaries"
+  = Documentaries:
+    "NOVA PBS": "https://www.youtube.com/@novapbs"
+    "National Geographic": "https://www.youtube.com/@NatGeo"
+    "Cosmos - What If": "https://www.youtube.com/playlist?list=PLZdXRHYAVxTJno6oFF9nLGuwXNGYHmE8U"
+
+  # Sets genre tag to "Kids", "TV-Y" for content rating
+  = Kids | = TV-Y:
+    "Jake Trains": "https://www.youtube.com/@JakeTrains"
+    "Kids Toys Play": "https://www.youtube.com/@KidsToysPlayChannel"
+
+  # Sets genre tag to "Music"
+  = Music:
+    # Subscriptions can support multiple urls and store in the same
+    # TV Show
+    "Rick Beato":
+      - "https://www.youtube.com/@RickBeato"
+      - "https://www.youtube.com/@rickbeato240"
+
+  # Set "News" for genre, use `Only Recent` preset to only store videos uploaded recently
+  = News | Only Recent:
+    "BBC News": "https://www.youtube.com/@BBCNews"
+
+# Sets URLs to be explicit seasons. If a video resides in multiple URLs, it will
+# only appear once in the higher-numbered season. This is how you can separate a channel's
+# videos and playlists you are interested in.
+#
+# Choose the player you intend to use by setting the top-level key to be either:
+#   - Plex TV Show Collection:
+#   - Jellyfin TV Show Collection:
+#   - Emby TV Show Collection:
+#   - Kodi TV Show Collection:
+Plex TV Show Collection:
+  = Music:
+    "~Beyond the Guitar":
+      s01_name: "Videos"
+      s01_url: "https://www.youtube.com/c/BeyondTheGuitar"
+
+      s02_name: "Music Videos"
+      s02_url: "https://www.youtube.com/playlist?list=PLE62gWlWZk5NWVAVuf0Lm9jdv_-_KXs0W"

pyproject.toml

@@ -1,10 +1,69 @@
-[tool.isort]
+[project]
-profile = "black"
+name ="ytdl-sub"
-line_length = 100
+dynamic = [ "version" ]
-force_single_line = true
+authors = [ { name = "Jesse Bannon" } ]
+description = "Automate downloading metadata generation with YoutubeDL"
+readme = "README.md"
+requires-python = ">=3.10"
+license = { file = "LICENSE" }
+classifiers = [
+    "Topic :: Multimedia :: Sound/Audio",
+    "Topic :: Multimedia :: Video",
+    "License :: Public Domain",
+    "Environment :: Console",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+]
+dependencies = [
+    "yt-dlp[default]==2026.6.9",
+    "colorama~=0.4",
+    "mergedeep~=1.3",
+    "mediafile~=0.12",
+    "PyYAML~=6.0",
+]
+urls = { Homepage = "https://github.com/jmbannon/ytdl-sub" }
 
-[tool.black]
+[build-system]
-line_length = 100
+requires = [ "setuptools >= 67.0" ]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+platforms = [ "Unix" ]
+
+[tool.setuptools.dynamic]
+version = { attr = "ytdl_sub.__pypi_version__" }
+[tool.setuptools.package-dir]
+"" = "src"
+[tool.setuptools.package-data]
+"*" = ["*.yaml"]
+[tool.setuptools.packages.find]
+where = ["src"]
+
+
+[project.optional-dependencies]
+test = [
+    "coverage[toml]>=6.3,<8.0",
+    "pytest>=7.2,<10.0",
+    "pytest-rerunfailures>=14,<17",
+]
+lint = [
+    "pylint==4.0.5",
+    "ruff==0.15.16",
+]
+docs = [
+    "sphinx>=7,<10",
+    "sphinx-rtd-theme>=2,<4",
+    "sphinx-book-theme~=1.0",
+    "sphinx-copybutton~=0.5",
+    "sphinx_design~=0.6",
+]
+build = [
+    "build~=1.2",
+    "twine>=5,<7",
+    "pyinstaller~=6.5",
+]
+[project.scripts]
+ytdl-sub = "ytdl_sub.main:main"
 
 [tool.pylint.MASTER]
 disable = [
@@ -15,30 +74,43 @@ disable = [
     "R0913", # Too many arguments
     "R0901", # too-many-ancestors
     "R0902", # too-many-instance-attributes
+    "R1711", # useless-return
+    "R0917", # too many positional arguments
     "W0511", # TODO
 ]
 
 load-plugins = "pylint.extensions.docparams"
 
-[tool.pydocstyle]
-inherit = false
-match = "[^test_].*\\.py"
-ignore = [
-    "D100", # docstring in public module
-    "D101", # Missing docstring in public class (covered by pylint)
-    "D104", # docstring in public package
-    "D107", # docstring in init
-    "D200", # One-line should fit on one line
-    "D203", # 1 blank line before class docstring
-    "D205", # 1 blank line between summary and description
-    "D212", # Multi-line should start at first line
-    "D400", # Should end with a period
-    "D401", # Return vs Returns
-    "D413", # Missing blank line after last section
-    "D415", # Should end with a period
-]
-
 [tool.coverage.run]
 include = [
     "src/*"
 ]
+
+[tool.coverage.report]
+exclude_also = [
+    "raise UNREACHABLE.*",
+]
+
+# ruff
+[tool.ruff]
+line-length = 100
+indent-width = 4
+
+# Assume Python 3.10
+target-version = "py310"
+
+[tool.ruff.lint]
+extend-select = ["I"]
+
+[tool.ruff.format]
+# Like Black, use double quotes for strings.
+quote-style = "double"
+
+# Like Black, indent with spaces, rather than tabs.
+indent-style = "space"
+
+# Like Black, respect magic trailing commas.
+skip-magic-trailing-comma = false
+
+# Like Black, automatically detect the appropriate line ending.
+line-ending = "auto"

setup.cfg

@@ -1,59 +0,0 @@
-[metadata]
-name = ytdl-sub
-version = attr:ytdl_sub.__pypi_version__
-author = Jesse Bannon
-description = Automate downloading and metadata generation with YoutubeDL
-long_description = file: README.md
-long_description_content_type= text/markdown
-author_email = use_github_issues@nope.com
-url = https://github.com/jmbannon/ytdl-sub
-license = GNUv3
-platforms = Unix
-classifiers =
-    Topic :: Multimedia :: Sound/Audio
-    Topic :: Multimedia :: Video
-    License :: Public Domain
-    Environment :: Console
-    Programming Language :: Python :: 3.10
-    Programming Language :: Python :: 3.11
-
-[options.entry_points]
-console_scripts =
-    ytdl-sub = ytdl_sub.main:main
-
-[options]
-package_dir =
-    = src
-packages=find:
-
-install_requires =
-    yt-dlp==2023.7.6
-    argparse==1.4.0
-    colorama==0.4.6
-    mergedeep==1.3.4
-    mediafile==0.12.0
-    PyYAML==5.3.1
-
-[options.package_data]
-* = *.yaml
-
-[options.packages.find]
-where=src
-
-[options.extras_require]
-test =
-    coverage[toml]==6.3.2
-    pytest==7.1.1
-lint =
-    black==22.3.0
-    isort==5.10.1
-    pylint==2.13.5
-    pydocstyle[toml]==6.1.1
-docs =
-    sphinx==4.5.0
-    sphinx-rtd-theme==1.0.0
-build =
-    build
-    twine
-    pyinstaller
-

src/ytdl_sub/__init__.py

@@ -1,2 +1,2 @@
-__pypi_version__ = "2023.03.24.post7"
+__pypi_version__ = "2023.10.22.post3"
-__local_version__ = "2023.03.24+14e4a4b"
+__local_version__ = "2023.10.22+bfba4f0"

src/ytdl_sub/cli/entrypoint.py

@@ -0,0 +1,346 @@
+import gc
+import os
+import random
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from yt_dlp.utils import sanitize_filename
+
+from ytdl_sub.cli.output_summary import output_summary
+from ytdl_sub.cli.output_transaction_log import (
+    _maybe_validate_transaction_log_file,
+    output_transaction_log,
+)
+from ytdl_sub.cli.parsers.cli_to_sub import print_cli_to_sub
+from ytdl_sub.cli.parsers.dl import DownloadArgsParser
+from ytdl_sub.cli.parsers.main import DEFAULT_CONFIG_FILE_NAME, parser
+from ytdl_sub.config.config_file import ConfigFile
+from ytdl_sub.config.validators.variable_validation import ResolutionLevel
+from ytdl_sub.subscriptions.subscription import Subscription
+from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled, ValidationException
+from ytdl_sub.utils.file_handler import FileHandler
+from ytdl_sub.utils.file_lock import working_directory_lock
+from ytdl_sub.utils.logger import Logger
+
+# pylint: disable=too-many-branches
+
+logger = Logger.get()
+
+# View is a command to run a simple dry-run on a URL using the `_view` preset.
+# Use ytdl-sub dl arguments to use the preset
+_VIEW_EXTRA_ARGS_FORMATTER = "--preset _view --overrides.url {}"
+
+
+def _log_time() -> str:
+    return datetime.now().strftime("%Y-%m-%d-%H%M%S")
+
+
+def _maybe_write_subscription_log_file(
+    config: ConfigFile,
+    subscription: Subscription,
+    dry_run: bool,
+    exception: Optional[Exception] = None,
+) -> None:
+    success: bool = exception is None
+
+    # If dry-run, do nothing
+    if dry_run:
+        return
+
+    # If persisting logs is disabled, do nothing
+    if not config.config_options.persist_logs:
+        return
+
+    # If persisting successful logs is disabled, do nothing
+    if success and not config.config_options.persist_logs.keep_successful_logs:
+        return
+
+    log_subscription_name = sanitize_filename(subscription.name).lower().replace(" ", "_")
+    log_success = "success" if success else "error"
+
+    log_filename = f"{_log_time()}.{log_subscription_name}.{log_success}.log"
+    persist_log_path = Path(config.config_options.persist_logs.logs_directory) / log_filename
+
+    if not success:
+        Logger.log_exception(exception=exception, log_filepath=persist_log_path)
+
+    os.makedirs(os.path.dirname(persist_log_path), exist_ok=True)
+    FileHandler.copy(Logger.debug_log_filename(), persist_log_path)
+
+
+def _download_subscriptions_from_yaml_files(
+    config: ConfigFile,
+    subscription_paths: List[str],
+    subscription_matches: List[str],
+    subscription_override_dict: Dict,
+    update_with_info_json: bool,
+    dry_run: bool,
+    shuffle: bool,
+) -> List[Subscription]:
+    """
+    Downloads all subscriptions from one or many subscription yaml files.
+
+    Parameters
+    ----------
+    config
+        Configuration file
+    subscription_paths
+        Path to subscription files to download
+    subscription_matches
+        Optional list of substrings to match subscription names to (only run if matched)
+    update_with_info_json
+        Whether to actually download or update using existing info json
+    dry_run
+        Whether to dry run or not
+    shuffle
+        Whether to shuffle the subscription download order
+
+    Returns
+    -------
+    List of subscriptions processed
+
+    Raises
+    ------
+    Exception
+        Any exception during download
+    """
+    subscriptions: List[Subscription] = []
+
+    # Load all the subscriptions first to perform all validation before downloading
+    for path in subscription_paths:
+        subscriptions += Subscription.from_file_path(
+            config=config,
+            subscription_path=path,
+            subscription_matches=subscription_matches,
+            subscription_override_dict=subscription_override_dict,
+        )
+
+    if shuffle:
+        logger.info("Shuffling subscriptions")
+        random.shuffle(subscriptions)
+
+    for subscription in subscriptions:
+        with subscription.exception_handling():
+            logger.info(
+                "Beginning subscription %s for %s",
+                ("dry run" if dry_run else "download"),
+                subscription.name,
+            )
+            logger.debug("Subscription full yaml:\n%s", subscription.as_yaml())
+
+            if update_with_info_json:
+                subscription.update_with_info_json(dry_run=dry_run)
+            else:
+                subscription.download(dry_run=dry_run)
+
+        _maybe_write_subscription_log_file(
+            config=config,
+            subscription=subscription,
+            dry_run=dry_run,
+            exception=subscription.exception,
+        )
+
+        Logger.cleanup(has_error=False)
+        gc.collect()  # Garbage collect after each subscription download
+
+    return subscriptions
+
+
+def _download_subscription_from_cli(
+    config: ConfigFile, dry_run: bool, extra_args: List[str]
+) -> Subscription:
+    """
+    Downloads a one-off subscription using the CLI
+
+    Parameters
+    ----------
+    config
+        Configuration file
+    dry_run
+        Whether this is a dry-run
+    extra_args
+        Extra arguments from argparse that contain dynamic subscription options
+
+    Returns
+    -------
+    Subscription and its download transaction log
+    """
+    dl_args_parser = DownloadArgsParser(
+        extra_arguments=extra_args, config_options=config.config_options
+    )
+    subscription_args_dict = dl_args_parser.to_subscription_dict()
+    subscription_name = dl_args_parser.get_dl_subscription_name()
+
+    subscription = Subscription.from_dict(
+        config=config, preset_name=subscription_name, preset_dict=subscription_args_dict
+    )
+
+    logger.info("Beginning CLI %s", ("dry run" if dry_run else "download"))
+    subscription.download(dry_run=dry_run)
+
+    return subscription
+
+
+def _view_url_from_cli(config: ConfigFile, url: str, split_chapters: bool) -> Subscription:
+    """
+    `ytdl-sub view` dry-runs a URL to print its source variables. Use the pre-built `_view` preset,
+    inject the URL argument, and dry-run.
+    """
+    preset = "_view_split_chapters" if split_chapters else "_view"
+    subscription = Subscription.from_dict(
+        config=config,
+        preset_name="ytdl-sub-view",
+        preset_dict={"preset": preset, "overrides": {"url": url}},
+    )
+
+    logger.info(
+        "Viewing source variables for URL '%s'%s",
+        url,
+        " with split chapters" if split_chapters else "",
+    )
+    subscription.download(dry_run=True)
+
+    return subscription
+
+
+def _parse_inspect_mocks(mocks: Optional[List[str]]) -> Dict[str, str]:
+    out: Dict[str, str] = {}
+    for mock in mocks or []:
+        spl = mock.split("=", 1)
+        if len(spl) == 1:
+            raise ValidationException("inspect mock must be in the form of VAR=VALUE")
+        out[spl[0].strip()] = spl[1]
+
+    return out
+
+
+def _inspect(
+    config: ConfigFile,
+    subscription_paths: List[str],
+    subscription_matches: List[str],
+    subscription_override_dict: Dict,
+    inspection_level: int,
+    mocks: Dict[str, str],
+) -> None:
+
+    subscriptions: List[Subscription] = []
+    for path in subscription_paths:
+        subscriptions += Subscription.from_file_path(
+            config=config,
+            subscription_path=path,
+            subscription_matches=subscription_matches,
+            subscription_override_dict=subscription_override_dict,
+        )
+
+    if len(subscriptions) > 1:
+        print(
+            "inspect can only inspect a single subscription. Use --match to filter for a single one"
+        )
+        return
+
+    print(subscriptions[0].resolved_yaml(resolution_level=inspection_level, mocks=mocks))
+
+
+def main() -> List[Subscription]:
+    """
+    Entrypoint for ytdl-sub, without the error handling
+    """
+    # If no args are provided, print help and exit
+    if len(sys.argv) < 2:
+        parser.print_help()
+        return []
+
+    args, extra_args = parser.parse_known_args()
+
+    if args.subparser == "cli-to-sub":
+        print_cli_to_sub(args=extra_args)
+        return []
+
+    # Load the config
+    if args.config:
+        config = ConfigFile.from_file_path(args.config)
+    elif os.path.isfile(DEFAULT_CONFIG_FILE_NAME):
+        config = ConfigFile.from_file_path(DEFAULT_CONFIG_FILE_NAME)
+    else:
+        logger.info("No config specified, using defaults.")
+        config = ConfigFile.default()
+
+    subscriptions: List[Subscription] = []
+
+    if args.subparser == "inspect":
+        subscription_override_dict = {}
+        if args.dl_override:
+            subscription_override_dict = DownloadArgsParser.from_dl_override(
+                override=args.dl_override, config=config
+            ).to_subscription_dict()
+
+        _inspect(
+            config=config,
+            subscription_paths=args.subscription_paths,
+            subscription_matches=args.match,
+            subscription_override_dict=subscription_override_dict,
+            inspection_level=ResolutionLevel.level_number(args.inspection_level),
+            mocks=_parse_inspect_mocks(args.mock),
+        )
+        return []
+
+    # If transaction log file is specified, make sure we can open it
+    _maybe_validate_transaction_log_file(transaction_log_file_path=args.transaction_log)
+
+    with working_directory_lock(config=config):
+        if args.subparser == "sub":
+            if (
+                args.update_with_info_json
+                and not config.config_options.experimental.enable_update_with_info_json
+            ):
+                raise ExperimentalFeatureNotEnabled(
+                    "--update-with-info-json requires setting"
+                    " configuration.experimental.enable_update_with_info_json to True. This"
+                    " feature is ",
+                    "still being tested and has the ability to destroy files. Ensure you have a ",
+                    "full backup before usage. You have been warned!",
+                )
+
+            subscription_override_dict = {}
+            if args.dl_override:
+                subscription_override_dict = DownloadArgsParser.from_dl_override(
+                    override=args.dl_override, config=config
+                ).to_subscription_dict()
+
+            logger.info("Validating subscriptions...")
+            subscriptions = _download_subscriptions_from_yaml_files(
+                config=config,
+                subscription_paths=args.subscription_paths,
+                subscription_matches=args.match,
+                subscription_override_dict=subscription_override_dict,
+                update_with_info_json=args.update_with_info_json,
+                dry_run=args.dry_run,
+                shuffle=args.shuffle,
+            )
+
+        # One-off download
+        elif args.subparser == "dl":
+            logger.info("Validating presets...")
+            subscriptions.append(
+                _download_subscription_from_cli(
+                    config=config, dry_run=args.dry_run, extra_args=extra_args
+                )
+            )
+        elif args.subparser == "view":
+            subscriptions.append(
+                _view_url_from_cli(config=config, url=args.url, split_chapters=args.split_chapters)
+            )
+        else:
+            raise ValidationException("Must provide one of the commands: sub, dl, view, cli-to-sub")
+
+    if not args.suppress_transaction_log:
+        output_transaction_log(
+            subscriptions=subscriptions,
+            transaction_log_file_path=args.transaction_log,
+        )
+
+    output_summary(subscriptions, suppress_colors=args.suppress_colors)
+
+    return subscriptions

src/ytdl_sub/cli/main.py

@@ -1,354 +0,0 @@
-import gc
-import os
-import sys
-from datetime import datetime
-from pathlib import Path
-from typing import List
-from typing import Optional
-from typing import Tuple
-
-from colorama import Fore
-from yt_dlp.utils import sanitize_filename
-
-from ytdl_sub.cli.download_args_parser import DownloadArgsParser
-from ytdl_sub.cli.main_args_parser import parser
-from ytdl_sub.config.config_file import ConfigFile
-from ytdl_sub.subscriptions.subscription import Subscription
-from ytdl_sub.utils.exceptions import ExperimentalFeatureNotEnabled
-from ytdl_sub.utils.exceptions import ValidationException
-from ytdl_sub.utils.file_handler import FileHandler
-from ytdl_sub.utils.file_handler import FileHandlerTransactionLog
-from ytdl_sub.utils.file_lock import working_directory_lock
-from ytdl_sub.utils.logger import Logger
-
-logger = Logger.get()
-
-# View is a command to run a simple dry-run on a URL using the `_view` preset.
-# Use ytdl-sub dl arguments to use the preset
-_VIEW_EXTRA_ARGS_FORMATTER = "--preset _view --overrides.url {}"
-
-
-def _maybe_write_subscription_log_file(
-    config: ConfigFile,
-    subscription: Subscription,
-    dry_run: bool,
-    exception: Optional[Exception] = None,
-) -> None:
-    success: bool = exception is None
-
-    # If dry-run, do nothing
-    if dry_run:
-        return
-
-    # If persisting logs is disabled, do nothing
-    if not config.config_options.persist_logs:
-        return
-
-    # If persisting successful logs is disabled, do nothing
-    if success and not config.config_options.persist_logs.keep_successful_logs:
-        return
-
-    log_time = datetime.now().strftime("%Y-%m-%d-%H%M%S")
-    log_subscription_name = sanitize_filename(subscription.name).lower().replace(" ", "_")
-    log_success = "success" if success else "error"
-
-    log_filename = f"{log_time}.{log_subscription_name}.{log_success}.log"
-    persist_log_path = Path(config.config_options.persist_logs.logs_directory) / log_filename
-
-    if not success:
-        Logger.log_exit_exception(exception=exception, log_filepath=persist_log_path)
-
-    os.makedirs(os.path.dirname(persist_log_path), exist_ok=True)
-    FileHandler.copy(Logger.debug_log_filename(), persist_log_path)
-
-
-def _download_subscriptions_from_yaml_files(
-    config: ConfigFile, subscription_paths: List[str], update_with_info_json: bool, dry_run: bool
-) -> List[Tuple[Subscription, FileHandlerTransactionLog]]:
-    """
-    Downloads all subscriptions from one or many subscription yaml files.
-
-    Parameters
-    ----------
-    config
-        Configuration file
-    subscription_paths
-        Path to subscription files to download
-    update_with_info_json
-        Whether to actually download or update using existing info json
-    dry_run
-        Whether to dry run or not
-
-    Returns
-    -------
-    List of (subscription, transaction_log)
-
-    Raises
-    ------
-    Exception
-        Any exception during download
-    """
-    subscriptions: List[Subscription] = []
-    output: List[Tuple[Subscription, FileHandlerTransactionLog]] = []
-
-    # Load all the subscriptions first to perform all validation before downloading
-    for path in subscription_paths:
-        subscriptions += Subscription.from_file_path(config=config, subscription_path=path)
-
-    for subscription in subscriptions:
-        logger.info(
-            "Beginning subscription %s for %s",
-            ("dry run" if dry_run else "download"),
-            subscription.name,
-        )
-        logger.debug("Subscription full yaml:\n%s", subscription.as_yaml())
-
-        try:
-            if update_with_info_json:
-                transaction_log = subscription.update_with_info_json(dry_run=dry_run)
-            else:
-                transaction_log = subscription.download(dry_run=dry_run)
-        except Exception as exc:  # pylint: disable=broad-except
-            _maybe_write_subscription_log_file(
-                config=config, subscription=subscription, dry_run=dry_run, exception=exc
-            )
-            raise
-        else:
-            output.append((subscription, transaction_log))
-            _maybe_write_subscription_log_file(
-                config=config, subscription=subscription, dry_run=dry_run
-            )
-            Logger.cleanup()  # Cleanup logger after each successful subscription download
-        finally:
-            gc.collect()  # Garbage collect after each subscription download
-
-    return output
-
-
-def _download_subscription_from_cli(
-    config: ConfigFile, dry_run: bool, extra_args: List[str]
-) -> Tuple[Subscription, FileHandlerTransactionLog]:
-    """
-    Downloads a one-off subscription using the CLI
-
-    Parameters
-    ----------
-    config
-        Configuration file
-    dry_run
-        Whether this is a dry-run
-    extra_args
-        Extra arguments from argparse that contain dynamic subscription options
-
-    Returns
-    -------
-    Subscription and its download transaction log
-    """
-    dl_args_parser = DownloadArgsParser(
-        extra_arguments=extra_args, config_options=config.config_options
-    )
-    subscription_args_dict = dl_args_parser.to_subscription_dict()
-    subscription_name = f"cli-dl-{dl_args_parser.get_args_hash()}"
-
-    subscription = Subscription.from_dict(
-        config=config, preset_name=subscription_name, preset_dict=subscription_args_dict
-    )
-
-    logger.info("Beginning CLI %s", ("dry run" if dry_run else "download"))
-    return subscription, subscription.download(dry_run=dry_run)
-
-
-def _view_url_from_cli(
-    config: ConfigFile, url: str, split_chapters: bool
-) -> Tuple[Subscription, FileHandlerTransactionLog]:
-    """
-    `ytdl-sub view` dry-runs a URL to print its source variables. Use the pre-built `_view` preset,
-    inject the URL argument, and dry-run.
-    """
-    preset = "_view_split_chapters" if split_chapters else "_view"
-    subscription = Subscription.from_dict(
-        config=config,
-        preset_name="ytdl-sub-view",
-        preset_dict={"preset": preset, "overrides": {"url": url}},
-    )
-
-    logger.info(
-        "Viewing source variables for URL '%s'%s",
-        url,
-        " with split chapters" if split_chapters else "",
-    )
-    return subscription, subscription.download(dry_run=True)
-
-
-def _maybe_validate_transaction_log_file(transaction_log_file_path: Optional[str]) -> None:
-    if transaction_log_file_path:
-        try:
-            with open(transaction_log_file_path, "w", encoding="utf-8"):
-                pass
-        except Exception as exc:
-            raise ValidationException(
-                f"Transaction log file '{transaction_log_file_path}' cannot be written to. "
-                f"Reason: {str(exc)}"
-            ) from exc
-
-
-def _output_transaction_log(
-    transaction_logs: List[Tuple[Subscription, FileHandlerTransactionLog]],
-    transaction_log_file_path: str,
-) -> None:
-    transaction_log_file_contents = ""
-    for subscription, transaction_log in transaction_logs:
-        if transaction_log.is_empty:
-            transaction_log_contents = f"\nNo files changed for {subscription.name}"
-        else:
-            transaction_log_contents = (
-                f"Transaction log for {subscription.name}:\n"
-                f"{transaction_log.to_output_message(subscription.output_directory)}"
-            )
-
-        if transaction_log_file_path:
-            transaction_log_file_contents += transaction_log_contents
-        else:
-            logger.info(transaction_log_contents)
-
-    if transaction_log_file_contents:
-        with open(transaction_log_file_path, "w", encoding="utf-8") as transaction_log_file:
-            transaction_log_file.write(transaction_log_file_contents)
-
-
-def _green(value: str) -> str:
-    return Fore.GREEN + value + Fore.RESET
-
-
-def _red(value: str) -> str:
-    return Fore.RED + value + Fore.RESET
-
-
-def _no_color(value: str) -> str:
-    return Fore.RESET + value + Fore.RESET
-
-
-def _str_int(value: int) -> str:
-    if value > 0:
-        return f"+{value}"
-    return str(value)
-
-
-def _color_int(value: int) -> str:
-    str_int = _str_int(value)
-    if value > 0:
-        return _green(str_int)
-    if value < 0:
-        return _red(str_int)
-    return _no_color(str_int)
-
-
-def _output_summary(transaction_logs: List[Tuple[Subscription, FileHandlerTransactionLog]]):
-    summary: List[str] = []
-
-    # Initialize widths to 0
-    width_sub_name: int = 0
-    width_num_entries_added: int = 0
-    width_num_entries_modified: int = 0
-    width_num_entries_removed: int = 0
-    width_num_entries: int = 0
-
-    # Calculate min width needed
-    for subscription, _ in transaction_logs:
-        width_sub_name = max(width_sub_name, len(subscription.name))
-        width_num_entries_added = max(
-            width_num_entries_added, len(_color_int(subscription.num_entries_added))
-        )
-        width_num_entries_modified = max(
-            width_num_entries_modified, len(_color_int(subscription.num_entries_modified))
-        )
-        width_num_entries_removed = max(
-            width_num_entries_removed, len(_color_int(subscription.num_entries_removed * -1))
-        )
-        width_num_entries = max(width_num_entries, len(str(subscription.num_entries)))
-
-    # Add spacing for aesthetics
-    width_sub_name += 4
-    width_num_entries += 4
-
-    # Build the summary
-    for subscription, _ in transaction_logs:
-        num_entries_added = _color_int(subscription.num_entries_added)
-        num_entries_modified = _color_int(subscription.num_entries_modified)
-        num_entries_removed = _color_int(subscription.num_entries_removed * -1)
-        num_entries = str(subscription.num_entries)
-        status = _green("success")
-
-        summary.append(
-            f"{subscription.name:<{width_sub_name}} "
-            f"{num_entries_added:>{width_num_entries_added}} "
-            f"{num_entries_modified:>{width_num_entries_modified}} "
-            f"{num_entries_removed:>{width_num_entries_removed}} "
-            f"{num_entries:>{width_num_entries}} "
-            f"{status}"
-        )
-
-    return "\n".join(summary)
-
-
-def main() -> List[Tuple[Subscription, FileHandlerTransactionLog]]:
-    """
-    Entrypoint for ytdl-sub, without the error handling
-    """
-    # If no args are provided, print help and exit
-    if len(sys.argv) < 2:
-        parser.print_help()
-        return []
-
-    args, extra_args = parser.parse_known_args()
-
-    # Load the config
-    config: ConfigFile = ConfigFile.from_file_path(args.config)
-    transaction_logs: List[Tuple[Subscription, FileHandlerTransactionLog]] = []
-
-    # If transaction log file is specified, make sure we can open it
-    _maybe_validate_transaction_log_file(transaction_log_file_path=args.transaction_log)
-
-    with working_directory_lock(config=config):
-        if args.subparser == "sub":
-            if (
-                args.update_with_info_json
-                and not config.config_options.experimental.enable_update_with_info_json
-            ):
-                raise ExperimentalFeatureNotEnabled(
-                    "--update-with-info-json requires setting"
-                    " configuration.experimental.enable_update_with_info_json to True. This"
-                    " feature is ",
-                    "still being tested and has the ability to destroy files. Ensure you have a ",
-                    "full backup before usage. You have been warned!",
-                )
-
-            transaction_logs = _download_subscriptions_from_yaml_files(
-                config=config,
-                subscription_paths=args.subscription_paths,
-                update_with_info_json=args.update_with_info_json,
-                dry_run=args.dry_run,
-            )
-
-        # One-off download
-        elif args.subparser == "dl":
-            transaction_logs.append(
-                _download_subscription_from_cli(
-                    config=config, dry_run=args.dry_run, extra_args=extra_args
-                )
-            )
-        elif args.subparser == "view":
-            transaction_logs.append(
-                _view_url_from_cli(config=config, url=args.url, split_chapters=args.split_chapters)
-            )
-
-    if not args.suppress_transaction_log:
-        _output_transaction_log(
-            transaction_logs=transaction_logs,
-            transaction_log_file_path=args.transaction_log,
-        )
-
-    # Hack to always show download summary, even if logs are set to quiet
-    logger.warning("Download Summary:\n%s", _output_summary(transaction_logs))
-
-    return transaction_logs

src/ytdl_sub/cli/output_summary.py

@@ -0,0 +1,117 @@
+from typing import List
+
+from colorama import Fore
+
+from ytdl_sub.subscriptions.subscription import Subscription
+from ytdl_sub.utils.logger import Logger
+
+logger = Logger.get()
+
+
+def _green(value: str, suppress_colors: bool = False) -> str:
+    return value if suppress_colors else Fore.GREEN + value + Fore.RESET
+
+
+def _red(value: str, suppress_colors: bool = False) -> str:
+    return value if suppress_colors else Fore.RED + value + Fore.RESET
+
+
+def _no_color(value: str, suppress_colors: bool = False) -> str:
+    return value if suppress_colors else Fore.RESET + value + Fore.RESET
+
+
+def _str_int(value: int) -> str:
+    if value > 0:
+        return f"+{value}"
+    return str(value)
+
+
+def _color_int(value: int, suppress_colors: bool = False) -> str:
+    str_int = _str_int(value)
+    if value > 0:
+        return _green(str_int, suppress_colors)
+    if value < 0:
+        return _red(str_int, suppress_colors)
+    return _no_color(str_int, suppress_colors)
+
+
+def output_summary(subscriptions: List[Subscription], suppress_colors: bool) -> None:
+    """
+    Parameters
+    ----------
+    subscriptions
+        Processed subscriptions
+    suppress_colors
+        Whether to have color or not
+
+    Returns
+    -------
+    Output summary to print
+    """
+    # many locals for proper output printing
+    # pylint: disable=too-many-locals
+    if len(subscriptions) == 0:
+        logger.info("No subscriptions ran")
+        return
+
+    summary: List[str] = []
+
+    # Initialize totals to 0
+    total_subs: int = len(subscriptions)
+    total_subs_str = f"Total: {total_subs}"
+    total_added: int = sum(sub.num_entries_added for sub in subscriptions)
+    total_modified: int = sum(sub.num_entries_modified for sub in subscriptions)
+    total_removed: int = sum(sub.num_entries_removed for sub in subscriptions)
+    total_entries: int = sum(sub.num_entries for sub in subscriptions)
+    total_errors: int = sum(sub.exception is not None for sub in subscriptions)
+
+    # Initialize widths to 0
+    width_sub_name: int = max(len(sub.name) for sub in subscriptions) + 4  # aesthetics
+    width_num_entries_added: int = len(_color_int(total_added, suppress_colors))
+    width_num_entries_modified: int = len(_color_int(total_modified, suppress_colors))
+    width_num_entries_removed: int = len(_color_int(total_removed, suppress_colors))
+    width_num_entries: int = len(str(total_entries)) + 4  # aesthetics
+
+    # Build the summary
+    for subscription in subscriptions:
+        num_entries_added = _color_int(subscription.num_entries_added, suppress_colors)
+        num_entries_modified = _color_int(subscription.num_entries_modified, suppress_colors)
+        num_entries_removed = _color_int(subscription.num_entries_removed * -1, suppress_colors)
+        num_entries = str(subscription.num_entries)
+        status = (
+            _red(subscription.exception.__class__.__name__, suppress_colors)
+            if subscription.exception
+            else _green("✔", suppress_colors)
+        )
+
+        summary.append(
+            f"{subscription.name:<{width_sub_name}} "
+            f"{num_entries_added:>{width_num_entries_added}} "
+            f"{num_entries_modified:>{width_num_entries_modified}} "
+            f"{num_entries_removed:>{width_num_entries_removed}} "
+            f"{num_entries:>{width_num_entries}} "
+            f"{status}"
+        )
+
+    total_errors_str = (
+        _green("Success", suppress_colors)
+        if total_errors == 0
+        else _red(f"Error{'s' if total_errors > 1 else ''}", suppress_colors)
+    )
+
+    summary.append(
+        f"{total_subs_str:<{width_sub_name}} "
+        f"{_color_int(total_added, suppress_colors):>{width_num_entries_added}} "
+        f"{_color_int(total_modified, suppress_colors):>{width_num_entries_modified}} "
+        f"{_color_int(total_removed * -1, suppress_colors):>{width_num_entries_removed}} "
+        f"{total_entries:>{width_num_entries}} "
+        f"{total_errors_str}"
+    )
+
+    if total_errors > 0:
+        summary.append("")
+        summary.append(f"See `{Logger.error_log_filename()}` for details on errors.")
+        summary.append("Consider making a GitHub issue including the uploaded log file.")
+
+    # Hack to always show download summary, even if logs are set to quiet
+    logger.warning("Download Summary:\n%s", "\n".join(summary))

src/ytdl_sub/cli/output_transaction_log.py

@@ -0,0 +1,53 @@
+from typing import List, Optional
+
+from ytdl_sub.subscriptions.subscription import Subscription
+from ytdl_sub.utils.exceptions import ValidationException
+from ytdl_sub.utils.logger import Logger
+
+logger = Logger.get()
+
+
+def _maybe_validate_transaction_log_file(transaction_log_file_path: Optional[str]) -> None:
+    if transaction_log_file_path:
+        try:
+            with open(transaction_log_file_path, "w", encoding="utf-8"):
+                pass
+        except Exception as exc:
+            raise ValidationException(
+                f"Transaction log file '{transaction_log_file_path}' cannot be written to. "
+                f"Reason: {str(exc)}"
+            ) from exc
+
+
+def output_transaction_log(
+    subscriptions: List[Subscription],
+    transaction_log_file_path: Optional[str],
+) -> None:
+    """
+    Maybe print and/or write transaction logs to a file
+
+    Parameters
+    ----------
+    subscriptions
+        Processed subscriptions
+    transaction_log_file_path
+        Optional file path to write to
+    """
+    transaction_log_file_contents = ""
+    for subscription in subscriptions:
+        if subscription.transaction_log.is_empty:
+            transaction_log_contents = f"\nNo files changed for {subscription.name}"
+        else:
+            transaction_log_contents = (
+                f"Transaction log for {subscription.name}:\n"
+                f"{subscription.transaction_log.to_output_message(subscription.output_directory)}"
+            )
+
+        if transaction_log_file_path:
+            transaction_log_file_contents += transaction_log_contents
+        else:
+            logger.info(transaction_log_contents)
+
+    if transaction_log_file_contents:
+        with open(transaction_log_file_path, "w", encoding="utf-8") as transaction_log_file:
+            transaction_log_file.write(transaction_log_file_contents)

src/ytdl_sub/cli/parsers/cli_to_sub.py

@@ -0,0 +1,63 @@
+from typing import List
+
+import yt_dlp
+import yt_dlp.options
+
+from ytdl_sub.utils.logger import Logger
+from ytdl_sub.utils.yaml import dump_yaml
+
+logger = Logger.get()
+
+# pylint: disable=missing-function-docstring
+
+##############################################################
+# --- BEGIN ----
+# Copy of https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py
+
+create_parser = yt_dlp.options.create_parser
+
+
+def parse_patched_options(opts):
+    patched_parser = create_parser()
+    patched_parser.defaults.update(
+        {
+            "ignoreerrors": False,
+            "retries": 0,
+            "fragment_retries": 0,
+            "extract_flat": False,
+            "concat_playlist": "never",
+            "update_self": False,
+        }
+    )
+    yt_dlp.options.create_parser = lambda: patched_parser
+    try:
+        return yt_dlp.parse_options(opts)
+    finally:
+        yt_dlp.options.create_parser = create_parser
+
+
+default_opts = parse_patched_options([]).ydl_opts
+
+
+def cli_to_api(opts, cli_defaults=False):
+    opts = (yt_dlp.parse_options if cli_defaults else parse_patched_options)(opts).ydl_opts
+
+    diff = {k: v for k, v in opts.items() if default_opts[k] != v}
+    if "postprocessors" in diff:
+        diff["postprocessors"] = [
+            pp for pp in diff["postprocessors"] if pp not in default_opts["postprocessors"]
+        ]
+    return diff
+
+
+# --- END ----
+##############################################################
+
+
+def print_cli_to_sub(args: List[str]) -> None:
+    api_args = cli_to_api(args)
+    if not api_args:
+        logger.info("Does not resolve to any yt-dlp args")
+        return
+
+    print(dump_yaml({"ytdl_options": api_args}))

src/ytdl_sub/cli/parsers/dl.py

@@ -1,13 +1,12 @@
 import hashlib
 import re
 import shlex
-from typing import Dict
+from typing import Any, Dict, List, Tuple
-from typing import List
-from typing import Tuple
 
 from mergedeep import mergedeep
 
-from ytdl_sub.cli.main_args_parser import MainArguments
+from ytdl_sub.cli.parsers.main import MainArguments
+from ytdl_sub.config.config_file import ConfigFile
 from ytdl_sub.config.config_validator import ConfigOptions
 from ytdl_sub.utils.exceptions import InvalidDlArguments
 
@@ -116,7 +115,7 @@ class DownloadArgsParser:
         return largest_consecutive + 1
 
     @classmethod
-    def _argument_name_and_value_to_dict(cls, arg_name: str, arg_value: str) -> Dict:
+    def _argument_name_and_value_to_dict(cls, arg_name: str, arg_value: Any) -> Dict:
         """
         :param arg_name: Argument name in the form of 'key1.key2.key3'
         :param arg_value: Argument value
@@ -134,11 +133,15 @@ class DownloadArgsParser:
 
         next_dict[arg_name_split[-1]] = arg_value
 
-        # TODO: handle ints/floats
+        if isinstance(arg_value, str):
-        if arg_value == "True":
+            if arg_value == "True":
-            next_dict[arg_name_split[-1]] = True
+                next_dict[arg_name_split[-1]] = True
-        elif arg_value == "False":
+            elif arg_value == "False":
-            next_dict[arg_name_split[-1]] = False
+                next_dict[arg_name_split[-1]] = False
+            elif arg_value.isdigit():
+                next_dict[arg_name_split[-1]] = int(arg_value)
+            elif arg_value.replace(".", "", 1).isdigit():
+                next_dict[arg_name_split[-1]] = float(arg_value)
 
         return argument_dict
 
@@ -236,9 +239,19 @@ class DownloadArgsParser:
 
         return subscription_dict
 
-    def get_args_hash(self) -> str:
+    def get_dl_subscription_name(self) -> str:
         """
-        :return: Hash of the arguments provided
+        Returns a deterministic name based on input args
         """
-        hash_string = str(sorted(self._unknown_arguments))
+        to_hash = str(sorted(self._unknown_arguments))
-        return hashlib.sha256(hash_string.encode()).hexdigest()[-8:]
+        hash_str = hashlib.sha256(to_hash.encode()).hexdigest()[-8:]
+        return f"cli-dl-{hash_str}"
+
+    @classmethod
+    def from_dl_override(cls, override: str, config: ConfigFile) -> "DownloadArgsParser":
+        """
+        Create a DownloadArgsParser from a sub --override argument value
+        """
+        return DownloadArgsParser(
+            extra_arguments=override.split(), config_options=config.config_options
+        )

src/ytdl_sub/cli/parsers/main.py

@@ -1,10 +1,12 @@
 import argparse
 import dataclasses
-from typing import List
+from typing import Dict, List
 
 from ytdl_sub import __local_version__
 from ytdl_sub.utils.logger import LoggerLevels
 
+DEFAULT_CONFIG_FILE_NAME: str = "config.yaml"
+
 
 @dataclasses.dataclass
 class CLIArgument:
@@ -38,6 +40,11 @@ class MainArguments:
         long="--suppress-transaction-log",
         is_positional=True,
     )
+    MATCH = CLIArgument(
+        short="-m",
+        long="--match",
+    )
+    SUPPRESS_COLORS = CLIArgument(short="-nc", long="--suppress-colors")
 
     @classmethod
     def all(cls) -> List[CLIArgument]:
@@ -52,6 +59,8 @@ class MainArguments:
             cls.LOG_LEVEL,
             cls.TRANSACTION_LOG,
             cls.SUPPRESS_TRANSACTION_LOG,
+            cls.MATCH,
+            cls.SUPPRESS_COLORS,
         ]
 
     @classmethod
@@ -86,8 +95,8 @@ def _add_shared_arguments(arg_parser: argparse.ArgumentParser, suppress_defaults
         MainArguments.CONFIG.long,
         metavar="CONFIGPATH",
         type=str,
-        help="path to the config yaml, uses config.yaml if not provided",
+        help=f"path to the config yaml, uses {DEFAULT_CONFIG_FILE_NAME} if not provided",
-        default=argparse.SUPPRESS if suppress_defaults else "config.yaml",
+        default=argparse.SUPPRESS if suppress_defaults else None,  # Default is set downstream
     )
     arg_parser.add_argument(
         MainArguments.DRY_RUN.short,
@@ -102,8 +111,8 @@ def _add_shared_arguments(arg_parser: argparse.ArgumentParser, suppress_defaults
         MainArguments.LOG_LEVEL.long,
         metavar="|".join(LoggerLevels.names()),
         type=str,
-        help="level of logs to print to console, defaults to info",
+        help="level of logs to print to console, defaults to verbose",
-        default=argparse.SUPPRESS if suppress_defaults else LoggerLevels.INFO.name,
+        default=argparse.SUPPRESS if suppress_defaults else LoggerLevels.VERBOSE.name,
         choices=LoggerLevels.names(),
         dest="ytdl_sub_log_level",
     )
@@ -122,6 +131,23 @@ def _add_shared_arguments(arg_parser: argparse.ArgumentParser, suppress_defaults
         help="do not output transaction logs to console or file",
         default=argparse.SUPPRESS if suppress_defaults else False,
     )
+    arg_parser.add_argument(
+        MainArguments.SUPPRESS_COLORS.short,
+        MainArguments.SUPPRESS_COLORS.long,
+        action="store_true",
+        help="do not use colors in ytdl-sub output",
+        default=argparse.SUPPRESS if suppress_defaults else False,
+    )
+    arg_parser.add_argument(
+        MainArguments.MATCH.short,
+        MainArguments.MATCH.long,
+        dest="match",
+        nargs="+",
+        action="extend",
+        type=str,
+        help="match subscription names to one or more substrings, and only run those subscriptions",
+        default=argparse.SUPPRESS if suppress_defaults else [],
+    )
 
 
 ###################################################################################################
@@ -133,6 +159,8 @@ parser.add_argument("-v", "--version", action="version", version="%(prog)s " + _
 _add_shared_arguments(parser, suppress_defaults=False)
 
 subparsers = parser.add_subparsers(dest="subparser")
+
+
 ###################################################################################################
 # SUBSCRIPTION PARSER
 class SubArguments:
@@ -140,6 +168,14 @@ class SubArguments:
         short="-u",
         long="--update-with-info-json",
     )
+    OVERRIDE = CLIArgument(
+        short="-o",
+        long="--dl-override",
+    )
+    SHUFFLE = CLIArgument(
+        short="-sh",
+        long="--shuffle",
+    )
 
 
 subscription_parser = subparsers.add_parser("sub")
@@ -158,6 +194,20 @@ subscription_parser.add_argument(
     help="update all subscriptions with the current config using info.json files",
     default=False,
 )
+subscription_parser.add_argument(
+    SubArguments.OVERRIDE.short,
+    SubArguments.OVERRIDE.long,
+    type=str,
+    help="override all subscription config values using `dl` syntax, "
+    "i.e. --dl-override='--ytdl_options.max_downloads 3'",
+)
+subscription_parser.add_argument(
+    SubArguments.SHUFFLE.short,
+    SubArguments.SHUFFLE.long,
+    action="store_true",
+    help="shuffle subscription order when downloading",
+    default=False,
+)
 
 ###################################################################################################
 # DOWNLOAD PARSER
@@ -182,3 +232,81 @@ view_parser.add_argument(
     help="View source variables after splitting by chapters",
 )
 view_parser.add_argument("url", help="URL to view source variables for")
+
+###################################################################################################
+# CLI-TO-SUB PARSER
+cli_to_sub_parser = subparsers.add_parser("cli-to-sub")
+
+###################################################################################################
+# INSPECT PARSER
+
+
+class InspectArguments:
+    LEVEL = CLIArgument(
+        short="-l",
+        long="--level",
+    )
+    LevelChoices: Dict[str, str] = {
+        "0": "original",
+        "1": "fill",
+        "2": "resolve",
+        "3": "internal",
+    }
+
+    MOCK = CLIArgument(
+        short="-k",
+        long="--mock",
+    )
+
+
+inspect_parser = subparsers.add_parser("inspect", formatter_class=argparse.RawTextHelpFormatter)
+inspect_parser.add_argument(
+    InspectArguments.LEVEL.short,
+    InspectArguments.LEVEL.long,
+    metavar=",".join(str(i) for i in range(4)),
+    type=str,
+    help="""level of inspection to perform:
+    0 - original   present the subscription as-is
+    1 - fill       fill in defined values
+    2 - resolve    resolve all possible variables (default)
+    3 - internal   resolve all variables to their internal representation
+    """,
+    default="resolve",
+    choices=list(InspectArguments.LevelChoices.keys())
+    + list(InspectArguments.LevelChoices.values()),
+    dest="inspection_level",
+)
+inspect_parser.add_argument(
+    MainArguments.MATCH.short,
+    MainArguments.MATCH.long,
+    dest="match",
+    nargs="+",
+    action="extend",
+    type=str,
+    help="match subscription names to one or more substrings, and only run those subscriptions",
+    default=[],
+)
+
+inspect_parser.add_argument(
+    "subscription_paths",
+    metavar="SUBPATH",
+    nargs="*",
+    help="path to subscription files, uses subscriptions.yaml if not provided",
+    default=["subscriptions.yaml"],
+)
+
+inspect_parser.add_argument(
+    SubArguments.OVERRIDE.short,
+    SubArguments.OVERRIDE.long,
+    type=str,
+    help="override all subscription config values using `dl` syntax, "
+    "i.e. --dl-override='--ytdl_options.max_downloads 3'",
+)
+
+inspect_parser.add_argument(
+    InspectArguments.MOCK.short,
+    InspectArguments.MOCK.long,
+    metavar="VAR=VALUE",
+    action="append",
+    help="ability to mock one or more variable values, i.e. --mock 'title=Lets Play'",
+)