Merge branch 'master' into jesse/reformat

.github/workflows/package-ubuntu.yaml

@@ -0,0 +1,223 @@
+name: ytld-sub Docker Ubuntu Build
+
+on:
+  push:
+    # Publish `master` as Docker `latest` image.
+    branches:
+      - master
+
+    # Publish `v1.2.3` tags as releases.
+    tags:
+      - v*
+
+  # Run tests for any PRs.
+  pull_request:
+
+env:
+  IMAGE_NAME: ytdl-sub
+
+jobs:
+  # Push image to GitHub Packages.
+  # See also https://docs.docker.com/docker-hub/builds/
+  build:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python-version: [ "3.10" ]
+
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Build Wheel
+        run: |
+          make docker_stage
+
+      - name: Save Python build cache
+        uses: actions/cache@v3
+        with:
+          path: docker/root
+          key: ${{github.sha}}
+
+
+  # Build ARM64 container, only on master branch to save time testing
+  package-arm64:
+    runs-on: ubuntu-22.04
+    needs: [
+      build
+    ]
+
+    permissions:
+      contents: read
+
+    if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Restore Python build cache
+        uses: actions/cache@v3
+        with:
+          path: docker/root
+          key: ${{github.sha}}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: linux/arm64
+
+      - name: Docker Setup Buildx
+        uses: docker/setup-buildx-action@v2.0.0
+
+      - name: Build Docker Image
+        run: |
+          docker buildx build \
+          --platform=linux/arm64 \
+          --cache-to=type=local,dest=/tmp/build-cache/arm64 \
+          --tag $IMAGE_NAME \
+          --label "runnumber=${GITHUB_RUN_ID}" \
+          --file docker/Dockerfile.ubuntu \
+          docker/
+
+      - name: Save ARM64 build cache
+        uses: actions/cache@v3
+        with:
+          path: /tmp/build-cache/arm64
+          key: ${{github.sha}}-arm64
+
+
+  # Build AMD64 container
+  package-amd64:
+    runs-on: ubuntu-22.04
+    needs: [
+      build
+    ]
+
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Restore Python build cache
+        uses: actions/cache@v3
+        with:
+          path: docker/root
+          key: ${{github.sha}}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: linux/amd64
+
+      - name: Docker Setup Buildx
+        uses: docker/setup-buildx-action@v2.0.0
+
+      - name: Build Docker Image
+        run: |
+          docker buildx build \
+          --platform=linux/amd64 \
+          --cache-to=type=local,dest=/tmp/build-cache/amd64 \
+          --tag $IMAGE_NAME \
+          --label "runnumber=${GITHUB_RUN_ID}" \
+          --file docker/Dockerfile.ubuntu \
+          docker/
+
+      - name: Save AMD64 build cache
+        uses: actions/cache@v3
+        with:
+          path: /tmp/build-cache/amd64
+          key: ${{github.sha}}-amd64
+
+
+  # On master branch, build the docker manifest file from the cached
+  # docker builds and push to the registry
+  deploy:
+    runs-on: ubuntu-22.04
+    needs: [
+      build,
+      package-arm64,
+      package-amd64
+    ]
+
+    permissions:
+      packages: write
+      contents: read
+
+    if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Restore Python build cache
+        uses: actions/cache@v3
+        with:
+          path: docker/root
+          key: ${{github.sha}}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: linux/amd64,linux/arm64
+
+      - name: Docker Setup Buildx
+        uses: docker/setup-buildx-action@v2.0.0
+
+      - name: login to GitHub Container Registry
+        uses: docker/login-action@v1
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Restore ARM64 build cache
+        uses: actions/cache@v3
+        with:
+          path: /tmp/build-cache/arm64
+          key: ${{github.sha}}-arm64
+
+      - name: Restore AMD64 build cache
+        uses: actions/cache@v3
+        with:
+          path: /tmp/build-cache/amd64
+          key: ${{github.sha}}-amd64
+
+      - name: Format image_id
+        id: formatted-image_id
+        run:  |
+            IMAGE_ID=ghcr.io/${{ github.repository_owner }}/${IMAGE_NAME}
+
+            # Change all uppercase to lowercase
+            IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+            echo IMAGE_ID=${IMAGE_ID}
+            
+            echo ::set-output name=IMAGE_ID::${IMAGE_ID}
+
+      - name: Get the version
+        id: formatted_version
+        run: |
+            # Strip git ref prefix from version
+            VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+            # Strip "v" prefix from tag name
+            [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+            # Use Docker `latest` tag convention
+            [ "$VERSION" == "master" ] && VERSION=latest
+            echo VERSION=${VERSION}
+            
+            echo ::set-output name=VERSION::${VERSION}
+
+      - name: Build Docker Image and push to registry
+        run: | 
+          docker buildx build --push \
+          --platform=linux/amd64,linux/arm64 \
+          --cache-from=type=local,src=/tmp/build-cache/amd64 \
+          --cache-from=type=local,src=/tmp/build-cache/arm64 \
+          --tag ${{ steps.formatted-image_id.outputs.IMAGE_ID }}:ubuntu-${{ steps.formatted_version.outputs.VERSION }} \
+          --label "runnumber=ubuntu-${GITHUB_RUN_ID}" \
+          --file docker/Dockerfile.ubuntu \
+          docker/

.github/workflows/package.yaml

@@ -88,7 +88,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: /tmp/build-cache/arm64
-          key: ${{github.sha}}
+          key: ${{github.sha}}-arm64
 
 
   # Build AMD64 container
@@ -131,7 +131,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: /tmp/build-cache/amd64
-          key: ${{github.sha}}
+          key: ${{github.sha}}-amd64
 
 
   # On master branch, build the docker manifest file from the cached
@@ -177,13 +177,13 @@ jobs:
         uses: actions/cache@v3
         with:
           path: /tmp/build-cache/arm64
-          key: ${{github.sha}}
+          key: ${{github.sha}}-arm64
 
       - name: Restore AMD64 build cache
         uses: actions/cache@v3
         with:
           path: /tmp/build-cache/amd64
-          key: ${{github.sha}}
+          key: ${{github.sha}}-amd64
 
       - name: Format image_id
         id: formatted-image_id

Makefile

@@ -32,7 +32,9 @@ docker_stage: wheel
 	cp dist/*.whl docker/root/
 	cp -R examples docker/root/defaults/
 docker: docker_stage
-	sudo docker build --no-cache -t ytdl-sub:local docker/
+	sudo docker build --progress=plain --no-cache -t ytdl-sub:local docker/
+docker_ubuntu: docker_stage
+	sudo docker build --progress=plain --no-cache -t ytdl-sub:local_ubuntu -f docker/Dockerfile.ubuntu docker/
 executable: clean
 	pyinstaller ytdl-sub.spec
 	mv dist/ytdl-sub dist/ytdl-sub${EXEC_SUFFIX}

README.md

@@ -238,6 +238,8 @@ player to recognize downloads as:
 `ytdl-sub` can be installed on the following platforms.
 
 - [Docker Compose](https://ytdl-sub.readthedocs.io/en/latest/install.html#docker-compose_)
+  - [with CPU passthrough](https://ytdl-sub.readthedocs.io/en/latest/install.html#cpu-passthrough)
+  - [with GPU passthrough](https://ytdl-sub.readthedocs.io/en/latest/install.html#nvidia-gpu-passthrough)
 - [Docker CLI](https://ytdl-sub.readthedocs.io/en/latest/install.html#docker)
 - [Windows](https://ytdl-sub.readthedocs.io/en/latest/install.html#windows)
 - [Linux](https://ytdl-sub.readthedocs.io/en/latest/install.html#linux)

docker/Dockerfile

@@ -45,8 +45,4 @@ RUN mkdir -p /config && \
 ENV EDITOR="nano" \
 HOME="/config"
 
-VOLUME /config
+VOLUME /config
-
-
-
-

docker/Dockerfile.ubuntu

@@ -0,0 +1,83 @@
+FROM ghcr.io/linuxserver/baseimage-ubuntu:jammy
+
+# https://askubuntu.com/questions/972516/debian-frontend-environment-variable
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Needed for phantomjs
+ENV OPENSSL_CONF=/etc/ssl
+
+###############################################################################
+# YTDL-SUB INSTALL
+
+SHELL ["/bin/bash", "-c"]
+COPY root/ /
+RUN mkdir -p /config && \
+    apt-get -y update && \
+    apt-get -y upgrade && \
+    apt-get install --no-install-recommends -y \
+      software-properties-common && \
+    apt-get -y update && \
+    apt-get -y upgrade && \
+    apt-get install --no-install-recommends -y \
+      vim \
+      g++ \
+      nano \
+      make \
+      python3.10-dev \
+      python3-pip \
+      fontconfig \
+      xz-utils \
+      bzip2 \
+      python3-venv && \
+    if [[ $(uname -m) == "x86_64" ]]; then \
+      curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz && \
+      tar -xf ffmpeg.tar.gz && \
+      chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffmpeg && \
+      chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffprobe && \
+      mv ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/bin/ffmpeg && \
+      mv ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/bin/ffprobe && \
+      rm ffmpeg.tar.gz && \
+      rm -rf ffmpeg-master-latest-linux64-gpl/ ; \
+    else \
+      curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz && \
+      tar -xf ffmpeg.tar.gz && \
+      chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg && \
+      chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe && \
+      mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg /usr/bin/ffmpeg && \
+      mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe /usr/bin/ffprobe && \
+      rm ffmpeg.tar.gz && \
+      rm -rf ffmpeg-master-latest-linuxarm64-gpl/ ; \
+    fi && \
+    # Ensure ffmpeg is installed
+    ffmpeg -version && \
+    # Install phantomjs if using x86_64, ensure it is properly installed
+    if [[ $(uname -m) == "x86_64" ]]; then \
+      curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 | tar xj && \
+      mv phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/bin/phantomjs && \
+      rm -rf phantomjs-2.1.1-linux-x86_64/ && \
+      echo "Phantom JS version:" && \
+      phantomjs --version ; \
+    fi && \
+    # Install ytdl-sub, ensure it is installed properly
+    pip install --no-cache-dir ytdl_sub-*.whl && \
+      ytdl-sub -h && \
+    # Delete unneeded packages after install
+    rm ytdl_sub-*.whl && \
+    apt-get remove -y \
+      g++ \
+      make \
+      xz-utils \
+      bzip2 \
+      python3.10-dev \
+      python3-venv && \
+    apt-get autoremove -y && \
+    apt-get purge -y --auto-remove && \
+    rm -rf /var/lib/apt/lists/*
+
+###############################################################################
+# CONTAINER CONFIGS
+
+ENV EDITOR="nano" \
+HOME="/config"
+
+VOLUME /config

docs/config.rst

@@ -62,13 +62,13 @@ download strategy has its own set of parameters.
 
 url
 '''
-.. autoclass:: ytdl_sub.downloaders.generic.url.UrlDownloadOptions()
+.. autoclass:: ytdl_sub.downloaders.url.url.UrlDownloadOptions()
   :members: url, playlist_thumbnails, source_thumbnails, download_reverse
   :member-order: bysource
 
 multi_url
 '''''''''
-.. autoclass:: ytdl_sub.downloaders.generic.multi_url.MultiUrlDownloadOptions()
+.. autoclass:: ytdl_sub.downloaders.url.multi_url.MultiUrlDownloadOptions()
   :members: urls, variables
 
 -------------------------------------------------------------------------------
@@ -79,7 +79,7 @@ output_options
 .. autoclass:: ytdl_sub.config.preset_options.OutputOptions()
   :members:
   :member-order: bysource
-  :exclude-members: get_upload_date_range_to_keep
+  :exclude-members: get_upload_date_range_to_keep, partial_validate
 
 -------------------------------------------------------------------------------
 

docs/install.rst

@@ -33,10 +33,65 @@ on how set up a cron job in the docker container.
          - DOCKER_MODS=linuxserver/mods:universal-cron
        volumes:
          - <path/to/ytdl-sub/config>:/config
-         - <path/to/tv_shows>:/tv_shows # optional
+         - <path/to/tv_shows>:/tv_shows  # optional
-         - <path/to/movies>:/movies # optional
+         - <path/to/movies>:/movies  # optional
-         - <path/to/music_videos>:/music_videos # optional
+         - <path/to/music_videos>:/music_videos  # optional
-         - <path/to/music>:/music # optional
+         - <path/to/music>:/music  # optional
+       restart: unless-stopped
+
+CPU Passthrough
+^^^^^^^^^^^^^^^^^^^^^^
+For CPU passthrough, you must use the ``ytdl-sub`` Ubuntu version with the following additions:
+
+.. code-block:: yaml
+
+   services:
+     ytdl-sub:
+       image: ghcr.io/jmbannon/ytdl-sub:ubuntu-latest
+       container_name: ytdl-sub
+       environment:
+         - PUID=1000
+         - PGID=1000
+         - TZ=America/Los_Angeles
+         - DOCKER_MODS=linuxserver/mods:universal-cron
+       volumes:
+         - <path/to/ytdl-sub/config>:/config
+         - <path/to/tv_shows>:/tv_shows  # optional
+         - <path/to/movies>:/movies  # optional
+         - <path/to/music_videos>:/music_videos  # optional
+         - <path/to/music>:/music  # optional
+       devices:
+         - /dev/dri:/dev/dri  # CPU passthrough
+       restart: unless-stopped
+
+Nvidia GPU Passthrough
+^^^^^^^^^^^^^^^^^^^^^^
+For GPU passthrough, you must use the ``ytdl-sub`` Ubuntu version with the following additions:
+
+.. code-block:: yaml
+
+   services:
+     ytdl-sub:
+       image: ghcr.io/jmbannon/ytdl-sub:ubuntu-latest
+       container_name: ytdl-sub
+       environment:
+         - PUID=1000
+         - PGID=1000
+         - TZ=America/Los_Angeles
+         - DOCKER_MODS=linuxserver/mods:universal-cron
+         - NVIDIA_DRIVER_CAPABILITIES=all  # Nvidia ENV args
+         - NVIDIA_VISIBLE_DEVICES=all
+       volumes:
+         - <path/to/ytdl-sub/config>:/config
+         - <path/to/tv_shows>:/tv_shows  # optional
+         - <path/to/movies>:/movies  # optional
+         - <path/to/music_videos>:/music_videos  # optional
+         - <path/to/music>:/music  # optional
+       deploy:
+         resources:
+           reservations:
+             devices:
+               - capabilities: [gpu]  # GPU passthrough
        restart: unless-stopped
 
 Docker

src/ytdl_sub/__init__.py

@@ -1,2 +1,2 @@
-__pypi_version__ = "2023.02.22"
+__pypi_version__ = "2023.03.15"
-__local_version__ = "2023.02.22+40d5517"
+__local_version__ = "2023.03.15+e69933d"