From 52fdc9a21357045144106211c7395aad72722ce2 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 15 Mar 2023 14:10:32 -0700 Subject: [PATCH 1/5] [DOCKER] Add ubuntu docker image for device passthrough (#545) --- .github/workflows/package-ubuntu.yaml | 222 ++++++++++++++++++++++++++ .github/workflows/package.yaml | 1 - Makefile | 4 +- docker/Dockerfile | 6 +- docker/Dockerfile.ubuntu | 83 ++++++++++ src/ytdl_sub/__init__.py | 4 +- 6 files changed, 311 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/package-ubuntu.yaml create mode 100644 docker/Dockerfile.ubuntu diff --git a/.github/workflows/package-ubuntu.yaml b/.github/workflows/package-ubuntu.yaml new file mode 100644 index 00000000..7f2e746b --- /dev/null +++ b/.github/workflows/package-ubuntu.yaml @@ -0,0 +1,222 @@ +name: ytld-sub Docker Ubuntu Build + +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - master + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + # Run tests for any PRs. + pull_request: + +env: + IMAGE_NAME: ytdl-sub + +jobs: + # Push image to GitHub Packages. + # See also https://docs.docker.com/docker-hub/builds/ + build: + runs-on: ubuntu-22.04 + strategy: + matrix: + python-version: [ "3.10" ] + + permissions: + contents: read + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Build Wheel + run: | + make docker_stage + + - name: Save Python build cache + uses: actions/cache@v3 + with: + path: docker/root + key: ${{github.sha}} + + + # Build ARM64 container, only on master branch to save time testing + package-arm64: + runs-on: ubuntu-22.04 + needs: [ + build + ] + + permissions: + contents: read + + if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') }} + steps: + - uses: actions/checkout@v3 + + - name: Restore Python build cache + uses: actions/cache@v3 + with: + path: docker/root + key: ${{github.sha}} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: linux/arm64 + + - name: Docker Setup Buildx + uses: docker/setup-buildx-action@v2.0.0 + + - name: Build Docker Image + run: | + docker buildx build \ + --platform=linux/arm64 \ + --cache-to=type=local,dest=/tmp/build-cache/arm64 \ + --tag $IMAGE_NAME \ + --label "runnumber=${GITHUB_RUN_ID}" \ + --file docker/Dockerfile.ubuntu \ + docker/ + + - name: Save ARM64 build cache + uses: actions/cache@v3 + with: + path: /tmp/build-cache/arm64 + key: ${{github.sha}} + + + # Build AMD64 container + package-amd64: + runs-on: ubuntu-22.04 + needs: [ + build + ] + + permissions: + contents: read + + steps: + - uses: actions/checkout@v3 + + - name: Restore Python build cache + uses: actions/cache@v3 + with: + path: docker/root + key: ${{github.sha}} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: linux/amd64 + + - name: Docker Setup Buildx + uses: docker/setup-buildx-action@v2.0.0 + + - name: Build Docker Image + run: | + docker buildx build \ + --platform=linux/amd64 \ + --cache-to=type=local,dest=/tmp/build-cache/amd64 \ + --tag $IMAGE_NAME \ + --label "runnumber=${GITHUB_RUN_ID}" \ + --file docker/Dockerfile.ubuntu \ + docker/ + + - name: Save AMD64 build cache + uses: actions/cache@v3 + with: + path: /tmp/build-cache/amd64 + key: ${{github.sha}} + + + # On master branch, build the docker manifest file from the cached + # docker builds and push to the registry + deploy: + runs-on: ubuntu-22.04 + needs: [ + package-arm64, + package-amd64 + ] + + permissions: + packages: write + contents: read + + if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') }} + steps: + - uses: actions/checkout@v3 + + - name: Restore Python build cache + uses: actions/cache@v3 + with: + path: docker/root + key: ${{github.sha}} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: linux/amd64,linux/arm64 + + - name: Docker Setup Buildx + uses: docker/setup-buildx-action@v2.0.0 + + - name: login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Restore ARM64 build cache + uses: actions/cache@v3 + with: + path: /tmp/build-cache/arm64 + key: ${{github.sha}} + + - name: Restore AMD64 build cache + uses: actions/cache@v3 + with: + path: /tmp/build-cache/amd64 + key: ${{github.sha}} + + - name: Format image_id + id: formatted-image_id + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/${IMAGE_NAME} + + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + echo IMAGE_ID=${IMAGE_ID} + + echo ::set-output name=IMAGE_ID::${IMAGE_ID} + + - name: Get the version + id: formatted_version + run: | + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + # Use Docker `latest` tag convention + [ "$VERSION" == "master" ] && VERSION=latest + echo VERSION=${VERSION} + + echo ::set-output name=VERSION::${VERSION} + + - name: Build Docker Image and push to registry + run: | + docker buildx build --push \ + --platform=linux/amd64,linux/arm64 \ + --cache-from=type=local,src=/tmp/build-cache/amd64 \ + --cache-from=type=local,src=/tmp/build-cache/arm64 \ + --tag ubuntu-${{ steps.formatted-image_id.outputs.IMAGE_ID }}:ubuntu-${{ steps.formatted_version.outputs.VERSION }} \ + --label "runnumber=ubuntu-${GITHUB_RUN_ID}" \ + --file docker/Dockerfile.ubuntu \ + docker/ \ No newline at end of file diff --git a/.github/workflows/package.yaml b/.github/workflows/package.yaml index cfb3984d..de6ef87a 100644 --- a/.github/workflows/package.yaml +++ b/.github/workflows/package.yaml @@ -139,7 +139,6 @@ jobs: deploy: runs-on: ubuntu-22.04 needs: [ - build, package-arm64, package-amd64 ] diff --git a/Makefile b/Makefile index 1242f47e..f61f8b34 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,9 @@ docker_stage: wheel cp dist/*.whl docker/root/ cp -R examples docker/root/defaults/ docker: docker_stage - sudo docker build --no-cache -t ytdl-sub:local docker/ + sudo docker build --progress=plain --no-cache -t ytdl-sub:local docker/ +docker_ubuntu: docker_stage + sudo docker build --progress=plain --no-cache -t ytdl-sub:local_ubuntu -f docker/Dockerfile.ubuntu docker/ executable: clean pyinstaller ytdl-sub.spec mv dist/ytdl-sub dist/ytdl-sub${EXEC_SUFFIX} diff --git a/docker/Dockerfile b/docker/Dockerfile index c80f584f..521775ec 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -45,8 +45,4 @@ RUN mkdir -p /config && \ ENV EDITOR="nano" \ HOME="/config" -VOLUME /config - - - - +VOLUME /config \ No newline at end of file diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu new file mode 100644 index 00000000..145bffa9 --- /dev/null +++ b/docker/Dockerfile.ubuntu @@ -0,0 +1,83 @@ +FROM ghcr.io/linuxserver/baseimage-ubuntu:jammy + +# https://askubuntu.com/questions/972516/debian-frontend-environment-variable +ARG DEBIAN_FRONTEND=noninteractive + +# Needed for phantomjs +ENV OPENSSL_CONF=/etc/ssl + +############################################################################### +# YTDL-SUB INSTALL + +SHELL ["/bin/bash", "-c"] +COPY root/ / +RUN mkdir -p /config && \ + apt-get -y update && \ + apt-get -y upgrade && \ + apt-get install --no-install-recommends -y \ + software-properties-common && \ + apt-get -y update && \ + apt-get -y upgrade && \ + apt-get install --no-install-recommends -y \ + vim \ + g++ \ + nano \ + make \ + python3.10-dev \ + python3-pip \ + fontconfig \ + xz-utils \ + bzip2 \ + python3-venv && \ + if [[ $(uname -m) == "x86_64" ]]; then \ + curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz && \ + tar -xf ffmpeg.tar.gz && \ + chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffmpeg && \ + chmod +x ffmpeg-master-latest-linux64-gpl/bin/ffprobe && \ + mv ffmpeg-master-latest-linux64-gpl/bin/ffmpeg /usr/bin/ffmpeg && \ + mv ffmpeg-master-latest-linux64-gpl/bin/ffprobe /usr/bin/ffprobe && \ + rm ffmpeg.tar.gz && \ + rm -rf ffmpeg-master-latest-linux64-gpl/ ; \ + else \ + curl -L -o ffmpeg.tar.gz https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz && \ + tar -xf ffmpeg.tar.gz && \ + chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg && \ + chmod +x ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe && \ + mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffmpeg /usr/bin/ffmpeg && \ + mv ffmpeg-master-latest-linuxarm64-gpl/bin/ffprobe /usr/bin/ffprobe && \ + rm ffmpeg.tar.gz && \ + rm -rf ffmpeg-master-latest-linuxarm64-gpl/ ; \ + fi && \ + # Ensure ffmpeg is installed + ffmpeg -version && \ + # Install phantomjs if using x86_64, ensure it is properly installed + if [[ $(uname -m) == "x86_64" ]]; then \ + curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2 | tar xj && \ + mv phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/bin/phantomjs && \ + rm -rf phantomjs-2.1.1-linux-x86_64/ && \ + echo "Phantom JS version:" && \ + phantomjs --version ; \ + fi && \ + # Install ytdl-sub, ensure it is installed properly + pip install --no-cache-dir ytdl_sub-*.whl && \ + ytdl-sub -h && \ + # Delete unneeded packages after install + rm ytdl_sub-*.whl && \ + apt-get remove -y \ + g++ \ + make \ + xz-utils \ + bzip2 \ + python3.10-dev \ + python3-venv && \ + apt-get autoremove -y && \ + apt-get purge -y --auto-remove && \ + rm -rf /var/lib/apt/lists/* + +############################################################################### +# CONTAINER CONFIGS + +ENV EDITOR="nano" \ +HOME="/config" + +VOLUME /config \ No newline at end of file diff --git a/src/ytdl_sub/__init__.py b/src/ytdl_sub/__init__.py index 38c8858f..2bf1fbf3 100644 --- a/src/ytdl_sub/__init__.py +++ b/src/ytdl_sub/__init__.py @@ -1,2 +1,2 @@ -__pypi_version__ = "2023.02.22" -__local_version__ = "2023.02.22+40d5517" +__pypi_version__ = "2023.03.15" +__local_version__ = "2023.03.15+e69933d" From c87b595f66de6407ed525b9a580c586aa7a1e93a Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 15 Mar 2023 14:59:52 -0700 Subject: [PATCH 2/5] [DOCKER] Properly tag ubuntu image (#546) * [DOCKER] Properly tag ubuntu image * unique keys --- .github/workflows/package-ubuntu.yaml | 11 ++++++----- .github/workflows/package.yaml | 9 +++++---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/package-ubuntu.yaml b/.github/workflows/package-ubuntu.yaml index 7f2e746b..6a8bc2f4 100644 --- a/.github/workflows/package-ubuntu.yaml +++ b/.github/workflows/package-ubuntu.yaml @@ -89,7 +89,7 @@ jobs: uses: actions/cache@v3 with: path: /tmp/build-cache/arm64 - key: ${{github.sha}} + key: ${{github.sha}}-arm64 # Build AMD64 container @@ -133,7 +133,7 @@ jobs: uses: actions/cache@v3 with: path: /tmp/build-cache/amd64 - key: ${{github.sha}} + key: ${{github.sha}}-amd64 # On master branch, build the docker manifest file from the cached @@ -141,6 +141,7 @@ jobs: deploy: runs-on: ubuntu-22.04 needs: [ + build, package-arm64, package-amd64 ] @@ -178,13 +179,13 @@ jobs: uses: actions/cache@v3 with: path: /tmp/build-cache/arm64 - key: ${{github.sha}} + key: ${{github.sha}}-arm64 - name: Restore AMD64 build cache uses: actions/cache@v3 with: path: /tmp/build-cache/amd64 - key: ${{github.sha}} + key: ${{github.sha}}-amd64 - name: Format image_id id: formatted-image_id @@ -216,7 +217,7 @@ jobs: --platform=linux/amd64,linux/arm64 \ --cache-from=type=local,src=/tmp/build-cache/amd64 \ --cache-from=type=local,src=/tmp/build-cache/arm64 \ - --tag ubuntu-${{ steps.formatted-image_id.outputs.IMAGE_ID }}:ubuntu-${{ steps.formatted_version.outputs.VERSION }} \ + --tag ${{ steps.formatted-image_id.outputs.IMAGE_ID }}:ubuntu-${{ steps.formatted_version.outputs.VERSION }} \ --label "runnumber=ubuntu-${GITHUB_RUN_ID}" \ --file docker/Dockerfile.ubuntu \ docker/ \ No newline at end of file diff --git a/.github/workflows/package.yaml b/.github/workflows/package.yaml index de6ef87a..835b75f2 100644 --- a/.github/workflows/package.yaml +++ b/.github/workflows/package.yaml @@ -88,7 +88,7 @@ jobs: uses: actions/cache@v3 with: path: /tmp/build-cache/arm64 - key: ${{github.sha}} + key: ${{github.sha}}-arm64 # Build AMD64 container @@ -131,7 +131,7 @@ jobs: uses: actions/cache@v3 with: path: /tmp/build-cache/amd64 - key: ${{github.sha}} + key: ${{github.sha}}-amd64 # On master branch, build the docker manifest file from the cached @@ -139,6 +139,7 @@ jobs: deploy: runs-on: ubuntu-22.04 needs: [ + build, package-arm64, package-amd64 ] @@ -176,13 +177,13 @@ jobs: uses: actions/cache@v3 with: path: /tmp/build-cache/arm64 - key: ${{github.sha}} + key: ${{github.sha}}-arm64 - name: Restore AMD64 build cache uses: actions/cache@v3 with: path: /tmp/build-cache/amd64 - key: ${{github.sha}} + key: ${{github.sha}}-amd64 - name: Format image_id id: formatted-image_id From 9c41a173756329fe668c187510a6b7749c5b27ff Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 15 Mar 2023 15:55:30 -0700 Subject: [PATCH 3/5] [DOCS] Fix download docs (#547) --- docs/config.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 79bb5266..7205e55d 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -62,13 +62,13 @@ download strategy has its own set of parameters. url ''' -.. autoclass:: ytdl_sub.downloaders.generic.url.UrlDownloadOptions() +.. autoclass:: ytdl_sub.downloaders.url.url.UrlDownloadOptions() :members: url, playlist_thumbnails, source_thumbnails, download_reverse :member-order: bysource multi_url ''''''''' -.. autoclass:: ytdl_sub.downloaders.generic.multi_url.MultiUrlDownloadOptions() +.. autoclass:: ytdl_sub.downloaders.url.multi_url.MultiUrlDownloadOptions() :members: urls, variables ------------------------------------------------------------------------------- @@ -79,7 +79,7 @@ output_options .. autoclass:: ytdl_sub.config.preset_options.OutputOptions() :members: :member-order: bysource - :exclude-members: get_upload_date_range_to_keep + :exclude-members: get_upload_date_range_to_keep, partial_validate ------------------------------------------------------------------------------- From ec51361728ddb58d687b35769cef93b40cc37c1d Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 15 Mar 2023 16:08:15 -0700 Subject: [PATCH 4/5] [DOCS] Docker passthrough install docs (#548) --- docs/install.rst | 63 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index 660cb6d4..9b6fef57 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -33,10 +33,65 @@ on how set up a cron job in the docker container. - DOCKER_MODS=linuxserver/mods:universal-cron volumes: - :/config - - :/tv_shows # optional - - :/movies # optional - - :/music_videos # optional - - :/music # optional + - :/tv_shows # optional + - :/movies # optional + - :/music_videos # optional + - :/music # optional + restart: unless-stopped + +Nvidia GPU Passthrough +^^^^^^^^^^^^^^^^^^^^^^ +For GPU passthrough, you must use the ``ytdl-sub`` Ubuntu version with the following additions: + +.. code-block:: yaml + + services: + ytdl-sub: + image: ghcr.io/jmbannon/ytdl-sub:ubuntu-latest + container_name: ytdl-sub + environment: + - PUID=1000 + - PGID=1000 + - TZ=America/Los_Angeles + - DOCKER_MODS=linuxserver/mods:universal-cron + - NVIDIA_DRIVER_CAPABILITIES=all # Nvidia ENV args + - NVIDIA_VISIBLE_DEVICES=all + volumes: + - :/config + - :/tv_shows # optional + - :/movies # optional + - :/music_videos # optional + - :/music # optional + deploy: + resources: + reservations: + devices: + - capabilities: [gpu] # GPU passthrough + restart: unless-stopped + +CPU Passthrough +^^^^^^^^^^^^^^^^^^^^^^ +For CPU passthrough, you must use the ``ytdl-sub`` Ubuntu version with the following additions: + +.. code-block:: yaml + + services: + ytdl-sub: + image: ghcr.io/jmbannon/ytdl-sub:ubuntu-latest + container_name: ytdl-sub + environment: + - PUID=1000 + - PGID=1000 + - TZ=America/Los_Angeles + - DOCKER_MODS=linuxserver/mods:universal-cron + volumes: + - :/config + - :/tv_shows # optional + - :/movies # optional + - :/music_videos # optional + - :/music # optional + devices: + - /dev/dri:/dev/dri # CPU passthrough restart: unless-stopped Docker From a96fcff55fffeb6c40d46d5d53bb7a20b6238263 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 15 Mar 2023 16:12:42 -0700 Subject: [PATCH 5/5] [DOCS] passthrough in README (#549) --- README.md | 2 ++ docs/install.rst | 50 ++++++++++++++++++++++++------------------------ 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 94ed04d7..1291ac37 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,8 @@ player to recognize downloads as: `ytdl-sub` can be installed on the following platforms. - [Docker Compose](https://ytdl-sub.readthedocs.io/en/latest/install.html#docker-compose_) + - [with CPU passthrough](https://ytdl-sub.readthedocs.io/en/latest/install.html#cpu-passthrough) + - [with GPU passthrough](https://ytdl-sub.readthedocs.io/en/latest/install.html#nvidia-gpu-passthrough) - [Docker CLI](https://ytdl-sub.readthedocs.io/en/latest/install.html#docker) - [Windows](https://ytdl-sub.readthedocs.io/en/latest/install.html#windows) - [Linux](https://ytdl-sub.readthedocs.io/en/latest/install.html#linux) diff --git a/docs/install.rst b/docs/install.rst index 9b6fef57..584d39cb 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -39,6 +39,31 @@ on how set up a cron job in the docker container. - :/music # optional restart: unless-stopped +CPU Passthrough +^^^^^^^^^^^^^^^^^^^^^^ +For CPU passthrough, you must use the ``ytdl-sub`` Ubuntu version with the following additions: + +.. code-block:: yaml + + services: + ytdl-sub: + image: ghcr.io/jmbannon/ytdl-sub:ubuntu-latest + container_name: ytdl-sub + environment: + - PUID=1000 + - PGID=1000 + - TZ=America/Los_Angeles + - DOCKER_MODS=linuxserver/mods:universal-cron + volumes: + - :/config + - :/tv_shows # optional + - :/movies # optional + - :/music_videos # optional + - :/music # optional + devices: + - /dev/dri:/dev/dri # CPU passthrough + restart: unless-stopped + Nvidia GPU Passthrough ^^^^^^^^^^^^^^^^^^^^^^ For GPU passthrough, you must use the ``ytdl-sub`` Ubuntu version with the following additions: @@ -69,31 +94,6 @@ For GPU passthrough, you must use the ``ytdl-sub`` Ubuntu version with the follo - capabilities: [gpu] # GPU passthrough restart: unless-stopped -CPU Passthrough -^^^^^^^^^^^^^^^^^^^^^^ -For CPU passthrough, you must use the ``ytdl-sub`` Ubuntu version with the following additions: - -.. code-block:: yaml - - services: - ytdl-sub: - image: ghcr.io/jmbannon/ytdl-sub:ubuntu-latest - container_name: ytdl-sub - environment: - - PUID=1000 - - PGID=1000 - - TZ=America/Los_Angeles - - DOCKER_MODS=linuxserver/mods:universal-cron - volumes: - - :/config - - :/tv_shows # optional - - :/movies # optional - - :/music_videos # optional - - :/music # optional - devices: - - /dev/dri:/dev/dri # CPU passthrough - restart: unless-stopped - Docker -------------- .. code-block:: bash