From c85373ad5f1e7644b88105fa799dc83b172cc72c Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 15 Mar 2023 11:15:11 -0700 Subject: [PATCH] separate build for ubuntu --- .github/workflows/package-ubuntu.yaml | 222 ++++++++++++++++++++++++++ .github/workflows/package.yaml | 1 - Makefile | 2 + docker/Dockerfile | 43 ++--- docker/Dockerfile.ubuntu | 65 ++++++++ 5 files changed, 302 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/package-ubuntu.yaml create mode 100644 docker/Dockerfile.ubuntu diff --git a/.github/workflows/package-ubuntu.yaml b/.github/workflows/package-ubuntu.yaml new file mode 100644 index 00000000..7b710238 --- /dev/null +++ b/.github/workflows/package-ubuntu.yaml @@ -0,0 +1,222 @@ +name: ytld-sub Docker Build + +on: + push: + # Publish `master` as Docker `latest` image. + branches: + - master + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + # Run tests for any PRs. + pull_request: + +env: + IMAGE_NAME: ytdl-sub + +jobs: + # Push image to GitHub Packages. + # See also https://docs.docker.com/docker-hub/builds/ + build: + runs-on: ubuntu-22.04 + strategy: + matrix: + python-version: [ "3.10" ] + + permissions: + contents: read + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Build Wheel + run: | + make docker_stage + + - name: Save Python build cache + uses: actions/cache@v3 + with: + path: docker/root + key: ${{github.sha}} + + + # Build ARM64 container, only on master branch to save time testing + package-arm64: + runs-on: ubuntu-22.04 + needs: [ + build + ] + + permissions: + contents: read + + if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') }} + steps: + - uses: actions/checkout@v3 + + - name: Restore Python build cache + uses: actions/cache@v3 + with: + path: docker/root + key: ${{github.sha}} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: linux/arm64 + + - name: Docker Setup Buildx + uses: docker/setup-buildx-action@v2.0.0 + + - name: Build Docker Image + run: | + docker buildx build \ + --platform=linux/arm64 \ + --cache-to=type=local,dest=/tmp/build-cache/arm64 \ + --tag $IMAGE_NAME \ + --label "runnumber=${GITHUB_RUN_ID}" \ + -f docker/Dockerfile.ubuntu + docker/ + + - name: Save ARM64 build cache + uses: actions/cache@v3 + with: + path: /tmp/build-cache/arm64 + key: ${{github.sha}} + + + # Build AMD64 container + package-amd64: + runs-on: ubuntu-22.04 + needs: [ + build + ] + + permissions: + contents: read + + steps: + - uses: actions/checkout@v3 + + - name: Restore Python build cache + uses: actions/cache@v3 + with: + path: docker/root + key: ${{github.sha}} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: linux/amd64 + + - name: Docker Setup Buildx + uses: docker/setup-buildx-action@v2.0.0 + + - name: Build Docker Image + run: | + docker buildx build \ + --platform=linux/amd64 \ + --cache-to=type=local,dest=/tmp/build-cache/amd64 \ + --tag $IMAGE_NAME \ + --label "runnumber=${GITHUB_RUN_ID}" \ + -f docker/Dockerfile.ubuntu + docker/ + + - name: Save AMD64 build cache + uses: actions/cache@v3 + with: + path: /tmp/build-cache/amd64 + key: ${{github.sha}} + + + # On master branch, build the docker manifest file from the cached + # docker builds and push to the registry + deploy: + runs-on: ubuntu-22.04 + needs: [ + package-arm64, + package-amd64 + ] + + permissions: + packages: write + contents: read + +# if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') }} + steps: + - uses: actions/checkout@v3 + + - name: Restore Python build cache + uses: actions/cache@v3 + with: + path: docker/root + key: ${{github.sha}} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: linux/amd64,linux/arm64 + + - name: Docker Setup Buildx + uses: docker/setup-buildx-action@v2.0.0 + + - name: login to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Restore ARM64 build cache + uses: actions/cache@v3 + with: + path: /tmp/build-cache/arm64 + key: ${{github.sha}} + + - name: Restore AMD64 build cache + uses: actions/cache@v3 + with: + path: /tmp/build-cache/amd64 + key: ${{github.sha}} + + - name: Format image_id + id: formatted-image_id + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/${IMAGE_NAME} + + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + echo IMAGE_ID=${IMAGE_ID} + + echo ::set-output name=IMAGE_ID::${IMAGE_ID} + + - name: Get the version + id: formatted_version + run: | + # Strip git ref prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + # Strip "v" prefix from tag name + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + # Use Docker `latest` tag convention + [ "$VERSION" == "master" ] && VERSION=latest + echo VERSION=${VERSION} + + echo ::set-output name=VERSION::${VERSION} + + - name: Build Docker Image and push to registry + run: | + docker buildx build --push \ + --platform=linux/amd64,linux/arm64 \ + --cache-from=type=local,src=/tmp/build-cache/amd64 \ + --cache-from=type=local,src=/tmp/build-cache/arm64 \ + --tag ubuntu-${{ steps.formatted-image_id.outputs.IMAGE_ID }}:ubuntu-${{ steps.formatted_version.outputs.VERSION }} \ + --label "runnumber=ubuntu-${GITHUB_RUN_ID}" \ + -f docker/Dockerfile.ubuntu \ + docker/ \ No newline at end of file diff --git a/.github/workflows/package.yaml b/.github/workflows/package.yaml index cfb3984d..de6ef87a 100644 --- a/.github/workflows/package.yaml +++ b/.github/workflows/package.yaml @@ -139,7 +139,6 @@ jobs: deploy: runs-on: ubuntu-22.04 needs: [ - build, package-arm64, package-amd64 ] diff --git a/Makefile b/Makefile index 6b21416b..f61f8b34 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,8 @@ docker_stage: wheel cp -R examples docker/root/defaults/ docker: docker_stage sudo docker build --progress=plain --no-cache -t ytdl-sub:local docker/ +docker_ubuntu: docker_stage + sudo docker build --progress=plain --no-cache -t ytdl-sub:local_ubuntu -f docker/Dockerfile.ubuntu docker/ executable: clean pyinstaller ytdl-sub.spec mv dist/ytdl-sub dist/ytdl-sub${EXEC_SUFFIX} diff --git a/docker/Dockerfile b/docker/Dockerfile index 2364876f..521775ec 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,41 +1,29 @@ -FROM ghcr.io/linuxserver/baseimage-ubuntu:jammy - -# https://askubuntu.com/questions/972516/debian-frontend-environment-variable -ARG DEBIAN_FRONTEND=noninteractive - -# https://github.com/NVIDIA/nvidia-docker/wiki/Installation-(Native-GPU-Support) -ENV NVIDIA_VISIBLE_DEVICES=all -ENV NVIDIA_DRIVER_CAPABILITIES="compute,video,utility" +FROM ghcr.io/linuxserver/baseimage-alpine:3.17 ############################################################################### # YTDL-SUB INSTALL COPY root/ / RUN mkdir -p /config && \ - apt-get -y update && \ - apt-get -y upgrade && \ - apt-get install --no-install-recommends -y \ - software-properties-common && \ - add-apt-repository ppa:savoury1/ffmpeg4 && \ - add-apt-repository ppa:savoury1/ffmpeg5 && \ - apt-get -y update && \ - apt-get -y upgrade && \ - apt-get install --no-install-recommends -y \ + apk update --no-cache && \ + apk upgrade --no-cache && \ + apk add --no-cache --repository=http://dl-3.alpinelinux.org/alpine/edge/main/ \ vim \ g++ \ nano \ make \ - python3.10-dev \ - python3-pip \ + python3=~3.10 \ + py3-pip \ fontconfig \ - python3-venv \ - # Ensure ffmpeg is properly installed - ffmpeg && \ + py3-setuptools && \ + # Install edge ffmpeg, ensure it is properly installed + apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \ + "ffmpeg>5.1" && \ ffmpeg -version && \ # Install phantomjs if using x86_64, ensure it is properly installed if [[ $(uname -m) == "x86_64" ]]; then \ cd /usr/share && \ - curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.8-linux-x86_64.tar.bz2 | tar xj && \ + curl -L https://github.com/Overbryd/docker-phantomjs-alpine/releases/download/2.11/phantomjs-alpine-x86_64.tar.bz2 | tar xj && \ ln -s /usr/share/phantomjs/phantomjs /usr/bin/phantomjs && \ echo "Phantom JS version:" && \ phantomjs --version && \ @@ -44,17 +32,12 @@ RUN mkdir -p /config && \ # Install ytdl-sub, ensure it is installed properly pip install --no-cache-dir ytdl_sub-*.whl && \ ytdl-sub -h && \ - ldconfig && \ # Delete unneeded packages after install rm ytdl_sub-*.whl && \ - apt-get remove -y \ + apk del \ g++ \ make \ - python3.10-dev \ - python3-venv && \ - apt-get autoremove -y && \ - apt-get purge -y --auto-remove && \ - rm -rf /var/lib/apt/lists/* + py3-setuptools ############################################################################### # CONTAINER CONFIGS diff --git a/docker/Dockerfile.ubuntu b/docker/Dockerfile.ubuntu new file mode 100644 index 00000000..2364876f --- /dev/null +++ b/docker/Dockerfile.ubuntu @@ -0,0 +1,65 @@ +FROM ghcr.io/linuxserver/baseimage-ubuntu:jammy + +# https://askubuntu.com/questions/972516/debian-frontend-environment-variable +ARG DEBIAN_FRONTEND=noninteractive + +# https://github.com/NVIDIA/nvidia-docker/wiki/Installation-(Native-GPU-Support) +ENV NVIDIA_VISIBLE_DEVICES=all +ENV NVIDIA_DRIVER_CAPABILITIES="compute,video,utility" + +############################################################################### +# YTDL-SUB INSTALL + +COPY root/ / +RUN mkdir -p /config && \ + apt-get -y update && \ + apt-get -y upgrade && \ + apt-get install --no-install-recommends -y \ + software-properties-common && \ + add-apt-repository ppa:savoury1/ffmpeg4 && \ + add-apt-repository ppa:savoury1/ffmpeg5 && \ + apt-get -y update && \ + apt-get -y upgrade && \ + apt-get install --no-install-recommends -y \ + vim \ + g++ \ + nano \ + make \ + python3.10-dev \ + python3-pip \ + fontconfig \ + python3-venv \ + # Ensure ffmpeg is properly installed + ffmpeg && \ + ffmpeg -version && \ + # Install phantomjs if using x86_64, ensure it is properly installed + if [[ $(uname -m) == "x86_64" ]]; then \ + cd /usr/share && \ + curl -L https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-1.9.8-linux-x86_64.tar.bz2 | tar xj && \ + ln -s /usr/share/phantomjs/phantomjs /usr/bin/phantomjs && \ + echo "Phantom JS version:" && \ + phantomjs --version && \ + cd -; \ + fi && \ + # Install ytdl-sub, ensure it is installed properly + pip install --no-cache-dir ytdl_sub-*.whl && \ + ytdl-sub -h && \ + ldconfig && \ + # Delete unneeded packages after install + rm ytdl_sub-*.whl && \ + apt-get remove -y \ + g++ \ + make \ + python3.10-dev \ + python3-venv && \ + apt-get autoremove -y && \ + apt-get purge -y --auto-remove && \ + rm -rf /var/lib/apt/lists/* + +############################################################################### +# CONTAINER CONFIGS + +ENV EDITOR="nano" \ +HOME="/config" + +VOLUME /config \ No newline at end of file