Abdulmohsen
GitHub
commit
98970da663
9 changed files with 91 additions and 42 deletions
7
.vscode/launch.json
vendored
7
.vscode/launch.json
vendored
|
|
@ -35,12 +35,7 @@
|
|||
"YTP_DOWNLOAD_PATH": "${workspaceFolder}/var/downloads",
|
||||
"YTP_TEMP_PATH": "${workspaceFolder}/var/tmp",
|
||||
"PYDEVD_DISABLE_FILE_VALIDATION": "1",
|
||||
"YTP_IGNORE_UI": "true",
|
||||
"YTP_PIP_IGNORE_UPDATES": "true",
|
||||
"YTP_LOG_LEVEL": "DEBUG",
|
||||
"YTP_DEBUG": "true",
|
||||
"YTP_YTDL_DEBUG": "false",
|
||||
"YTP_ACCESS_LOG": "true",
|
||||
"YTP_IGNORE_UI": "true"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
|
|
|||
3
.vscode/settings.json
vendored
3
.vscode/settings.json
vendored
|
|
@ -17,10 +17,13 @@
|
|||
"aiocron",
|
||||
"arrowless",
|
||||
"copyts",
|
||||
"daterange",
|
||||
"dotenv",
|
||||
"finaldir",
|
||||
"getpid",
|
||||
"gpac",
|
||||
"httpx",
|
||||
"levelno",
|
||||
"libcurl",
|
||||
"libx",
|
||||
"matroska",
|
||||
|
|
|
|||
13
Dockerfile
13
Dockerfile
|
|
@ -13,7 +13,7 @@ ENV PYTHONFAULTHANDLER=1
|
|||
|
||||
# Use sed to strip carriage-return characters from the entrypoint script (in case building on Windows)
|
||||
# Install dependencies
|
||||
RUN apk add --update coreutils curl gcc g++ musl-dev libffi-dev openssl-dev && pip install pipenv
|
||||
RUN apk add --update coreutils curl gcc g++ musl-dev libffi-dev openssl-dev curl make && pip install pipenv
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
|
@ -21,6 +21,13 @@ ARG PIPENV_FLAGS="--deploy"
|
|||
COPY ./Pipfile* .
|
||||
RUN PIPENV_VENV_IN_PROJECT=1 pipenv install ${PIPENV_FLAGS}
|
||||
|
||||
# Bundle mp4box into the image
|
||||
WORKDIR /tmp
|
||||
RUN mkdir /gpac-master && curl -k -L https://github.com/gpac/gpac/archive/refs/tags/v2.4.0.zip -o /tmp/gpac.zip && \
|
||||
unzip /tmp/gpac.zip && mv /tmp/gpac-*/* /gpac-master
|
||||
WORKDIR /gpac-master
|
||||
RUN ./configure --static-mp4box --use-zlib=no && make -j4
|
||||
|
||||
FROM python:3.11-alpine
|
||||
|
||||
ARG TZ=UTC
|
||||
|
|
@ -46,13 +53,15 @@ RUN sed -i 's/\r$//g' /entrypoint.sh && chmod +x /entrypoint.sh
|
|||
COPY --chown=app:app ./app /app/app
|
||||
COPY --chown=app:app --from=node_builder /app/exported /app/ui/exported
|
||||
COPY --chown=app:app --from=python_builder /app/.venv /opt/python
|
||||
COPY --chown=app:app --from=python_builder /gpac-master/bin/gcc/MP4Box /usr/bin/mp4box
|
||||
COPY --chown=app:app ./healthcheck.sh /usr/local/bin/healthcheck
|
||||
|
||||
ENV PATH="/opt/python/bin:$PATH"
|
||||
|
||||
RUN chown -R app:app /config /downloads && chmod +x /usr/local/bin/healthcheck && \
|
||||
sed -i 's$#!\/app\/\.venv\/bin\/python$#!/opt/python/bin/python$' /opt/python/bin/* && \
|
||||
sed -i "s%'\/app\/\.venv'%'/opt/python'%" /opt/python/bin/activate*
|
||||
sed -i "s%'\/app\/\.venv'%'/opt/python'%" /opt/python/bin/activate* && \
|
||||
chmod +x /usr/bin/mp4box
|
||||
|
||||
VOLUME /config
|
||||
VOLUME /downloads
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ YTPTube started as a fork of [meTube](https://github.com/alexta69/metube), Since
|
|||
* Basic Authentication support.
|
||||
* Support for curl_cffi, see [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#impersonation)
|
||||
* Support for both advanced and basic mode for WebUI.
|
||||
* Bundled tools in container: curl-cffi, ffmpeg, ffprobe, aria2, rtmpdump, mkvtoolsnix, mp4box.
|
||||
|
||||
For more API endpoints, please refer to the [API documentation](API.md).
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import hashlib
|
|||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
from email.utils import formatdate
|
||||
|
|
@ -16,7 +17,24 @@ from .ffprobe import ffprobe
|
|||
from .ItemDTO import ItemDTO
|
||||
from .YTDLPOpts import YTDLPOpts
|
||||
|
||||
LOG = logging.getLogger("download")
|
||||
LOG = logging.getLogger("Download")
|
||||
|
||||
|
||||
class NestedLogger:
|
||||
debug_messages = ["[debug] ", "[download] "]
|
||||
|
||||
def __init__(self, logger: logging.Logger):
|
||||
self.logger = logger
|
||||
|
||||
def debug(self, msg: str):
|
||||
levelno = logging.DEBUG if any(msg.startswith(x) for x in self.debug_messages) else logging.INFO
|
||||
self.logger.log(level=levelno, msg=re.sub(r"^\[(debug|info)\] ", "", msg, flags=re.IGNORECASE))
|
||||
|
||||
def error(self, msg):
|
||||
self.logger.error(msg)
|
||||
|
||||
def warning(self, msg):
|
||||
self.logger.warning(msg)
|
||||
|
||||
|
||||
class Download:
|
||||
|
|
@ -66,7 +84,7 @@ class Download:
|
|||
temp_keep: bool = False
|
||||
"Keep temp folder after download."
|
||||
|
||||
def __init__(self, info: ItemDTO, info_dict: dict = None, debug: bool = False):
|
||||
def __init__(self, info: ItemDTO, info_dict: dict = None):
|
||||
config = Config.get_instance()
|
||||
|
||||
self.download_dir = info.download_dir
|
||||
|
|
@ -78,7 +96,8 @@ class Download:
|
|||
self.info = info
|
||||
self.id = info._id
|
||||
self.default_ytdl_opts = config.ytdl_options
|
||||
self.debug = debug
|
||||
self.debug = bool(config.debug)
|
||||
self.debug_ytdl = bool(config.ytdl_debug)
|
||||
self.cancelled = False
|
||||
self.tmpfilename = None
|
||||
self.status_queue = None
|
||||
|
|
@ -89,6 +108,7 @@ class Download:
|
|||
self.is_live = bool(info.is_live) or info.live_in is not None
|
||||
self.is_manifestless = "is_manifestless" in self.info.options and self.info.options["is_manifestless"] is True
|
||||
self.info_dict = info_dict
|
||||
self.logger = logging.getLogger(f"Download.{info.id if info.id else info._id}")
|
||||
|
||||
def _progress_hook(self, data: dict):
|
||||
dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields}
|
||||
|
|
@ -99,14 +119,14 @@ class Download:
|
|||
self.status_queue.put({"id": self.id, **dataDict})
|
||||
|
||||
def _postprocessor_hook(self, data: dict):
|
||||
if self.debug:
|
||||
self.logger.debug(f"Postprocessor hook: {data}")
|
||||
|
||||
if "MoveFiles" != data.get("postprocessor") or "finished" != data.get("status"):
|
||||
dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields}
|
||||
self.status_queue.put({"id": self.id, **dataDict, "status": "postprocessing"})
|
||||
return
|
||||
|
||||
if self.debug:
|
||||
LOG.debug(f"Postprocessor hook: {data}")
|
||||
|
||||
if "__finaldir" in data["info_dict"]:
|
||||
filename = os.path.join(data["info_dict"]["__finaldir"], os.path.basename(data["info_dict"]["filepath"]))
|
||||
else:
|
||||
|
|
@ -147,19 +167,21 @@ class Download:
|
|||
}
|
||||
)
|
||||
|
||||
if self.debug:
|
||||
if self.debug_ytdl:
|
||||
params["verbose"] = True
|
||||
params["noprogress"] = False
|
||||
|
||||
if self.info.cookies:
|
||||
try:
|
||||
cookie_file = os.path.join(self.temp_path, f"cookie_{self.info._id}.txt")
|
||||
LOG.debug(f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'.")
|
||||
self.logger.debug(
|
||||
f"Creating cookie file for '{self.info.id}: {self.info.title}' - '{cookie_file}'."
|
||||
)
|
||||
with open(cookie_file, "w") as f:
|
||||
f.write(self.info.cookies)
|
||||
params["cookiefile"] = f.name
|
||||
except ValueError as e:
|
||||
LOG.error(f"Failed to create cookie file for '{self.info.id}: {self.info.title}'. '{e!s}'.")
|
||||
self.logger.error(f"Failed to create cookie file for '{self.info.id}: {self.info.title}'. '{e!s}'.")
|
||||
|
||||
if self.is_live or self.is_manifestless:
|
||||
hasDeletedOptions = False
|
||||
|
|
@ -171,37 +193,39 @@ class Download:
|
|||
deletedOpts.append(opt)
|
||||
|
||||
if hasDeletedOptions:
|
||||
LOG.warning(
|
||||
self.logger.warning(
|
||||
f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted which are known to cause issues with live stream and post stream manifestless mode."
|
||||
)
|
||||
|
||||
LOG.info(
|
||||
self.logger.info(
|
||||
f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" preset="{self.preset}" started.'
|
||||
)
|
||||
|
||||
LOG.debug("Params before passing to yt-dlp.", extra=params)
|
||||
self.logger.debug("Params before passing to yt-dlp.", extra=params)
|
||||
|
||||
if "impersonate" in params:
|
||||
from yt_dlp.networking.impersonate import ImpersonateTarget
|
||||
|
||||
params["impersonate"] = ImpersonateTarget.from_str(params["impersonate"])
|
||||
|
||||
params["logger"] = NestedLogger(self.logger)
|
||||
|
||||
cls = yt_dlp.YoutubeDL(params=params)
|
||||
|
||||
if isinstance(self.info_dict, dict) and len(self.info_dict) > 1:
|
||||
LOG.debug(f"Downloading '{self.info.url}' using pre-info.")
|
||||
self.logger.debug(f"Downloading '{self.info.url}' using pre-info.")
|
||||
cls.process_ie_result(self.info_dict, download=True)
|
||||
ret = cls._download_retcode
|
||||
else:
|
||||
LOG.debug(f"Downloading using url: {self.info.url}")
|
||||
self.logger.debug(f"Downloading using url: {self.info.url}")
|
||||
ret = cls.download([self.info.url])
|
||||
|
||||
self.status_queue.put({"id": self.id, "status": "finished" if ret == 0 else "error"})
|
||||
except Exception as exc:
|
||||
LOG.exception(exc)
|
||||
self.logger.exception(exc)
|
||||
self.status_queue.put({"id": self.id, "status": "error", "msg": str(exc), "error": str(exc)})
|
||||
|
||||
LOG.info(f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" completed.')
|
||||
self.logger.info(f'Task id="{self.info.id}" PID="{os.getpid()}" title="{self.info.title}" completed.')
|
||||
|
||||
async def start(self, emitter: Emitter):
|
||||
self.emitter = emitter
|
||||
|
|
@ -244,28 +268,28 @@ class Download:
|
|||
self.cancel_in_progress = True
|
||||
procId = self.proc.ident
|
||||
|
||||
LOG.info(f"Closing PID='{procId}' download process.")
|
||||
self.logger.info(f"Closing PID='{procId}' download process.")
|
||||
|
||||
try:
|
||||
try:
|
||||
if "update_task" in self.__dict__ and self.update_task.done() is False:
|
||||
self.update_task.cancel()
|
||||
except Exception as e:
|
||||
LOG.error(f"Failed to close status queue: '{procId}'. {e}")
|
||||
self.logger.error(f"Failed to close status queue: '{procId}'. {e}")
|
||||
|
||||
self.kill()
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
if self.proc.is_alive():
|
||||
LOG.debug(f"Waiting for PID='{procId}' to close.")
|
||||
self.logger.debug(f"Waiting for PID='{procId}' to close.")
|
||||
await loop.run_in_executor(None, self.proc.join)
|
||||
LOG.debug(f"PID='{procId}' closed.")
|
||||
self.logger.debug(f"PID='{procId}' closed.")
|
||||
|
||||
if self.status_queue:
|
||||
LOG.debug(f"Closing status queue for PID='{procId}'.")
|
||||
self.logger.debug(f"Closing status queue for PID='{procId}'.")
|
||||
self.status_queue.put(Terminator())
|
||||
LOG.debug(f"Closed status queue for PID='{procId}'.")
|
||||
self.logger.debug(f"Closed status queue for PID='{procId}'.")
|
||||
|
||||
if self.proc:
|
||||
self.proc.close()
|
||||
|
|
@ -273,11 +297,11 @@ class Download:
|
|||
|
||||
self.delete_temp()
|
||||
|
||||
LOG.debug(f"Closed PID='{procId}' download process.")
|
||||
self.logger.debug(f"Closed PID='{procId}' download process.")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
LOG.error(f"Failed to close process: '{procId}'. {e}")
|
||||
self.logger.error(f"Failed to close process: '{procId}'. {e}")
|
||||
|
||||
return False
|
||||
|
||||
|
|
@ -295,11 +319,11 @@ class Download:
|
|||
return False
|
||||
|
||||
try:
|
||||
LOG.info(f"Killing download process: '{self.proc.ident}'.")
|
||||
self.logger.info(f"Killing download process: '{self.proc.ident}'.")
|
||||
self.proc.kill()
|
||||
return True
|
||||
except Exception as e:
|
||||
LOG.error(f"Failed to kill process: '{self.proc.ident}'. {e}")
|
||||
self.logger.error(f"Failed to kill process: '{self.proc.ident}'. {e}")
|
||||
|
||||
return False
|
||||
|
||||
|
|
@ -308,7 +332,7 @@ class Download:
|
|||
return
|
||||
|
||||
if "finished" != self.info.status and self.is_live:
|
||||
LOG.warning(
|
||||
self.logger.warning(
|
||||
f"Keeping live temp folder '{self.temp_path}', as the reported status is not finished '{self.info.status}'."
|
||||
)
|
||||
return
|
||||
|
|
@ -317,12 +341,12 @@ class Download:
|
|||
return
|
||||
|
||||
if self.temp_path == self.temp_dir:
|
||||
LOG.warning(
|
||||
self.logger.warning(
|
||||
f"Attempted to delete video temp folder: {self.temp_path}, but it is the same as main temp folder."
|
||||
)
|
||||
return
|
||||
|
||||
LOG.info(f"Deleting Temp folder '{self.temp_path}'.")
|
||||
self.logger.info(f"Deleting Temp folder '{self.temp_path}'.")
|
||||
shutil.rmtree(self.temp_path, ignore_errors=True)
|
||||
|
||||
async def progress_update(self):
|
||||
|
|
@ -343,7 +367,7 @@ class Download:
|
|||
continue
|
||||
|
||||
if self.debug:
|
||||
LOG.debug(f"Status Update: {self.info._id=} {status=}")
|
||||
self.logger.debug(f"Status Update: {self.info._id=} {status=}")
|
||||
|
||||
if isinstance(status, str):
|
||||
asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}")
|
||||
|
|
@ -400,7 +424,7 @@ class Download:
|
|||
except Exception as e:
|
||||
self.info.extras["is_video"] = True
|
||||
self.info.extras["is_audio"] = True
|
||||
LOG.exception(e)
|
||||
LOG.error(f"Failed to ffprobe: {status.get}. {e}")
|
||||
self.logger.exception(e)
|
||||
self.logger.error(f"Failed to ffprobe: {status.get}. {e}")
|
||||
|
||||
asyncio.create_task(self.emitter.updated(dl=self.info), name=f"emitter-u-{self.id}")
|
||||
|
|
|
|||
|
|
@ -310,7 +310,7 @@ class DownloadQueue(metaclass=Singleton):
|
|||
if property.startswith("playlist"):
|
||||
dl.template = str(dl.template).replace(f"%({property})s", str(value))
|
||||
|
||||
dlInfo: Download = Download(info=dl, info_dict=entry, debug=bool(self.config.ytdl_debug))
|
||||
dlInfo: Download = Download(info=dl, info_dict=entry)
|
||||
|
||||
if dlInfo.info.live_in or "is_upcoming" == entry.get("live_status"):
|
||||
dlInfo.info.status = "not_live"
|
||||
|
|
|
|||
|
|
@ -424,6 +424,7 @@ def arg_converter(args: str) -> dict:
|
|||
|
||||
"""
|
||||
import yt_dlp.options
|
||||
from yt_dlp.utils import DateRange
|
||||
|
||||
create_parser = yt_dlp.options.create_parser
|
||||
|
||||
|
|
@ -443,7 +444,9 @@ def arg_converter(args: str) -> dict:
|
|||
if "postprocessors" in diff:
|
||||
diff["postprocessors"] = [pp for pp in diff["postprocessors"] if pp not in default_opts["postprocessors"]]
|
||||
|
||||
return json.loads(json.dumps(diff))
|
||||
from .encoder import Encoder
|
||||
|
||||
return json.loads(json.dumps(diff, cls=Encoder))
|
||||
|
||||
|
||||
def validate_uuid(uuid_str: str, version: int = 4) -> bool:
|
||||
|
|
|
|||
|
|
@ -132,4 +132,9 @@ class YTDLPOpts(metaclass=Singleton):
|
|||
if "format" in data and data["format"] in ["not_set", "default"]:
|
||||
data["format"] = None
|
||||
|
||||
if "daterange" in data and isinstance(data["daterange"], dict):
|
||||
from yt_dlp.utils import DateRange
|
||||
|
||||
data["daterange"] = DateRange(data["daterange"]["start"], data["daterange"]["end"])
|
||||
|
||||
return data
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from yt_dlp.utils import DateRange
|
||||
|
||||
|
||||
class Encoder(json.JSONEncoder):
|
||||
"""
|
||||
|
|
@ -13,6 +16,12 @@ class Encoder(json.JSONEncoder):
|
|||
if isinstance(o, Path):
|
||||
return str(o)
|
||||
|
||||
if isinstance(o, DateRange):
|
||||
return {"start": str(o.start).replace("-", ""), "end": str(o.end).replace("-", "")}
|
||||
|
||||
if isinstance(o, date):
|
||||
return str(o)
|
||||
|
||||
if isinstance(o, object):
|
||||
if hasattr(o, "serialize"):
|
||||
return o.serialize()
|
||||
|
|
|
|||
Loading…
Reference in a new issue