diff --git a/README.md b/README.md index 95853cae..cc944ec1 100644 --- a/README.md +++ b/README.md @@ -274,41 +274,42 @@ If you feel like donating and appreciate my work, you can do so by donating to c Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line, or the `environment:` section in `compose.yaml` file. -| Environment Variable | Description | Default | -| ------------------------ | ---------------------------------------------------------------- | ---------------------------------- | -| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` | -| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` | -| YTP_INSTANCE_TITLE | The title of the instance | `empty string` | -| YTP_FILE_LOGGING | Whether to log to file | `false` | -| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` | -| YTP_MAX_WORKERS | How many works to use for downloads | `1` | -| YTP_AUTH_USERNAME | Username for basic authentication | `empty string` | -| YTP_AUTH_PASSWORD | Password for basic authentication | `empty string` | -| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` | -| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` | -| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` | -| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` | -| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | -| YTP_KEEP_ARCHIVE | Keep history of downloaded videos | `true` | -| YTP_YTDL_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | -| YTP_HOST | Which IP address to bind to | `0.0.0.0` | -| YTP_PORT | Which port to bind to | `8081` | -| YTP_LOG_LEVEL | Log level | `info` | -| YTP_STREAMER_VCODEC | The video codec to use for in-browser streaming | `libx264` | -| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` | -| YTP_ACCESS_LOG | Whether to log access to the web server | `true` | -| YTP_DEBUG | Whether to turn on debug mode | `false` | -| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` | -| YTP_SOCKET_TIMEOUT | The timeout for the yt-dlp socket connection variable | `30` | -| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` | -| YTP_DB_FILE | The path to the SQLite database file | `{config_path}/ytptube.db` | -| YTP_MANUAL_ARCHIVE | The path to the manual archive file | `{config_path}/manual_archive.log` | -| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` | -| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `empty string` | -| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` | -| YTP_BASIC_MODE | Whether to run WebUI in basic mode | `false` | -| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use. | `empty string` | -| YTP_BROWSER_ENABLED | Whether to enable the file browser | `false` | -| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | -| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | +| Environment Variable | Description | Default | +| ------------------------- | ------------------------------------------------------------------ | ---------------------------------- | +| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` | +| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` | +| YTP_INSTANCE_TITLE | The title of the instance | `empty string` | +| YTP_FILE_LOGGING | Whether to log to file | `false` | +| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` | +| YTP_MAX_WORKERS | How many works to use for downloads | `1` | +| YTP_AUTH_USERNAME | Username for basic authentication | `empty string` | +| YTP_AUTH_PASSWORD | Password for basic authentication | `empty string` | +| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` | +| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` | +| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` | +| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` | +| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | +| YTP_KEEP_ARCHIVE | Keep history of downloaded videos | `true` | +| YTP_YTDL_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | +| YTP_HOST | Which IP address to bind to | `0.0.0.0` | +| YTP_PORT | Which port to bind to | `8081` | +| YTP_LOG_LEVEL | Log level | `info` | +| YTP_STREAMER_VCODEC | The video codec to use for in-browser streaming | `libx264` | +| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` | +| YTP_ACCESS_LOG | Whether to log access to the web server | `true` | +| YTP_DEBUG | Whether to turn on debug mode | `false` | +| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` | +| YTP_SOCKET_TIMEOUT | The timeout for the yt-dlp socket connection variable | `30` | +| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` | +| YTP_DB_FILE | The path to the SQLite database file | `{config_path}/ytptube.db` | +| YTP_MANUAL_ARCHIVE | The path to the manual archive file | `{config_path}/manual_archive.log` | +| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` | +| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `empty string` | +| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` | +| YTP_BASIC_MODE | Whether to run WebUI in basic mode | `false` | +| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use. | `empty string` | +| YTP_BROWSER_ENABLED | Whether to enable the file browser | `false` | +| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | +| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | +| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index 0716d527..5b7a6456 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -18,6 +18,7 @@ from .config import Config from .DataStore import DataStore from .Download import Download from .Events import EventBus, Events +from .Events import info as event_info from .Events import warning as event_warning from .ItemDTO import Item, ItemDTO from .Presets import Presets @@ -239,6 +240,7 @@ class DownloadQueue(metaclass=Singleton): options: dict = {} error: str | None = None live_in: str | None = None + is_premiere: bool = bool(entry.get("is_premiere", False)) # check if the video is live stream. if "live_status" in entry and "is_upcoming" == entry.get("live_status"): @@ -246,7 +248,7 @@ class DownloadQueue(metaclass=Singleton): live_in = formatdate(entry.get("release_timestamp"), usegmt=True) item.extras.update({"live_in": live_in}) else: - error = "Live stream not yet started. And no date is set." + error = f"No start time is set for {'premiere' if is_premiere else 'live stream'}." else: error = entry.get("msg") @@ -274,7 +276,7 @@ class DownloadQueue(metaclass=Singleton): is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status) try: - download_dir = calc_download_path(base_path=self.config.download_path, folder=item.folder) + download_dir: str = calc_download_path(base_path=self.config.download_path, folder=item.folder) except Exception as e: LOG.exception(e) return {"status": "error", "msg": str(e)} @@ -287,6 +289,14 @@ class DownloadQueue(metaclass=Singleton): if isinstance(key, str) and key.startswith("playlist") and entry.get(key): item.extras[key] = entry.get(key) + item.extras["duration"] = entry.get("duration", item.extras.get("duration")) + + if not item.extras.get("live_in") and live_in: + item.extras["live_in"] = live_in + + if not item.extras.get("live_in") and is_premiere: + item.extras["is_premiere"] = is_premiere + dl = ItemDTO( id=str(entry.get("id")), title=str(entry.get("title")), @@ -310,18 +320,20 @@ class DownloadQueue(metaclass=Singleton): try: dlInfo: Download = Download(info=dl, info_dict=entry, logs=logs) - text_logs = "" + text_logs: str = "" if filtered_logs := extract_ytdlp_logs(logs): text_logs = f" Logs: {', '.join(filtered_logs)}" if live_in or "is_upcoming" == entry.get("live_status"): NotifyEvent = Events.COMPLETED dlInfo.info.status = "not_live" - dlInfo.info.msg = "Stream is not live yet." + text_logs - itemDownload = self.done.put(dlInfo) + dlInfo.info.msg = ( + f"{'Premiere video' if is_premiere else 'Live Stream' } is not available yet." + text_logs + ) + itemDownload: Download = self.done.put(dlInfo) elif len(entry.get("formats", [])) < 1: - availability = entry.get("availability", "public") - msg = "No formats found." + availability: str = entry.get("availability", "public") + msg: str = "No formats found." if availability and availability not in ("public",): msg += f" Availability is set for '{availability}'." @@ -329,7 +341,27 @@ class DownloadQueue(metaclass=Singleton): dlInfo.info.status = "error" itemDownload = self.done.put(dlInfo) NotifyEvent = Events.COMPLETED - await self._notify.emit(Events.LOG_WARNING, data=event_warning(msg)) + await self._notify.emit(Events.LOG_WARNING, data=event_warning(f"No formats found for '{dl.title}'.")) + elif is_premiere and self.config.prevent_live_premiere: + dlInfo.info.error = ( + f"Premiering right now. Delaying download by '{300+dl.extras.get('duration',0)}' seconds." + ) + _live_in = live_in or item.extras.get("live_in", None) + if _live_in: + starts_in = parsedate_to_datetime(live_in) + starts_in = starts_in.replace(tzinfo=UTC) if starts_in.tzinfo is None else starts_in.astimezone(UTC) + starts_in = starts_in + timedelta(minutes=5, seconds=dl.extras.get("duration", 0)) + dlInfo.info.error += f" Starts in {starts_in.isoformat()}." + + dlInfo.info.status = "not_live" + itemDownload = self.done.put(dlInfo) + NotifyEvent = Events.COMPLETED + await self._notify.emit( + Events.LOG_INFO, + data=event_info( + f"'{dl.title}' is premiering. Download delayed by '{300+dl.extras.get('duration')}'s." + ), + ) else: NotifyEvent = Events.ADDED itemDownload = self.queue.put(dlInfo) @@ -470,12 +502,10 @@ class DownloadQueue(metaclass=Singleton): if not entry: return {"status": "error", "msg": "Unable to extract info." + "\n".join(logs)} - if not item.requeued: - condition = Conditions.get_instance().match(info=entry) - if condition is not None: - already.pop() - LOG.info(f"Condition '{condition}' matched for '{item.url}'.") - return await self.add(item=item.new_with(requeued=True, cli=condition.cli), already=already) + if not item.requeued and (condition := Conditions.get_instance().match(info=entry)): + already.pop() + LOG.info(f"Condition '{condition.name}' matched for '{item.url}'.") + return await self.add(item=item.new_with(requeued=True, cli=condition.cli), already=already) end_time = time.perf_counter() - started LOG.debug(f"extract_info: for 'URL: {item.url}' is done in '{end_time:.3f}'. Length: '{len(entry)}/keys'.") @@ -833,32 +863,44 @@ class DownloadQueue(metaclass=Singleton): if self.is_paused() or self.done.empty(): return - LOG.debug("Checking for live stream items in the history queue.") + LOG.debug("Checking history queue for queued live stream links.") time_now = datetime.now(tz=UTC) - status = ["not_live", "is_upcoming", "is_live"] + status: list[str] = ["not_live", "is_upcoming", "is_live"] for id, item in list(self.done.items()): if item.info.status not in status: continue - item_ref = f"{id=} {item.info.id=} {item.info.title=}" + item_ref: str = f"{id=} {item.info.id=} {item.info.title=}" if not item.is_live: LOG.debug(f"Item '{item_ref}' is not a live stream.") continue - if not item.info.live_in: + live_in: str | None = item.info.live_in or item.info.extras.get("live_in", None) + if not live_in: LOG.debug(f"Item '{item_ref}' marked as live stream, but no date is set.") continue - starts_in = parsedate_to_datetime(item.info.live_in) + starts_in = parsedate_to_datetime(live_in) starts_in = starts_in.replace(tzinfo=UTC) if starts_in.tzinfo is None else starts_in.astimezone(UTC) if time_now < (starts_in + timedelta(minutes=1)): LOG.debug(f"Item '{item_ref}' is not yet live. will start in '{dt_delta(starts_in-time_now)}'.") continue + duration: int | None = item.info.extras.get("duration", None) + is_premiere: bool = item.info.extras.get("is_premiere", False) + + if is_premiere and duration and self.config.prevent_live_premiere: + premiere_ends: datetime = starts_in + timedelta(minutes=5, seconds=duration) + if time_now < premiere_ends: + LOG.debug( + f"Item '{item_ref}' is premiering and download is delayed by '{300+duration}' seconds. Will start at '{premiere_ends.isoformat()}'" + ) + continue + LOG.info(f"Re-queuing item '{item_ref} {item.info.extras=}' for download.") try: @@ -868,17 +910,18 @@ class DownloadQueue(metaclass=Singleton): continue try: - info = item.info - new_queue = Item( - url=info.url, - preset=info.preset, - folder=info.folder, - cookies=info.cookies, - template=info.template, - cli=item.info.cli, - extras=item.info.extras, + await self.add( + item=Item( + url=item.info.url, + preset=item.info.preset, + folder=item.info.folder, + cookies=item.info.cookies, + template=item.info.template, + cli=item.info.cli, + extras=item.info.extras, + ) ) - await self.add(item=new_queue) except Exception as e: - LOG.error(f"Failed to re-queue item '{item_ref}'. {e!s}") + self.done.put(item) LOG.exception(e) + LOG.error(f"Failed to re-queue item '{item_ref}'. {e!s}") diff --git a/app/library/Utils.py b/app/library/Utils.py index a70290d9..95682407 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -17,6 +17,7 @@ from typing import TypeVar import yt_dlp from Crypto.Cipher import AES +from yt_dlp.utils import match_str from .LogWrapper import LogWrapper @@ -59,7 +60,8 @@ FILES_TYPE: list = [ {"rx": re.compile(r"\.(nfo|json|jpg|torrent|\.info\.json)$", re.IGNORECASE), "type": "metadata"}, ] -DATETIME_PATTERN = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:\d{2}))\s?") +TAG_REGEX: re.Pattern[str] = re.compile(r"%{([^:}]+)(?::([^}]*))?}c") +DT_PATTERN: re.Pattern[str] = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:\d{2}))\s?") T = TypeVar("T") @@ -204,6 +206,10 @@ def extract_info( if not data: return data + data["is_premiere"] = match_str("media_type=video & duration & is_live", data) + if not data["is_premiere"]: + data["is_premiere"] = "video" == data.get("media_type") and "is_upcoming" == data.get("live_status") + return yt_dlp.YoutubeDL.sanitize_info(data) if sanitize_info else data @@ -963,7 +969,7 @@ async def read_logfile(file: Path, offset: int = 0, limit: int = 50) -> dict: for line in lines[-(offset + limit) : -offset] if offset else lines[-limit:]: line_bytes = line if isinstance(line, bytes) else line.encode() msg = line.decode(errors="replace") - dt_match = DATETIME_PATTERN.match(msg) + dt_match = DT_PATTERN.match(msg) result.append( { "id": sha256(line_bytes).hexdigest(), @@ -1005,7 +1011,7 @@ async def tail_log(file: Path, emitter: callable, sleep_time: float = 0.5): continue msg = line.decode(errors="replace") - dt_match = DATETIME_PATTERN.match(msg) + dt_match = DT_PATTERN.match(msg) await emitter( { @@ -1228,3 +1234,25 @@ def load_modules(root_path: Path, directory: Path): importlib.import_module(full_name) except ImportError as e: LOG.error(f"Failed to import module '{full_name}': {e}") + + +def parse_tags(text: str) -> tuple[str, dict[str, str | bool]]: + """ + Parse tags from a string formatted with %{tag_name[:value]}c. + + Args: + text (str): The input string containing tags. + + Returns: + tuple[str, dict[str, str | bool]]: A tuple containing the string with tags removed and a dictionary of tags. + + """ + tags: dict[str, str | bool] = {} + + def replacer(match: re.Match) -> str: + name = match.group(1) + value = match.group(2) + tags[name] = value if value is not None else True + return "" + + return TAG_REGEX.sub(replacer, text).strip(), tags diff --git a/app/library/config.py b/app/library/config.py index bf3ab7da..9c0de57d 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -158,6 +158,9 @@ class Config: is_native: bool = False "Is the application running in webview." + prevent_live_premiere: bool = False + """Prevent downloading of the initial premiere live broadcast.""" + pictures_backends: list[str] = [ "https://unsplash.it/1920/1080?random", "https://picsum.photos/1920/1080", @@ -209,6 +212,7 @@ class Config: "console_enabled", "browser_enabled", "ytdlp_auto_update", + "prevent_premiere_live", ) "The variables that are booleans." diff --git a/ui/components/History.vue b/ui/components/History.vue index be126043..05a86072 100644 --- a/ui/components/History.vue +++ b/ui/components/History.vue @@ -537,6 +537,9 @@ const setIcon = item => { if (!item.filename) { return 'fa-solid fa-exclamation' } + if (item.extras?.is_premiere) { + return 'fa-solid fa-star' + } return item.is_live ? 'fa-solid fa-globe' : 'fa-solid fa-circle-check' } @@ -549,7 +552,7 @@ const setIcon = item => { } if ('not_live' === item.status) { - return 'fa-solid fa-headset' + return item.extras?.is_premiere ? 'fa-solid fa-star' : 'fa-solid fa-headset' } return 'fa-solid fa-circle' @@ -579,6 +582,11 @@ const setStatus = item => { if (!item.filename) { return 'Skipped?' } + + if (item.extras?.is_premiere) { + return 'Premiere' + } + return item.is_live ? 'Live Ended' : 'Completed' } @@ -591,6 +599,9 @@ const setStatus = item => { } if ('not_live' === item.status) { + if (item.extras?.is_premiere){ + return 'Premiere' + } return display_style.value === 'cards' ? 'Live Stream' : 'Live' } diff --git a/ui/components/NotifyDropdown.vue b/ui/components/NotifyDropdown.vue index 71e330d1..dd1d1dda 100644 --- a/ui/components/NotifyDropdown.vue +++ b/ui/components/NotifyDropdown.vue @@ -32,7 +32,7 @@
-