Merge pull request #269 from arabcoders/dev

Communicate more error logs to user when yt-dlp throws them.

.vscode/settings.json

@@ -36,6 +36,8 @@
     "libcurl",
     "libx",
     "matroska",
+    "Microformat",
+    "microformats",
     "mkvtoolsnix",
     "mpegts",
     "msvideo",

app/library/Download.py

@@ -16,7 +16,7 @@ from .config import Config
 from .Events import EventBus, Events
 from .ffprobe import ffprobe
 from .ItemDTO import ItemDTO
-from .Utils import extract_info, load_cookies
+from .Utils import extract_info, extract_ytdlp_logs, load_cookies
 from .YTDLPOpts import YTDLPOpts
 
 
@@ -81,7 +81,19 @@ class Download:
     temp_keep: bool = False
     "Keep temp folder after download."
 
-    def __init__(self, info: ItemDTO, info_dict: dict = None):
+    logs: list = []
+    "Logs from yt-dlp."
+
+    def __init__(self, info: ItemDTO, info_dict: dict = None, logs: list | None = None):
+        """
+        Initialize download task.
+
+        Args:
+            info (ItemDTO): ItemDTO object.
+            info_dict (dict): yt-dlp metadata dict.
+            logs (list): Logs from yt-dlp.
+
+        """
         config = Config.get_instance()
 
         self.download_dir = info.download_dir
@@ -106,6 +118,7 @@ class Download:
         self.logger = logging.getLogger(f"Download.{info.id if info.id else info._id}")
         self.started_time = 0
         self.queue_time = datetime.now(tz=UTC)
+        self.logs = logs if logs else []
 
     def _progress_hook(self, data: dict):
         dataDict = {k: v for k, v in data.items() if k in self._ytdlp_fields}
@@ -193,6 +206,14 @@ class Download:
 
             if not self.info_dict:
                 self.logger.info(f"Extracting info for '{self.info.url}'.")
+                self.logs = []
+                ie_params = params.copy()
+                ie_params["callback"] = {
+                    "func": lambda _, msg: self.logs.append(msg),
+                    "level": logging.WARNING,
+                    "name": "callback-logger",
+                }
+
                 info = extract_info(
                     config=params,
                     url=self.info.url,
@@ -219,7 +240,10 @@ class Download:
                     )
 
             if isinstance(self.info_dict, dict) and len(self.info_dict.get("formats", [])) < 1:
-                msg = f"Failed to extract formats for '{self.info.url}'. The extracted info dict is empty."
+                msg = f"Failed to extract any formats for '{self.info.url}'."
+                if filtered_logs := extract_ytdlp_logs(self.logs):
+                    msg += f" Logs: {', '.join(filtered_logs)}"
+
                 raise ValueError(msg)  # noqa: TRY301
 
             self.logger.info(

app/library/DownloadQueue.py

@@ -24,7 +24,15 @@ from .ItemDTO import Item, ItemDTO
 from .Presets import Presets
 from .Scheduler import Scheduler
 from .Singleton import Singleton
-from .Utils import arg_converter, calc_download_path, dt_delta, extract_info, is_downloaded, load_cookies
+from .Utils import (
+    arg_converter,
+    calc_download_path,
+    dt_delta,
+    extract_info,
+    extract_ytdlp_logs,
+    is_downloaded,
+    load_cookies,
+)
 from .YTDLPOpts import YTDLPOpts
 
 LOG = logging.getLogger("DownloadQueue")
@@ -184,7 +192,7 @@ class DownloadQueue(metaclass=Singleton):
             except Exception as e:
                 LOG.error(f"Failed to cancel downloads. {e!s}")
 
-    async def __add_entry(self, entry: dict, item: Item, already=None):
+    async def __add_entry(self, entry: dict, item: Item, already=None, logs: list | None = None):
         """
         Add an entry to the download queue.
 
@@ -192,11 +200,15 @@ class DownloadQueue(metaclass=Singleton):
             entry (dict): The entry to add to the download queue.
             item (Item): The item to add to the download queue.
             already (set): The set of already downloaded items.
+            logs (list): The list of logs generated during information extraction.
 
         Returns:
             dict: The status of the operation.
 
         """
+        if not logs:
+            logs = []
+
         if item.has_extras():
             for key in item.extras.copy():
                 if not self.keep_extra_key(key):
@@ -320,12 +332,15 @@ class DownloadQueue(metaclass=Singleton):
             )
 
             try:
-                dlInfo: Download = Download(info=dl, info_dict=entry)
+                dlInfo: Download = Download(info=dl, info_dict=entry, logs=logs)
+                text_logs = ""
+                if filtered_logs := extract_ytdlp_logs(logs):
+                    text_logs = f" Logs: {', '.join(filtered_logs)}"
 
                 if live_in or "is_upcoming" == entry.get("live_status"):
                     NotifyEvent = Events.COMPLETED
                     dlInfo.info.status = "not_live"
-                    dlInfo.info.msg = "Stream is not live yet."
+                    dlInfo.info.msg = "Stream is not live yet." + text_logs
                     itemDownload = self.done.put(dlInfo)
                 elif len(entry.get("formats", [])) < 1:
                     availability = entry.get("availability", "public")
@@ -333,14 +348,15 @@ class DownloadQueue(metaclass=Singleton):
                     if availability and availability not in ("public",):
                         msg += f" Availability is set for '{availability}'."
 
+                    dlInfo.info.error = msg + text_logs
                     dlInfo.info.status = "error"
-                    dlInfo.info.error = msg
                     itemDownload = self.done.put(dlInfo)
                     NotifyEvent = Events.COMPLETED
                     await self._notify.emit(Events.LOG_WARNING, data=event_warning(msg))
                 elif self.config.allow_manifestless is False and is_manifestless is True:
                     dlInfo.info.status = "error"
-                    dlInfo.info.error = "Video is in post-live manifestless mode."
+                    dlInfo.info.error = "Video is in post-live manifestless mode." + text_logs
+
                     itemDownload = self.done.put(dlInfo)
                     NotifyEvent = Events.COMPLETED
                 else:
@@ -485,7 +501,7 @@ class DownloadQueue(metaclass=Singleton):
                 except Exception as e:
                     LOG.error(f"Failed to remove cookie file '{yt_conf['cookiefile']}'. {e!s}")
 
-        return await self.__add_entry(entry=entry, item=item, already=already)
+        return await self.__add_entry(entry=entry, item=item, already=already, logs=logs)
 
     async def cancel(self, ids: list[str]) -> dict[str, str]:
         """

app/library/Utils.py

@@ -1069,3 +1069,33 @@ def dt_delta(delta: timedelta) -> str:
         parts.append("<1s")
 
     return " ".join(parts)
+
+
+def extract_ytdlp_logs(logs: list[str], filters: list[str | re.Pattern] = None) -> list[str]:
+    """
+    Extract yt-dlp log lines matching built-in filters plus any extras.
+
+    Args:
+        logs (list): log strings.
+        filters (list[str|Re.Pattern]): Optional extra filters of strings and/or regex.
+
+    Returns:
+        (list): List of matching log lines.
+
+    """
+    all_patterns: list[str | re.Pattern] = [
+        "This live event will begin",
+        "Video unavailable. This video is private",
+        "This video is available to this channel",
+        "Private video. Sign in if you've been granted access to this video",
+        "[youtube] Premieres in",
+    ] + (filters or [])
+
+    compiled: list[re.Pattern] = [
+        p if isinstance(p, re.Pattern) else re.compile(re.escape(p), re.IGNORECASE) for p in all_patterns
+    ]
+
+    matched: list[str] = []
+    matched.extend(line for line in logs if line and any(p.search(line) for p in compiled))
+
+    return list(dict.fromkeys(matched))

app/main.py

@@ -22,6 +22,80 @@ from library.Tasks import Tasks
 LOG = logging.getLogger("app")
 MIME = magic.Magic(mime=True)
 
+# Monkey patch yt-dlp to fix live_from_start mpd issues
+
+from yt_dlp.extractor.youtube import YoutubeIE
+
+
+def _patched_prepare_live_from_start_formats(
+    self, formats, video_id, live_start_time, url, webpage_url, smuggled_data, is_live
+):
+    import functools
+    import threading
+    import time
+
+    from yt_dlp.utils import LazyList, bug_reports_message, traverse_obj
+
+    lock = threading.Lock()
+    start_time = time.time()
+    formats = [f for f in formats if f.get("is_from_start")]
+
+    def refetch_manifest(format_id, delay):  # noqa: ARG001
+        nonlocal formats, start_time, is_live
+        if time.time() <= start_time + delay:
+            return
+        # re-download player responses & re-list formats
+        _, _, prs, player_url = self._download_player_responses(url, smuggled_data, video_id, webpage_url)
+        video_details = traverse_obj(prs, (..., "videoDetails"), expected_type=dict)
+        microformats = traverse_obj(prs, (..., "microformat", "playerMicroformatRenderer"), expected_type=dict)
+        _, live_status, _, formats, _ = self._list_formats(video_id, microformats, video_details, prs, player_url)
+        is_live = live_status == "is_live"
+        start_time = time.time()
+
+    def mpd_feed(format_id, delay):
+        """
+        @returns (manifest_url, manifest_stream_number, is_live) or None
+        """
+        for retry in self.RetryManager(fatal=False):
+            with lock:
+                refetch_manifest(format_id, delay)
+
+            f = next((f for f in formats if f.get("format_id") == format_id), None)
+            if not f:
+                if not is_live:
+                    retry.error = f"{video_id}: Video is no longer live"
+                else:
+                    retry.error = f"Cannot find refreshed manifest for format {format_id}{bug_reports_message()}"
+                continue
+
+            if not isinstance(f, dict) or not f.get("manifest_url"):
+                break
+
+            return f["manifest_url"], f["manifest_stream_number"], is_live
+
+        return None
+
+    for f in formats:
+        f["is_live"] = is_live
+        gen = functools.partial(
+            self._live_dash_fragments,
+            video_id,
+            f.get("format_id"),
+            live_start_time,
+            mpd_feed,
+            (not is_live) and f.copy(),
+        )
+        if is_live:
+            f["fragments"] = gen
+            f["protocol"] = "http_dash_segments_generator"
+        else:
+            f["fragments"] = LazyList(gen({}))
+            f.pop("is_from_start", None)
+
+
+YoutubeIE._prepare_live_from_start_formats = _patched_prepare_live_from_start_formats
+# End
+
 
 class Main:
     def __init__(self):

app/upgrader.py

@@ -75,11 +75,10 @@ if __name__ == "__main__":
     try:
         import coloredlogs
 
-        logging.basicConfig(level=logging.DEBUG)
         coloredlogs.install(
             level=logging.INFO, fmt="%(asctime)s [%(name)s] [%(levelname)-5.5s] %(message)s", datefmt="%H:%M:%S"
         )
     except ImportError:
-        pass
+        logging.basicConfig(level=logging.DEBUG)
 
     Upgrader()

entrypoint.sh

@@ -32,4 +32,10 @@ if [ ! -w "${YTP_DOWNLOAD_PATH}" ]; then
   exit 1
 fi
 
+###########
+# Run yt-dlp upgrader
+# This will update yt-dlp to the latest version
+###########
+/opt/python/bin/python /app/app/upgrader.py
+
 exec "${@}"

ui/components/History.vue

@@ -110,10 +110,10 @@
                   <div class="is-text-overflow" v-tooltip="item.title">
                     <NuxtLink target="_blank" :href="item.url">{{ item.title }}</NuxtLink>
                   </div>
-                  <div v-if="item.error">
+                  <div v-if="item.error" class="is-text-overflow is-pointer" @click="toggle_class($event)">
                     <span class="has-text-danger">{{ item.error }}</span>
                   </div>
-                  <div v-if="showMessage(item)">
+                  <div v-if="showMessage(item)" class="is-text-overflow is-pointer" @click="toggle_class($event)">
                     <span class="has-text-danger">{{ item.msg }}</span>
                   </div>
                 </td>
@@ -242,10 +242,14 @@
           <div class="card-content">
             <div class="columns is-mobile is-multiline">
               <div class="column is-12" v-if="item.error">
-                <span class="has-text-danger">{{ item.error }}</span>
+                <div class="is-text-overflow is-pointer" @click="toggle_class($event)">
+                  <span class="has-text-danger">{{ item.error }}</span>
+                </div>
               </div>
               <div class="column is-12" v-if="showMessage(item)">
-                <span class="has-text-danger">{{ item.msg }}</span>
+                <div class="is-text-overflow is-pointer" @click="toggle_class($event)">
+                  <span class="has-text-danger">{{ item.msg }}</span>
+                </div>
               </div>
               <div class="column is-half-mobile has-text-centered is-text-overflow is-unselectable">
                 <span class="icon-text">
@@ -686,4 +690,6 @@ const downloadSelected = () => {
     body.removeChild(link);
   }
 }
+
+const toggle_class = e => e.currentTarget.classList.toggle('is-text-overflow')
 </script>

ui/package.json

@@ -12,17 +12,17 @@
   "web-types": "./web-types.json",
   "dependencies": {
     "@pinia/nuxt": "^0.11.0",
-    "@sentry/nuxt": "^9.14.0",
+    "@sentry/nuxt": "^9.15.0",
     "@vueuse/core": "^13.1.0",
     "@vueuse/nuxt": "^13.1.0",
     "@xterm/addon-fit": "^0.10.0",
     "@xterm/xterm": "^5.5.0",
     "cron-parser": "^5.1.1",
-    "cronstrue": "^2.59.0",
+    "cronstrue": "^2.60.0",
     "floating-vue": "^5.2.2",
     "hls.js": "^1.6.2",
     "moment": "^2.30.1",
-    "nuxt": "^3.16.2",
+    "nuxt": "^3.17.1",
     "pinia": "^3.0.2",
     "socket.io-client": "^4.7.2",
     "vue": "^3.4",