Merge pull request #60 from arabcoders/dev

Fixed checking already in queue item message.

README.md

@@ -163,12 +163,6 @@ The `config/ytdlp.json`, is a json file which can be used to alter the default `
   "windowsfilenames": true,
   // Write subtitles if the stream has them.
   "writesubtitles": true,
-  // How many parts to download at the same time.
-  "concurrent_fragment_downloads": 10,
-  // If we encounter error during downloading how many times should we retry.
-  "fragment_retries": 10,
-  // Should we skip fragments? sometimes skipping fragments leads to missing live stream fragments.
-  "skip_unavailable_fragments": false,
   // Write info.json file for each download. It can be used by many tools to generate info etc.
   "writeinfojson": true,
   // Write thumbnail if available.

app/DataStore.py

@@ -3,7 +3,6 @@ import copy
 from datetime import datetime, timezone
 from email.utils import formatdate
 import json
-import logging
 import sqlite3
 from Utils import calcDownloadPath
 from Config import Config

app/Download.py

@@ -15,6 +15,9 @@ log = logging.getLogger('download')
 
 
 class Download:
+    """
+    Download task.
+    """
     id: str = None
     manager = None
     download_dir: str = None
@@ -31,6 +34,13 @@ class Download:
     canceled: bool = False
     is_live: bool = False
 
+    bad_live_options: list = [
+        "concurrent_fragment_downloads",
+        "fragment_retries",
+        "skip_unavailable_fragments",
+    ]
+    "Bad yt-dlp options which are known to cause issues with live stream and post manifestless mode."
+
     _ytdlp_fields: tuple = (
         'tmpfilename',
         'filename',
@@ -75,6 +85,8 @@ class Download:
         self.max_workers = int(config.max_workers)
         self.tempKeep = bool(config.temp_keep)
         self.is_live = bool(info.is_live) or info.live_in is not None
+        self.is_manifestless = 'is_manifestless' in self.info.options and self.info.options[
+            'is_manifestless'] is True
 
     def _progress_hook(self, data: dict):
         dicto = {k: v for k, v in data.items() if k in self._ytdlp_fields}
@@ -139,6 +151,13 @@ class Download:
                     log.error(
                         f'Invalid cookies: was provided for {self.info.title} - {str(e)}')
 
+            if self.is_live or self.is_manifestless:
+                log.debug(
+                    f'Live stream or post manifestless mode detected, disabling options: {self.bad_live_options}')
+                for opt in self.bad_live_options:
+                    if opt in params:
+                        del params[opt]
+
             log.info(
                 f'Downloading {os.getpid()=} id="{self.info.id}" title="{self.info.title}".')
 

app/DownloadQueue.py

@@ -57,6 +57,8 @@ class DownloadQueue:
         if not entry:
             return {'status': 'error', 'msg': 'Invalid/empty data was given.'}
 
+        options: dict = {}
+
         error: str = None
         live_in: str = None
 
@@ -113,10 +115,20 @@ class DownloadQueue:
 
                 await self.clear([item.info._id])
 
-            if self.queue.exists(key=entry.get('id'), url=entry.get('webpage_url') or entry.get('url')):
-                log.info(
-                    f'Item [{item.info.title}] already in download queue')
-                return {'status': 'error', 'msg': 'Link already queued for downloading.'}
+            try:
+                item = self.queue.get(key=entry.get('id'), url=entry.get(
+                    'webpage_url') or entry.get('url'))
+                if item is not None:
+                    err_message = f'Item [{item.info.id}: {item.info.title}] already in download queue.'
+                    log.info(err_message)
+                    return {'status': 'error', 'msg': err_message}
+            except KeyError:
+                pass
+
+            is_manifestless = 'post_live' == entry.get(
+                'live_status') if 'live_status' in entry else None
+
+            options.update({'is_manifestless': is_manifestless})
 
             dl = ItemDTO(
                 id=entry.get('id'),
@@ -132,6 +144,7 @@ class DownloadQueue:
                 error=error,
                 is_live=entry.get('is_live', None) or live_in is not None,
                 live_in=live_in,
+                options=options
             )
 
             try:
@@ -162,7 +175,7 @@ class DownloadQueue:
                 dlInfo.info.status = 'not_live'
                 itemDownload = self.done.put(dlInfo)
                 NotifiyEvent = 'completed'
-            elif self.config.allow_manifestless is False and 'live_status' in entry and 'post_live' == entry.get('live_status'):
+            elif self.config.allow_manifestless is False and is_manifestless is True:
                 dlInfo.info.status = 'error'
                 dlInfo.info.error = 'Video is in Post-Live Manifestless mode.'
                 itemDownload = self.done.put(dlInfo)

app/ItemDTO.py

@@ -25,6 +25,7 @@ class ItemDTO:
     datetime: str = field(default_factory=lambda: str(formatdate(time.time())))
     live_in: str = None
     file_size: int = None
+    options: dict = field(default_factory=dict)
 
     # yt-dlp injected fields.
     tmpfilename: str = None