Merge pull request #74 from arabcoders/dev

Locally check id before requesting extract_info

.vscode/launch.json

@@ -27,6 +27,22 @@
       "request": "launch",
       "program": "app/main.py",
       "console": "internalConsole",
+      "justMyCode": true,
+      "env": {
+        "YTP_CONFIG_PATH": "${workspaceFolder}/var/config",
+        "YTP_DOWNLOAD_PATH": "${workspaceFolder}/var/downloads",
+        "YTP_TEMP_PATH": "${workspaceFolder}/var/tmp",
+        "YTP_URL_HOST": "http://localhost:8081",
+        "YTP_LOG_LEVEL": "DEBUG"
+      }
+    },
+    {
+      "name": "Python: main.py (Deep)",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "app/main.py",
+      "console": "internalConsole",
+      "justMyCode": false,
       "env": {
         "YTP_CONFIG_PATH": "${workspaceFolder}/var/config",
         "YTP_DOWNLOAD_PATH": "${workspaceFolder}/var/downloads",

app/Config.py

@@ -44,6 +44,8 @@ class Config:
     debug: bool = False
 
     new_version_available: bool = False
+    
+    extract_info_timeout: int = 70
 
     _int_vars: tuple = ('port', 'max_workers',)
     _immutable: tuple = ('version', '__instance', 'ytdl_options', 'new_version_available')

app/DownloadQueue.py

@@ -211,57 +211,43 @@ class DownloadQueue:
         LOG.info(f'Adding {url=} {quality=} {format=} {folder=} {output_template=} {ytdlp_cookies=} {ytdlp_config=}')
 
         already = set() if already is None else already
+
         if url in already:
-            LOG.info('recursion detected, skipping')
+            LOG.info('recursion detected, skipping.')
             return {'status': 'ok'}
-        else:
-            already.add(url)
+
+        already.add(url)
+
         try:
-            with ThreadPoolExecutor(thread_name_prefix='extract_info') as pool:
-                LOG.debug(f'extracting info from {url=}')
-                entry = await asyncio.get_running_loop().run_in_executor(
-                    pool,
+            downloaded, id_dict = self.isDownloaded(url)
+            if downloaded is True:
+                message = f"[ { id_dict.get('id') } ]: has been downloaded already."
+                LOG.info(message)
+                return {'status': 'error', 'msg': message}
+
+            started = time.perf_counter()
+            LOG.debug(f'extract_info: checking {url=}')
+
+            entry = await asyncio.wait_for(
+                fut=asyncio.get_running_loop().run_in_executor(
+                    None,
                     ExtractInfo,
                     mergeConfig(self.config.ytdl_options, ytdlp_config),
                     url,
                     bool(self.config.ytdl_debug)
-                )
+                ),
+                timeout=self.config.extract_info_timeout)
 
-                if not entry:
-                    if not self.config.keep_archive:
-                        return {
-                            'status': 'error',
-                            'msg': 'No metadata, most likely video has been downloaded before.' if self.config.keep_archive else 'Unable to extract info check logs.'
-                        }
+            if not entry:
+                return {'status': 'error', 'msg': 'Unable to extract info check logs.'}
 
-                    LOG.debug(f'No metadata, Rechecking with archive disabled. {url=}')
-                    entry = await asyncio.get_running_loop().run_in_executor(
-                        pool,
-                        ExtractInfo,
-                        mergeConfig(self.config.ytdl_options, ytdlp_config),
-                        url,
-                        bool(self.config.ytdl_debug),
-                        True
-                    )
-
-                    if not entry:
-                        return {'status': 'error', 'msg': 'Unable to extract info check logs.'}
-
-                    if self.isDownloaded(entry):
-                        LOG.info(f'[{entry.get("id")}: {entry.get("title")}]: has been downloaded already.')
-                        return {
-                            'status': 'error',
-                            'msg': f'[{entry.get("id")}: {entry.get("title")}]: has been downloaded already.'
-                        }
-
-            if self.isDownloaded(entry):
-                raise yt_dlp.utils.ExistingVideoReached()
-
-            LOG.debug(f'extract_info length: {len(entry)}')
+            LOG.debug(f'extract_info: for [{url=}] is done in {time.perf_counter() - started}. Length: {len(entry)}')
         except yt_dlp.utils.ExistingVideoReached:
             return {'status': 'error', 'msg': 'Video has been downloaded already and recorded in archive.log file.'}
         except yt_dlp.utils.YoutubeDLError as exc:
             return {'status': 'error', 'msg': str(exc)}
+        except asyncio.exceptions.TimeoutError as exc:
+            return {'status': 'error', 'msg': 'TimeoutError: Unable to extract info.'}
 
         return await self.__add_entry(
             entry=entry,
@@ -282,31 +268,37 @@ class DownloadQueue:
                 LOG.warning(f'Requested cancel for non-existent download {id=}. {str(e)}')
                 continue
 
+            itemMessage = f"{id=} {item.info.id=} {item.info.title=}"
+
             if item.started() is True:
-                LOG.info(f'Canceling {id=} {item.info.title=}')
+                LOG.debug(f'Canceling {itemMessage}')
                 item.cancel()
+                LOG.info(f'Cancelled {itemMessage}')
+
             else:
                 item.close()
-                LOG.info(f'Deleting from queue {id=} {item.info.title=}')
+                LOG.debug(f'Deleting from queue {itemMessage}')
                 self.queue.delete(id)
                 await self.notifier.canceled(id)
                 self.done.put(item)
                 await self.notifier.completed(item)
+                LOG.info(f'Deleted from queue {itemMessage}')
 
         return {'status': 'ok'}
 
     async def clear(self, ids):
         for id in ids:
-
             try:
                 item = self.done.get(key=id)
             except KeyError as e:
                 LOG.warning(f'Requested delete for non-existent download {id=}. {str(e)}')
                 continue
 
-            LOG.info(f'Deleting completed download {id=} {item.info.title=}')
+            itemMessage = f"{id=} {item.info.id=} {item.info.title=}"
+            LOG.debug(f'Deleting completed download {itemMessage}')
             self.done.delete(id)
             await self.notifier.cleared(id)
+            LOG.info(f'Deleted completed download {itemMessage}')
 
         return {'status': 'ok'}
 
@@ -348,10 +340,8 @@ class DownloadQueue:
                         LOG.info(f'Waiting for worker to be free.')
                     await asyncio.sleep(1)
 
-                LOG.debug(f"Has '{executor.get_available_workers()}' free workers.")
-
                 while not self.queue.hasDownloads():
-                    LOG.info('Waiting for item to download.')
+                    LOG.info(f"Waiting for item to download. '{executor.get_available_workers()}' free workers.")
                     await self.event.wait()
                     self.event.clear()
                     LOG.debug(f"Cleared wait event.")
@@ -409,5 +399,8 @@ class DownloadQueue:
 
         self.event.set()
 
-    def isDownloaded(self, info: dict) -> bool:
-        return self.config.keep_archive and isDownloaded(self.config.ytdl_options.get('download_archive', None), info)
+    def isDownloaded(self, url: str) -> tuple[bool, dict[str | None, str | None, str | None]]:
+        if not url or not self.config.keep_archive:
+            return False, None
+
+        return isDownloaded(self.config.ytdl_options.get('download_archive', None), url)

app/Utils.py

@@ -14,6 +14,8 @@ LOG = logging.getLogger('Utils')
 AUDIO_FORMATS: tuple = ('m4a', 'mp3', 'opus', 'wav')
 IGNORED_KEYS: tuple = ('cookiefile', 'paths', 'outtmpl', 'progress_hooks', 'postprocessor_hooks',)
 
+YTDLP_INFO_CLS = None
+
 
 def get_format(format: str, quality: str) -> str:
     """
@@ -197,24 +199,53 @@ def mergeConfig(config: dict, new_config: dict) -> dict:
     return mergeDict(new_config, config)
 
 
-def isDownloaded(archive_file: str, info: dict) -> bool:
-    if not info or not archive_file or not os.path.exists(archive_file):
-        return False
+def isDownloaded(archive_file: str, url: str) -> tuple[bool, dict[str | None, str | None, str | None]]:
+    global YTDLP_INFO_CLS
 
-    try:
-        id = yt_dlp.YoutubeDL()._make_archive_id(info)
-        if not id:
-            return False
-    except Exception as e:
-        LOG.error(f'Error generating archive id: {e}')
-        return False
+    idDict = {
+        'id': None,
+        'ie_key': None,
+        'archive_id': None,
+    }
+
+    if not url or not archive_file or not os.path.exists(archive_file):
+        return False, idDict,
+
+    if not YTDLP_INFO_CLS:
+        YTDLP_INFO_CLS = yt_dlp.YoutubeDL(params={
+            'color': 'no_color',
+            'extract_flat': True,
+            'skip_download': True,
+            'ignoreerrors': True,
+            'ignore_no_formats_error': True,
+            'quiet': True,
+        })
+
+    for key, ie in YTDLP_INFO_CLS._ies.items():
+        if not ie.suitable(url):
+            continue
+
+        if not ie.working():
+            break
+
+        temp_id = ie.get_temp_id(url)
+        if not temp_id:
+            break
+
+        idDict['id'] = temp_id
+        idDict['ie_key'] = key
+        idDict['archive_id'] = YTDLP_INFO_CLS._make_archive_id(idDict)
+        break
+
+    if not idDict['archive_id']:
+        return False, idDict,
 
     with open(archive_file, 'r') as f:
         for line in f.readlines():
-            if id in line:
-                return True
+            if idDict['archive_id'] in line:
+                return True, idDict,
 
-    return False
+    return False, idDict,
 
 
 def jsonCookie(cookies: dict[dict[str, any]]) -> str | None:
@@ -241,7 +272,7 @@ def jsonCookie(cookies: dict[dict[str, any]]) -> str | None:
                 cookieDict = cookies[domain][subDomain][cookie]
 
                 if 0 == int(cookieDict['expirationDate']):
-                    cookieDict['expirationDate']: float = datetime.now(timezone.utc).timestamp() + (86400 * 1000)
+                    cookieDict['expirationDate'] = datetime.now(timezone.utc).timestamp() + (86400 * 1000)
 
                 hasCookies = True
                 netscapeCookies += "\t".join([

frontend/src/App.vue

@@ -1,5 +1,6 @@
 <template>
-  <PageHeader :config="config" @toggleForm="addForm = !addForm" @toggleTasks="showTasks = !showTasks" />
+  <PageHeader :config="config" @toggleForm="addForm = !addForm" @toggleTasks="showTasks = !showTasks"
+    @reload="reloadWindow" />
   <formAdd v-if="addForm" :config="config" @addItem="addItem" />
   <pageTasks v-if="showTasks" :tasks="config.tasks" />
   <DownloadingList :config="config" :queue="downloading" @deleteItem="deleteItem" />
@@ -176,6 +177,8 @@ const playItem = (item) => {
   video_link.value = config.app.url_host + config.app.url_prefix + baseDir + encodeURIComponent(item.filename);
 };
 
+const reloadWindow = () => window.location.reload();
+
 bus.on((event, data) => {
   if (!['show_form'].includes(event)) {
     return true;
@@ -187,4 +190,3 @@ bus.on((event, data) => {
 });
 
 </script>
-

frontend/src/components/Page-Header.vue

@@ -17,11 +17,24 @@
           <font-awesome-icon icon="fa-solid fa-tasks" />
         </button>
       </div>
+      <div class="navbar-item">
+        <button v-tooltip="'Reload window'" class="button is-dark has-tooltip-bottom" @click="$emit('reload')">
+          <font-awesome-icon icon="fa-solid fa-rotate-right" />
+        </button>
+      </div>
       <div class="navbar-item">
         <button v-tooltip="'Switch to Light theme'" class="button is-dark has-tooltip-bottom"
-          @click="selectedTheme = 'light'" v-if="selectedTheme == 'dark'">🌞</button>
+          @click="selectedTheme = 'light'" v-if="selectedTheme == 'dark'">
+          <span class="icon is-small is-left has-text-warning">
+            <font-awesome-icon icon="fa-solid fa-sun" />
+          </span>
+        </button>
         <button v-tooltip="'Switch to Dark theme'" class="button is-dark  has-tooltip-bottom"
-          @click="selectedTheme = 'dark'" v-if="selectedTheme == 'light'">🌚</button>
+          @click="selectedTheme = 'dark'" v-if="selectedTheme == 'light'">
+          <span class="icon is-small is-left">
+            <font-awesome-icon icon="fa-solid fa-moon" />
+          </span>
+        </button>
       </div>
     </div>
   </nav>
@@ -45,36 +58,39 @@ defineProps({
 const applyPreferredColorScheme = (scheme) => {
   for (var s = 0; s < document.styleSheets.length; s++) {
     for (var i = 0; i < document.styleSheets[s].cssRules.length; i++) {
-      const rule = document.styleSheets[s].cssRules[i];
-
-      if (rule && rule.media && rule.media.mediaText.includes("prefers-color-scheme")) {
-        switch (scheme) {
-          case "light":
-            rule.media.appendMedium("original-prefers-color-scheme");
-            if (rule.media.mediaText.includes("light")) {
-              rule.media.deleteMedium("(prefers-color-scheme: light)");
-            }
-            if (rule.media.mediaText.includes("dark")) {
-              rule.media.deleteMedium("(prefers-color-scheme: dark)");
-            }
-            break;
-          case "dark":
-            rule.media.appendMedium("(prefers-color-scheme: light)");
-            rule.media.appendMedium("(prefers-color-scheme: dark)");
-            if (rule.media.mediaText.includes("original")) {
-              rule.media.deleteMedium("original-prefers-color-scheme");
-            }
-            break;
-          default:
-            rule.media.appendMedium("(prefers-color-scheme: dark)");
-            if (rule.media.mediaText.includes("light")) {
-              rule.media.deleteMedium("(prefers-color-scheme: light)");
-            }
-            if (rule.media.mediaText.includes("original")) {
-              rule.media.deleteMedium("original-prefers-color-scheme");
-            }
-            break;
+      try {
+        const rule = document.styleSheets[s].cssRules[i];
+        if (rule && rule.media && rule.media.mediaText.includes("prefers-color-scheme")) {
+          switch (scheme) {
+            case "light":
+              rule.media.appendMedium("original-prefers-color-scheme");
+              if (rule.media.mediaText.includes("light")) {
+                rule.media.deleteMedium("(prefers-color-scheme: light)");
+              }
+              if (rule.media.mediaText.includes("dark")) {
+                rule.media.deleteMedium("(prefers-color-scheme: dark)");
+              }
+              break;
+            case "dark":
+              rule.media.appendMedium("(prefers-color-scheme: light)");
+              rule.media.appendMedium("(prefers-color-scheme: dark)");
+              if (rule.media.mediaText.includes("original")) {
+                rule.media.deleteMedium("original-prefers-color-scheme");
+              }
+              break;
+            default:
+              rule.media.appendMedium("(prefers-color-scheme: dark)");
+              if (rule.media.mediaText.includes("light")) {
+                rule.media.deleteMedium("(prefers-color-scheme: light)");
+              }
+              if (rule.media.mediaText.includes("original")) {
+                rule.media.deleteMedium("original-prefers-color-scheme");
+              }
+              break;
+          }
         }
+      } catch (e) {
+        console.debug(e);
       }
     }
   }

frontend/src/main.js

@@ -6,7 +6,7 @@ import { library } from '@fortawesome/fontawesome-svg-core'
 import { FontAwesomeIcon } from '@fortawesome/vue-fontawesome'
 import {
   faCog, faTrash, faLink, faPlus, faTrashCan, faCircleXmark, faCircleCheck, faRotateRight, faDownload, faUpRightFromSquare,
-  faSpinner, faArrowUp, faArrowDown, faTasks, faCalendar, faArrowUpAZ, faArrowDownAZ, faEject, faGlobe
+  faSpinner, faArrowUp, faArrowDown, faTasks, faCalendar, faArrowUpAZ, faArrowDownAZ, faEject, faGlobe, faMoon, faSun
 } from '@fortawesome/free-solid-svg-icons'
 
 import { faSquare, faSquareCheck } from '@fortawesome/free-regular-svg-icons'
@@ -17,8 +17,9 @@ import './assets/css/bulma-dark.css'
 import './assets/css/style.css'
 import 'floating-vue/dist/style.css'
 
-library.add(faCog, faTrash, faLink, faPlus, faTrashCan, faCircleXmark, faCircleCheck, faRotateRight, faDownload, faUpRightFromSquare,
-  faSquare, faSquareCheck, faSpinner, faArrowUp, faArrowDown, faTasks, faCalendar, faArrowUpAZ, faArrowDownAZ, faEject, faGlobe)
+library.add(faCog, faTrash, faLink, faPlus, faTrashCan, faCircleXmark, faCircleCheck, faRotateRight, faDownload,
+  faUpRightFromSquare, faSquare, faSquareCheck, faSpinner, faArrowUp, faArrowDown, faTasks, faCalendar, faArrowUpAZ,
+  faArrowDownAZ, faEject, faGlobe, faMoon, faSun)
 const app = createApp(App);
 
 app.config.globalProperties.capitalize = s => s && s[0].toUpperCase() + s.slice(1);