handle download archive in the app instead of yt-dlp.

.vscode/launch.json

@@ -35,6 +35,8 @@
         "YTP_YTDL_OPTIONS_FILE": "${workspaceFolder}/var/config/ytdlp.json",
         "YTP_URL_HOST": "http://localhost:8081",
         "YTP_YTDL_DEBUG": "true",
+        "YTP_TEMP_KEEP": "true",
+        "YTP_KEEP_ARCHIVE": "true",
       }
     }
   ]

README.md

@@ -60,14 +60,17 @@ services:
 
 Certain values can be set via environment variables, using the `-e` parameter on the docker command line, or the `environment:` section in docker-compose.
 
-* __UMASK__: umask value used by YTPTube. Defaults to `022`.
 * __YTP_CONFIG_PATH__: path to where the queue persistence files will be saved. Defaults to `/config` in the docker image, and `./var/config` otherwise.
 * __YTP_DOWNLOAD_PATH__: path to where the downloads will be saved. Defaults to `/downloads` in the docker image, and `./var/downloads` otherwise.
 * __YTP_TEMP_PATH__: path where intermediary download files will be saved. Defaults to `/tmp` in the docker image, and `./var/tmp` otherwise.
+* __YTP_TEMP_KEEP__: Whether to keep the Individual video temp directory or remove it. Useful for live streams that don't keep archive. Defaults to `false`.
 * __YTP_URL_PREFIX__: base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`.
 * __YTP_OUTPUT_TEMPLATE__: the template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`. This will be the default for all downloads unless the request include output template.
-* __YTP_YTDL_OPTIONS_FILE__: A path to a JSON file that will be loaded and used for populating `ytdlp options`.
 * __YTP_KEEP_ARCHIVE__: Whether to keep history of downloaded videos to prevent downloading same file multiple times. Defaults to `false`.
+* __YTP_YTDL_DEBUG__: Whether to turn debug logging for the internal `yt-dlp` package. Defaults to `false`.
+* __YTP_HOST__: Which ip address to bind to. Defaults to `0.0.0.0`.
+* __YTP_PORT__: Which port to bind to. Defaults to `8081`.
+* __YTP_LOGGING_LEVEL__: Logging level. Defaults to `info`.
 
 ## Running behind a reverse proxy
 
@@ -146,25 +149,94 @@ A Docker image can be built locally (it will build the UI too):
 docker build . -t ytptube
 ```
 
+### ytdlp.json File
+
+The `config/ytdlp.json`, is a json file which can be used to alter the default `yt-dlp` config settings. For example these are the options i personally use,
+
+```json5
+{
+  // Make the final filename windows compatible.
+  "windowsfilenames": true,
+  // Write subtitles if the stream has them.
+  "writesubtitles": true,
+  // How many parts to download at the same time.
+  "concurrent_fragment_downloads": 10,
+  // If we encounter error during downloading how many times should we retry.
+  "fragment_retries": 10,
+  // Should we skip fragments? sometimes skipping fragments leads to missing live stream fragments.
+  "skip_unavailable_fragments": false,
+  // Write info.json file for each download. It can be used by many tools to generate info etc.
+  "writeinfojson": true,
+  // Write thumbnail if available.
+  "writethumbnail": true,
+  // Do not download automatically generated subtitles. 
+  "writeautomaticsub": false,
+  // MP4 is limited with the codecs we use, so "mkv" make sense.
+  "merge_output_format": "mkv",
+  // Record live stream from the start.
+  "live_from_start": true,
+  // For YouTube try to force H264 video codec & AAC audio.
+  "format_sort": [
+      "codec:avc:m4a"
+  ],
+  // Your Choice of subtitle languages to download.
+  "subtitleslangs": [ "en", "ar" ],
+  // postprocessors to run on the file
+  "postprocessors": [
+      // this processor convert the downloaded thumbnail to JPG.
+      {
+          "key": "FFmpegThumbnailsConvertor",
+          "format": "jpg"
+      },
+      // This processor convert subtitles to SRT
+      {
+          "key": "FFmpegSubtitlesConvertor",
+          "format": "srt"
+      },
+      // This processor embed metadata & info.json file into the final MKV file.
+      {
+          "key": "FFmpegMetadata",
+          "add_infojson": true,
+          "add_metadata": true
+      },
+      // This process embed subtitles into the final file if it doesn't subtitles embedded.
+      {
+          "key": "FFmpegEmbedSubtitle",
+          "already_have_subtitle": false
+      }
+  ]
+}
+``` 
+
 ### tasks.json File
 
 The `config/tasks.json`, is a json file, which can be used to queue URLs for downloading, it's mainly useful if you follow specific channels and you want it downloaded automatically, The schema for the file is as the following, Only the `URL` key is required.
 
-```json
+```json5
 [
   {
-    "url": "", // (URL: string) **REQUIRED**, URL to the content.
+    // (URL: string) **REQUIRED**, URL to the content.
-    "name": "My super secret channel", // (Name: string) Optional field. Mainly used for logging. If omitted, random GUID will be shown.
+    "url": "", 
-    "timer": "1 */1 * * *", // (Timer: string) Optional field. Using regular cronjob timer, if the field is omitted, it will run every hour in random minute.
+    // (Name: string) Optional field. Mainly used for logging. If omitted, random GUID will be shown.
-    "ytdlp_cookies": {}, // (yt-dlp cookies: object) Optional field. A JSON cookies exported by flagCookies.
+    "name": "My super secret channel", 
-    "ytdlp_config": {}, // (yt-dlp config: object) Optional field. A JSON yt-dlp config.
+    // (Timer: string) Optional field. Using regular cronjob timer, if the field is omitted, it will run every hour in random minute.
-    "output_template": "", // (Output Template: string) Optional field. A File output format,
+    "timer": "1 */1 * * *", 
-    "folder":"", // (Folder: string) Optional field. Where to store the downloads relative to the main download path.
+    // (yt-dlp cookies: object) Optional field. A JSON cookies exported by flagCookies.
-    "format": "", // (Format: string) Optional field. Format as specified in Web GUI. Defaults to "any".
+    "ytdlp_cookies": {}, 
-    "quality": "", // (Quality: string), Optional field. Quality as specified in Web GUI. Defaults to "best".
+    // (yt-dlp config: object) Optional field. A JSON yt-dlp config.
+    "ytdlp_config": {},
+    // (Output Template: string) Optional field. A File output format,
+    "output_template": "",
+    // (Folder: string) Optional field. Where to store the downloads relative to the main download path.
+    "folder":"",
+    // (Format: string) Optional field. Format as specified in Web GUI. Defaults to "any".
+    "format": "",
+    // (Quality: string), Optional field. Quality as specified in Web GUI. Defaults to "best".
+    "quality": "", 
   },
   {
-    "url": "https://..." // This is valid config, it will queue the channel for downloading every hour at random minute.
+   // (URL: string) **REQUIRED**, URL to the content.
+   "url": "https://..." // This is valid config, it will queue the channel for downloading every hour at random minute.
   },
   ...
 ]

app/Config.py

@@ -8,10 +8,11 @@ from version import APP_VERSION
 
 
 class Config:
+    __instance = None
     config_path: str = '.'
     download_path: str = '.'
     temp_path: str = '{download_path}'
-
+    temp_keep: bool = False
     db_file: str = '{config_path}/ytptube.db'
 
     url_host: str = ''
@@ -22,7 +23,6 @@ class Config:
     output_template_chapter: str = '%(title)s - %(section_number)s %(section_title)s.%(ext)s'
 
     ytdl_options: dict | str = {}
-    ytdl_options_file: str = ''
     ytdl_debug: bool = False
 
     host: str = '0.0.0.0'
@@ -36,17 +36,29 @@ class Config:
 
     version: str = APP_VERSION
 
-    _boolean_vars: tuple = ('keep_archive', 'ytdl_debug',)
+    _boolean_vars: tuple = ('keep_archive', 'ytdl_debug', 'temp_keep',)
-    _immutable: tuple = ('version',)
+    _immutable: tuple = ('version', '__instance', 'ytdl_options',)
+
+    @staticmethod
+    def get_instance():
+        """ Static access method. """
+        return Config() if not Config.__instance else Config.__instance
 
     def __init__(self):
+        """ Virtually private constructor. """
+        if Config.__instance is not None:
+            raise Exception(
+                "This class is a singleton!. Use Config.getInstance() instead.")
+        else:
+            Config.__instance = self
+
         baseDefualtPath: str = os.path.dirname(os.path.dirname(__file__))
 
         self.config_path = os.path.join(baseDefualtPath, 'var', 'config')
         self.download_path = os.path.join(baseDefualtPath, 'var', 'downloads')
         self.temp_path = os.path.join(baseDefualtPath, 'var', 'tmp')
 
-        for k, v in self.getAttributes().items():
+        for k, v in self._getAttributes().items():
             if k.startswith('_'):
                 continue
 
@@ -102,37 +114,28 @@ class Config:
 
         coloredlogs.install()
 
-        if isinstance(self.ytdl_options, str):
+        optsFile: str = os.path.join(self.config_path, 'ytdlp.json')
-            try:
+        if os.path.exists(optsFile) and os.path.getsize(optsFile) > 0:
-                self.ytdl_options = json.loads(self.ytdl_options)
+            log.info(f'Loading yt-dlp custom options from "{optsFile}"')
-                assert isinstance(self.ytdl_options, dict)
-            except (json.decoder.JSONDecodeError, AssertionError) as e:
-                log.error(f'JSON error in "YTP_YTDL_OPTIONS": {e}')
-                sys.exit(1)
 
-        if self.ytdl_options_file:
+            try:
-            log.info(
+                with open(optsFile) as json_data:
-                f'Loading yt-dlp custom options from "{self.ytdl_options_file}"')
+                    opts = json.load(json_data)
-            if not os.path.exists(self.ytdl_options_file):
+                assert isinstance(opts, dict)
+                self.ytdl_options.update(opts)
+            except (json.decoder.JSONDecodeError, AssertionError) as e:
                 log.error(
-                    f'"YTP_YTDL_OPTIONS_FILE" ENV points to non-existent file: "{self.ytdl_options_file}"')
+                    f'JSON error in "{optsFile}": {e}')
-            else:
+                sys.exit(1)
-                try:
+        else:
-                    with open(self.ytdl_options_file) as json_data:
+            log.info(f'No custom yt-dlp options found in "{self.config_path}"')
-                        opts = json.load(json_data)
-                    assert isinstance(opts, dict)
-                    self.ytdl_options.update(opts)
-                except (json.decoder.JSONDecodeError, AssertionError) as e:
-                    log.error(
-                        f'JSON error in "{self.ytdl_options_file}": {e}')
-                    sys.exit(1)
 
         if self.keep_archive:
             log.info(f'keep archive: {self.keep_archive}')
             self.ytdl_options['download_archive'] = os.path.join(
                 self.config_path, 'archive.log')
 
-    def getAttributes(self) -> dict:
+    def _getAttributes(self) -> dict:
         attrs: dict = {}
         vclass: str = self.__class__
 

app/Download.py

@@ -9,6 +9,7 @@ import shutil
 import yt_dlp
 from Utils import Notifier, get_format, get_opts, jsonCookie, mergeConfig
 from ItemDTO import ItemDTO
+from Config import Config
 
 log = logging.getLogger('download')
 
@@ -37,6 +38,7 @@ class Download:
         'speed',
         'eta',
     )
+    tempKeep: bool = False
 
     def __init__(
         self,
@@ -64,6 +66,7 @@ class Download:
         self.proc = None
         self.loop = None
         self.notifier = None
+        self.tempKeep = bool(Config.get_instance().temp_keep)
 
     def _download(self):
         try:
@@ -88,7 +91,8 @@ class Download:
                     )
 
             # Create temp dir for each download.
-            self.tempPath = os.path.join(self.temp_dir, self.info._id)
+            self.tempPath = os.path.join(
+                self.temp_dir, self.info.id if self.info.id else self.info._id)
             if not os.path.exists(self.tempPath):
                 os.makedirs(self.tempPath, exist_ok=True)
 
@@ -144,9 +148,13 @@ class Download:
                 'error': str(exc)
             })
         finally:
-            if self.tempPath and self.info._id and os.path.exists(self.tempPath):
+            if self.tempKeep is False and self.tempPath and os.path.exists(self.tempPath):
-                log.debug(f'Deleting Temp directory: {self.tempPath}')
+                if self.tempPath == self.temp_dir:
-                shutil.rmtree(self.tempPath, ignore_errors=True)
+                    log.warning(
+                        f'Attempted to delete video temp directory: {self.tempPath}, but it is the same as main temp directory.')
+                else:
+                    log.debug(f'Deleting Temp directory: {self.tempPath}')
+                    shutil.rmtree(self.tempPath, ignore_errors=True)
 
     async def start(self, notifier: Notifier):
         if self.manager is None:

app/DownloadQueue.py

@@ -1,5 +1,6 @@
 import asyncio
 from email.utils import formatdate
+import json
 import logging
 import os
 import time
@@ -9,7 +10,7 @@ from Config import Config
 from Download import Download
 from ItemDTO import ItemDTO
 from DataStore import DataStore
-from Utils import Notifier, ObjectSerializer, calcDownloadPath, ExtractInfo, mergeConfig
+from Utils import Notifier, ObjectSerializer, calcDownloadPath, ExtractInfo, isDownloaded, mergeConfig
 
 log = logging.getLogger('DownloadQueue')
 
@@ -216,6 +217,10 @@ class DownloadQueue:
                     'status': 'error',
                     'msg': 'No metadata, most likely video has been downloaded before.' if self.config.keep_archive else 'Unable to extract info check logs.'
                 }
+
+            if self.isDownloaded(entry):
+                raise yt_dlp.utils.ExistingVideoReached()
+
             log.debug(f'entry: extract info says: {entry}')
         except yt_dlp.utils.ExistingVideoReached:
             return {'status': 'error', 'msg': 'Video has been downloaded already and recorded in archive.log file.'}
@@ -313,3 +318,6 @@ class DownloadQueue:
                 else:
                     self.done.put(entry)
                     await self.notifier.completed(entry.info)
+
+    def isDownloaded(self, info: dict) -> bool:
+        return self.config.keep_archive and isDownloaded(self.config.ytdl_options.get('download_archive', None), info)

app/Utils.py

@@ -173,7 +173,7 @@ def ExtractInfo(config: dict, url: str, debug: bool = False) -> dict:
     }
 
     # Remove keys that are not needed for info extraction as those keys generate files when used with extract_info.
-    for key in ('writeinfojson', 'writethumbnail', 'writedescription', 'writeautomaticsub'):
+    for key in ('writeinfojson', 'writethumbnail', 'writedescription', 'writeautomaticsub', 'download_archive',):
         if key in params:
             del params[key]
 
@@ -224,6 +224,18 @@ def mergeConfig(config: dict, new_config: dict) -> dict:
     return mergeDict(new_config, config)
 
 
+def isDownloaded(archive_file: str, info: dict) -> bool:
+    if not info or not archive_file or not os.path.exists(archive_file):
+        return False
+
+    id = yt_dlp.YoutubeDL()._make_archive_id(info)
+    if not id:
+        return False
+
+    with open(archive_file, 'r') as f:
+        return id in f.read()
+
+
 def jsonCookie(cookies: dict[dict[str, any]]) -> str | None:
     """Converts JSON cookies to Netscape cookies
 

app/main.py

@@ -18,6 +18,7 @@ import caribou
 import sqlite3
 from aiocron import crontab
 
+
 class Main:
     config: Config = None
     serializer: ObjectSerializer = None
@@ -30,7 +31,7 @@ class Main:
     logger: logging.Logger = None
 
     def __init__(self):
-        self.config = Config()
+        self.config = Config.get_instance()
         self.logger = logging.getLogger('main')
 
         try:
@@ -42,6 +43,7 @@ class Main:
             self.logger.error(
                 f'Could not create download folder at {self.config.download_path}')
             raise e
+
         try:
             if not os.path.exists(self.config.temp_path):
                 self.logger.info(