Merge pull request #152 from arabcoders/dev

Document env variables and added remove_files env option.

.vscode/launch.json

@@ -35,9 +35,9 @@
         "YTP_TEMP_PATH": "${workspaceFolder}/var/tmp",
         "YTP_URL_HOST": "http://localhost:8081",
         "YTP_LOG_LEVEL": "DEBUG",
-        "YTP_DEBUG": "false",
+        "YTP_DEBUG": "true",
-        "YTP_YTDL_DEBUG": "false",
+        "YTP_YTDL_DEBUG": "true",
-        "YTP_MAX_WORKERS": "2",
+        "YTP_MAX_WORKERS": "2"
       }
     },
     {

README.md

@@ -71,6 +71,7 @@ Certain values can be set via environment variables, using the `-e` parameter on
 * __YTP_TEMP_KEEP__: Whether to keep the Individual video temp directory or remove it. Defaults to `false`.
 * __YTP_URL_PREFIX__: base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`.
 * __YTP_OUTPUT_TEMPLATE__: the template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`. This will be the default for all downloads unless the request include output template.
+* __YTP_OUTPUT_TEMPLATE_CHAPTER__: the template for the filenames of the downloaded videos, when split into chapters via postprocessors, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s - %(section_number)s %(section_title)s.%(ext)s.`
 * __YTP_KEEP_ARCHIVE__: Whether to keep history of downloaded videos to prevent downloading same file multiple times. Defaults to `true`.
 * __YTP_YTDL_DEBUG__: Whether to turn debug logging for the internal `yt-dlp` package. Defaults to `false`.
 * __YTP_ALLOW_MANIFESTLESS__: Allow `yt-dlp` to download live streams videos which are yet to be processed by YouTube. Defaults to `false`
@@ -82,6 +83,14 @@ Certain values can be set via environment variables, using the `-e` parameter on
 * __YTP_STREAMER_ACODEC__: The audio codec to use for in-browser streaming. Defaults to `aac`.
 * __YTP_AUTH_USERNAME__: Username for basic authentication. Defaults open for all
 * __YTP_AUTH_PASSWORD__: Password for basic authentication. Defaults open for all.
+* __YTP_REMOVE_FILES__: Whether to remove the actual downloaded file when clicking the remove button. Defaults to `false`.
+* __YTP_ACCESS_LOG__: Whether to log access to the web server. Defaults to `true`.
+* __YTP_DEBUG__: Whether to turn on debug mode. Defaults to `false`.
+* __YTP_DEBUGPY_PORT__: The port to use for the debugpy debugger. Defaults to `5678`.
+* __YTP_SOCKET_TIMEOUT__: The timeout for the yt-dlp socket connection variable. Defaults to `30`.
+* __YTP_EXTRACT_INFO_TIMEOUT__: The timeout for extracting video information. Defaults to `70`.
+* __YTP_DB_FILE__: The path to the SQLite database file. Defaults to `{config_path}/ytptube.db`.
+* __YTP_MANUAL_ARCHIVE__: The path to the manual archive file. Defaults to `{config_path}/manual_archive.log`.
 
 ## Running behind a reverse proxy
 

app/library/DownloadQueue.py

@@ -112,7 +112,7 @@ class DownloadQueue:
             if self.done.exists(key=entry["id"], url=entry.get("webpage_url") or entry.get("url")):
                 item = self.done.get(key=entry["id"], url=entry.get("webpage_url") or entry["url"])
                 LOG.warning(f"Item '{item.info.id}' - '{item.info.title}' already downloaded. Removing from history.")
-                await self.clear([item.info._id])
+                await self.clear([item.info._id], remove_file=False)
 
             try:
                 item = self.queue.get(key=entry.get("id"), url=entry.get("webpage_url") or entry.get("url"))
@@ -306,7 +306,7 @@ class DownloadQueue:
 
         return status
 
-    async def clear(self, ids: list[str]) -> dict[str, str]:
+    async def clear(self, ids: list[str], remove_file: bool = False) -> dict[str, str]:
         status: dict[str, str] = {"status": "ok"}
 
         for id in ids:
@@ -318,11 +318,36 @@ class DownloadQueue:
                 LOG.warning(f"Requested delete for non-existent download {id=}. {str(e)}")
                 continue
 
-            itemMessage = f"{id=} {item.info.id=} {item.info.title=}"
+            itemRef: str = f"{id=} {item.info.id=} {item.info.title=}"
-            LOG.debug(f"Deleting completed download {itemMessage}")
+            fileDeleted: bool = False
+            filename: str = ""
+
+            if remove_file and self.config.remove_files and "finished" == item.info.status:
+                filename = item.info.filename
+                if item.info.folder:
+                    filename = f"{item.info.folder}/{item.info.filename}"
+
+                try:
+                    realFile: str = calcDownloadPath(
+                        basePath=self.config.download_path,
+                        folder=filename,
+                        createPath=False,
+                    )
+                    if realFile and os.path.exists(realFile):
+                        os.remove(realFile)
+                        fileDeleted = True
+                    else:
+                        LOG.warning(f"File '{filename}' does not exist.")
+                except Exception as e:
+                    LOG.error(f"Unable to remove '{itemRef}' local file '{filename}'. {str(e)}")
+
             self.done.delete(id)
             asyncio.create_task(self.emitter.cleared(id, dl=item), name=f"notifier-c-{id}")
-            LOG.info(f"Deleted completed download {itemMessage}")
+            msg = f"Deleted completed download '{itemRef}'."
+            if fileDeleted and filename:
+                msg += f" and removed local file '{filename}'."
+
+            LOG.info(msg=msg)
             status[id] = "ok"
 
         return status

app/library/HttpAPI.py

@@ -293,14 +293,17 @@ class HttpAPI(common):
         post = await request.json()
         ids = post.get("ids")
         where = post.get("where")
-
         if not ids or where not in ["queue", "done"]:
             return web.json_response(
                 data={"error": "ids and where are required."}, status=web.HTTPBadRequest.status_code
             )
 
+        remove_file: bool = bool(post.get("remove_file", True))
+
         return web.json_response(
-            data=await (self.queue.cancel(ids) if where == "queue" else self.queue.clear(ids)),
+            data=await (
+                self.queue.cancel(ids) if where == "queue" else self.queue.clear(ids, remove_file=remove_file)
+            ),
             status=web.HTTPOk.status_code,
             dumps=self.encoder.encode,
         )

app/library/HttpSocket.py

@@ -193,13 +193,18 @@ class HttpSocket(common):
         await self.emitter.emit("item_cancel", status)
 
     @ws_event
-    async def item_delete(self, sid: str, id: str):
+    async def item_delete(self, sid: str, data: dict):
+        if not data:
+            await self.emitter.warning("Invalid request.", to=sid)
+            return
+
+        id: str = data.get("id")
         if not id:
             await self.emitter.warning("Invalid request.", to=sid)
             return
 
         status: dict[str, str] = {}
-        status = await self.queue.clear([id])
+        status = await self.queue.clear([id], remove_file=bool(data.get("remove_file", False)))
         status.update({"identifier": id})
 
         await self.emitter.emit("item_delete", status)

app/library/Utils.py

@@ -4,6 +4,7 @@ import json
 import logging
 import os
 import pathlib
+import posixpath
 import re
 import socket
 import uuid
@@ -23,6 +24,7 @@ IGNORED_KEYS: tuple[str] = (
     "postprocessor_hooks",
 )
 YTDLP_INFO_CLS: yt_dlp.YoutubeDL = None
+OS_ALT_SEP: list[str] = list(sep for sep in [os.sep, os.path.altsep] if sep is not None and sep != "/")
 
 
 class StreamingError(Exception):

app/library/config.py

@@ -14,13 +14,10 @@ from .version import APP_VERSION
 
 
 class Config:
-    __instance = None
     config_path: str = "."
     download_path: str = "."
     temp_path: str = "/tmp"
     temp_keep: bool = False
-    db_file: str = "{config_path}/ytptube.db"
-    manual_archive: str = "{config_path}/archive.manual.log"
 
     url_host: str = ""
     url_prefix: str = ""
@@ -29,69 +26,72 @@ class Config:
     output_template: str = "%(title)s.%(ext)s"
     output_template_chapter: str = "%(title)s - %(section_number)s %(section_title)s.%(ext)s"
 
-    ytdl_options: dict | str = {}
+    keep_archive: bool = True
+
     ytdl_debug: bool = False
+    allow_manifestless: bool = False
 
     host: str = "0.0.0.0"
     port: int = 8081
 
-    keep_archive: bool = True
-
-    base_path: str = ""
-
     log_level: str = "info"
+    max_workers: int = 1
+
+    streamer_vcodec: str = "libx264"
+    streamer_acodec: str = "aac"
+
+    auth_username: str | None = None
+    auth_password: str | None = None
+
+    remove_files: bool = False
 
     access_log: bool = True
 
-    allow_manifestless: bool = False
-
-    max_workers: int = 1
-
-    version: str = APP_VERSION
-
     debug: bool = False
-
     debugpy_port: int = 5678
 
-    new_version_available: bool = False
+    socket_timeout: int = 30
-
     extract_info_timeout: int = 70
 
-    socket_timeout: int = 30
+    db_file: str = "{config_path}/ytptube.db"
+    manual_archive: str = "{config_path}/archive.manual.log"
 
-    started: int = 0
+    # immutable config vars.
-
+    version: str = APP_VERSION
-    streamer_vcodec = "libx264"
+    __instance = None
-    streamer_acodec = "aac"
+    ytdl_options: dict | str = {}
-
-    auth_username: str = None
-    auth_password: str = None
-
-    ytdlp_version: str = YTDLP_VERSION
     tasks: list = []
+    new_version_available: bool = False
+    ytdlp_version: str = YTDLP_VERSION
+    started: int = 0
     presets: list = [
         {"name": "default", "format": "default", "postprocessors": [], "args": {}},
         {
-            "name": "Video - 1080p h264/acc",
+            "name": "Best video and audio",
+            "format": "bv+ba/b",
+            "args": {},
+        },
+        {
+            "name": "1080p H264/m4a or best available",
             "format": "bv[height<=1080][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]",
             "args": {
                 "format_sort": ["vcodec:h264"],
             },
         },
         {
-            "name": "Video - 720p h264/acc",
+            "name": "720p h264/m4a or best available",
             "format": "bv[height<=720][ext=mp4]+ba[ext=m4a]/b[ext=mp4]/b[ext=webm]",
             "args": {
                 "format_sort": ["vcodec:h264"],
             },
         },
         {
-            "name": "Audio - Best audio [MP3]",
+            "name": "Audio only",
             "format": "bestaudio/best",
             "postprocessors": [
                 {
                     "key": "FFmpegExtractAudio",
-                    "preferredcodec": "mp3",
+                    "preferredcodec": "best",
                     "preferredquality": "5",
                     "nopostoverwrites": False,
                 },
@@ -106,7 +106,6 @@ class Config:
                 "fragment_retries": 10,
                 "writethumbnail": True,
                 "extract_flat": "discard_in_playlist",
-                "final_ext": "mp3",
             },
         },
     ]
@@ -116,6 +115,7 @@ class Config:
         "config_path",
         "download_path",
     )
+
     _immutable: tuple = (
         "version",
         "__instance",
@@ -141,6 +141,7 @@ class Config:
         "temp_keep",
         "allow_manifestless",
         "access_log",
+        "remove_files",
     )
 
     _frontend_vars: tuple = (
@@ -152,6 +153,7 @@ class Config:
         "url_host",
         "started",
         "url_prefix",
+        "remove_files",
     )
 
     @staticmethod

ui/components/History.vue

@@ -199,7 +199,7 @@
                 </a>
               </div>
               <div class="column is-half-mobile">
-                <a class="button is-danger is-fullwidth" @click="socket.emit('item_delete', item._id)">
+                <a class="button is-danger is-fullwidth" @click="removeItem(item)">
                   <span class="icon-text is-block">
                     <span class="icon"><i class="fa-solid fa-trash-can" /></span>
                     <span>Remove</span>
@@ -350,40 +350,51 @@ const deleteSelectedItems = () => {
     return;
   }
 
-  if (false === confirm('Are you sure you want to delete selected items?')) {
+  let msg = `Are you sure you want to delete '${selectedElms.value.length}' items?`
+  if (true === config.app.remove_files) {
+    msg += '\nThis will delete the files from the server if they exist.';
+  }
+
+  if (false === confirm(msg)) {
     return;
   }
 
   for (const key in selectedElms.value) {
     const item = stateStore.history[selectedElms.value[key]];
-    if (item.status === 'finished') {
+    if ('finished' === item.status) {
-      archiveItem(item);
+      socket.emit('archive_item', item);
     }
-    socket.emit('item_delete', item._id);
+    socket.emit('item_delete', {
+      id: item._id,
+      remove_file: config.app.remove_files,
+    });
   }
 }
 
 const clearCompleted = () => {
-  const state = confirm('Are you sure you want to clear all completed downloads?');
+  let msg = 'Are you sure you want to clear all completed downloads?';
-  if (false === state) {
+  if (false === confirm(msg)) {
     return;
   }
 
   for (const key in stateStore.history) {
-    if (ag(stateStore.get('history', key, {}), 'status') === 'finished') {
+    if ('finished' === ag(stateStore.get('history', key, {}), 'status')) {
-      socket.emit('item_delete', stateStore.history[key]._id);
+      socket.emit('item_delete', { id: stateStore.history[key]._id, remove_file: false, });
     }
   }
 }
 
 const clearIncomplete = () => {
-  if (false === confirm('Are you sure you want to clear all incomplete downloads?')) {
+  if (false === confirm('Are you sure you want to clear all in-complete downloads?')) {
     return;
   }
 
   for (const key in stateStore.history) {
     if (stateStore.history[key].status !== 'finished') {
-      socket.emit('item_delete', stateStore.history[key]._id);
+      socket.emit('item_delete', {
+        id: stateStore.history[key]._id,
+        remove_file: false,
+      });
     }
   }
 }
@@ -427,11 +438,23 @@ const archiveItem = item => {
     return
   }
   socket.emit('archive_item', item);
-  socket.emit('item_delete', item._id);
+  socket.emit('item_delete', { id: item._id, remove_file: false });
+}
+
+const removeItem = item => {
+  const msg = `Remove '${item.title ?? item.id ?? item.url ?? '??'}'?\n this will delete the file from the server.`;
+  if (config.app.remove_files && !confirm(msg)) {
+    return false
+  }
+
+  socket.emit('item_delete', {
+    id: item._id,
+    remove_file: config.app.remove_files
+  });
 }
 
 const reQueueItem = item => {
-  socket.emit('item_delete', item._id)
+  socket.emit('item_delete', { id: item._id, remove_file: false })
   socket.emit('add_url', {
     url: item.url,
     preset: item.preset,

ui/stores/ConfigStore.js

@@ -3,6 +3,7 @@ const CONFIG_KEYS = {
   app: {
     download_path: '/downloads',
     keep_archive: false,
+    remove_files: false,
     output_template: '',
     ytdlp_version: '',
     version: '',