Fixes

.github/workflows/main.yml

@@ -14,16 +14,16 @@ on:
           - debug
   push:
     branches:
-      - "*"
+      - '*'
     paths-ignore:
-      - "**.md"
+      - '**.md'
-      - ".github/**"
+      - '.github/**'
   pull_request:
     branches:
-      - "master"
+      - 'master'
     paths-ignore:
-      - "**.md"
+      - '**.md'
-      - ".github/ISSUE_TEMPLATE/**"
+      - '.github/ISSUE_TEMPLATE/**'
 
 env:
   PLATFORMS: linux/amd64

.vscode/launch.json

@@ -34,6 +34,7 @@
         "YTP_TEMP_PATH": "${workspaceFolder}/var/tmp",
         "YTP_YTDL_OPTIONS_FILE": "${workspaceFolder}/var/config/ytdlp.json",
         "YTP_URL_HOST": "http://localhost:8081",
+        "YTP_KEEP_ARCHIVE": "true",
       }
     }
   ]

README.md

@@ -41,6 +41,7 @@ Certain values can be set via environment variables, using the `-e` parameter on
 * __YTP_OUTPUT_TEMPLATE__: the template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`.
 * __YTP_YTDL_OPTIONS__: Additional options to pass to yt-dlp, in JSON format. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L183). They roughly correspond to command-line options, though some do not have exact equivalents here, for example `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores.
 * __YTP_YTDL_OPTIONS_FILE__: A path to a JSON file that will be loaded and used for populating `YTDL_OPTIONS` above.
+* __YTP_KEEP_ARCHIVE__: Boolean. Whether to keep history of downloaded videos to prevent downloading same file multiple times.
 
 The following example value for `YTDL_OPTIONS` embeds English subtitles and chapter markers (for videos that have them), and also changes the permissions on the downloaded video and sets the file modification timestamp to the date of when it was downloaded:
 

app/migrations/20231115152938_initial.py

@@ -11,6 +11,7 @@ def upgrade(connection):
     CREATE TABLE "history" (
         "id" TEXT PRIMARY KEY UNIQUE NOT NULL,
         "type" TEXT NOT NULL,
+        "url" TEXT NOT NULL,
         "data" JSON NOT NULL,
         "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
     );
@@ -22,6 +23,11 @@ def upgrade(connection):
     """
     connection.execute(sql)
 
+    sql = """
+    CREATE UNIQUE INDEX "history_url" ON "history" ("url");
+    """
+    connection.execute(sql)
+
     connection.commit()
 
 

app/src/Config.py

@@ -27,9 +27,11 @@ class Config:
     host: str = '0.0.0.0'
     port: int = 8081
 
+    keep_archive: bool = False
+
     base_path: str = ''
 
-    _boolean_vars: tuple = ()
+    _boolean_vars: tuple = ('keep_archive')
 
     def __init__(self):
         baseDefualtPath: str = os.path.dirname(os.path.dirname(__file__))
@@ -64,11 +66,11 @@ class Config:
                 setattr(self, k, v)
 
             if k in self._boolean_vars:
-                if v not in (True, False, 'True', 'false', 'true', 'false', 'on', 'off', '1', '0'):
+                if str(v).lower() not in (True, False, 'true', 'false', 'on', 'off', '1', '0'):
                     log.error(
                         f'Config variable "{k}" is set to a non-boolean value "{v}".')
                     sys.exit(1)
-                setattr(self, k, v in (True, 'true', 'True', 'on', '1'))
+                setattr(self, k, str(v).lower() in (True, 'True', 'on', '1'))
 
         if not self.url_prefix.endswith('/'):
             self.url_prefix += '/'
@@ -85,7 +87,8 @@ class Config:
             log.info(
                 f'Loading yt-dlp custom options from "{self.ytdl_options_file}"')
             if not os.path.exists(self.ytdl_options_file):
-                log.error(f'"YTP_YTDL_OPTIONS_FILE" ENV points to non-existent file: "{self.ytdl_options_file}"')
+                log.error(
+                    f'"YTP_YTDL_OPTIONS_FILE" ENV points to non-existent file: "{self.ytdl_options_file}"')
             else:
                 try:
                     with open(self.ytdl_options_file) as json_data:
@@ -96,6 +99,11 @@ class Config:
                     log.error(f'JSON error in "{self.ytdl_options_file}": {e}')
                     sys.exit(1)
 
+        logging.info(f'keep archive: {self.keep_archive}')
+        if self.keep_archive:
+            self.ytdl_options['download_archive'] = os.path.join(
+                self.config_path, 'archive.log')
+
     def getAttributes(self) -> dict:
         attrs: dict = {}
         vclass: str = self.__class__

app/src/DataStore.py

@@ -1,5 +1,6 @@
 from collections import OrderedDict
 import json
+import logging
 import sqlite3
 from src.Utils import calcDownloadPath
 from src.Config import Config
@@ -47,12 +48,14 @@ class DataStore:
     def saved_items(self) -> list[tuple[str, ItemDTO]]:
         items: list[tuple[str, ItemDTO]] = []
         with sqlite3.connect(self.db_file) as db:
+            db.row_factory = sqlite3.Row
             cursor = db.execute(
                 'SELECT "id", "data" FROM "history" WHERE "type" = ? ORDER BY "created_at" ASC',
                 (self.type,)
             )
 
             for row in cursor:
+                logging.debug(row)
                 data: dict = json.loads(row['data'])
                 key: str = data.pop('_id')
                 item: ItemDTO = ItemDTO(**data)
@@ -80,17 +83,21 @@ class DataStore:
         return not bool(self.dict)
 
     def _updateStoreItem(self, type: str, item: ItemDTO) -> None:
+        sqlStatement = """
+        INSERT INTO "history" ("id", "type", "url", "data")
+        VALUES (?, ?, ?, ?)
+        ON CONFLICT DO UPDATE SET "type" = ?, "url" = ?, "data" = ?
+        """
         with sqlite3.connect(self.db_file) as db:
-            db.execute(
+            db.execute(sqlStatement.strip(), (
-                'INSERT INTO "history" ("id", "type", "data") VALUES (?, ?, ?) ON CONFLICT ("id") DO UPDATE SET "type" = ?, "data" = ?',
+                item._id,
-                (
+                type,
-                    item._id,
+                item.url,
-                    type,
+                item.json(),
-                    item.json(),
+                type,
-                    type,
+                item.url,
-                    item.json(),
+                item.json(),
-                )
+            ))
-            )
 
     def _deleteStoreItem(self, key: str) -> None:
         with sqlite3.connect(self.db_file) as db: