From af6af4ac3743594a8a6a9f13161597359fa81148 Mon Sep 17 00:00:00 2001 From: ArabCoders Date: Fri, 24 Nov 2023 22:56:36 +0300 Subject: [PATCH] Fixes --- .github/workflows/main.yml | 12 +++++------ .vscode/launch.json | 1 + README.md | 1 + app/migrations/20231115152938_initial.py | 6 ++++++ app/src/Config.py | 16 ++++++++++---- app/src/DataStore.py | 27 +++++++++++++++--------- 6 files changed, 43 insertions(+), 20 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 69732dd8..d9f1ebd1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,16 +14,16 @@ on: - debug push: branches: - - "*" + - '*' paths-ignore: - - "**.md" - - ".github/**" + - '**.md' + - '.github/**' pull_request: branches: - - "master" + - 'master' paths-ignore: - - "**.md" - - ".github/ISSUE_TEMPLATE/**" + - '**.md' + - '.github/ISSUE_TEMPLATE/**' env: PLATFORMS: linux/amd64 diff --git a/.vscode/launch.json b/.vscode/launch.json index 95dc7b47..e2cd4cf5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -34,6 +34,7 @@ "YTP_TEMP_PATH": "${workspaceFolder}/var/tmp", "YTP_YTDL_OPTIONS_FILE": "${workspaceFolder}/var/config/ytdlp.json", "YTP_URL_HOST": "http://localhost:8081", + "YTP_KEEP_ARCHIVE": "true", } } ] diff --git a/README.md b/README.md index 6ad04c09..78b85867 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ Certain values can be set via environment variables, using the `-e` parameter on * __YTP_OUTPUT_TEMPLATE__: the template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`. * __YTP_YTDL_OPTIONS__: Additional options to pass to yt-dlp, in JSON format. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L183). They roughly correspond to command-line options, though some do not have exact equivalents here, for example `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores. * __YTP_YTDL_OPTIONS_FILE__: A path to a JSON file that will be loaded and used for populating `YTDL_OPTIONS` above. +* __YTP_KEEP_ARCHIVE__: Boolean. Whether to keep history of downloaded videos to prevent downloading same file multiple times. The following example value for `YTDL_OPTIONS` embeds English subtitles and chapter markers (for videos that have them), and also changes the permissions on the downloaded video and sets the file modification timestamp to the date of when it was downloaded: diff --git a/app/migrations/20231115152938_initial.py b/app/migrations/20231115152938_initial.py index 64754f4c..119de811 100644 --- a/app/migrations/20231115152938_initial.py +++ b/app/migrations/20231115152938_initial.py @@ -11,6 +11,7 @@ def upgrade(connection): CREATE TABLE "history" ( "id" TEXT PRIMARY KEY UNIQUE NOT NULL, "type" TEXT NOT NULL, + "url" TEXT NOT NULL, "data" JSON NOT NULL, "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); @@ -22,6 +23,11 @@ def upgrade(connection): """ connection.execute(sql) + sql = """ + CREATE UNIQUE INDEX "history_url" ON "history" ("url"); + """ + connection.execute(sql) + connection.commit() diff --git a/app/src/Config.py b/app/src/Config.py index 1a2a06cf..536d25ac 100644 --- a/app/src/Config.py +++ b/app/src/Config.py @@ -27,9 +27,11 @@ class Config: host: str = '0.0.0.0' port: int = 8081 + keep_archive: bool = False + base_path: str = '' - _boolean_vars: tuple = () + _boolean_vars: tuple = ('keep_archive') def __init__(self): baseDefualtPath: str = os.path.dirname(os.path.dirname(__file__)) @@ -64,11 +66,11 @@ class Config: setattr(self, k, v) if k in self._boolean_vars: - if v not in (True, False, 'True', 'false', 'true', 'false', 'on', 'off', '1', '0'): + if str(v).lower() not in (True, False, 'true', 'false', 'on', 'off', '1', '0'): log.error( f'Config variable "{k}" is set to a non-boolean value "{v}".') sys.exit(1) - setattr(self, k, v in (True, 'true', 'True', 'on', '1')) + setattr(self, k, str(v).lower() in (True, 'True', 'on', '1')) if not self.url_prefix.endswith('/'): self.url_prefix += '/' @@ -85,7 +87,8 @@ class Config: log.info( f'Loading yt-dlp custom options from "{self.ytdl_options_file}"') if not os.path.exists(self.ytdl_options_file): - log.error(f'"YTP_YTDL_OPTIONS_FILE" ENV points to non-existent file: "{self.ytdl_options_file}"') + log.error( + f'"YTP_YTDL_OPTIONS_FILE" ENV points to non-existent file: "{self.ytdl_options_file}"') else: try: with open(self.ytdl_options_file) as json_data: @@ -96,6 +99,11 @@ class Config: log.error(f'JSON error in "{self.ytdl_options_file}": {e}') sys.exit(1) + logging.info(f'keep archive: {self.keep_archive}') + if self.keep_archive: + self.ytdl_options['download_archive'] = os.path.join( + self.config_path, 'archive.log') + def getAttributes(self) -> dict: attrs: dict = {} vclass: str = self.__class__ diff --git a/app/src/DataStore.py b/app/src/DataStore.py index f43010e0..76107ac7 100644 --- a/app/src/DataStore.py +++ b/app/src/DataStore.py @@ -1,5 +1,6 @@ from collections import OrderedDict import json +import logging import sqlite3 from src.Utils import calcDownloadPath from src.Config import Config @@ -47,12 +48,14 @@ class DataStore: def saved_items(self) -> list[tuple[str, ItemDTO]]: items: list[tuple[str, ItemDTO]] = [] with sqlite3.connect(self.db_file) as db: + db.row_factory = sqlite3.Row cursor = db.execute( 'SELECT "id", "data" FROM "history" WHERE "type" = ? ORDER BY "created_at" ASC', (self.type,) ) for row in cursor: + logging.debug(row) data: dict = json.loads(row['data']) key: str = data.pop('_id') item: ItemDTO = ItemDTO(**data) @@ -80,17 +83,21 @@ class DataStore: return not bool(self.dict) def _updateStoreItem(self, type: str, item: ItemDTO) -> None: + sqlStatement = """ + INSERT INTO "history" ("id", "type", "url", "data") + VALUES (?, ?, ?, ?) + ON CONFLICT DO UPDATE SET "type" = ?, "url" = ?, "data" = ? + """ with sqlite3.connect(self.db_file) as db: - db.execute( - 'INSERT INTO "history" ("id", "type", "data") VALUES (?, ?, ?) ON CONFLICT ("id") DO UPDATE SET "type" = ?, "data" = ?', - ( - item._id, - type, - item.json(), - type, - item.json(), - ) - ) + db.execute(sqlStatement.strip(), ( + item._id, + type, + item.url, + item.json(), + type, + item.url, + item.json(), + )) def _deleteStoreItem(self, key: str) -> None: with sqlite3.connect(self.db_file) as db: