diff --git a/API.md b/API.md index ba5d5a29..963c251a 100644 --- a/API.md +++ b/API.md @@ -27,6 +27,9 @@ This document describes the available endpoints and their usage. All endpoints r - [GET /api/history](#get-apihistory) - [DELETE /api/history/{id}/archive](#delete-apihistoryidarchive) - [POST /api/history/{id}/archive](#post-apihistoryidarchive) + - [GET /api/archiver](#get-apiarchiver) + - [POST /api/archiver](#post-apiarchiver) + - [DELETE /api/archiver](#delete-apiarchiver) - [GET /api/tasks](#get-apitasks) - [PUT /api/tasks](#put-apitasks) - [POST /api/tasks/{id}/mark](#post-apitasksidmark) @@ -401,6 +404,81 @@ or an error: --- +### GET /api/archiver +**Purpose**: Read entries from the download archive associated with a preset. + +**Query Parameters**: +- `preset` (required): Preset name. +- `ids` (optional): Comma-separated list of archive IDs to filter. If omitted, returns all entries. + +**Response**: +```json +{ + "file": "/path/to/archive.log", + "items": ["youtube ABC123", "vimeo XYZ789"], + "count": 2 +} +``` +or an error: +```json +{ "error": "Preset '' does not provide a download_archive." } +``` + +- `400 Bad Request` if `preset` is missing/invalid or the preset has no `download_archive`. + +--- + +### POST /api/archiver +**Purpose**: Append archive IDs to the download archive resolved by a preset. + +**Body**: +```json +{ + "preset": "", + "items": ["youtube ABC123", "vimeo XYZ789"], + "skip_check": false +} +``` + +**Response**: +```json +{ + "file": "/path/to/archive.log", + "status": true, + "items": ["youtube ABC123", "vimeo XYZ789"] +} +``` +Notes: +- Returns `200 OK` if any new entries were added, `304 Not Modified` if no-op (all already present or invalid). +- `400 Bad Request` if `preset` is missing/invalid, the preset has no `download_archive`, or `items` is empty. + +--- + +### DELETE /api/archiver +**Purpose**: Remove archive IDs from the download archive resolved by a preset. + +**Body**: +```json +{ + "preset": "", + "items": ["youtube ABC123", "vimeo XYZ789"] +} +``` + +**Response**: +```json +{ + "file": "/path/to/archive.log", + "status": true, + "items": ["youtube ABC123", "vimeo XYZ789"] +} +``` +Notes: +- Returns `200 OK` if the archive file changed, `304 Not Modified` if it was unchanged. +- `400 Bad Request` if `preset` is missing/invalid, the preset has no `download_archive`, or `items` is empty. + +--- + ### GET /api/tasks **Purpose**: Retrieves the scheduled tasks from the internal `Tasks` manager. diff --git a/FAQ.md b/FAQ.md index a5bbf97f..52b725ee 100644 --- a/FAQ.md +++ b/FAQ.md @@ -3,48 +3,48 @@ Certain configuration values can be set via environment variables, using the `-e` parameter on the docker command line, or the `environment:` section in `compose.yaml` file. -| Environment Variable | Description | Default | -| ------------------------------ | ------------------------------------------------------------------ | ---------------------------------- | -| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` | -| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` | -| YTP_INSTANCE_TITLE | The title of the instance | `empty string` | -| YTP_FILE_LOGGING | Whether to log to file | `false` | -| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` | -| YTP_MAX_WORKERS | How many works to use for downloads | `1` | -| YTP_AUTH_USERNAME | Username for basic authentication | `empty string` | -| YTP_AUTH_PASSWORD | Password for basic authentication | `empty string` | -| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` | -| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` | -| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` | -| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` | -| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | -| YTP_HOST | Which IP address to bind to | `0.0.0.0` | -| YTP_PORT | Which port to bind to | `8081` | -| YTP_LOG_LEVEL | Log level | `info` | -| YTP_STREAMER_VCODEC | The video codec to use for in-browser streaming | `libx264` | -| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` | -| YTP_ACCESS_LOG | Whether to log access to the web server | `true` | -| YTP_DEBUG | Whether to turn on debug mode | `false` | -| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` | -| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` | -| YTP_DB_FILE | The path to the SQLite database file | `{config_path}/ytptube.db` | -| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` | -| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `empty string` | -| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` | -| YTP_BASIC_MODE | Whether to run WebUI in basic mode | `false` | -| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use | `empty string` | -| YTP_BROWSER_ENABLED | Whether to enable the file browser | `false` | -| YTP_BROWSER_CONTROL_ENABLED | Whether to enable the file browser actions | `false` | -| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | -| YTP_YTDLP_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | -| YTP_YTDLP_VERSION | The version of yt-dlp to use. Defaults to latest version | `empty string` | -| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | -| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | -| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer | `15 */1 * * *` | -| YTP_PLAYLIST_ITEMS_CONCURRENCY | The number of playlist items be to processed at same time | `1` | -| YTP_TEMP_DISABLED | Disable temp files handling. | `false` | -| YTP_DOWNLOAD_PATH_DEPTH | How many subdirectories to show in auto complete. | `1` | - +| Environment Variable | Description | Default | +| ------------------------------ | ------------------------------------------------------------------ | -------------------------- | +| TZ | The timezone to use for the application | `(not_set)` | +| YTP_OUTPUT_TEMPLATE | The template for the filenames of the downloaded videos | `%(title)s.%(ext)s` | +| YTP_DEFAULT_PRESET | The default preset to use for the download | `default` | +| YTP_INSTANCE_TITLE | The title of the instance | `empty string` | +| YTP_FILE_LOGGING | Whether to log to file | `false` | +| YTP_DOWNLOAD_PATH | Path to where the downloads will be saved | `/downloads` | +| YTP_MAX_WORKERS | How many works to use for downloads | `1` | +| YTP_AUTH_USERNAME | Username for basic authentication | `empty string` | +| YTP_AUTH_PASSWORD | Password for basic authentication | `empty string` | +| YTP_CONSOLE_ENABLED | Whether to enable the console | `false` | +| YTP_REMOVE_FILES | Remove the actual file when clicking the remove button | `false` | +| YTP_CONFIG_PATH | Path to where the config files will be stored. | `/config` | +| YTP_TEMP_PATH | Path to where tmp files are stored. | `/tmp` | +| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` | +| YTP_HOST | Which IP address to bind to | `0.0.0.0` | +| YTP_PORT | Which port to bind to | `8081` | +| YTP_LOG_LEVEL | Log level | `info` | +| YTP_STREAMER_VCODEC | The video codec to use for in-browser streaming | `libx264` | +| YTP_STREAMER_ACODEC | The audio codec to use for in-browser streaming | `aac` | +| YTP_ACCESS_LOG | Whether to log access to the web server | `true` | +| YTP_DEBUG | Whether to turn on debug mode | `false` | +| YTP_DEBUGPY_PORT | The port to use for the debugpy debugger | `5678` | +| YTP_EXTRACT_INFO_TIMEOUT | The timeout for extracting video information | `70` | +| YTP_DB_FILE | The path to the SQLite database file | `{config_path}/ytptube.db` | +| YTP_UI_UPDATE_TITLE | Whether to update the title of the page with the current stats | `true` | +| YTP_PIP_PACKAGES | A space separated list of pip packages to install | `empty string` | +| YTP_PIP_IGNORE_UPDATES | Do not update the custom pip packages | `false` | +| YTP_BASIC_MODE | Whether to run WebUI in basic mode | `false` | +| YTP_PICTURES_BACKENDS | A comma separated list of pictures urls to use | `empty string` | +| YTP_BROWSER_ENABLED | Whether to enable the file browser | `true` | +| YTP_BROWSER_CONTROL_ENABLED | Whether to enable the file browser actions | `false` | +| YTP_YTDLP_AUTO_UPDATE | Whether to enable the auto update for yt-dlp | `true` | +| YTP_YTDLP_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` | +| YTP_YTDLP_VERSION | The version of yt-dlp to use. Defaults to latest version | `empty string` | +| YTP_BASE_PATH | Set this if you are serving YTPTube from sub-folder | `/` | +| YTP_PREVENT_LIVE_PREMIERE | Prevents the initial youtube premiere stream from being downloaded | `false` | +| YTP_TASKS_HANDLER_TIMER | The cron expression for the tasks handler timer | `15 */1 * * *` | +| YTP_PLAYLIST_ITEMS_CONCURRENCY | The number of playlist items be to processed at same time | `1` | +| YTP_TEMP_DISABLED | Disable temp files handling. | `false` | +| YTP_DOWNLOAD_PATH_DEPTH | How many subdirectories to show in auto complete. | `1` | # Browser extensions & bookmarklets diff --git a/app/library/DataStore.py b/app/library/DataStore.py index c08c9722..753ce0dc 100644 --- a/app/library/DataStore.py +++ b/app/library/DataStore.py @@ -96,6 +96,32 @@ class DataStore: msg: str = f"{key=} or {url=} not found." raise KeyError(msg) + def get_item(self, **kwargs) -> Download | None: + """ + Get a specific item from the datastore based on provided attributes. + + Args: + **kwargs: Arbitrary keyword arguments representing attributes of the ItemDTO. + If no attributes are provided, the method returns None. + If any attribute matches, the corresponding Download object is returned. + + Returns: + Download | None: The requested item if found, otherwise None. + + """ + if not kwargs: + return None + + for i in self._dict: + if not self._dict[i].info: + continue + + info = self._dict[i].info.__dict__ + if any((key in info and info == value) for key, value in kwargs.items()): + return self._dict[i] + + return None + def get_by_id(self, id: str) -> Download | None: return self._dict.get(id, None) diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index fd169a10..5652f549 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -676,11 +676,43 @@ class DownloadQueue(metaclass=Singleton): item.extras.update({"external_downloader": True}) if item.is_archived(): + if archive_id := item.get_archive_id(): + store_type, _ = self.get_item(archive_id=archive_id) + if not store_type: + dlInfo = Download( + info=ItemDTO( + id=archive_id.split()[1], + title=archive_id, + url=item.url, + preset=item.preset, + folder=item.folder, + status="skip", + cookies=item.cookies, + template=item.template, + msg="URL is already downloaded.", + extras=item.extras, + ) + ) + + if archive_file := dlInfo.info.get_ytdlp_opts().get_all().get("download_archive"): + dlInfo.info.msg += f" Found in archive '{archive_file}'." + + self.done.put(dlInfo) + + await self._notify.emit( + Events.ITEM_MOVED, + data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info}, + title="Download History Update", + message=f"Download history updated with '{item.url}'.", + ) + return {"status": "ok"} + message: str = f"The URL '{item.url}' is already downloaded and recorded in archive." LOG.error(message) await self._notify.emit( Events.LOG_INFO, data={"preset": item.preset}, title="Already Downloaded", message=message ) + return {"status": "error", "msg": message} started: float = time.perf_counter() @@ -908,26 +940,24 @@ class DownloadQueue(metaclass=Singleton): return items - def get_item(self, id: str) -> Download | None: + def get_item(self, **kwargs) -> tuple[StoreType, Download] | tuple[None, None]: """ Get a specific item from the download queue or history. Args: - id (str): The ID of the item to retrieve. + **kwargs: The key-value pair to search for. Supported keys are 'id', 'url'. Returns: - Download | None: The requested item if found, otherwise None. + (StoreType, Download) | None: The requested item if found, otherwise None. """ - try: - return self.queue.get(key=id) - except KeyError: - pass + if item := self.queue.get_item(**kwargs): + return (StoreType.QUEUE, item) - try: - return self.done.get(key=id) - except KeyError: - return None + if item := self.done.get_item(**kwargs): + return (StoreType.HISTORY, item) + + return (None, None) async def _download_pool(self) -> None: """ diff --git a/app/library/config.py b/app/library/config.py index d70ca70c..a45fb16c 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -164,7 +164,7 @@ class Config: console_enabled: bool = False "Enable direct access to yt-dlp console." - browser_enabled: bool = False + browser_enabled: bool = True "Enable file browser access." browser_control_enabled: bool = False diff --git a/app/routes/api/archiver.py b/app/routes/api/archiver.py new file mode 100644 index 00000000..c732f5a9 --- /dev/null +++ b/app/routes/api/archiver.py @@ -0,0 +1,210 @@ +import logging +from collections.abc import Iterable + +from aiohttp import web +from aiohttp.web import Request, Response + +from app.library.Archiver import Archiver +from app.library.Presets import Presets +from app.library.router import route +from app.library.YTDLPOpts import YTDLPOpts + +LOG: logging.Logger = logging.getLogger(__name__) + + +def _get_archive_file_from_preset(preset: str) -> str | None: + """ + Resolve the archive file path for a given preset. + + Validates that the preset exists and that applying the preset results + in yt-dlp options that contain a 'download_archive' path. + """ + if not preset or not Presets.get_instance().has(preset): + return None + + try: + opts: dict = YTDLPOpts.get_instance().preset(preset).get_all() + except Exception as e: + LOG.error(f"Failed to build yt-dlp opts for preset '{preset}'. {e!s}") + return None + + if not (archive_file := opts.get("download_archive")): + return None + + if not isinstance(archive_file, str) or len(archive_file.strip()) < 1: + return None + + return archive_file.strip() + + +def _normalize_ids(items: Iterable[str] | None) -> tuple[list[str], list[str]]: + """ + Validate and normalize archive IDs. + + - Trims whitespace + - Enforces that each ID has at least two whitespace-separated tokens + (e.g., "youtube ABC123") as required by yt-dlp's archive format + - De-duplicates while preserving order + + Returns a tuple: (valid_ids, invalid_inputs) + """ + if not items: + return ([], []) + + seen: set[str] = set() + valid: list[str] = [] + invalid: list[str] = [] + + for raw in items: + if raw is None: + continue + + s = str(raw).strip() + if not s: + continue + + if len(s.split()) < 2: + invalid.append(s) + continue + + if s in seen: + continue + + seen.add(s) + valid.append(s) + + return (valid, invalid) + + +@route("GET", "api/archiver/", "archiver") +async def archiver_get(request: Request) -> Response: + """ + Read IDs from the download archive for a given preset. + + Query params: + - preset: required preset name/id + - ids: optional comma-separated list to filter; when omitted, returns all + """ + preset: str | None = request.query.get("preset") + if not preset: + return web.json_response(data={"error": "preset is required."}, status=web.HTTPBadRequest.status_code) + + archive_file: str | None = _get_archive_file_from_preset(preset) + if not archive_file: + return web.json_response( + data={"error": f"Preset '{preset}' does not provide a download_archive."}, + status=web.HTTPBadRequest.status_code, + ) + + ids_param: str | None = request.query.get("ids") + ids: list[str] = [] + if ids_param: + ids_list = [s.strip() for s in ids_param.split(",") if s and s.strip()] + ids, invalid = _normalize_ids(ids_list) + if invalid: + return web.json_response( + data={"error": "invalid ids provided.", "invalid_items": invalid}, + status=web.HTTPBadRequest.status_code, + ) + + try: + data: list[str] = Archiver.get_instance().read(archive_file, ids or None) + return web.json_response( + data={"file": archive_file, "items": data, "count": len(data)}, status=web.HTTPOk.status_code + ) + except Exception as e: + LOG.exception(e) + return web.json_response( + data={"error": f"Failed to read archive file for preset '{preset}'.", "message": str(e)}, + status=web.HTTPInternalServerError.status_code, + ) + + +@route("POST", "api/archiver/", "archiver_add") +async def archiver_add(request: Request) -> Response: + """ + Append IDs to the download archive for a given preset. + + Body: { "preset": string, "items": [string, ...], "skip_check": bool? } + """ + post = await request.json() + preset: str | None = post.get("preset") if isinstance(post, dict) else None + if not preset: + return web.json_response(data={"error": "preset is required."}, status=web.HTTPBadRequest.status_code) + + archive_file: str | None = _get_archive_file_from_preset(preset) + if not archive_file: + return web.json_response( + data={"error": f"Preset '{preset}' does not provide a download_archive."}, + status=web.HTTPBadRequest.status_code, + ) + + items, invalid = _normalize_ids((post or {}).get("items", [])) if isinstance(post, dict) else ([], []) + if invalid: + return web.json_response( + data={"error": "invalid ids provided.", "invalid_items": invalid}, + status=web.HTTPBadRequest.status_code, + ) + if len(items) < 1: + return web.json_response( + data={"error": "items is required and must be a non-empty list."}, status=web.HTTPBadRequest.status_code + ) + + skip_check: bool = bool(post.get("skip_check", False)) if isinstance(post, dict) else False + + try: + status: bool = Archiver.get_instance().add(archive_file, items, skip_check=skip_check) + return web.json_response( + data={"file": archive_file, "status": status, "items": items}, + status=web.HTTPOk.status_code if status else web.HTTPNotModified.status_code, + ) + except Exception as e: + LOG.exception(e) + return web.json_response( + data={"error": f"Failed to add items to archive for preset '{preset}'.", "message": str(e)}, + status=web.HTTPInternalServerError.status_code, + ) + + +@route("DELETE", "api/archiver/", "archiver_delete") +async def archiver_delete(request: Request) -> Response: + """ + Remove IDs from the download archive for a given preset. + + Body: { "preset": string, "items": [string, ...] } + """ + post = await request.json() + preset: str | None = post.get("preset") if isinstance(post, dict) else None + if not preset: + return web.json_response(data={"error": "preset is required."}, status=web.HTTPBadRequest.status_code) + + archive_file: str | None = _get_archive_file_from_preset(preset) + if not archive_file: + return web.json_response( + data={"error": f"Preset '{preset}' does not provide a download_archive."}, + status=web.HTTPBadRequest.status_code, + ) + + items, invalid = _normalize_ids((post or {}).get("items", [])) if isinstance(post, dict) else ([], []) + if invalid: + return web.json_response( + data={"error": "invalid ids provided.", "invalid_items": invalid}, + status=web.HTTPBadRequest.status_code, + ) + if len(items) < 1: + return web.json_response( + data={"error": "items is required and must be a non-empty list."}, status=web.HTTPBadRequest.status_code + ) + + try: + status: bool = Archiver.get_instance().delete(archive_file, items) + return web.json_response( + data={"file": archive_file, "status": status, "items": items}, + status=web.HTTPOk.status_code if status else web.HTTPNotModified.status_code, + ) + except Exception as e: + LOG.exception(e) + return web.json_response( + data={"error": f"Failed to delete items from archive for preset '{preset}'.", "message": str(e)}, + status=web.HTTPInternalServerError.status_code, + ) diff --git a/ui/app/components/NewDownload.vue b/ui/app/components/NewDownload.vue index 0e5e6fa0..d15be8ca 100644 --- a/ui/app/components/NewDownload.vue +++ b/ui/app/components/NewDownload.vue @@ -64,7 +64,7 @@
@@ -105,7 +105,7 @@ + :placeholder="getDefault('template', config.app.output_template || '%(title)s.%(ext)s')">