diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index 5652f549..bcfbfebd 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -24,6 +24,7 @@ from .Presets import Presets from .Scheduler import Scheduler from .Singleton import Singleton from .Utils import ( + archive_add, arg_converter, calc_download_path, dt_delta, @@ -675,8 +676,10 @@ class DownloadQueue(metaclass=Singleton): LOG.warning(f"Using external downloader '{yt_conf.get('external_downloader')}' for '{item.url}'.") item.extras.update({"external_downloader": True}) + archive_id = item.get_archive_id() + if item.is_archived(): - if archive_id := item.get_archive_id(): + if archive_id: store_type, _ = self.get_item(archive_id=archive_id) if not store_type: dlInfo = Download( @@ -750,7 +753,55 @@ class DownloadQueue(metaclass=Singleton): if not item.requeued and (condition := Conditions.get_instance().match(info=entry)): already.pop() - LOG.info(f"Matched '{condition.name}' for '{item.url}' Adding '{condition.cli}' to request.") + + item_title = entry.get("title") or entry.get("id") or item.url + message = f"Condition '{condition.name}' matched for '{item_title}'." + + if condition.cli: + message += f" Re-queuing with '{condition.cli}'." + + LOG.info(message) + + if condition.extras.get("ignore_download", False): + extra_msg: str = "" + if yt_conf.get("download_archive") and not condition.extras.get("no_archive", False): + archive_add(yt_conf.get("download_archive"), [archive_id]) + extra_msg = f" and added to archive '{yt_conf.get('download_archive')}'" + + log_message = f"Ignoring download of '{item_title}' as per condition '{condition.name}'{extra_msg}." + + store_type, _ = self.get_item(archive_id=archive_id) + if not store_type: + dlInfo = Download( + info=ItemDTO( + id=entry.get("id"), + title=item_title, + url=item.url, + preset=item.preset, + folder=item.folder, + status="skip", + cookies=item.cookies, + template=item.template, + msg=log_message, + extras=item.extras, + ) + ) + self.done.put(dlInfo) + + LOG.info(log_message) + await asyncio.gather( + *[ + self._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message), + self._notify.emit( + Events.ITEM_MOVED, + data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info}, + title="Download History Update", + message=f"Download history updated with '{item.url}'.", + ), + ] + ) + return {"status": "ok"} + return await self.add(item=item.new_with(requeued=True, cli=condition.cli), already=already) _status, _msg = ytdlp_reject(entry=entry, yt_params=yt_conf) diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index b45d5bcb..f629d777 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -263,14 +263,14 @@ class HttpAPI: response: Response = await handler(request) contentType: str | None = response.headers.get("content-type", None) - if contentType and "/" != base_path and contentType.startswith("text/html"): + if contentType and contentType.startswith("text/html"): rewrite_path: str = base_path.rstrip("/") async with await anyio.open_file(response._path, "rb") as f: content = await f.read() content: str = ( content.decode("utf-8") .replace('', f'') - .replace('baseURL:""', f'baseURL:"{rewrite_path}/"') + .replace("/_base_path/", f"{rewrite_path}/") ) new_response = web.Response(text=content, content_type="text/html") diff --git a/app/library/conditions.py b/app/library/conditions.py index 76eed61d..822cf0ff 100644 --- a/app/library/conditions.py +++ b/app/library/conditions.py @@ -27,9 +27,12 @@ class Condition: filter: str """The filter to run on info dict.""" - cli: str + cli: str = "" """If matched append this to the download request and retry.""" + extras: dict[str, Any] = field(default_factory=dict) + """Any extra data to store with the condition.""" + def serialize(self) -> dict: return self.__dict__ @@ -137,6 +140,10 @@ class Conditions(metaclass=Singleton): item["id"] = str(uuid.uuid4()) need_save = True + if "extras" not in item: + item["extras"] = {} + need_save = True + item: Condition = init_class(Condition, item) self._items.append(item) @@ -145,7 +152,7 @@ class Conditions(metaclass=Singleton): continue if need_save: - LOG.info("Saving conditions due to format, or id change.") + LOG.info("Saving conditions due changes.") self.save(self._items) return self @@ -203,15 +210,16 @@ class Conditions(metaclass=Singleton): msg = f"Invalid filter. '{e!s}'." raise ValueError(msg) from e - if not item.get("cli"): - msg = "No command options for yt-dlp were found." - raise ValueError(msg) + if item.get("cli"): + try: + arg_converter(args=item.get("cli")) + except Exception as e: + msg = f"Invalid command options for yt-dlp. '{e!s}'." + raise ValueError(msg) from e - try: - arg_converter(args=item.get("cli")) - except Exception as e: - msg = f"Invalid command options for yt-dlp. '{e!s}'." - raise ValueError(msg) from e + if not isinstance(item.get("extras"), dict): + msg = "Extras must be a dictionary." + raise ValueError(msg) # noqa: TRY004 return True diff --git a/app/library/config.py b/app/library/config.py index 7331629b..8a83578b 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -359,6 +359,9 @@ class Config: if isinstance(self.pictures_backends, str) and self.pictures_backends: self.pictures_backends = self.pictures_backends.split(",") + if not self.base_path.endswith("/"): + self.base_path += "/" + numeric_level = getattr(logging, self.log_level.upper(), None) if not isinstance(numeric_level, int): msg = f"Invalid log level '{self.log_level}' specified." @@ -371,7 +374,7 @@ class Config: encoding="utf-8", ) - LOG = logging.getLogger("config") + LOG: logging.Logger = logging.getLogger("config") if self.debug: try: diff --git a/app/library/task_handlers/youtube.py b/app/library/task_handlers/youtube.py index 834bf83d..e4e71967 100644 --- a/app/library/task_handlers/youtube.py +++ b/app/library/task_handlers/youtube.py @@ -34,10 +34,10 @@ class YoutubeHandler(BaseHandler): @staticmethod def can_handle(task: Task) -> bool: if not task.get_ytdlp_opts().get_all().get("download_archive"): - LOG.debug(f"Task '{task.name}' does not have an archive file configured.") + LOG.debug(f"'{task.name}': Task does not have an archive file configured.") return False - LOG.debug(f"Checking if task '{task.name}' is using parsable YouTube URL: {task.url}") + LOG.debug(f"'{task.name}': Checking if task URL is parsable YouTube URL: {task.url}") return YoutubeHandler.parse(task.url) is not None @staticmethod @@ -56,16 +56,15 @@ class YoutubeHandler(BaseHandler): parsed: dict[str, str] | None = YoutubeHandler.parse(task.url) if not parsed: - LOG.error(f"Cannot parse '{task.name}' URL: {task.url}") + LOG.error(f"'{task.name}': Cannot parse task URL: {task.url}") return params: dict = task.get_ytdlp_opts().get_all() feed_url: str = YoutubeHandler.FEED.format(type=parsed["type"], id=parsed["id"]) - LOG.debug(f"Fetching '{task.name}' feed.") + LOG.info(f"'{task.name}': Fetching feed.") items: list = [] - has_items = False response = await YoutubeHandler.request(url=feed_url, ytdlp_opts=params) response.raise_for_status() @@ -76,11 +75,12 @@ class YoutubeHandler(BaseHandler): "yt": "http://www.youtube.com/xml/schemas/2015", } + real_count: int = 0 for entry in root.findall("atom:entry", ns): vid_elem: Element[str] | None = entry.find("yt:videoId", ns) vid: str | None = vid_elem.text if vid_elem is not None else "" if not vid: - LOG.warning(f"Entry in '{task.name}' feed is missing a video ID. Skipping entry.") + LOG.warning(f"'{task.name}': Entry in the feed is missing a video ID. Skipping.") continue url: str = f"https://www.youtube.com/watch?v={vid}" @@ -88,7 +88,7 @@ class YoutubeHandler(BaseHandler): id_dict: dict[str, str | None] = get_archive_id(url) archive_id: str | None = id_dict.get("archive_id") if not archive_id: - LOG.warning(f"Could not compute archive ID for video '{vid}' in '{task.name}' feed. Skipping entry.") + LOG.warning(f"'{task.name}': Could not compute archive ID for video '{vid}' in feed. Skipping.") continue title_elem: Element[str] | None = entry.find("atom:title", ns) @@ -96,7 +96,7 @@ class YoutubeHandler(BaseHandler): pub_elem: Element[str] | None = entry.find("atom:published", ns) published: str | None = pub_elem.text if pub_elem is not None else "" - has_items = True + real_count += 1 if archive_id in YoutubeHandler.queued: continue @@ -104,10 +104,10 @@ class YoutubeHandler(BaseHandler): items.append({"id": vid, "url": url, "title": title, "published": published, "archive_id": archive_id}) if len(items) < 1: - if not has_items: - LOG.warning(f"No entries found in '{task.name}' feed. URL: {feed_url}") + if real_count < 1: + LOG.warning(f"'{task.name}': No entries found the RSS feed. URL: {feed_url}") else: - LOG.debug(f"No new items found in '{task.name}' feed.") + LOG.info(f"'{task.name}': Feed has '{real_count}' entries, all already downloaded/queued.") return filtered: list = [] @@ -135,10 +135,10 @@ class YoutubeHandler(BaseHandler): filtered.append(item) if len(filtered) < 1: - LOG.debug(f"No new items found in '{task.name}' feed.") + LOG.info(f"'{task.name}': Feed has '{real_count}' entries, all already downloaded/queued.") return - LOG.info(f"Found '{len(filtered)}' new items from '{task.name}' feed.") + LOG.info(f"'{task.name}': Found '{len(filtered)}/{real_count}' new items from feed.") rItem: Item = Item.format( { @@ -158,7 +158,7 @@ class YoutubeHandler(BaseHandler): ) except Exception as e: LOG.exception(e) - LOG.error(f"Error while adding items from '{task.name}'. {e!s}") + LOG.error(f"'{task.name}': Error while adding items from task feed. {e!s}") return @staticmethod diff --git a/app/routes/api/conditions.py b/app/routes/api/conditions.py index f9550f61..c6cccc6a 100644 --- a/app/routes/api/conditions.py +++ b/app/routes/api/conditions.py @@ -71,19 +71,25 @@ async def conditions_add(request: Request, encoder: Encoder, notify: EventBus) - if not item.get("name"): return web.json_response( - {"error": "name is required.", "data": item}, status=web.HTTPBadRequest.status_code + {"error": "Name is required.", "data": item}, status=web.HTTPBadRequest.status_code ) if not item.get("filter"): return web.json_response( - {"error": "filter is required.", "data": item}, status=web.HTTPBadRequest.status_code + {"error": "Filter is required.", "data": item}, status=web.HTTPBadRequest.status_code + ) + + if not item.get("cli") and len(item.get("extras", {}).keys()) < 1: + return web.json_response( + {"error": "A Condition Must have cli options or extras", "data": item}, + status=web.HTTPBadRequest.status_code, ) if not item.get("cli"): - return web.json_response( - {"error": "command options for yt-dlp is required.", "data": item}, - status=web.HTTPBadRequest.status_code, - ) + item["cli"] = "" + + if not item.get("extras"): + item["extras"] = {} if not item.get("id", None) or not validate_uuid(item.get("id"), version=4): item["id"] = str(uuid.uuid4()) diff --git a/ui/app/components/ConditionForm.vue b/ui/app/components/ConditionForm.vue index b29bc385..4e8d13bd 100644 --- a/ui/app/components/ConditionForm.vue +++ b/ui/app/components/ConditionForm.vue @@ -107,6 +107,60 @@ + +
+
+ +
+
+
+
+ +
+
+ +
+
+ +
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+
+ + + + For advanced users only. This feature is meant to be expanded later. the only supported key right + now. ignore_download with value of true. Keys must be lowercase with + underscores (e.g., custom_field). + +
+
@@ -231,6 +285,8 @@ const config = useConfigStore() const form = reactive(JSON.parse(JSON.stringify(props.item))) const import_string = ref('') +const newExtraKey = ref('') +const newExtraValue = ref('') const test_data = ref<{ show: boolean, url: string, @@ -241,10 +297,14 @@ const test_data = ref<{ const showOptions = ref(false) const ytDlpOpt = ref([]) +// Initialize extras if not present +if (!form.extras) { + form.extras = {} +} + watch(() => config.ytdlp_options, newOptions => ytDlpOpt.value = newOptions .filter(opt => !opt.ignored) - .flatMap(opt => opt.flags - .filter(flag => flag.startsWith('--')) + .flatMap(opt => opt.flags.filter(flag => flag.startsWith('--')) .map(flag => ({ value: flag, description: opt.description || '' }))), { immediate: true } ) @@ -252,7 +312,7 @@ watch(() => config.ytdlp_options, newOptions => ytDlpOpt.value = newOptions watch(() => form.filter, () => test_data.value.changed = true) const checkInfo = async (): Promise => { - const required: (keyof ConditionItem)[] = ['name', 'filter', 'cli'] + const required: (keyof ConditionItem)[] = ['name', 'filter'] for (const key of required) { if (!form[key]) { @@ -261,6 +321,11 @@ const checkInfo = async (): Promise => { } } + if ((!form.cli || '' === form.cli.trim()) && Object.keys(form.extras).length < 1) { + toast.error('Either command options for yt-dlp or at least one extra option is required.') + return + } + if (form.cli && '' !== form.cli.trim()) { const options = await convertOptions(form.cli) if (options === null) { @@ -272,10 +337,10 @@ const checkInfo = async (): Promise => { const copy: ConditionItem = JSON.parse(JSON.stringify(form)) for (const key in copy) { - if (typeof copy[key] !== 'string') { + if (typeof (copy as any)[key] !== 'string') { continue } - copy[key] = copy[key].trim() + (copy as any)[key] = (copy as any)[key].trim() } emitter('submit', { reference: toRaw(props.reference), item: toRaw(copy) }) @@ -343,7 +408,7 @@ const importItem = async (): Promise => { return } - if ((form.filter || form.cli) && !(await box.confirm('Overwrite the current form fields?', true))) { + if ((form.filter || form.cli || Object.keys(form.extras).length > 0) && !(await box.confirm('Overwrite the current form fields?', true))) { return } @@ -359,6 +424,10 @@ const importItem = async (): Promise => { form.cli = item.cli } + if (item.extras) { + form.extras = { ...item.extras } + } + import_string.value = '' showImport.value = false } catch (e: any) { @@ -391,4 +460,82 @@ const logic_test = computed(() => { return false } }) + +const validateKey = (key: string): boolean => /^[a-z][a-z0-9_]*$/.test(key) + +const parseValue = (value: string): string | number | boolean => { + if (!isNaN(Number(value)) && !isNaN(parseFloat(value))) { + return Number(value) + } + + if ('true' === value.toLowerCase()) { + return true + } + if ('false' === value.toLowerCase()) { + return false + } + + return value +} + +const addExtra = (): void => { + const key = newExtraKey.value.trim() + const value = newExtraValue.value.trim() + + if (!key || !value) { + toast.error('Both key and value are required.') + return + } + + if (!validateKey(key)) { + toast.error('Key must be lower_case.') + return + } + + if (form.extras[key]) { + toast.error(`Key '${key}' already exists.`) + return + } + + form.extras[key] = parseValue(value) + newExtraKey.value = '' + newExtraValue.value = '' +} + +const removeExtra = (key: string): void => { + delete form.extras[key] +} + +const updateExtraKey = (event: Event, oldKey: string): void => { + const target = event.target as HTMLInputElement + const newKey = target.value.trim() + + if (!newKey) { + return + } + + if (!validateKey(newKey)) { + toast.error('Key must be lowercase and contain only letters, numbers, and underscores.') + target.value = oldKey + return + } + + if (newKey !== oldKey) { + if (form.extras[newKey]) { + toast.error(`Key '${newKey}' already exists.`) + target.value = oldKey + return + } + + const value = form.extras[oldKey] + delete form.extras[oldKey] + form.extras[newKey] = value + } +} + +const updateExtraValue = (key: string, event: Event): void => { + const target = event.target as HTMLInputElement + const value = target.value.trim() + form.extras[key] = value ? parseValue(value) : '' +} diff --git a/ui/app/pages/conditions.vue b/ui/app/pages/conditions.vue index 9dca9c90..adbb38ce 100644 --- a/ui/app/pages/conditions.vue +++ b/ui/app/pages/conditions.vue @@ -64,6 +64,14 @@ {{ cond.cli }}

+

+ + Extras: + + {{ key }}: {{ value }} + + +

@@ -119,13 +127,15 @@