Merge pull request #414 from arabcoders/dev

Make it possible to ignore item download via conditions
This commit is contained in:
Abdulmohsen 2025-09-08 19:20:05 +03:00 committed by GitHub
commit 125a1a81a0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 852 additions and 617 deletions

View file

@ -24,6 +24,7 @@ from .Presets import Presets
from .Scheduler import Scheduler
from .Singleton import Singleton
from .Utils import (
archive_add,
arg_converter,
calc_download_path,
dt_delta,
@ -675,8 +676,10 @@ class DownloadQueue(metaclass=Singleton):
LOG.warning(f"Using external downloader '{yt_conf.get('external_downloader')}' for '{item.url}'.")
item.extras.update({"external_downloader": True})
archive_id = item.get_archive_id()
if item.is_archived():
if archive_id := item.get_archive_id():
if archive_id:
store_type, _ = self.get_item(archive_id=archive_id)
if not store_type:
dlInfo = Download(
@ -750,7 +753,55 @@ class DownloadQueue(metaclass=Singleton):
if not item.requeued and (condition := Conditions.get_instance().match(info=entry)):
already.pop()
LOG.info(f"Matched '{condition.name}' for '{item.url}' Adding '{condition.cli}' to request.")
item_title = entry.get("title") or entry.get("id") or item.url
message = f"Condition '{condition.name}' matched for '{item_title}'."
if condition.cli:
message += f" Re-queuing with '{condition.cli}'."
LOG.info(message)
if condition.extras.get("ignore_download", False):
extra_msg: str = ""
if yt_conf.get("download_archive") and not condition.extras.get("no_archive", False):
archive_add(yt_conf.get("download_archive"), [archive_id])
extra_msg = f" and added to archive '{yt_conf.get('download_archive')}'"
log_message = f"Ignoring download of '{item_title}' as per condition '{condition.name}'{extra_msg}."
store_type, _ = self.get_item(archive_id=archive_id)
if not store_type:
dlInfo = Download(
info=ItemDTO(
id=entry.get("id"),
title=item_title,
url=item.url,
preset=item.preset,
folder=item.folder,
status="skip",
cookies=item.cookies,
template=item.template,
msg=log_message,
extras=item.extras,
)
)
self.done.put(dlInfo)
LOG.info(log_message)
await asyncio.gather(
*[
self._notify.emit(Events.LOG_INFO, data={}, title="Ignored Download", message=log_message),
self._notify.emit(
Events.ITEM_MOVED,
data={"to": "history", "preset": dlInfo.info.preset, "item": dlInfo.info},
title="Download History Update",
message=f"Download history updated with '{item.url}'.",
),
]
)
return {"status": "ok"}
return await self.add(item=item.new_with(requeued=True, cli=condition.cli), already=already)
_status, _msg = ytdlp_reject(entry=entry, yt_params=yt_conf)

View file

@ -263,14 +263,14 @@ class HttpAPI:
response: Response = await handler(request)
contentType: str | None = response.headers.get("content-type", None)
if contentType and "/" != base_path and contentType.startswith("text/html"):
if contentType and contentType.startswith("text/html"):
rewrite_path: str = base_path.rstrip("/")
async with await anyio.open_file(response._path, "rb") as f:
content = await f.read()
content: str = (
content.decode("utf-8")
.replace('<base href="/">', f'<base href="{rewrite_path}/">')
.replace('baseURL:""', f'baseURL:"{rewrite_path}/"')
.replace("/_base_path/", f"{rewrite_path}/")
)
new_response = web.Response(text=content, content_type="text/html")

View file

@ -27,9 +27,12 @@ class Condition:
filter: str
"""The filter to run on info dict."""
cli: str
cli: str = ""
"""If matched append this to the download request and retry."""
extras: dict[str, Any] = field(default_factory=dict)
"""Any extra data to store with the condition."""
def serialize(self) -> dict:
return self.__dict__
@ -137,6 +140,10 @@ class Conditions(metaclass=Singleton):
item["id"] = str(uuid.uuid4())
need_save = True
if "extras" not in item:
item["extras"] = {}
need_save = True
item: Condition = init_class(Condition, item)
self._items.append(item)
@ -145,7 +152,7 @@ class Conditions(metaclass=Singleton):
continue
if need_save:
LOG.info("Saving conditions due to format, or id change.")
LOG.info("Saving conditions due changes.")
self.save(self._items)
return self
@ -203,15 +210,16 @@ class Conditions(metaclass=Singleton):
msg = f"Invalid filter. '{e!s}'."
raise ValueError(msg) from e
if not item.get("cli"):
msg = "No command options for yt-dlp were found."
raise ValueError(msg)
if item.get("cli"):
try:
arg_converter(args=item.get("cli"))
except Exception as e:
msg = f"Invalid command options for yt-dlp. '{e!s}'."
raise ValueError(msg) from e
try:
arg_converter(args=item.get("cli"))
except Exception as e:
msg = f"Invalid command options for yt-dlp. '{e!s}'."
raise ValueError(msg) from e
if not isinstance(item.get("extras"), dict):
msg = "Extras must be a dictionary."
raise ValueError(msg) # noqa: TRY004
return True

View file

@ -359,6 +359,9 @@ class Config:
if isinstance(self.pictures_backends, str) and self.pictures_backends:
self.pictures_backends = self.pictures_backends.split(",")
if not self.base_path.endswith("/"):
self.base_path += "/"
numeric_level = getattr(logging, self.log_level.upper(), None)
if not isinstance(numeric_level, int):
msg = f"Invalid log level '{self.log_level}' specified."
@ -371,7 +374,7 @@ class Config:
encoding="utf-8",
)
LOG = logging.getLogger("config")
LOG: logging.Logger = logging.getLogger("config")
if self.debug:
try:

View file

@ -34,10 +34,10 @@ class YoutubeHandler(BaseHandler):
@staticmethod
def can_handle(task: Task) -> bool:
if not task.get_ytdlp_opts().get_all().get("download_archive"):
LOG.debug(f"Task '{task.name}' does not have an archive file configured.")
LOG.debug(f"'{task.name}': Task does not have an archive file configured.")
return False
LOG.debug(f"Checking if task '{task.name}' is using parsable YouTube URL: {task.url}")
LOG.debug(f"'{task.name}': Checking if task URL is parsable YouTube URL: {task.url}")
return YoutubeHandler.parse(task.url) is not None
@staticmethod
@ -56,16 +56,15 @@ class YoutubeHandler(BaseHandler):
parsed: dict[str, str] | None = YoutubeHandler.parse(task.url)
if not parsed:
LOG.error(f"Cannot parse '{task.name}' URL: {task.url}")
LOG.error(f"'{task.name}': Cannot parse task URL: {task.url}")
return
params: dict = task.get_ytdlp_opts().get_all()
feed_url: str = YoutubeHandler.FEED.format(type=parsed["type"], id=parsed["id"])
LOG.debug(f"Fetching '{task.name}' feed.")
LOG.info(f"'{task.name}': Fetching feed.")
items: list = []
has_items = False
response = await YoutubeHandler.request(url=feed_url, ytdlp_opts=params)
response.raise_for_status()
@ -76,11 +75,12 @@ class YoutubeHandler(BaseHandler):
"yt": "http://www.youtube.com/xml/schemas/2015",
}
real_count: int = 0
for entry in root.findall("atom:entry", ns):
vid_elem: Element[str] | None = entry.find("yt:videoId", ns)
vid: str | None = vid_elem.text if vid_elem is not None else ""
if not vid:
LOG.warning(f"Entry in '{task.name}' feed is missing a video ID. Skipping entry.")
LOG.warning(f"'{task.name}': Entry in the feed is missing a video ID. Skipping.")
continue
url: str = f"https://www.youtube.com/watch?v={vid}"
@ -88,7 +88,7 @@ class YoutubeHandler(BaseHandler):
id_dict: dict[str, str | None] = get_archive_id(url)
archive_id: str | None = id_dict.get("archive_id")
if not archive_id:
LOG.warning(f"Could not compute archive ID for video '{vid}' in '{task.name}' feed. Skipping entry.")
LOG.warning(f"'{task.name}': Could not compute archive ID for video '{vid}' in feed. Skipping.")
continue
title_elem: Element[str] | None = entry.find("atom:title", ns)
@ -96,7 +96,7 @@ class YoutubeHandler(BaseHandler):
pub_elem: Element[str] | None = entry.find("atom:published", ns)
published: str | None = pub_elem.text if pub_elem is not None else ""
has_items = True
real_count += 1
if archive_id in YoutubeHandler.queued:
continue
@ -104,10 +104,10 @@ class YoutubeHandler(BaseHandler):
items.append({"id": vid, "url": url, "title": title, "published": published, "archive_id": archive_id})
if len(items) < 1:
if not has_items:
LOG.warning(f"No entries found in '{task.name}' feed. URL: {feed_url}")
if real_count < 1:
LOG.warning(f"'{task.name}': No entries found the RSS feed. URL: {feed_url}")
else:
LOG.debug(f"No new items found in '{task.name}' feed.")
LOG.info(f"'{task.name}': Feed has '{real_count}' entries, all already downloaded/queued.")
return
filtered: list = []
@ -135,10 +135,10 @@ class YoutubeHandler(BaseHandler):
filtered.append(item)
if len(filtered) < 1:
LOG.debug(f"No new items found in '{task.name}' feed.")
LOG.info(f"'{task.name}': Feed has '{real_count}' entries, all already downloaded/queued.")
return
LOG.info(f"Found '{len(filtered)}' new items from '{task.name}' feed.")
LOG.info(f"'{task.name}': Found '{len(filtered)}/{real_count}' new items from feed.")
rItem: Item = Item.format(
{
@ -158,7 +158,7 @@ class YoutubeHandler(BaseHandler):
)
except Exception as e:
LOG.exception(e)
LOG.error(f"Error while adding items from '{task.name}'. {e!s}")
LOG.error(f"'{task.name}': Error while adding items from task feed. {e!s}")
return
@staticmethod

View file

@ -71,19 +71,25 @@ async def conditions_add(request: Request, encoder: Encoder, notify: EventBus) -
if not item.get("name"):
return web.json_response(
{"error": "name is required.", "data": item}, status=web.HTTPBadRequest.status_code
{"error": "Name is required.", "data": item}, status=web.HTTPBadRequest.status_code
)
if not item.get("filter"):
return web.json_response(
{"error": "filter is required.", "data": item}, status=web.HTTPBadRequest.status_code
{"error": "Filter is required.", "data": item}, status=web.HTTPBadRequest.status_code
)
if not item.get("cli") and len(item.get("extras", {}).keys()) < 1:
return web.json_response(
{"error": "A Condition Must have cli options or extras", "data": item},
status=web.HTTPBadRequest.status_code,
)
if not item.get("cli"):
return web.json_response(
{"error": "command options for yt-dlp is required.", "data": item},
status=web.HTTPBadRequest.status_code,
)
item["cli"] = ""
if not item.get("extras"):
item["extras"] = {}
if not item.get("id", None) or not validate_uuid(item.get("id"), version=4):
item["id"] = str(uuid.uuid4())

View file

@ -107,6 +107,60 @@
</span>
</div>
</div>
<div class="column is-12">
<div class="field">
<label class="label is-inline">
<span class="icon"><i class="fa-solid fa-plus-square" /></span>
Extra Options
</label>
<div class="content">
<div v-if="form.extras && Object.keys(form.extras).length > 0" class="mb-3">
<div v-for="(value, key) in form.extras" :key="key" class="field is-grouped mb-2">
<div class="control">
<input type="text" class="input" :value="key" @input="updateExtraKey($event, key)"
:disabled="addInProgress" placeholder="key_name">
</div>
<div class="control is-expanded">
<input type="text" class="input" :value="value" @input="updateExtraValue(key, $event)"
:disabled="addInProgress" placeholder="value">
</div>
<div class="control">
<button type="button" class="button is-danger" @click="removeExtra(key)"
:disabled="addInProgress">
<span class="icon"><i class="fa-solid fa-times" /></span>
</button>
</div>
</div>
</div>
<div class="field is-grouped">
<div class="control">
<input type="text" class="input" v-model="newExtraKey" :disabled="addInProgress"
placeholder="new_key" @keyup.enter="addExtra">
</div>
<div class="control is-expanded">
<input type="text" class="input" v-model="newExtraValue" :disabled="addInProgress"
placeholder="new_value" @keyup.enter="addExtra">
</div>
<div class="control">
<button type="button" class="button is-primary" @click="addExtra"
:disabled="addInProgress || !newExtraKey || !newExtraValue">
<span class="icon"><i class="fa-solid fa-plus" /></span>
<span>Add</span>
</button>
</div>
</div>
</div>
<span class="help is-bold">
<span class="icon"><i class="fa-solid fa-info" /></span>
<span>
For advanced users only. This feature is meant to be expanded later. the only supported key right
now. <code>ignore_download</code> with value of <code>true</code>. Keys must be lowercase with
underscores (e.g., custom_field).</span>
</span>
</div>
</div>
</div>
</div>
@ -231,6 +285,8 @@ const config = useConfigStore()
const form = reactive<ConditionItem>(JSON.parse(JSON.stringify(props.item)))
const import_string = ref('')
const newExtraKey = ref('')
const newExtraValue = ref('')
const test_data = ref<{
show: boolean,
url: string,
@ -241,10 +297,14 @@ const test_data = ref<{
const showOptions = ref<boolean>(false)
const ytDlpOpt = ref<AutoCompleteOptions>([])
// Initialize extras if not present
if (!form.extras) {
form.extras = {}
}
watch(() => config.ytdlp_options, newOptions => ytDlpOpt.value = newOptions
.filter(opt => !opt.ignored)
.flatMap(opt => opt.flags
.filter(flag => flag.startsWith('--'))
.flatMap(opt => opt.flags.filter(flag => flag.startsWith('--'))
.map(flag => ({ value: flag, description: opt.description || '' }))),
{ immediate: true }
)
@ -252,7 +312,7 @@ watch(() => config.ytdlp_options, newOptions => ytDlpOpt.value = newOptions
watch(() => form.filter, () => test_data.value.changed = true)
const checkInfo = async (): Promise<void> => {
const required: (keyof ConditionItem)[] = ['name', 'filter', 'cli']
const required: (keyof ConditionItem)[] = ['name', 'filter']
for (const key of required) {
if (!form[key]) {
@ -261,6 +321,11 @@ const checkInfo = async (): Promise<void> => {
}
}
if ((!form.cli || '' === form.cli.trim()) && Object.keys(form.extras).length < 1) {
toast.error('Either command options for yt-dlp or at least one extra option is required.')
return
}
if (form.cli && '' !== form.cli.trim()) {
const options = await convertOptions(form.cli)
if (options === null) {
@ -272,10 +337,10 @@ const checkInfo = async (): Promise<void> => {
const copy: ConditionItem = JSON.parse(JSON.stringify(form))
for (const key in copy) {
if (typeof copy[key] !== 'string') {
if (typeof (copy as any)[key] !== 'string') {
continue
}
copy[key] = copy[key].trim()
(copy as any)[key] = (copy as any)[key].trim()
}
emitter('submit', { reference: toRaw(props.reference), item: toRaw(copy) })
@ -343,7 +408,7 @@ const importItem = async (): Promise<void> => {
return
}
if ((form.filter || form.cli) && !(await box.confirm('Overwrite the current form fields?', true))) {
if ((form.filter || form.cli || Object.keys(form.extras).length > 0) && !(await box.confirm('Overwrite the current form fields?', true))) {
return
}
@ -359,6 +424,10 @@ const importItem = async (): Promise<void> => {
form.cli = item.cli
}
if (item.extras) {
form.extras = { ...item.extras }
}
import_string.value = ''
showImport.value = false
} catch (e: any) {
@ -391,4 +460,82 @@ const logic_test = computed(() => {
return false
}
})
const validateKey = (key: string): boolean => /^[a-z][a-z0-9_]*$/.test(key)
const parseValue = (value: string): string | number | boolean => {
if (!isNaN(Number(value)) && !isNaN(parseFloat(value))) {
return Number(value)
}
if ('true' === value.toLowerCase()) {
return true
}
if ('false' === value.toLowerCase()) {
return false
}
return value
}
const addExtra = (): void => {
const key = newExtraKey.value.trim()
const value = newExtraValue.value.trim()
if (!key || !value) {
toast.error('Both key and value are required.')
return
}
if (!validateKey(key)) {
toast.error('Key must be lower_case.')
return
}
if (form.extras[key]) {
toast.error(`Key '${key}' already exists.`)
return
}
form.extras[key] = parseValue(value)
newExtraKey.value = ''
newExtraValue.value = ''
}
const removeExtra = (key: string): void => {
delete form.extras[key]
}
const updateExtraKey = (event: Event, oldKey: string): void => {
const target = event.target as HTMLInputElement
const newKey = target.value.trim()
if (!newKey) {
return
}
if (!validateKey(newKey)) {
toast.error('Key must be lowercase and contain only letters, numbers, and underscores.')
target.value = oldKey
return
}
if (newKey !== oldKey) {
if (form.extras[newKey]) {
toast.error(`Key '${newKey}' already exists.`)
target.value = oldKey
return
}
const value = form.extras[oldKey]
delete form.extras[oldKey]
form.extras[newKey] = value
}
}
const updateExtraValue = (key: string, event: Event): void => {
const target = event.target as HTMLInputElement
const value = target.value.trim()
form.extras[key] = value ? parseValue(value) : ''
}
</script>

View file

@ -64,6 +64,14 @@
<span class="icon"><i class="fa-solid fa-terminal" /></span>
<span>{{ cond.cli }}</span>
</p>
<p class="is-text-overflow" v-if="cond.extras && Object.keys(cond.extras).length > 0">
<span class="icon"><i class="fa-solid fa-list" /></span>
<span>Extras:
<span v-for="(value, key) in cond.extras" :key="key" class="tag is-info mr-2">
<strong>{{ key }}</strong>: {{ value }}
</span>
</span>
</p>
</div>
</div>
<div class="card-content" v-if="cond?.raw">
@ -119,13 +127,15 @@
<script setup lang="ts">
import type { ConditionItem, ImportedConditionItem } from '~/types/conditions'
type ConditionItemWithUI = ConditionItem & { raw?: boolean }
const toast = useNotification()
const config = useConfigStore()
const socket = useSocketStore()
const box = useConfirm()
const isMobile = useMediaQuery({ maxWidth: 1024 })
const items = ref<ConditionItem[]>([])
const items = ref<ConditionItemWithUI[]>([])
const item = ref<Partial<ConditionItem>>({})
const itemRef = ref<string | null | undefined>("")
const toggleForm = ref(false)
@ -188,12 +198,11 @@ const updateItems = async (newItems: ConditionItem[]): Promise<boolean> => {
try {
addInProgress.value = true
const validItems = newItems.map(({ id, name, filter, cli }) => {
if (!name || !filter || !cli) {
toast.error('Name, filter and cli are required.')
throw new Error('Missing fields')
const validItems = newItems.map(({ id, name, filter, cli, extras }) => {
if (!name || !filter) {
throw new Error('Name and filter are required.')
}
return { id, name, filter, cli }
return { id, name, filter, cli, extras }
})
const response = await request('/api/conditions', {
@ -276,7 +285,7 @@ const exportItem = (cond: ConditionItem): void => {
const userData: ImportedConditionItem = {
...Object.fromEntries(Object.entries(clone).filter(([_, v]) => !!v)),
_type: 'condition',
_version: '1.0',
_version: '1.1',
} as ImportedConditionItem
copyText(encode(userData))

View file

@ -3,7 +3,7 @@ export type ConditionItem = {
name: string
filter: string
cli: string
[key: string]: any
extras: Record<string, any>
}
export type ImportedConditionItem = ConditionItem & {

View file

@ -37,7 +37,7 @@ export default defineNuxtConfig({
transpile: ['vue-toastification'],
},
app: {
baseURL: 'production' == process.env.NODE_ENV ? '' : '/',
baseURL: 'production' == process.env.NODE_ENV ? '/_base_path/' : '/',
buildAssetsDir: "assets",
head: {
"meta": [

View file

@ -21,7 +21,7 @@
"floating-vue": "^5.2.2",
"hls.js": "^1.6.11",
"moment": "^2.30.1",
"nuxt": "4.0.3",
"nuxt": "^4.1.1",
"pinia": "^3.0.3",
"socket.io-client": "^4.8.1",
"vue": "^3.5.21",

File diff suppressed because it is too large Load diff

12
uv.lock
View file

@ -640,11 +640,11 @@ wheels = [
[[package]]
name = "markdown"
version = "3.8.2"
version = "3.9"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/d7/c2/4ab49206c17f75cb08d6311171f2d65798988db4360c4d1485bd0eedd67c/markdown-3.8.2.tar.gz", hash = "sha256:247b9a70dd12e27f67431ce62523e675b866d254f900c4fe75ce3dda62237c45", size = 362071 }
sdist = { url = "https://files.pythonhosted.org/packages/8d/37/02347f6d6d8279247a5837082ebc26fc0d5aaeaf75aa013fcbb433c777ab/markdown-3.9.tar.gz", hash = "sha256:d2900fe1782bd33bdbbd56859defef70c2e78fc46668f8eb9df3128138f2cb6a", size = 364585 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/96/2b/34cc11786bc00d0f04d0f5fdc3a2b1ae0b6239eef72d3d345805f9ad92a1/markdown-3.8.2-py3-none-any.whl", hash = "sha256:5c83764dbd4e00bdd94d85a19b8d55ccca20fe35b2e678a1422b380324dd5f24", size = 106827 },
{ url = "https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl", hash = "sha256:9f4d91ed810864ea88a6f32c07ba8bee1346c0cc1f6b1f9f6c822f2a9667d280", size = 107441 },
]
[[package]]
@ -1381,11 +1381,11 @@ wheels = [
[[package]]
name = "yt-dlp"
version = "2025.8.27"
version = "2025.9.5"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/f4/d4/d9dd231b03f09fdfb5f0fe70f30de0b5f59454aa54fa6b2b2aea49404988/yt_dlp-2025.8.27.tar.gz", hash = "sha256:ed74768d2a93b29933ab14099da19497ef571637f7aa375140dd3d882b9c1854", size = 3038374 }
sdist = { url = "https://files.pythonhosted.org/packages/50/b2/fb255d991857a6a8b2539487ed6063e7bf318f19310d81f039dedb3c2ad6/yt_dlp-2025.9.5.tar.gz", hash = "sha256:9ce080f80b2258e872fe8a75f4707ea2c644e697477186e20b9a04d9a9ea37cf", size = 3045982 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/cb/9c/b69fc0c800f80b94ea2f8eff1d1f473fecee6aa337681d297ba7c7c5d3fd/yt_dlp-2025.8.27-py3-none-any.whl", hash = "sha256:0b8fd3bb7c54bc2e7ecb5cdac7d64c30e2503ea4d3dd9ae24d4f09e22aaa95f4", size = 3267059 },
{ url = "https://files.pythonhosted.org/packages/06/64/b3cc116e4f209c493f23d6af033c60ba32df74e086190fbed2bdc0073d12/yt_dlp-2025.9.5-py3-none-any.whl", hash = "sha256:68a03b5c50e3d0f6af7244bd4bf491c1b12e4e2112b051cde05cdfd2647eb9a8", size = 3272317 },
]
[[package]]