danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Convert json config in tasks to use cli arguments.

Browse source
This commit is contained in:
arabcoders 2025-03-31 00:12:35 +03:00
parent 8f313d7f58
commit 1bda1893e4
6 changed files with 154 additions and 236 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
app/library/HttpAPI.py
View file

@ -872,15 +872,12 @@ class HttpAPI(Common):
if not item.get("timer", None) or str(item.get("timer")).strip() == "":
item["timer"] = f"{random.randint(1,59)} */1 * * *" # noqa: S311
if not item.get("cookies", None):
item["cookies"] = ""
if not item.get("config", None) or str(item.get("config")).strip() == "":
item["config"] = {}
if not item.get("template", None):
item["template"] = ""
if not item.get("cli", None):
item["cli"] = ""
try:
ins.validate(item)
except ValueError as e:

68
app/library/Tasks.py
View file

@ -3,13 +3,15 @@ import json
import logging
import os
import time
from dataclasses import dataclass, field
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import Any
import httpx
from aiohttp import web
from app.library.Utils import arg_converter
from .config import Config
from .encoder import Encoder
from .Events import EventBus, Events, error, info, success
@ -28,8 +30,7 @@ class Task:
preset: str = ""
timer: str = ""
template: str = ""
cookies: str = ""
config: dict[str, str] = field(default_factory=dict)
cli: str = ""
def serialize(self) -> dict:
return self.__dict__
@ -134,16 +135,20 @@ class Tasks(metaclass=Singleton):
with open(self._file) as f:
tasks = json.load(f)
except Exception as e:
LOG.error(f"Failed to parse tasks from '{self._file}'. '{e}'.")
LOG.error(f"Failed to parse tasks from '{self._file}'. '{e!s}'.")
return self
if not tasks or len(tasks) < 1:
LOG.info(f"No tasks were defined in '{self._file}'.")
return self
need_update = False
for i, task in enumerate(tasks):
try:
task, task_status = self.clean_task(task)
task = Task(**task)
if task_status:
need_update = True
except Exception as e:
LOG.error(f"Failed to parse task at list position '{i}'. '{e!s}'.")
continue
@ -157,6 +162,10 @@ class Tasks(metaclass=Singleton):
LOG.exception(e)
LOG.error(f"Failed to queue task '{i}: {task.name}'. '{e!s}'.")
if need_update:
LOG.info("Updating tasks file to remove old keys.")
self.save(self.get_all())
return self
def clear(self, shutdown: bool = False) -> "Tasks":
@ -213,17 +222,12 @@ class Tasks(metaclass=Singleton):
msg = "No URL found."
raise ValueError(msg)
if not isinstance(task.get("cookies"), str):
msg = "Invalid cookies type."
raise ValueError(msg) # noqa: TRY004
if not isinstance(task.get("config"), dict):
msg = "Invalid config type."
raise ValueError(msg) # noqa: TRY004
if not isinstance(task.get("template"), str):
msg = "Invalid template type."
raise ValueError(msg) # noqa: TRY004
if task.get("cli"):
try:
arg_converter(args=task.get("cli"))
except Exception as e:
msg = f"Invalid cli options. '{e!s}'."
raise ValueError(msg) from e
return True
@ -263,6 +267,27 @@ class Tasks(metaclass=Singleton):
return self
def clean_task(self, task: dict) -> tuple[dict, bool]:
"""
Clean the task from old keys.
Args:
task (dict): The task to clean.
Returns:
tuple[dict, bool]: The cleaned task and a status if the task was cleaned.
"""
status = False
removedKeys = ["cookies", "config"]
for key in removedKeys:
if key in task:
status = True
task.pop(key)
return task, status
async def _runner(self, task: Task):
"""
Run the task.
@ -283,16 +308,8 @@ class Tasks(metaclass=Singleton):
preset: str = str(task.preset or self._default_preset)
folder: str = task.folder if task.folder else ""
cookies: str = str(task.cookies) if task.cookies else ""
template: str = task.template if task.template else ""
config = task.config if task.config else {}
if isinstance(config, str) and config:
try:
config = json.loads(config)
except Exception as e:
LOG.error(f"Failed to parse json yt-dlp config for '{task.name}'. {e!s}")
return
cli: str = task.cli if task.cli else ""
LOG.info(f"Task '{task.id}: {task.name}' dispatched at '{timeNow}'.")
@ -307,9 +324,8 @@ class Tasks(metaclass=Singleton):
"url": task.url,
"preset": preset,
"folder": folder,
"cookies": cookies,
"config": config,
"template": template,
"cli": cli,
},
id=task.id,
),

11
app/library/Utils.py
View file

@ -434,12 +434,13 @@ def validate_url(url: str) -> bool:
return True
def arg_converter(args: str) -> dict:
def arg_converter(args: str, remove_options: bool = False) -> dict:
"""
Convert yt-dlp options to a dictionary.
Args:
args (str): yt-dlp options string.
remove_options (bool): Remove default options.
Returns:
dict: yt-dlp options dictionary.
@ -472,6 +473,14 @@ def arg_converter(args: str) -> dict:
if isinstance(matchFilter, set):
diff["match_filter"] = {"filters": list(matchFilter)}
if remove_options:
for key in diff.copy():
if key in IGNORED_KEYS:
diff.pop(key, None)
if "_warnings" in diff:
diff.pop("_warnings", None)
from .encoder import Encoder
return json.loads(json.dumps(diff, cls=Encoder))

9
app/library/common.py
View file

@ -4,6 +4,7 @@ import logging
from .config import Config
from .DownloadQueue import DownloadQueue
from .encoder import Encoder
from .Utils import arg_converter
LOG = logging.getLogger("common")
@ -94,6 +95,14 @@ class Common:
msg = f"Failed to parse json yt-dlp config for '{url}'. {e!s}"
raise ValueError(msg) from e
cli = item.get("cli")
if cli and len(cli) > 1:
try:
config = arg_converter(args=cli, remove_options=True)
except Exception as e:
msg = f"Failed to parse yt-dlp cli options. {e!s}"
raise ValueError(msg) from e
return {
"url": url,
"preset": preset,

183
ui/components/TaskForm.vue
View file

@ -50,7 +50,6 @@
</span>
</div>
<div class="column is-6-tablet is-12-mobile">
<div class="field">
<label class="label is-inline" for="name" v-text="'Name'" />
@ -81,14 +80,11 @@
<div class="column is-6-tablet is-12-mobile">
<div class="field">
<label class="label is-inline" for="folder">
Preset
</label>
<label class="label is-inline" for="preset">Preset</label>
<div class="control has-icons-left">
<div class="select is-fullwidth">
<select id="preset" class="is-fullwidth" v-model="form.preset"
:disabled="addInProgress || hasFormatInConfig"
v-tooltip.bottom="hasFormatInConfig ? 'Presets are disabled. Format key is present in the config.' : ''">
:disabled="addInProgress || hasFormatInConfig">
<option v-for="item in config.presets" :key="item.name" :value="item.name">
{{ item.name }}
</option>
@ -98,8 +94,10 @@
</div>
<span class="help">
<span class="icon"><i class="fa-solid fa-info" /></span>
<span>Select the preset to use for this URL. The preset will be ignored if format key is present in
config.</span>
<span>Select the preset to use for this URL. <span class="text-has-danger">If the
<code>-f, --format</code> argument is present in the command line options, the preset and all
it's options will be ignored.</span>
</span>
</span>
</div>
</div>
@ -126,7 +124,7 @@
</div>
</div>
<div class="column is-6-tablet is-12-mobile" v-if="showAdvanced">
<div class="column is-6-tablet is-12-mobile">
<div class="field">
<label class="label is-inline" for="folder">
Download path
@ -138,24 +136,25 @@
</div>
<span class="help">
<span class="icon"><i class="fa-solid fa-info" /></span>
<span>Downloads are relative to download path, defaults to root path if not set.</span>
<span>Paths are relative to global download path, defaults to preset download path if set otherwise,
fallback root path if not set.</span>
</span>
</div>
</div>
<div class="column is-6-tablet is-12-mobile" v-if="showAdvanced">
<div class="column is-6-tablet is-12-mobile">
<div class="field">
<label class="label is-inline" for="output_template">
Output template
</label>
<div class="control has-icons-left">
<input type="text" class="input" id="output_template" placeholder="The output template to use"
v-model="form.template" :disabled="addInProgress">
<input type="text" class="input" id="output_template" :disabled="addInProgress"
placeholder="Leave empty to use default template." v-model="form.template">
<span class="icon is-small is-left"><i class="fa-solid fa-file" /></span>
</div>
<span class="help">
<span class="icon"><i class="fa-solid fa-info" /></span>
<span>The output template to use, if not set, it will defaults to
<span>Use this output template if non are given with URL. if not set, it will defaults to
<code>{{ config.app.output_template }}</code>.
For more information <NuxtLink href="https://github.com/yt-dlp/yt-dlp#output-template"
target="_blank">visit this url</NuxtLink>.
@ -164,41 +163,24 @@
</div>
</div>
<div class="column is-6-tablet is-12-mobile" v-if="showAdvanced">
<div class="column is-12">
<div class="field">
<label class="label is-inline" for="config"
v-tooltip="'Extends current global yt-dlp config. (JSON)'">
JSON yt-dlp config or CLI options. <NuxtLink
v-if="form.config && (typeof form.config === 'string') && !form.config.trim().startsWith('{')"
@click="convertOptions()">Convert to JSON</NuxtLink>
<label class="label is-inline" for="cli_options">
Command arguments for yt-dlp
</label>
<div class="control">
<textarea class="textarea" id="config" v-model="form.config" :disabled="addInProgress"
placeholder="--no-embed-metadata --no-embed-thumbnail"></textarea>
<input type="text" class="input" v-model="form.cli" id="cli_options" :disabled="addInProgress"
placeholder="command options to use, e.g. --no-embed-metadata --no-embed-thumbnail">
</div>
<span class="help">
<span class="icon"><i class="fa-solid fa-info" /></span>
<span> Extends current global yt-dlp config with given options. Some fields are ignored like
<code>cookiefile</code>, <code>paths</code>, and <code>outtmpl</code> etc. Warning: Use with
caution
some of those options can break yt-dlp or the frontend.</span>
</span>
</div>
</div>
<div class="column is-6-tablet is-12-mobile" v-if="showAdvanced">
<div class="field">
<label class="label is-inline" for="cookies"
v-tooltip="'Netscape HTTP Cookie format.'">Cookies</label>
<div class="control">
<textarea class="textarea is-pre" id="cookies" v-model="form.cookies" :disabled="addInProgress" />
</div>
<span class="help">
<span class="icon"><i class="fa-solid fa-info" /></span>
<span>Use the <NuxtLink target="_blank"
to="https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp">
Recommended addon</NuxtLink> by yt-dlp to export cookies. The cookies MUST be in Netscape HTTP
Cookie format.</span>
<span>yt-dlp cli arguments. Check <NuxtLink target="_blank"
to="https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#general-options">this page</NuxtLink>.
For more info. Some arguments are ignored by default. Warning: Use with
caution some of those options can break yt-dlp or the frontend. <span class="has-text-danger">If
<code>-f, --format</code> argument is present, the preset and all it's options will be
ignored</span>.
</span>
</span>
</div>
</div>
@ -221,12 +203,6 @@
<span>Cancel</span>
</button>
</p>
<p class="card-footer-item">
<button class="button is-fullwidth is-info" type="button" @click="showAdvanced = !showAdvanced">
<span class="icon"><i class="fa-solid fa-cog" /></span>
<span>Advanced</span>
</button>
</p>
</div>
</div>
</form>
@ -244,8 +220,6 @@ import { CronExpressionParser } from 'cron-parser'
const emitter = defineEmits(['cancel', 'submit']);
const toast = useToast();
const config = useConfigStore();
const convertInProgress = ref(false);
const showAdvanced = useStorage('task.showAdvanced', false);
const showImport = useStorage('showImport', false);
const import_string = ref('');
@ -269,10 +243,6 @@ const props = defineProps({
const form = reactive(props.task);
onMounted(() => {
if (props.task?.config && (typeof props.task.config === 'object')) {
form.config = JSON.stringify(props.task.config, null, 4);
}
if (!props.task?.preset || '' === props.task.preset) {
form.preset = toRaw(config.app.default_preset);
}
@ -304,60 +274,17 @@ const checkInfo = async () => {
return;
}
if (typeof form.config === 'object') {
form.config = JSON.stringify(form.config, null, 4);
}
if (form.config && form.config && !form.config.trim().startsWith('{')) {
await convertOptions();
}
if (form.config) {
try {
form.config = JSON.parse(form.config)
} catch (e) {
toast.error(`Invalid JSON yt-dlp config. ${e.message}`)
return;
if (form?.cli && '' !== form.cli) {
const options = await convertOptions(form.cli);
if (null === options) {
return
}
form.cli = form.cli.trim(" ")
}
emitter('submit', { reference: toRaw(props.reference), task: toRaw(form) });
}
const convertOptions = async () => {
if (convertInProgress.value) {
return
}
try {
convertInProgress.value = true
const response = await convertCliOptions(form.config)
form.config = JSON.stringify(response.opts, null, 2)
if (response.output_template) {
form.template = response.output_template
}
if (response.download_path) {
form.folder = response.download_path
}
} catch (e) {
toast.error(e.message)
} finally {
convertInProgress.value = false
}
}
const hasFormatInConfig = computed(() => {
if (!form.config) {
return false
}
try {
const config = JSON.parse(form.config)
return "format" in config
} catch (e) {
return false
}
})
const importItem = async () => {
let val = import_string.value.trim()
if (!val) {
@ -384,7 +311,7 @@ const importItem = async () => {
return
}
if (form.config || form.url || form.timer) {
if (form.url || form.timer) {
if (false === confirm('This will overwrite the current form fields. Are you sure?')) {
return
}
@ -398,8 +325,8 @@ const importItem = async () => {
form.url = item.url
}
if (item.preset) {
form.preset = item.preset
if (item.template) {
form.template = item.template
}
if (item.timer) {
@ -410,12 +337,19 @@ const importItem = async () => {
form.folder = item.folder
}
if (item.template) {
form.template = item.template
if (item.cli) {
form.cli = item.cli
}
if (item.config) {
form.config = JSON.stringify(item.config, null, 2)
if (item.preset) {
// -- check if the preset exists in config.presets
const preset = config.presets.find(p => p.name === item.preset)
if (!preset) {
toast.warning(`Preset '${item.preset}' not found. Preset will be set to default.`)
form.preset = 'default'
} else {
form.preset = item.preset
}
}
import_string.value = ''
@ -424,4 +358,33 @@ const importItem = async () => {
toast.error(`Failed to import string. ${e.message}`)
}
}
const convertOptions = async args => {
try {
const response = await convertCliOptions(args)
if (response.output_template) {
form.template = response.output_template
}
if (response.download_path) {
form.folder = response.download_path
}
return response.opts
} catch (e) {
toast.error(e.message)
}
return null;
}
const hasFormatInConfig = computed(() => {
if (!form?.cli) {
return false
}
if (form.cli.includes('-f') || form.cli.includes('--format')) {
return true
}
})
</script>

110
ui/pages/tasks.vue
View file

@ -39,8 +39,7 @@ div.is-centered {
</div>
<div class="is-hidden-mobile">
<span class="subtitle">
The task runner is simple queue system that allows you to schedule downloads. The tasks are run at the
scheduled time.
The task runner is simple queue system that allows you to schedule downloads to run at the specific time.
</span>
</div>
</div>
@ -62,10 +61,6 @@ div.is-centered {
<a class="has-text-primary" v-tooltip="'Export task.'" @click.prevent="exportItem(item)">
<span class="icon"><i class="fa-solid fa-file-export" /></span>
</a>
<button @click="item.raw = !item.raw">
<span class="icon"><i class="fa-solid"
:class="{ 'fa-arrow-down': !item?.raw, 'fa-arrow-up': item?.raw }" /></span>
</button>
</div>
</header>
<div class="card-content">
@ -86,11 +81,10 @@ div.is-centered {
<span class="icon"><i class="fa-solid fa-tv" /></span>
<span>{{ item.preset ?? config.app.default_preset }}</span>
</p>
</div>
</div>
<div class="card-content" v-if="item?.raw">
<div class="content">
<pre><code>{{ filterItem(item) }}</code></pre>
<p class="is-text-overflow">
<span class="icon"><i class="fa-solid fa-terminal" /></span>
<span>{{ item.cli }}</span>
</p>
</div>
</div>
<div class="card-footer">
@ -120,17 +114,6 @@ div.is-centered {
icon="fas fa-exclamation-circle" v-if="!tasks || tasks.length < 1" />
</div>
</div>
<div class="column is-12" v-if="tasks && tasks.length > 0 && !toggleForm">
<Message message_class="has-background-info-90 has-text-dark" title="Tips" icon="fas fa-info-circle">
<ul>
<li>
When you export task, the preset settings is automatically merged into the task and the preset set to
<code>default</code> to be more portable. The exporter doesn't include <code>Cookies</code> field for
security reasons.
</li>
</ul>
</Message>
</div>
</div>
</template>
@ -202,6 +185,8 @@ const resetForm = (closeForm = false) => {
}
const updateTasks = async items => {
let data = {}
try {
addInProgress.value = true
@ -213,7 +198,7 @@ const updateTasks = async items => {
body: JSON.stringify(items),
})
const data = await response.json()
data = await response.json()
if (200 !== response.status) {
toast.error(`Failed to update task. ${data.error}`);
@ -224,7 +209,7 @@ const updateTasks = async items => {
resetForm(true)
return true
} catch (e) {
toast.error(`Failed to update task. ${data.error}`);
toast.error(`Failed to update task. ${data?.error ?? e.message}`);
} finally {
addInProgress.value = false
}
@ -272,11 +257,6 @@ const updateItem = async ({ reference, task }) => {
resetForm(true)
}
const filterItem = item => {
const { raw, ...rest } = item
return JSON.stringify(rest, null, 2)
}
const editItem = item => {
task.value = item
taskRef.value = item.id
@ -327,21 +307,8 @@ const runNow = item => {
data.template = item.template
}
if (item.cookies) {
data.cookies = item.cookies
}
if (item.config) {
if (typeof item.config === 'object') {
data.config = JSON.stringify(item.config)
}
try {
JSON.parse(data.config)
} catch (e) {
toast.error(`Invalid JSON yt-dlp config. ${e.message}`)
return
}
if (item.cli) {
data.cli = item.cli
}
socket.emit('add_url', data)
@ -359,69 +326,26 @@ const statusHandler = async stream => {
}
const exportItem = async item => {
let preset = config.presets.find(p => p.name === item.preset)
if (!preset) {
toast.error('Preset not found.')
return
}
const info = JSON.parse(JSON.stringify(item))
preset = JSON.parse(JSON.stringify(preset))
let data = {
name: info.name,
url: info.url,
preset: 'default',
preset: info.preset,
timer: info.timer,
folder: info.folder,
template: info.template,
config: info.config,
}
// -- merge preset options with task args.
let args = {}
if (preset.args && Object.keys(preset.args).length > 0) {
for (const key of Object.keys(preset.args)) {
args[key] = preset.args[key]
}
}
const defaults = ['default', 'not_set']
if (preset.format && !defaults.includes(preset.format)) {
args.format = preset.format
if (info.template) {
data.template = info.template
}
if (preset.postprocessors && preset.postprocessors.length > 0) {
args.postprocessors = preset.postprocessors
}
if (preset.folder && !info.folder) {
data.folder = preset.folder
}
if (preset.template && !info.template) {
data.template = preset.template
}
if (!data.config || Object.keys(data.config).length < 1) {
data.config = {}
}
for (const key of Object.keys(args)) {
if (key in data.config && Array.isArray(args[key]) && Array.isArray(data.config[key])) {
data.config[key] = data.config[key].concat(args[key])
continue
}
if (data?.config[key]) {
continue
}
data.config[key] = args[key]
if (info.cli) {
data.cli = info.cli
}
data['_type'] = 'task'
data['_version'] = '1.1'
data['_version'] = '2.0'
return copyText(base64UrlEncode(JSON.stringify(data)));
}