Further updates on how we handle archive status
This commit is contained in:
parent
e6aa56a008
commit
da8358bad6
11 changed files with 284 additions and 137 deletions
|
|
@ -121,8 +121,8 @@ class DataStore:
|
||||||
|
|
||||||
return items
|
return items
|
||||||
|
|
||||||
def put(self, value: Download) -> Download:
|
def put(self, value: Download, no_notify: bool = False) -> Download:
|
||||||
if "error" == value.info.status:
|
if "error" == value.info.status and not no_notify:
|
||||||
from app.library.Events import EventBus, Events
|
from app.library.Events import EventBus, Events
|
||||||
|
|
||||||
asyncio.create_task(EventBus.get_instance().emit(Events.ITEM_ERROR, value.info), name="emit_item_error")
|
asyncio.create_task(EventBus.get_instance().emit(Events.ITEM_ERROR, value.info), name="emit_item_error")
|
||||||
|
|
@ -181,16 +181,18 @@ class DataStore:
|
||||||
except AttributeError:
|
except AttributeError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
encoded: str = stored.json()
|
||||||
|
|
||||||
self._connection.execute(
|
self._connection.execute(
|
||||||
sqlStatement.strip(),
|
sqlStatement.strip(),
|
||||||
(
|
(
|
||||||
stored._id,
|
stored._id,
|
||||||
str(type),
|
str(type),
|
||||||
stored.url,
|
stored.url,
|
||||||
stored.json(),
|
encoded,
|
||||||
str(type),
|
str(type),
|
||||||
stored.url,
|
stored.url,
|
||||||
stored.json(),
|
encoded,
|
||||||
datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
|
datetime.now(UTC).strftime("%Y-%m-%d %H:%M:%S"),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -24,18 +24,15 @@ from .Presets import Presets
|
||||||
from .Scheduler import Scheduler
|
from .Scheduler import Scheduler
|
||||||
from .Singleton import Singleton
|
from .Singleton import Singleton
|
||||||
from .Utils import (
|
from .Utils import (
|
||||||
archive_read,
|
|
||||||
arg_converter,
|
arg_converter,
|
||||||
calc_download_path,
|
calc_download_path,
|
||||||
dt_delta,
|
dt_delta,
|
||||||
extract_info,
|
extract_info,
|
||||||
extract_ytdlp_logs,
|
extract_ytdlp_logs,
|
||||||
get_archive_id,
|
|
||||||
load_cookies,
|
load_cookies,
|
||||||
str_to_dt,
|
str_to_dt,
|
||||||
ytdlp_reject,
|
ytdlp_reject,
|
||||||
)
|
)
|
||||||
from .YTDLPOpts import YTDLPOpts
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from app.library.Presets import Preset
|
from app.library.Presets import Preset
|
||||||
|
|
@ -659,22 +656,18 @@ class DownloadQueue(metaclass=Singleton):
|
||||||
"level": logging.WARNING,
|
"level": logging.WARNING,
|
||||||
"name": "callback-logger",
|
"name": "callback-logger",
|
||||||
},
|
},
|
||||||
**YTDLPOpts.get_instance().preset(name=item.preset).add_cli(args=item.cli, from_user=True).get_all(),
|
**item.get_ytdlp_opts().get_all(),
|
||||||
}
|
}
|
||||||
|
|
||||||
if yt_conf.get("external_downloader"):
|
if yt_conf.get("external_downloader"):
|
||||||
LOG.warning(f"Using external downloader '{yt_conf.get('external_downloader')}' for '{item.url}'.")
|
LOG.warning(f"Using external downloader '{yt_conf.get('external_downloader')}' for '{item.url}'.")
|
||||||
item.extras.update({"external_downloader": True})
|
item.extras.update({"external_downloader": True})
|
||||||
|
|
||||||
if archive_file := yt_conf.get("download_archive"):
|
if item.is_archived():
|
||||||
idDict: dict = get_archive_id(item.url)
|
message: str = f"The URL '{item.url}' is already downloaded and recorded in archive."
|
||||||
if (archive_id := idDict.get("archive_id")) and len(archive_read(archive_file, [archive_id])) > 0:
|
LOG.error(message)
|
||||||
message: str = (
|
await self._notify.emit(Events.LOG_INFO, title="Already Downloaded", message=message)
|
||||||
f"'{idDict.get('id')}': The URL '{item.url}' is already downloaded and recorded in archive."
|
return {"status": "error", "msg": message}
|
||||||
)
|
|
||||||
LOG.error(message)
|
|
||||||
await self._notify.emit(Events.LOG_INFO, title="Already Downloaded", message=message)
|
|
||||||
return {"status": "error", "msg": message}
|
|
||||||
|
|
||||||
started: float = time.perf_counter()
|
started: float = time.perf_counter()
|
||||||
|
|
||||||
|
|
@ -1016,7 +1009,8 @@ class DownloadQueue(metaclass=Singleton):
|
||||||
nMessage = f"Completed '{entry.info.title}' download."
|
nMessage = f"Completed '{entry.info.title}' download."
|
||||||
_tasks.append(self._notify.emit(Events.ITEM_COMPLETED, data=entry.info, title=nTitle, message=nMessage))
|
_tasks.append(self._notify.emit(Events.ITEM_COMPLETED, data=entry.info, title=nTitle, message=nMessage))
|
||||||
|
|
||||||
self.done.put(value=entry)
|
await asyncio.sleep(0.2)
|
||||||
|
self.done.put(entry)
|
||||||
_tasks.append(
|
_tasks.append(
|
||||||
self._notify.emit(
|
self._notify.emit(
|
||||||
Events.ITEM_MOVED,
|
Events.ITEM_MOVED,
|
||||||
|
|
@ -1122,6 +1116,7 @@ class DownloadQueue(metaclass=Singleton):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
item.info.archive_status()
|
||||||
self.done.put(item)
|
self.done.put(item)
|
||||||
LOG.exception(e)
|
LOG.exception(e)
|
||||||
LOG.error(f"Failed to retry item '{item_ref}'. {e!s}")
|
LOG.error(f"Failed to retry item '{item_ref}'. {e!s}")
|
||||||
|
|
|
||||||
|
|
@ -97,7 +97,7 @@ class HttpAPI:
|
||||||
|
|
||||||
app.on_shutdown.append(self.on_shutdown)
|
app.on_shutdown.append(self.on_shutdown)
|
||||||
|
|
||||||
def add_routes(self, app: web.Application):
|
def add_routes(self, app: web.Application) -> None:
|
||||||
"""
|
"""
|
||||||
Add the routes to the application.
|
Add the routes to the application.
|
||||||
|
|
||||||
|
|
@ -133,7 +133,8 @@ class HttpAPI:
|
||||||
elif "" == base_path or not routePath.rstrip("/").startswith(base_path.rstrip("/")):
|
elif "" == base_path or not routePath.rstrip("/").startswith(base_path.rstrip("/")):
|
||||||
route.path = f"{base_path}/{route.path.lstrip('/')}"
|
route.path = f"{base_path}/{route.path.lstrip('/')}"
|
||||||
|
|
||||||
LOG.debug(f"Add ({route.name}) {route.method}: {route.path}.")
|
if self.config.debug:
|
||||||
|
LOG.debug(f"Add ({route.name}) {route.method}: {route.path}.")
|
||||||
|
|
||||||
app.router.add_route(route.method, route.path, handler=_handle(route.handler), name=route.name)
|
app.router.add_route(route.method, route.path, handler=_handle(route.handler), name=route.name)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -110,9 +110,10 @@ class HttpSocket:
|
||||||
load_modules(self.rootPath, self.rootPath / "routes" / "socket")
|
load_modules(self.rootPath, self.rootPath / "routes" / "socket")
|
||||||
|
|
||||||
for route in get_routes(RouteType.SOCKET).values():
|
for route in get_routes(RouteType.SOCKET).values():
|
||||||
LOG.debug(
|
if self.config.debug:
|
||||||
f"Add ({route.name}) {route.method.value if isinstance(route.method,RouteType) else route.method}: {route.path}."
|
LOG.debug(
|
||||||
)
|
f"Add ({route.name}) {route.method.value if isinstance(route.method, RouteType) else route.method}: {route.path}."
|
||||||
|
)
|
||||||
self.sio.on(route.path)(HttpSocket._injector(route.handler, route.path))
|
self.sio.on(route.path)(HttpSocket._injector(route.handler, route.path))
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
|
||||||
|
|
@ -6,9 +6,10 @@ from dataclasses import dataclass, field
|
||||||
from email.utils import formatdate
|
from email.utils import formatdate
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.library.Utils import clean_item, get_archive_id
|
from app.library.Utils import archive_add, archive_delete, archive_read, clean_item, get_archive_id
|
||||||
|
from app.library.YTDLPOpts import YTDLPOpts
|
||||||
|
|
||||||
LOG = logging.getLogger("ItemDTO")
|
LOG: logging.Logger = logging.getLogger("ItemDTO")
|
||||||
|
|
||||||
|
|
||||||
@dataclass(kw_only=True)
|
@dataclass(kw_only=True)
|
||||||
|
|
@ -92,29 +93,6 @@ class Item:
|
||||||
"""
|
"""
|
||||||
return Item.format({**self.serialize(), **kwargs})
|
return Item.format({**self.serialize(), **kwargs})
|
||||||
|
|
||||||
def __repr__(self):
|
|
||||||
from .config import Config
|
|
||||||
from .Utils import calc_download_path, strip_newline
|
|
||||||
|
|
||||||
data = {}
|
|
||||||
for k, v in self.serialize().items():
|
|
||||||
if not v and k not in ("auto_start"):
|
|
||||||
continue
|
|
||||||
|
|
||||||
if k == "cli":
|
|
||||||
data[k] = strip_newline(v)
|
|
||||||
elif k == "extras":
|
|
||||||
data[k] = f"{len(v)} items"
|
|
||||||
elif k == "cookies":
|
|
||||||
data[k] = f"{len(v)}/chars"
|
|
||||||
elif k == "folder":
|
|
||||||
data[k] = calc_download_path(base_path=Config.get_instance().download_path, folder=v)
|
|
||||||
else:
|
|
||||||
data[k] = v
|
|
||||||
|
|
||||||
items = "".join(f'{k}="{v}", ' for k, v in data.items() if v)
|
|
||||||
return f"Item({items.strip(', ')})"
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def format(item: dict) -> "Item":
|
def format(item: dict) -> "Item":
|
||||||
"""
|
"""
|
||||||
|
|
@ -137,16 +115,14 @@ class Item:
|
||||||
msg = "url param is required."
|
msg = "url param is required."
|
||||||
raise ValueError(msg)
|
raise ValueError(msg)
|
||||||
|
|
||||||
data = {
|
data: dict[str, str] = {"url": url}
|
||||||
"url": url,
|
|
||||||
}
|
|
||||||
|
|
||||||
preset = item.get("preset")
|
preset: str | None = item.get("preset")
|
||||||
if preset and isinstance(preset, str) and preset != Item._default_preset():
|
if preset and isinstance(preset, str) and preset != Item._default_preset():
|
||||||
from .Presets import Presets
|
from .Presets import Presets
|
||||||
|
|
||||||
if not Presets.get_instance().has(preset):
|
if not Presets.get_instance().has(preset):
|
||||||
msg = f"Preset '{preset}' does not exist."
|
msg: str = f"Preset '{preset}' does not exist."
|
||||||
raise ValueError(msg)
|
raise ValueError(msg)
|
||||||
|
|
||||||
data["preset"] = preset
|
data["preset"] = preset
|
||||||
|
|
@ -163,19 +139,19 @@ class Item:
|
||||||
if "auto_start" in item and isinstance(item.get("auto_start"), bool):
|
if "auto_start" in item and isinstance(item.get("auto_start"), bool):
|
||||||
data["auto_start"] = bool(item.get("auto_start"))
|
data["auto_start"] = bool(item.get("auto_start"))
|
||||||
|
|
||||||
extras = item.get("extras")
|
extras: dict | None = item.get("extras")
|
||||||
if extras and isinstance(extras, dict) and len(extras) > 0:
|
if extras and isinstance(extras, dict) and len(extras) > 0:
|
||||||
data["extras"] = extras
|
data["extras"] = extras
|
||||||
|
|
||||||
if item.get("requeued") and isinstance(item.get("requeued"), bool):
|
if item.get("requeued") and isinstance(item.get("requeued"), bool):
|
||||||
data["requeued"] = item.get("requeued")
|
data["requeued"] = item.get("requeued")
|
||||||
|
|
||||||
cli = item.get("cli")
|
cli: str | None = item.get("cli")
|
||||||
if cli and len(cli) > 2:
|
if cli and len(cli) > 2:
|
||||||
from .Utils import arg_converter
|
from .Utils import arg_converter
|
||||||
|
|
||||||
try:
|
try:
|
||||||
removed_options = []
|
removed_options: list = []
|
||||||
arg_converter(args=cli, level=True, removed_options=removed_options)
|
arg_converter(args=cli, level=True, removed_options=removed_options)
|
||||||
if len(removed_options) > 0:
|
if len(removed_options) > 0:
|
||||||
LOG.warning("Removed the following options '%s' for '%s'.", ", ".join(removed_options), url)
|
LOG.warning("Removed the following options '%s' for '%s'.", ", ".join(removed_options), url)
|
||||||
|
|
@ -187,6 +163,55 @@ class Item:
|
||||||
|
|
||||||
return Item(**data)
|
return Item(**data)
|
||||||
|
|
||||||
|
def get_archive_id(self) -> str | None:
|
||||||
|
if not self.url:
|
||||||
|
return None
|
||||||
|
|
||||||
|
idDict: dict = get_archive_id(self.url)
|
||||||
|
return idDict.get("archive_id")
|
||||||
|
|
||||||
|
def get_ytdlp_opts(self) -> YTDLPOpts:
|
||||||
|
params: YTDLPOpts = YTDLPOpts.get_instance()
|
||||||
|
|
||||||
|
if self.preset:
|
||||||
|
params = params.preset(name=self.preset)
|
||||||
|
|
||||||
|
if self.cli:
|
||||||
|
params = params.add_cli(self.cli, from_user=True)
|
||||||
|
|
||||||
|
return params
|
||||||
|
|
||||||
|
def get_archive_file(self) -> str | None:
|
||||||
|
return self.get_ytdlp_opts().get_all().get("download_archive")
|
||||||
|
|
||||||
|
def is_archived(self) -> bool:
|
||||||
|
archive_id: str | None = self.get_archive_id()
|
||||||
|
archive_file: str | None = self.get_archive_file()
|
||||||
|
return len(archive_read(archive_file, [archive_id])) > 0 if archive_file and archive_id else False
|
||||||
|
|
||||||
|
def __repr__(self) -> str:
|
||||||
|
from .config import Config
|
||||||
|
from .Utils import calc_download_path, strip_newline
|
||||||
|
|
||||||
|
data = {}
|
||||||
|
for k, v in self.serialize().items():
|
||||||
|
if not v and k not in ("auto_start"):
|
||||||
|
continue
|
||||||
|
|
||||||
|
if k == "cli":
|
||||||
|
data[k] = strip_newline(v)
|
||||||
|
elif k == "extras":
|
||||||
|
data[k] = f"{len(v)} items"
|
||||||
|
elif k == "cookies":
|
||||||
|
data[k] = f"{len(v)}/chars"
|
||||||
|
elif k == "folder":
|
||||||
|
data[k] = calc_download_path(base_path=Config.get_instance().download_path, folder=v)
|
||||||
|
else:
|
||||||
|
data[k] = v
|
||||||
|
|
||||||
|
items = "".join(f'{k}="{v}", ' for k, v in data.items() if v)
|
||||||
|
return f"Item({items.strip(', ')})"
|
||||||
|
|
||||||
|
|
||||||
@dataclass(kw_only=True)
|
@dataclass(kw_only=True)
|
||||||
class ItemDTO:
|
class ItemDTO:
|
||||||
|
|
@ -196,30 +221,55 @@ class ItemDTO:
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_id: str = field(default_factory=lambda: str(uuid.uuid4()), init=False)
|
_id: str = field(default_factory=lambda: str(uuid.uuid4()), init=False)
|
||||||
|
""" Unique identifier for the item. """
|
||||||
error: str | None = None
|
error: str | None = None
|
||||||
|
""" Error message if the item failed. """
|
||||||
id: str
|
id: str
|
||||||
|
""" The ID of the item yt-dlp """
|
||||||
title: str
|
title: str
|
||||||
|
""" The title of the item. """
|
||||||
url: str
|
url: str
|
||||||
quality: str | None = None
|
""" The URL of the item. """
|
||||||
format: str | None = None
|
|
||||||
preset: str = "default"
|
preset: str = "default"
|
||||||
|
""" The preset to be used for the item. """
|
||||||
folder: str
|
folder: str
|
||||||
|
""" The folder to save the item to. """
|
||||||
download_dir: str | None = None
|
download_dir: str | None = None
|
||||||
|
""" The full path to the download directory. """
|
||||||
temp_dir: str | None = None
|
temp_dir: str | None = None
|
||||||
|
""" The full path to the temporary directory. """
|
||||||
status: str | None = None
|
status: str | None = None
|
||||||
|
""" The status of the item. """
|
||||||
cookies: str | None = None
|
cookies: str | None = None
|
||||||
|
""" The cookies to be used for the item. """
|
||||||
template: str | None = None
|
template: str | None = None
|
||||||
|
""" The output template to be used for the item. """
|
||||||
template_chapter: str | None = None
|
template_chapter: str | None = None
|
||||||
|
""" The output template for chapters to be used for the item. """
|
||||||
timestamp: float = field(default_factory=lambda: time.time_ns())
|
timestamp: float = field(default_factory=lambda: time.time_ns())
|
||||||
|
""" The timestamp of the item. """
|
||||||
is_live: bool | None = None
|
is_live: bool | None = None
|
||||||
|
""" If the item is a live stream. """
|
||||||
datetime: str = field(default_factory=lambda: str(formatdate(time.time())))
|
datetime: str = field(default_factory=lambda: str(formatdate(time.time())))
|
||||||
|
""" The datetime of the item. """
|
||||||
live_in: str | None = None
|
live_in: str | None = None
|
||||||
|
""" The time until the live stream starts. """
|
||||||
file_size: int | None = None
|
file_size: int | None = None
|
||||||
|
""" The file size of the item. """
|
||||||
options: dict = field(default_factory=dict)
|
options: dict = field(default_factory=dict)
|
||||||
|
""" The options used for the item. """
|
||||||
extras: dict = field(default_factory=dict)
|
extras: dict = field(default_factory=dict)
|
||||||
|
""" Extra data associated with the item. """
|
||||||
cli: str = ""
|
cli: str = ""
|
||||||
|
""" The command options for yt-dlp to be used for this download. """
|
||||||
auto_start: bool = True
|
auto_start: bool = True
|
||||||
|
""" If the item should be started automatically. """
|
||||||
|
is_archivable: bool | None = None
|
||||||
|
""" If the item can be archived. """
|
||||||
|
is_archived: bool | None = None
|
||||||
|
""" If the item has been archived. """
|
||||||
|
archive_id: str | None = None
|
||||||
|
""" The archive ID of the item. """
|
||||||
|
|
||||||
# yt-dlp injected fields.
|
# yt-dlp injected fields.
|
||||||
tmpfilename: str | None = None
|
tmpfilename: str | None = None
|
||||||
|
|
@ -232,7 +282,13 @@ class ItemDTO:
|
||||||
speed: str | None = None
|
speed: str | None = None
|
||||||
eta: str | None = None
|
eta: str | None = None
|
||||||
|
|
||||||
|
_recomputed: bool = False
|
||||||
|
_archive_file: str | None = None
|
||||||
|
|
||||||
def serialize(self) -> dict:
|
def serialize(self) -> dict:
|
||||||
|
if "finished" == self.status and not self._recomputed:
|
||||||
|
self.archive_status()
|
||||||
|
|
||||||
item, _ = clean_item(self.__dict__.copy(), ItemDTO.removed_fields())
|
item, _ = clean_item(self.__dict__.copy(), ItemDTO.removed_fields())
|
||||||
return item
|
return item
|
||||||
|
|
||||||
|
|
@ -256,11 +312,95 @@ class ItemDTO:
|
||||||
str | None: The archive ID if available, None otherwise.
|
str | None: The archive ID if available, None otherwise.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
if not self.info:
|
if self.archive_id:
|
||||||
|
return self.archive_id
|
||||||
|
|
||||||
|
if not self.url:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
idDict: dict = get_archive_id(self.url)
|
idDict: dict = get_archive_id(self.url)
|
||||||
return idDict.get("archive_id")
|
self.archive_id = idDict.get("archive_id")
|
||||||
|
|
||||||
|
return self.archive_id
|
||||||
|
|
||||||
|
def get_ytdlp_opts(self) -> YTDLPOpts:
|
||||||
|
"""
|
||||||
|
Get the yt-dlp options for the item.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
YTDLPOpts: The yt-dlp options for the item.
|
||||||
|
|
||||||
|
"""
|
||||||
|
params: YTDLPOpts = YTDLPOpts.get_instance()
|
||||||
|
|
||||||
|
if self.preset:
|
||||||
|
params = params.preset(name=self.preset)
|
||||||
|
|
||||||
|
if self.cli:
|
||||||
|
params = params.add_cli(self.cli, from_user=True)
|
||||||
|
|
||||||
|
return params
|
||||||
|
|
||||||
|
def archive_status(self, force: bool = False) -> None:
|
||||||
|
if not force and (self._recomputed or not self.archive_id):
|
||||||
|
return
|
||||||
|
|
||||||
|
if "finished" == self.status:
|
||||||
|
self._recomputed = True
|
||||||
|
|
||||||
|
self.is_archivable = bool(self._archive_file)
|
||||||
|
if not self.is_archivable:
|
||||||
|
self.is_archived = False
|
||||||
|
else:
|
||||||
|
self.is_archived = len(archive_read(self._archive_file, [self.archive_id])) > 0
|
||||||
|
|
||||||
|
def get_archive_file(self) -> str | None:
|
||||||
|
"""
|
||||||
|
Get the archive file path from the yt-dlp options.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str | None: The archive file path if available, None otherwise.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if self._archive_file or self._recomputed or not self.archive_id:
|
||||||
|
return self._archive_file
|
||||||
|
|
||||||
|
self._archive_file = self.get_ytdlp_opts().get_all().get("download_archive")
|
||||||
|
if self._archive_file:
|
||||||
|
self._archive_file = self._archive_file.strip()
|
||||||
|
|
||||||
|
return self._archive_file
|
||||||
|
|
||||||
|
def archive_add(self) -> bool:
|
||||||
|
"""
|
||||||
|
Archive the item by adding its archive ID to the download archive file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
bool: True if the item was archived, False otherwise.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if self.is_archived or not self.is_archivable or not self.archive_id or not self._archive_file:
|
||||||
|
return False
|
||||||
|
|
||||||
|
self.is_archived = archive_add(self._archive_file, [self.archive_id])
|
||||||
|
|
||||||
|
return self.is_archived
|
||||||
|
|
||||||
|
def archive_delete(self) -> bool:
|
||||||
|
"""
|
||||||
|
Remove the item's archive ID from the download archive file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
bool: True if the item was removed from the archive, False otherwise.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if not self.is_archivable or not self.is_archived or not self.archive_id or not self._archive_file:
|
||||||
|
return False
|
||||||
|
|
||||||
|
archive_delete(self._archive_file, [self.archive_id])
|
||||||
|
self.is_archived = False
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def removed_fields() -> tuple:
|
def removed_fields() -> tuple:
|
||||||
|
|
@ -275,4 +415,11 @@ class ItemDTO:
|
||||||
"output_template_chapter",
|
"output_template_chapter",
|
||||||
"config",
|
"config",
|
||||||
"temp_path",
|
"temp_path",
|
||||||
|
"_recomputed",
|
||||||
|
"_archive_file",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def __post_init__(self):
|
||||||
|
self.get_archive_id()
|
||||||
|
self.get_archive_file()
|
||||||
|
self.archive_status()
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,6 @@ from typing import Any
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
|
|
||||||
from .config import Config
|
from .config import Config
|
||||||
from .encoder import Encoder
|
|
||||||
from .Events import EventBus, Events
|
from .Events import EventBus, Events
|
||||||
from .Singleton import Singleton
|
from .Singleton import Singleton
|
||||||
from .Utils import arg_converter, init_class
|
from .Utils import arg_converter, init_class
|
||||||
|
|
@ -92,6 +91,7 @@ class Preset:
|
||||||
return self.__dict__
|
return self.__dict__
|
||||||
|
|
||||||
def json(self) -> str:
|
def json(self) -> str:
|
||||||
|
from .encoder import Encoder
|
||||||
return Encoder().encode(self.serialize())
|
return Encoder().encode(self.serialize())
|
||||||
|
|
||||||
def get(self, key: str, default: Any = None) -> Any:
|
def get(self, key: str, default: Any = None) -> Any:
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@ from datetime import UTC, datetime, timedelta
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from http.cookiejar import MozillaCookieJar
|
from http.cookiejar import MozillaCookieJar
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TypeVar
|
from typing import Any, TypeVar
|
||||||
|
|
||||||
from Crypto.Cipher import AES
|
from Crypto.Cipher import AES
|
||||||
from yt_dlp.utils import age_restricted, match_str
|
from yt_dlp.utils import age_restricted, match_str
|
||||||
|
|
@ -203,7 +203,7 @@ def extract_info(
|
||||||
if no_archive and "download_archive" in params:
|
if no_archive and "download_archive" in params:
|
||||||
del params["download_archive"]
|
del params["download_archive"]
|
||||||
|
|
||||||
data = YTDLP(params=params).extract_info(url, download=False)
|
data: dict[str, Any] | None = YTDLP(params=params).extract_info(url, download=False)
|
||||||
|
|
||||||
if data and follow_redirect and "_type" in data and "url" == data["_type"]:
|
if data and follow_redirect and "_type" in data and "url" == data["_type"]:
|
||||||
return extract_info(
|
return extract_info(
|
||||||
|
|
@ -262,6 +262,7 @@ def merge_dict(source: dict, destination: dict) -> dict:
|
||||||
|
|
||||||
return destination_copy
|
return destination_copy
|
||||||
|
|
||||||
|
|
||||||
def check_id(file: Path) -> bool | str:
|
def check_id(file: Path) -> bool | str:
|
||||||
"""
|
"""
|
||||||
Check if we are able to get an id from the file name.
|
Check if we are able to get an id from the file name.
|
||||||
|
|
@ -1185,14 +1186,11 @@ def load_modules(root_path: Path, directory: Path):
|
||||||
|
|
||||||
package_name: str = str(directory.relative_to(root_path).as_posix()).replace("/", ".")
|
package_name: str = str(directory.relative_to(root_path).as_posix()).replace("/", ".")
|
||||||
|
|
||||||
LOG.debug(f"Loading routes from '{directory}' with package name '{package_name}'.")
|
|
||||||
|
|
||||||
for _, name, _ in pkgutil.iter_modules([directory]):
|
for _, name, _ in pkgutil.iter_modules([directory]):
|
||||||
full_name: str = f"{package_name}.{name}"
|
full_name: str = f"{package_name}.{name}"
|
||||||
if name.startswith("_"):
|
if name.startswith("_"):
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
LOG.debug(f"Loading module '{full_name}'.")
|
|
||||||
importlib.import_module(full_name)
|
importlib.import_module(full_name)
|
||||||
except ImportError as e:
|
except ImportError as e:
|
||||||
LOG.error(f"Failed to import module '{full_name}': {e}")
|
LOG.error(f"Failed to import module '{full_name}': {e}")
|
||||||
|
|
@ -1367,7 +1365,7 @@ def archive_add(file: str | Path, ids: list[str], skip_check: bool = False) -> b
|
||||||
skip_check (bool): If True, skip checking for existing IDs.
|
skip_check (bool): If True, skip checking for existing IDs.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
if not ids:
|
if not ids or not file:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
path: Path = Path(file) if not isinstance(file, Path) else file
|
path: Path = Path(file) if not isinstance(file, Path) else file
|
||||||
|
|
@ -1423,12 +1421,15 @@ def archive_read(file: str | Path, ids: list[str] | None = None) -> list[str]:
|
||||||
list[str]: List of ids found in the archive file filtered by `ids` if provided.
|
list[str]: List of ids found in the archive file filtered by `ids` if provided.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
if not file:
|
||||||
|
return []
|
||||||
|
|
||||||
path: Path = Path(file) if not isinstance(file, Path) else file
|
path: Path = Path(file) if not isinstance(file, Path) else file
|
||||||
if not path.exists():
|
if not file or not path.exists():
|
||||||
return []
|
return []
|
||||||
|
|
||||||
ids_set: set[str] | None = (
|
ids_set: set[str] | None = (
|
||||||
{s.strip() for s in ids if str(s).strip() and len(str(s).strip().split()) < 2} if ids else None
|
{s.strip() for s in ids if str(s).strip() and len(str(s).strip().split()) >= 2} if ids else None
|
||||||
)
|
)
|
||||||
|
|
||||||
found: list[str] = []
|
found: list[str] = []
|
||||||
|
|
@ -1457,12 +1458,15 @@ def archive_delete(file: str | Path, ids: list[str]) -> bool:
|
||||||
bool: True if deletion succeeded (or nothing to do), False on error.
|
bool: True if deletion succeeded (or nothing to do), False on error.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
path: Path = Path(file) if not isinstance(file, Path) else file
|
if not file or not ids:
|
||||||
|
|
||||||
if not path.exists() or not ids:
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
remove_ids: set[str] = {x.strip() for x in ids if str(x).strip() and len(str(x).strip().split()) < 2}
|
path: Path = Path(file) if not isinstance(file, Path) else file
|
||||||
|
|
||||||
|
if not path.exists():
|
||||||
|
return False
|
||||||
|
|
||||||
|
remove_ids: set[str] = {x.strip() for x in ids if str(x).strip() and len(str(x).strip().split()) >= 2}
|
||||||
if not remove_ids:
|
if not remove_ids:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -200,7 +200,8 @@ class YTDLPOpts:
|
||||||
if data["format"] == "-best":
|
if data["format"] == "-best":
|
||||||
data["format"] = data["format"][1:]
|
data["format"] = data["format"][1:]
|
||||||
|
|
||||||
LOG.debug(f"Final yt-dlp options: '{data!s}'.")
|
if self._config.debug:
|
||||||
|
LOG.debug(f"Final yt-dlp options: '{data!s}'.")
|
||||||
|
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,6 @@ from app.library.Events import EventBus, Events
|
||||||
from app.library.ItemDTO import Item
|
from app.library.ItemDTO import Item
|
||||||
from app.library.Presets import Preset, Presets
|
from app.library.Presets import Preset, Presets
|
||||||
from app.library.router import route
|
from app.library.router import route
|
||||||
from app.library.Utils import archive_add, archive_delete, archive_read, get_archive_id
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from library.Download import Download
|
from library.Download import Download
|
||||||
|
|
@ -100,14 +99,8 @@ async def item_view(request: Request, queue: DownloadQueue, encoder: Encoder, co
|
||||||
if not item.info:
|
if not item.info:
|
||||||
return web.json_response(data={"error": "item has no info."}, status=web.HTTPNotFound.status_code)
|
return web.json_response(data={"error": "item has no info."}, status=web.HTTPNotFound.status_code)
|
||||||
|
|
||||||
is_archived = False
|
info: dict = {
|
||||||
params: dict = item.get_ytdlp_opts().get_all()
|
|
||||||
if (archive_file := params.get("download_archive")) and (archive_id := item.get_archive_id()):
|
|
||||||
is_archived: bool = len(archive_read(archive_file, [archive_id])) > 0
|
|
||||||
|
|
||||||
info = {
|
|
||||||
**item.info.serialize(),
|
**item.info.serialize(),
|
||||||
"is_archived": is_archived,
|
|
||||||
"ffprobe": {},
|
"ffprobe": {},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -267,14 +260,14 @@ async def items_add(request: Request, queue: DownloadQueue, encoder: Encoder) ->
|
||||||
|
|
||||||
|
|
||||||
@route("POST", r"api/history/{id}/archive", "history.item.archive.add")
|
@route("POST", r"api/history/{id}/archive", "history.item.archive.add")
|
||||||
async def item_archive_add(request: Request, queue: DownloadQueue) -> Response:
|
async def item_archive_add(request: Request, queue: DownloadQueue, notify: EventBus) -> Response:
|
||||||
"""
|
"""
|
||||||
Manually mark an item as archived.
|
Manually mark an item as archived.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
request (Request): The request object.
|
request (Request): The request object.
|
||||||
queue (DownloadQueue): The download queue instance.
|
queue (DownloadQueue): The download queue instance.
|
||||||
config (Config): The configuration instance.
|
notify (EventBus): The event bus instance.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Response: The response object.
|
Response: The response object.
|
||||||
|
|
@ -290,28 +283,33 @@ async def item_archive_add(request: Request, queue: DownloadQueue) -> Response:
|
||||||
except KeyError:
|
except KeyError:
|
||||||
return web.json_response(data={"error": f"item '{id}' not found."}, status=web.HTTPNotFound.status_code)
|
return web.json_response(data={"error": f"item '{id}' not found."}, status=web.HTTPNotFound.status_code)
|
||||||
|
|
||||||
params: dict = item.get_ytdlp_opts().get_all()
|
if not item.info.is_archivable:
|
||||||
|
|
||||||
if not (archive_file := params.get("download_archive")):
|
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
data={"error": f"item '{item.info.title}' does not have an archive file."},
|
data={"error": f"item '{item.info.title}' does not have an archive file."},
|
||||||
status=web.HTTPBadRequest.status_code,
|
status=web.HTTPBadRequest.status_code,
|
||||||
)
|
)
|
||||||
|
|
||||||
idDict = get_archive_id(url=item.info.url)
|
if not item.info.archive_id:
|
||||||
if not (archive_id := idDict.get("archive_id")):
|
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
data={"error": f"item '{item.info.title}' does not have an archive ID."},
|
data={"error": f"item '{item.info.title}' does not have an archive ID."},
|
||||||
status=web.HTTPBadRequest.status_code,
|
status=web.HTTPBadRequest.status_code,
|
||||||
)
|
)
|
||||||
|
|
||||||
if len(archive_read(archive_file, [archive_id])) > 0:
|
if item.info.is_archived:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
data={"error": f"item '{item.info.title}' already archived."},
|
data={"error": f"item '{item.info.title}' already archived."},
|
||||||
status=web.HTTPConflict.status_code,
|
status=web.HTTPConflict.status_code,
|
||||||
)
|
)
|
||||||
|
|
||||||
archive_add(archive_file, [archive_id])
|
if not item.info.archive_add():
|
||||||
|
return web.json_response(
|
||||||
|
data={"error": f"item '{item.info.title}' could not be added to archive."},
|
||||||
|
status=web.HTTPInternalServerError.status_code,
|
||||||
|
)
|
||||||
|
|
||||||
|
item.info.archive_status(force=True)
|
||||||
|
queue.done.put(item, no_notify=True)
|
||||||
|
await notify.emit(Events.ITEM_UPDATED, data=item.info)
|
||||||
|
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
data={"message": f"item '{item.info.title}' archived."},
|
data={"message": f"item '{item.info.title}' archived."},
|
||||||
|
|
@ -320,13 +318,14 @@ async def item_archive_add(request: Request, queue: DownloadQueue) -> Response:
|
||||||
|
|
||||||
|
|
||||||
@route("DELETE", r"api/history/{id}/archive", "history.item.archive.delete")
|
@route("DELETE", r"api/history/{id}/archive", "history.item.archive.delete")
|
||||||
async def item_archive_delete(request: Request, queue: DownloadQueue) -> Response:
|
async def item_archive_delete(request: Request, queue: DownloadQueue, notify: EventBus) -> Response:
|
||||||
"""
|
"""
|
||||||
Remove an item from the archive.
|
Remove an item from the archive.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
request (Request): The request object.
|
request (Request): The request object.
|
||||||
queue (DownloadQueue): The download queue instance.
|
queue (DownloadQueue): The download queue instance.
|
||||||
|
notify (EventBus): The event bus instance.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Response: The response object.
|
Response: The response object.
|
||||||
|
|
@ -342,38 +341,35 @@ async def item_archive_delete(request: Request, queue: DownloadQueue) -> Respons
|
||||||
except KeyError:
|
except KeyError:
|
||||||
return web.json_response(data={"error": f"item '{id}' not found."}, status=web.HTTPNotFound.status_code)
|
return web.json_response(data={"error": f"item '{id}' not found."}, status=web.HTTPNotFound.status_code)
|
||||||
|
|
||||||
url: str = item.info.url
|
if not item.info.is_archivable:
|
||||||
title: str = f" '{item.info.title}'"
|
|
||||||
params: dict = item.get_ytdlp_opts().get_all()
|
|
||||||
|
|
||||||
if not (archive_file := params.get("download_archive")):
|
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
data={"error": "Archive file is not configured."},
|
data={"error": f"item '{item.info.title}' does not have an archive file."},
|
||||||
status=web.HTTPBadRequest.status_code,
|
status=web.HTTPBadRequest.status_code,
|
||||||
)
|
)
|
||||||
|
|
||||||
archive_file = Path(archive_file)
|
if not item.info.archive_id:
|
||||||
|
|
||||||
if not archive_file.exists():
|
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
data={"error": f"Archive file '{archive_file}' does not exist."},
|
data={"error": f"item '{item.info.title}' does not have an archive ID."},
|
||||||
status=web.HTTPNotFound.status_code,
|
|
||||||
)
|
|
||||||
|
|
||||||
idDict = get_archive_id(url=url)
|
|
||||||
if not (archive_id := idDict.get("archive_id")):
|
|
||||||
return web.json_response(
|
|
||||||
data={"error": "item does not have an archive ID."},
|
|
||||||
status=web.HTTPBadRequest.status_code,
|
status=web.HTTPBadRequest.status_code,
|
||||||
)
|
)
|
||||||
|
|
||||||
if not archive_delete(archive_file, [archive_id]):
|
if not item.info.is_archived:
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
data={"error": f"item{title} not found in '{archive_file}' archive."},
|
data={"error": f"item '{item.info.title}' not archived."},
|
||||||
status=web.HTTPNotFound.status_code,
|
status=web.HTTPConflict.status_code,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if not item.info.archive_delete():
|
||||||
|
return web.json_response(
|
||||||
|
data={"error": f"item '{item.info.title}' not found in archive file."},
|
||||||
|
status=web.HTTPInternalServerError.status_code,
|
||||||
|
)
|
||||||
|
|
||||||
|
item.info.archive_status(force=True)
|
||||||
|
queue.done.put(item, no_notify=True)
|
||||||
|
await notify.emit(Events.ITEM_UPDATED, data=item.info)
|
||||||
|
|
||||||
return web.json_response(
|
return web.json_response(
|
||||||
data={"message": f"item{title} removed from '{archive_file}' archive."},
|
data={"message": f"item '{item.info.title}' removed from archive."},
|
||||||
status=web.HTTPOk.status_code,
|
status=web.HTTPOk.status_code,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -203,25 +203,23 @@
|
||||||
<span>Local Information</span>
|
<span>Local Information</span>
|
||||||
</NuxtLink>
|
</NuxtLink>
|
||||||
|
|
||||||
<template v-if="item.status != 'finished' || !item.filename">
|
<hr class="dropdown-divider" />
|
||||||
<hr class="dropdown-divider" />
|
<NuxtLink class="dropdown-item" @click="retryItem(item, true)">
|
||||||
<NuxtLink class="dropdown-item" @click="retryItem(item, true)">
|
<span class="icon"><i class="fa-solid fa-rotate-right" /></span>
|
||||||
<span class="icon"><i class="fa-solid fa-rotate-right" /></span>
|
<span>Add to download form</span>
|
||||||
<span>Add to download form</span>
|
</NuxtLink>
|
||||||
</NuxtLink>
|
|
||||||
</template>
|
|
||||||
|
|
||||||
<template v-if="'finished' !== item.status && config.app?.keep_archive">
|
<template v-if="item.is_archivable && !item.is_archived">
|
||||||
<hr class="dropdown-divider" />
|
<hr class="dropdown-divider" />
|
||||||
<NuxtLink class="dropdown-item has-text-danger" @click="addArchiveDialog(item)">
|
<NuxtLink class="dropdown-item has-text-danger" @click="addArchiveDialog(item)">
|
||||||
<span class="icon"><i class="fa-solid fa-box-archive" /></span>
|
<span class="icon"><i class="fa-solid fa-box-archive" /></span>
|
||||||
<span>Archive Item</span>
|
<span>Add to archive</span>
|
||||||
</NuxtLink>
|
</NuxtLink>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<template v-if="'finished' === item.status && item.filename && config.app?.keep_archive">
|
<template v-if="item.is_archivable && item.is_archived">
|
||||||
<hr class="dropdown-divider" />
|
<hr class="dropdown-divider" />
|
||||||
<NuxtLink class="dropdown-item" @click="removeFromArchiveDialog(item)">
|
<NuxtLink class="dropdown-item has-text-danger" @click="removeFromArchiveDialog(item)">
|
||||||
<span class="icon"><i class="fa-solid fa-box-archive" /></span>
|
<span class="icon"><i class="fa-solid fa-box-archive" /></span>
|
||||||
<span>Remove from archive</span>
|
<span>Remove from archive</span>
|
||||||
</NuxtLink>
|
</NuxtLink>
|
||||||
|
|
@ -380,15 +378,13 @@
|
||||||
<span>Local Information</span>
|
<span>Local Information</span>
|
||||||
</NuxtLink>
|
</NuxtLink>
|
||||||
|
|
||||||
<template v-if="item.status != 'finished' || !item.filename">
|
<hr class="dropdown-divider" />
|
||||||
<hr class="dropdown-divider" />
|
<NuxtLink class="dropdown-item" @click="retryItem(item, true)">
|
||||||
<NuxtLink class="dropdown-item" @click="retryItem(item, true)">
|
<span class="icon"><i class="fa-solid fa-rotate-right" /></span>
|
||||||
<span class="icon"><i class="fa-solid fa-rotate-right" /></span>
|
<span>Add to download form</span>
|
||||||
<span>Add to download form</span>
|
</NuxtLink>
|
||||||
</NuxtLink>
|
|
||||||
</template>
|
|
||||||
|
|
||||||
<template v-if="'finished' !== item.status && config.app?.keep_archive && !config.app.basic_mode">
|
<template v-if="item.is_archivable && !item.is_archived">
|
||||||
<hr class="dropdown-divider" />
|
<hr class="dropdown-divider" />
|
||||||
<NuxtLink class="dropdown-item has-text-danger" @click="addArchiveDialog(item)">
|
<NuxtLink class="dropdown-item has-text-danger" @click="addArchiveDialog(item)">
|
||||||
<span class="icon"><i class="fa-solid fa-box-archive" /></span>
|
<span class="icon"><i class="fa-solid fa-box-archive" /></span>
|
||||||
|
|
@ -396,10 +392,9 @@
|
||||||
</NuxtLink>
|
</NuxtLink>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<template
|
<template v-if="item.is_archivable && item.is_archived">
|
||||||
v-if="'finished' === item.status && item.filename && config.app?.keep_archive && !config.app.basic_mode">
|
|
||||||
<hr class="dropdown-divider" />
|
<hr class="dropdown-divider" />
|
||||||
<NuxtLink class="dropdown-item" @click="removeFromArchiveDialog(item)">
|
<NuxtLink class="dropdown-item has-text-danger" @click="removeFromArchiveDialog(item)">
|
||||||
<span class="icon"><i class="fa-solid fa-box-archive" /></span>
|
<span class="icon"><i class="fa-solid fa-box-archive" /></span>
|
||||||
<span>Remove from archive</span>
|
<span>Remove from archive</span>
|
||||||
</NuxtLink>
|
</NuxtLink>
|
||||||
|
|
@ -862,7 +857,6 @@ const removeFromArchiveDialog = (item: StoreItem) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
const removeFromArchive = async (item: StoreItem, opts?: { re_add?: boolean, remove_history?: boolean }) => {
|
const removeFromArchive = async (item: StoreItem, opts?: { re_add?: boolean, remove_history?: boolean }) => {
|
||||||
console.log('Removing from archive:', item, opts)
|
|
||||||
try {
|
try {
|
||||||
const req = await request(`/api/history/${item._id}/archive`, { method: 'DELETE' })
|
const req = await request(`/api/history/${item._id}/archive`, { method: 'DELETE' })
|
||||||
const data = await req.json()
|
const data = await req.json()
|
||||||
|
|
|
||||||
6
ui/app/types/store.d.ts
vendored
6
ui/app/types/store.d.ts
vendored
|
|
@ -82,6 +82,12 @@ type StoreItem = {
|
||||||
speed?: number | null
|
speed?: number | null
|
||||||
/** Time remaining for the item download if available */
|
/** Time remaining for the item download if available */
|
||||||
eta?: number | null
|
eta?: number | null
|
||||||
|
/** If the item can be archived */
|
||||||
|
is_archivable?: boolean
|
||||||
|
/** If the item is archived */
|
||||||
|
is_archived?: boolean
|
||||||
|
/** Item archive ID */
|
||||||
|
archive_id?: string | null
|
||||||
}
|
}
|
||||||
|
|
||||||
export type { ItemStatus, StoreItem }
|
export type { ItemStatus, StoreItem }
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue