diff --git a/README.md b/README.md
index a236a028..ff9f493a 100644
--- a/README.md
+++ b/README.md
@@ -270,7 +270,6 @@ Certain configuration values can be set via environment variables, using the `-e
| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` |
| YTP_KEEP_ARCHIVE | Keep history of downloaded videos | `true` |
| YTP_YTDL_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` |
-| YTP_ALLOW_MANIFESTLESS | Allow `yt-dlp` to download unprocessed streams | `false` |
| YTP_HOST | Which IP address to bind to | `0.0.0.0` |
| YTP_PORT | Which port to bind to | `8081` |
| YTP_LOG_LEVEL | Log level | `info` |
diff --git a/app/library/Download.py b/app/library/Download.py
index fc705f93..7bfaf4b4 100644
--- a/app/library/Download.py
+++ b/app/library/Download.py
@@ -63,7 +63,7 @@ class Download:
"fragment_retries",
"skip_unavailable_fragments",
]
- "Bad yt-dlp options which are known to cause issues with live stream and post manifestless mode."
+ "Bad yt-dlp options which are known to cause issues with live stream."
_ytdlp_fields: tuple = (
"tmpfilename",
@@ -113,7 +113,6 @@ class Download:
self.max_workers = int(config.max_workers)
self.temp_keep = bool(config.temp_keep)
self.is_live = bool(info.is_live) or info.live_in is not None
- self.is_manifestless = "is_manifestless" in self.info.options and self.info.options["is_manifestless"] is True
self.info_dict = info_dict
self.logger = logging.getLogger(f"Download.{info.id if info.id else info._id}")
self.started_time = 0
@@ -225,7 +224,7 @@ class Download:
if info:
self.info_dict = info
- if self.is_live or self.is_manifestless:
+ if self.is_live:
hasDeletedOptions = False
deletedOpts: list = []
for opt in self.bad_live_options:
@@ -236,7 +235,7 @@ class Download:
if hasDeletedOptions:
self.logger.warning(
- f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted which are known to cause issues with live stream and post stream manifestless mode."
+ f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted."
)
if isinstance(self.info_dict, dict) and len(self.info_dict.get("formats", [])) < 1:
@@ -263,7 +262,7 @@ class Download:
cls.process_ie_result(
ie_result=self.info_dict,
download=True,
- extra_info={k: v for k, v in self.info.extras.items() if k.startswith("playlist")},
+ extra_info={k: v for k, v in self.info.extras.items() if k not in self.info_dict},
)
ret = cls._download_retcode
else:
diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py
index 18c1d11b..fc15b78f 100644
--- a/app/library/DownloadQueue.py
+++ b/app/library/DownloadQueue.py
@@ -193,7 +193,153 @@ class DownloadQueue(metaclass=Singleton):
except Exception as e:
LOG.error(f"Failed to cancel downloads. {e!s}")
- async def __add_entry(self, entry: dict, item: Item, already=None, logs: list | None = None):
+ async def _process_playlist(self, entry: dict, item: Item, already=None):
+ LOG.info(f"Processing playlist '{entry.get('id')}: {entry.get('title')}'.")
+ entries = entry.get("entries", [])
+ playlistCount = int(entry.get("playlist_count", len(entries)))
+ results = []
+
+ for i, etr in enumerate(entries, start=1):
+ extras = {
+ "playlist": entry.get("id"),
+ "playlist_index": f"{{0:0{len(str(playlistCount))}d}}".format(i),
+ "playlist_autonumber": i,
+ }
+
+ for property in ("id", "title", "uploader", "uploader_id"):
+ if property in entry:
+ extras[f"playlist_{property}"] = entry.get(property)
+
+ if "thumbnail" not in etr and "youtube:" in entry.get("extractor", ""):
+ extras["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg"
+
+ results.append(
+ await self.add(
+ item=item.new_with(url=etr.get("url") or etr.get("webpage_url"), extras=extras),
+ already=already,
+ )
+ )
+
+ if any("error" == res["status"] for res in results):
+ return {
+ "status": "error",
+ "msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res),
+ }
+
+ return {"status": "ok"}
+
+ async def _add_video(self, entry: dict, item: Item, logs: list[str] | None = None):
+ if not logs:
+ logs: list[str] = []
+
+ options: dict = {}
+ error: str | None = None
+ live_in: str | None = None
+
+ # check if the video is live stream.
+ if "live_status" in entry and "is_upcoming" == entry.get("live_status"):
+ if entry.get("release_timestamp"):
+ live_in = formatdate(entry.get("release_timestamp"), usegmt=True)
+ item.extras.update({"live_in": live_in})
+ else:
+ error = "Live stream not yet started. And no date is set."
+ else:
+ error = entry.get("msg")
+
+ LOG.debug(f"Entry id '{entry.get('id')}' url '{entry.get('webpage_url')} - {entry.get('url')}'.")
+
+ try:
+ _item = self.done.get(key=entry.get("id"), url=entry.get("webpage_url") or entry.get("url"))
+ if _item is not None:
+ err_msg = f"Item '{_item.info.id}' - '{_item.info.title}' already exists. Removing from history."
+ LOG.warning(err_msg)
+ await self.clear([_item.info._id], remove_file=False)
+ except KeyError:
+ pass
+
+ try:
+ _item = self.queue.get(key=str(entry.get("id")), url=str(entry.get("webpage_url") or entry.get("url")))
+ if _item is not None:
+ err_msg = f"Item ID '{_item.info.id}' - '{_item.info.title}' already in download queue."
+ LOG.info(err_msg)
+ return {"status": "error", "msg": err_msg}
+ except KeyError:
+ pass
+
+ live_status: list = ["is_live", "is_upcoming"]
+ is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status)
+
+ try:
+ download_dir = calc_download_path(base_path=self.config.download_path, folder=item.folder)
+ except Exception as e:
+ LOG.exception(e)
+ return {"status": "error", "msg": str(e)}
+
+ for field in ("uploader", "channel", "thumbnail"):
+ if entry.get(field):
+ item.extras[field] = entry.get(field)
+
+ for key in entry:
+ if isinstance(key, str) and key.startswith("playlist") and entry.get(key):
+ item.extras[key] = entry.get(key)
+
+ dl = ItemDTO(
+ id=str(entry.get("id")),
+ title=str(entry.get("title")),
+ url=str(entry.get("webpage_url") or entry.get("url")),
+ preset=item.preset,
+ folder=item.folder,
+ download_dir=download_dir,
+ temp_dir=self.config.temp_path,
+ cookies=item.cookies,
+ template=item.template if item.template else self.config.output_template,
+ template_chapter=self.config.output_template_chapter,
+ datetime=formatdate(time.time()),
+ error=error,
+ is_live=is_live,
+ live_in=live_in if live_in else item.extras.get("live_in", None),
+ options=options,
+ cli=item.cli,
+ extras=item.extras,
+ )
+
+ try:
+ dlInfo: Download = Download(info=dl, info_dict=entry, logs=logs)
+
+ text_logs = ""
+ if filtered_logs := extract_ytdlp_logs(logs):
+ text_logs = f" Logs: {', '.join(filtered_logs)}"
+
+ if live_in or "is_upcoming" == entry.get("live_status"):
+ NotifyEvent = Events.COMPLETED
+ dlInfo.info.status = "not_live"
+ dlInfo.info.msg = "Stream is not live yet." + text_logs
+ itemDownload = self.done.put(dlInfo)
+ elif len(entry.get("formats", [])) < 1:
+ availability = entry.get("availability", "public")
+ msg = "No formats found."
+ if availability and availability not in ("public",):
+ msg += f" Availability is set for '{availability}'."
+
+ dlInfo.info.error = msg + text_logs
+ dlInfo.info.status = "error"
+ itemDownload = self.done.put(dlInfo)
+ NotifyEvent = Events.COMPLETED
+ await self._notify.emit(Events.LOG_WARNING, data=event_warning(msg))
+ else:
+ NotifyEvent = Events.ADDED
+ itemDownload = self.queue.put(dlInfo)
+ self.event.set()
+
+ await self._notify.emit(NotifyEvent, data=itemDownload.info.serialize())
+
+ return {"status": "ok"}
+ except Exception as e:
+ LOG.exception(e)
+ LOG.error(f"Failed to download item. '{e!s}'")
+ return {"status": "error", "msg": str(e)}
+
+ async def _add_item(self, entry: dict, item: Item, already=None, logs: list | None = None):
"""
Add an entry to the download queue.
@@ -207,176 +353,21 @@ class DownloadQueue(metaclass=Singleton):
dict: The status of the operation.
"""
- if not logs:
- logs = []
-
- if item.has_extras():
- for key in item.extras.copy():
- if not self.keep_extra_key(key):
- item.extras.pop(key)
- else:
- item.extras = {}
-
if not entry:
return {"status": "error", "msg": "Invalid/empty data was given."}
- options: dict = {}
+ event_type = entry.get("_type", "video")
- error: str | None = None
- live_in: str | None = None
+ if event_type.startswith("playlist"):
+ return await self._process_playlist(entry=entry, item=item, already=already)
- eventType = entry.get("_type") or "video"
-
- if "playlist" == eventType:
- LOG.info(f"Processing playlist '{entry.get('id')}: {entry.get('title')}'.")
- entries = entry.get("entries", [])
- playlistCount = int(entry.get("playlist_count", len(entries)))
- results = []
-
- for i, etr in enumerate(entries, start=1):
- etr["playlist"] = entry.get("id")
- etr["playlist_index"] = f"{{0:0{len(str(playlistCount)):d}d}}".format(i)
- etr["playlist_autonumber"] = i
-
- for property in ("id", "title", "uploader", "uploader_id"):
- if property in entry:
- etr[f"playlist_{property}"] = entry.get(property)
-
- if "thumbnail" not in etr and "youtube:" in entry.get("extractor", ""):
- etr["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg"
-
- results.append(
- await self.__add_entry(
- entry=etr,
- item=item.new_with(url=etr.get("url") or etr.get("webpage_url")),
- already=already,
- )
- )
-
- if any("error" == res["status"] for res in results):
- return {
- "status": "error",
- "msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res),
- }
-
- return {"status": "ok"}
-
- if ("video" == eventType or eventType.startswith("url")) and "id" in entry and "title" in entry:
- # check if the video is live stream.
- if "live_status" in entry and "is_upcoming" == entry.get("live_status"):
- if entry.get("release_timestamp"):
- live_in = formatdate(entry.get("release_timestamp"), usegmt=True)
- item.extras.update({"live_in": live_in})
- else:
- error = "Live stream not yet started. And no date is set."
- else:
- error = entry.get("msg")
-
- LOG.debug(f"Entry id '{entry.get('id')}' url '{entry.get('webpage_url')} - {entry.get('url')}'.")
-
- try:
- _item = self.done.get(key=entry.get("id"), url=entry.get("webpage_url") or entry.get("url"))
- if _item is not None:
- err_msg = f"Item '{_item.info.id}' - '{_item.info.title}' already exists. Removing from history."
- LOG.warning(err_msg)
- await self.clear([_item.info._id], remove_file=False)
- except KeyError:
- pass
-
- try:
- _item = self.queue.get(key=str(entry.get("id")), url=str(entry.get("webpage_url") or entry.get("url")))
- if _item is not None:
- err_msg = f"Item ID '{_item.info.id}' - '{_item.info.title}' already in download queue."
- LOG.info(err_msg)
- return {"status": "error", "msg": err_msg}
- except KeyError:
- pass
-
- is_manifestless = entry.get("is_manifestless", False)
- options.update({"is_manifestless": is_manifestless})
-
- live_status: list = ["is_live", "is_upcoming"]
- is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status)
-
- try:
- download_dir = calc_download_path(base_path=self.config.download_path, folder=item.folder)
- except Exception as e:
- LOG.exception(e)
- return {"status": "error", "msg": str(e)}
-
- for field in ("uploader", "channel", "thumbnail"):
- if entry.get(field):
- item.extras[field] = entry.get(field)
-
- for key in entry:
- if isinstance(key, str) and key.startswith("playlist") and entry.get(key):
- item.extras[key] = entry.get(key)
-
- dl = ItemDTO(
- id=str(entry.get("id")),
- title=str(entry.get("title")),
- url=str(entry.get("webpage_url") or entry.get("url")),
- preset=item.preset,
- folder=item.folder,
- download_dir=download_dir,
- temp_dir=self.config.temp_path,
- cookies=item.cookies,
- template=item.template if item.template else self.config.output_template,
- template_chapter=self.config.output_template_chapter,
- datetime=formatdate(time.time()),
- error=error,
- is_live=is_live,
- live_in=live_in if live_in else item.extras.get("live_in", None),
- options=options,
- cli=item.cli,
- extras=item.extras,
- )
-
- try:
- dlInfo: Download = Download(info=dl, info_dict=entry, logs=logs)
- text_logs = ""
- if filtered_logs := extract_ytdlp_logs(logs):
- text_logs = f" Logs: {', '.join(filtered_logs)}"
-
- if live_in or "is_upcoming" == entry.get("live_status"):
- NotifyEvent = Events.COMPLETED
- dlInfo.info.status = "not_live"
- dlInfo.info.msg = "Stream is not live yet." + text_logs
- itemDownload = self.done.put(dlInfo)
- elif len(entry.get("formats", [])) < 1:
- availability = entry.get("availability", "public")
- msg = "No formats found."
- if availability and availability not in ("public",):
- msg += f" Availability is set for '{availability}'."
-
- dlInfo.info.error = msg + text_logs
- dlInfo.info.status = "error"
- itemDownload = self.done.put(dlInfo)
- NotifyEvent = Events.COMPLETED
- await self._notify.emit(Events.LOG_WARNING, data=event_warning(msg))
- elif self.config.allow_manifestless is False and is_manifestless is True:
- dlInfo.info.status = "error"
- dlInfo.info.error = "Video is in post-live manifestless mode." + text_logs
-
- itemDownload = self.done.put(dlInfo)
- NotifyEvent = Events.COMPLETED
- else:
- NotifyEvent = Events.ADDED
- itemDownload = self.queue.put(dlInfo)
- self.event.set()
-
- await self._notify.emit(NotifyEvent, data=itemDownload.info.serialize())
-
- return {"status": "ok"}
- except Exception as e:
- LOG.exception(e)
- LOG.error(f"Failed to download item. '{e!s}'")
- return {"status": "error", "msg": str(e)}
-
- if eventType.startswith("url"):
+ if event_type.startswith("url"):
return await self.add(item=item.new_with(url=entry.get("url")), already=already)
- return {"status": "error", "msg": f'Unsupported resource "{eventType}"'}
+ if not event_type.startswith("video"):
+ return {"status": "error", "msg": f'Unsupported event type "{event_type}".'}
+
+ return await self._add_video(entry=entry, item=item, logs=logs)
async def add(self, item: Item, already: set | None = None):
"""
@@ -485,7 +476,7 @@ class DownloadQueue(metaclass=Singleton):
if condition is not None:
already.pop()
LOG.info(f"Condition '{condition}' matched for '{item.url}'.")
- return await self.add(item=item.new_with(requeued=True, cli=condition.cli),already=already)
+ return await self.add(item=item.new_with(requeued=True, cli=condition.cli), already=already)
end_time = time.perf_counter() - started
LOG.debug(f"extract_info: for 'URL: {item.url}' is done in '{end_time:.3f}'. Length: '{len(entry)}/keys'.")
@@ -504,12 +495,12 @@ class DownloadQueue(metaclass=Singleton):
finally:
if cookie_file and os.path.exists(cookie_file):
try:
- os.remove(yt_conf["cookiefile"])
+ os.remove(cookie_file)
del yt_conf["cookiefile"]
except Exception as e:
LOG.error(f"Failed to remove cookie file '{yt_conf['cookiefile']}'. {e!s}")
- return await self.__add_entry(entry=entry, item=item, already=already, logs=logs)
+ return await self._add_item(entry=entry, item=item, already=already, logs=logs)
async def cancel(self, ids: list[str]) -> dict[str, str]:
"""
@@ -767,20 +758,6 @@ class DownloadQueue(metaclass=Singleton):
return is_downloaded(file, url)
- def keep_extra_key(self, key: str) -> bool:
- """
- Check if the extra key should be kept.
-
- Args:
- key (str): The extra key to check.
-
- Returns:
- bool: True if the extra key should be kept, False otherwise.
-
- """
- keys = ("playlist", "external_downloader", "live_in")
- return any(key == k or key.startswith(k) for k in keys)
-
async def monitor_stale(self):
"""
Monitor the queue and pool for stale downloads and cancel them if needed.
diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py
index d824354c..00337da0 100644
--- a/app/library/HttpAPI.py
+++ b/app/library/HttpAPI.py
@@ -268,6 +268,8 @@ class HttpAPI(Common):
HttpAPI: The instance of the HttpAPI.
"""
+ registered_options = []
+
for attr_name in dir(self):
method = getattr(self, attr_name)
if hasattr(method, "_http_method") and hasattr(method, "_http_path"):
@@ -277,6 +279,15 @@ class HttpAPI(Common):
self.routes.route(method._http_method, f"/{http_path}")(method)
+ if http_path in registered_options:
+ continue
+
+ async def options_handler(_: Request) -> Response:
+ return web.Response(status=204)
+
+ self.routes.route("OPTIONS", f"/{http_path}")(options_handler)
+ registered_options.append(http_path)
+
self.routes.static("/api/download/", self.config.download_path)
self._preload_static(app)
@@ -391,7 +402,16 @@ class HttpAPI(Common):
},
)
- response = await handler(request)
+ try:
+ response = await handler(request)
+ except web.HTTPException as e:
+ return web.json_response(data={"error": str(e)}, status=e.status_code)
+ except Exception as e:
+ LOG.exception(e)
+ response = web.json_response(
+ data={"error": "Internal Server Error"},
+ status=web.HTTPInternalServerError.status_code,
+ )
if isinstance(response, web.FileResponse):
try:
@@ -405,20 +425,6 @@ class HttpAPI(Common):
return middleware_handler
- @route("OPTIONS", "/{path:.*}")
- async def add_coors(self, _: Request) -> Response:
- """
- Add CORS headers to the response.
-
- Args:
- _: The request object.
-
- Returns:
- Response: The response object.
-
- """
- return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code)
-
@route("GET", "api/ping")
async def ping(self, _: Request) -> Response:
"""
diff --git a/app/library/ItemDTO.py b/app/library/ItemDTO.py
index 7934aa8f..cd90223f 100644
--- a/app/library/ItemDTO.py
+++ b/app/library/ItemDTO.py
@@ -68,6 +68,7 @@ class Item:
"""
return self.cli and len(self.cli) > 2
+ @staticmethod
def _default_preset() -> str:
from .config import Config
diff --git a/app/library/Utils.py b/app/library/Utils.py
index e901f3a3..cc5f5018 100644
--- a/app/library/Utils.py
+++ b/app/library/Utils.py
@@ -941,7 +941,7 @@ async def tail_log(file: str, emitter: callable, sleep_time: float = 0.5):
from anyio import open_file
- if not pathlib(file).exists():
+ if not pathlib.Path(file).exists():
return
async with await open_file(file, "rb") as f:
@@ -1020,20 +1020,23 @@ def get_archive_id(url: str) -> tuple[bool, dict[str | None, str | None, str | N
)
for key, ie in YTDLP_INFO_CLS._ies.items():
- if not ie.suitable(url):
- continue
+ try:
+ if not ie.suitable(url):
+ continue
- if not ie.working():
+ if not ie.working():
+ break
+
+ temp_id = ie.get_temp_id(url)
+ if not temp_id:
+ break
+
+ idDict["id"] = temp_id
+ idDict["ie_key"] = key
+ idDict["archive_id"] = YTDLP_INFO_CLS._make_archive_id(idDict)
break
-
- temp_id = ie.get_temp_id(url)
- if not temp_id:
- break
-
- idDict["id"] = temp_id
- idDict["ie_key"] = key
- idDict["archive_id"] = YTDLP_INFO_CLS._make_archive_id(idDict)
- break
+ except Exception as e:
+ LOG.error(f"Error getting archive ID: {e}")
return idDict
diff --git a/app/library/YTDLPOpts.py b/app/library/YTDLPOpts.py
index 4c4c1894..da61ea70 100644
--- a/app/library/YTDLPOpts.py
+++ b/app/library/YTDLPOpts.py
@@ -131,7 +131,7 @@ class YTDLPOpts(metaclass=Singleton):
with open(file, "w") as f:
f.write(preset.cookies)
- load_cookies(file)
+ load_cookies(str(file))
self._preset_opts["cookiefile"] = str(file)
diff --git a/app/library/config.py b/app/library/config.py
index da933847..805dc11a 100644
--- a/app/library/config.py
+++ b/app/library/config.py
@@ -43,9 +43,6 @@ class Config:
ytdl_debug: bool = False
"""Enable yt-dlp debugging."""
- allow_manifestless: bool = False
- """Allow downloading videos without manifest."""
-
host: str = "0.0.0.0"
"""The host to bind the server to."""
@@ -194,7 +191,6 @@ class Config:
"ytdl_debug",
"debug",
"temp_keep",
- "allow_manifestless",
"access_log",
"remove_files",
"ignore_ui",
diff --git a/ui/components/History.vue b/ui/components/History.vue
index 3e6bb0fe..a5954de4 100644
--- a/ui/components/History.vue
+++ b/ui/components/History.vue
@@ -639,7 +639,7 @@ const reQueueItem = (item, event = null) => {
cookies: item.cookies,
template: item.template,
cli: item?.cli,
- extras: item?.extras && Object.keys(item.extras) > 0 ? item.extras : {},
+ extras: toRaw(item.extras ?? {}) ?? {},
};
socket.emit('item_delete', { id: item._id, remove_file: false })
diff --git a/ui/pages/tasks.vue b/ui/pages/tasks.vue
index 2cf46b27..0f965f67 100644
--- a/ui/pages/tasks.vue
+++ b/ui/pages/tasks.vue
@@ -81,7 +81,7 @@ div.is-centered {
{{ item.preset ?? config.app.default_preset }}
-
+
{{ item.cli }}