danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix problem with processing playlist

Browse source
This commit is contained in:
arabcoders 2025-05-23 18:19:06 +03:00
parent 413707950e
commit 647acc488d
10 changed files with 203 additions and 222 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -270,7 +270,6 @@ Certain configuration values can be set via environment variables, using the `-e
| YTP_TEMP_KEEP | Whether to keep the Individual video temp directory or remove it | `false` |
| YTP_KEEP_ARCHIVE | Keep history of downloaded videos | `true` |
| YTP_YTDL_DEBUG | Whether to turn debug logging for the internal `yt-dlp` package | `false` |
| YTP_ALLOW_MANIFESTLESS | Allow `yt-dlp` to download unprocessed streams | `false` |
| YTP_HOST | Which IP address to bind to | `0.0.0.0` |
| YTP_PORT | Which port to bind to | `8081` |
| YTP_LOG_LEVEL | Log level | `info` |

9
app/library/Download.py
View file

@ -63,7 +63,7 @@ class Download:
"fragment_retries",
"skip_unavailable_fragments",
]
"Bad yt-dlp options which are known to cause issues with live stream and post manifestless mode."
"Bad yt-dlp options which are known to cause issues with live stream."
_ytdlp_fields: tuple = (
"tmpfilename",
@ -113,7 +113,6 @@ class Download:
self.max_workers = int(config.max_workers)
self.temp_keep = bool(config.temp_keep)
self.is_live = bool(info.is_live) or info.live_in is not None
self.is_manifestless = "is_manifestless" in self.info.options and self.info.options["is_manifestless"] is True
self.info_dict = info_dict
self.logger = logging.getLogger(f"Download.{info.id if info.id else info._id}")
self.started_time = 0
@ -225,7 +224,7 @@ class Download:
if info:
self.info_dict = info
if self.is_live or self.is_manifestless:
if self.is_live:
hasDeletedOptions = False
deletedOpts: list = []
for opt in self.bad_live_options:
@ -236,7 +235,7 @@ class Download:
if hasDeletedOptions:
self.logger.warning(
f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted which are known to cause issues with live stream and post stream manifestless mode."
f"Live stream detected for '{self.info.title}', The following opts '{deletedOpts=}' have been deleted."
)
if isinstance(self.info_dict, dict) and len(self.info_dict.get("formats", [])) < 1:
@ -263,7 +262,7 @@ class Download:
cls.process_ie_result(
ie_result=self.info_dict,
download=True,
extra_info={k: v for k, v in self.info.extras.items() if k.startswith("playlist")},
extra_info={k: v for k, v in self.info.extras.items() if k not in self.info_dict},
)
ret = cls._download_retcode
else:

339
app/library/DownloadQueue.py
View file

@ -193,7 +193,153 @@ class DownloadQueue(metaclass=Singleton):
except Exception as e:
LOG.error(f"Failed to cancel downloads. {e!s}")
async def __add_entry(self, entry: dict, item: Item, already=None, logs: list | None = None):
async def _process_playlist(self, entry: dict, item: Item, already=None):
LOG.info(f"Processing playlist '{entry.get('id')}: {entry.get('title')}'.")
entries = entry.get("entries", [])
playlistCount = int(entry.get("playlist_count", len(entries)))
results = []
for i, etr in enumerate(entries, start=1):
extras = {
"playlist": entry.get("id"),
"playlist_index": f"{{0:0{len(str(playlistCount))}d}}".format(i),
"playlist_autonumber": i,
}
for property in ("id", "title", "uploader", "uploader_id"):
if property in entry:
extras[f"playlist_{property}"] = entry.get(property)
if "thumbnail" not in etr and "youtube:" in entry.get("extractor", ""):
extras["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg"
results.append(
await self.add(
item=item.new_with(url=etr.get("url") or etr.get("webpage_url"), extras=extras),
already=already,
)
)
if any("error" == res["status"] for res in results):
return {
"status": "error",
"msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res),
}
return {"status": "ok"}
async def _add_video(self, entry: dict, item: Item, logs: list[str] | None = None):
if not logs:
logs: list[str] = []
options: dict = {}
error: str | None = None
live_in: str | None = None
# check if the video is live stream.
if "live_status" in entry and "is_upcoming" == entry.get("live_status"):
if entry.get("release_timestamp"):
live_in = formatdate(entry.get("release_timestamp"), usegmt=True)
item.extras.update({"live_in": live_in})
else:
error = "Live stream not yet started. And no date is set."
else:
error = entry.get("msg")
LOG.debug(f"Entry id '{entry.get('id')}' url '{entry.get('webpage_url')} - {entry.get('url')}'.")
try:
_item = self.done.get(key=entry.get("id"), url=entry.get("webpage_url") or entry.get("url"))
if _item is not None:
err_msg = f"Item '{_item.info.id}' - '{_item.info.title}' already exists. Removing from history."
LOG.warning(err_msg)
await self.clear([_item.info._id], remove_file=False)
except KeyError:
pass
try:
_item = self.queue.get(key=str(entry.get("id")), url=str(entry.get("webpage_url") or entry.get("url")))
if _item is not None:
err_msg = f"Item ID '{_item.info.id}' - '{_item.info.title}' already in download queue."
LOG.info(err_msg)
return {"status": "error", "msg": err_msg}
except KeyError:
pass
live_status: list = ["is_live", "is_upcoming"]
is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status)
try:
download_dir = calc_download_path(base_path=self.config.download_path, folder=item.folder)
except Exception as e:
LOG.exception(e)
return {"status": "error", "msg": str(e)}
for field in ("uploader", "channel", "thumbnail"):
if entry.get(field):
item.extras[field] = entry.get(field)
for key in entry:
if isinstance(key, str) and key.startswith("playlist") and entry.get(key):
item.extras[key] = entry.get(key)
dl = ItemDTO(
id=str(entry.get("id")),
title=str(entry.get("title")),
url=str(entry.get("webpage_url") or entry.get("url")),
preset=item.preset,
folder=item.folder,
download_dir=download_dir,
temp_dir=self.config.temp_path,
cookies=item.cookies,
template=item.template if item.template else self.config.output_template,
template_chapter=self.config.output_template_chapter,
datetime=formatdate(time.time()),
error=error,
is_live=is_live,
live_in=live_in if live_in else item.extras.get("live_in", None),
options=options,
cli=item.cli,
extras=item.extras,
)
try:
dlInfo: Download = Download(info=dl, info_dict=entry, logs=logs)
text_logs = ""
if filtered_logs := extract_ytdlp_logs(logs):
text_logs = f" Logs: {', '.join(filtered_logs)}"
if live_in or "is_upcoming" == entry.get("live_status"):
NotifyEvent = Events.COMPLETED
dlInfo.info.status = "not_live"
dlInfo.info.msg = "Stream is not live yet." + text_logs
itemDownload = self.done.put(dlInfo)
elif len(entry.get("formats", [])) < 1:
availability = entry.get("availability", "public")
msg = "No formats found."
if availability and availability not in ("public",):
msg += f" Availability is set for '{availability}'."
dlInfo.info.error = msg + text_logs
dlInfo.info.status = "error"
itemDownload = self.done.put(dlInfo)
NotifyEvent = Events.COMPLETED
await self._notify.emit(Events.LOG_WARNING, data=event_warning(msg))
else:
NotifyEvent = Events.ADDED
itemDownload = self.queue.put(dlInfo)
self.event.set()
await self._notify.emit(NotifyEvent, data=itemDownload.info.serialize())
return {"status": "ok"}
except Exception as e:
LOG.exception(e)
LOG.error(f"Failed to download item. '{e!s}'")
return {"status": "error", "msg": str(e)}
async def _add_item(self, entry: dict, item: Item, already=None, logs: list | None = None):
"""
Add an entry to the download queue.
@ -207,176 +353,21 @@ class DownloadQueue(metaclass=Singleton):
dict: The status of the operation.
"""
if not logs:
logs = []
if item.has_extras():
for key in item.extras.copy():
if not self.keep_extra_key(key):
item.extras.pop(key)
else:
item.extras = {}
if not entry:
return {"status": "error", "msg": "Invalid/empty data was given."}
options: dict = {}
event_type = entry.get("_type", "video")
error: str | None = None
live_in: str | None = None
if event_type.startswith("playlist"):
return await self._process_playlist(entry=entry, item=item, already=already)
eventType = entry.get("_type") or "video"
if "playlist" == eventType:
LOG.info(f"Processing playlist '{entry.get('id')}: {entry.get('title')}'.")
entries = entry.get("entries", [])
playlistCount = int(entry.get("playlist_count", len(entries)))
results = []
for i, etr in enumerate(entries, start=1):
etr["playlist"] = entry.get("id")
etr["playlist_index"] = f"{{0:0{len(str(playlistCount)):d}d}}".format(i)
etr["playlist_autonumber"] = i
for property in ("id", "title", "uploader", "uploader_id"):
if property in entry:
etr[f"playlist_{property}"] = entry.get(property)
if "thumbnail" not in etr and "youtube:" in entry.get("extractor", ""):
etr["thumbnail"] = f"https://img.youtube.com/vi/{etr['id']}/maxresdefault.jpg"
results.append(
await self.__add_entry(
entry=etr,
item=item.new_with(url=etr.get("url") or etr.get("webpage_url")),
already=already,
)
)
if any("error" == res["status"] for res in results):
return {
"status": "error",
"msg": ", ".join(res["msg"] for res in results if res["status"] == "error" and "msg" in res),
}
return {"status": "ok"}
if ("video" == eventType or eventType.startswith("url")) and "id" in entry and "title" in entry:
# check if the video is live stream.
if "live_status" in entry and "is_upcoming" == entry.get("live_status"):
if entry.get("release_timestamp"):
live_in = formatdate(entry.get("release_timestamp"), usegmt=True)
item.extras.update({"live_in": live_in})
else:
error = "Live stream not yet started. And no date is set."
else:
error = entry.get("msg")
LOG.debug(f"Entry id '{entry.get('id')}' url '{entry.get('webpage_url')} - {entry.get('url')}'.")
try:
_item = self.done.get(key=entry.get("id"), url=entry.get("webpage_url") or entry.get("url"))
if _item is not None:
err_msg = f"Item '{_item.info.id}' - '{_item.info.title}' already exists. Removing from history."
LOG.warning(err_msg)
await self.clear([_item.info._id], remove_file=False)
except KeyError:
pass
try:
_item = self.queue.get(key=str(entry.get("id")), url=str(entry.get("webpage_url") or entry.get("url")))
if _item is not None:
err_msg = f"Item ID '{_item.info.id}' - '{_item.info.title}' already in download queue."
LOG.info(err_msg)
return {"status": "error", "msg": err_msg}
except KeyError:
pass
is_manifestless = entry.get("is_manifestless", False)
options.update({"is_manifestless": is_manifestless})
live_status: list = ["is_live", "is_upcoming"]
is_live = bool(entry.get("is_live") or live_in or entry.get("live_status") in live_status)
try:
download_dir = calc_download_path(base_path=self.config.download_path, folder=item.folder)
except Exception as e:
LOG.exception(e)
return {"status": "error", "msg": str(e)}
for field in ("uploader", "channel", "thumbnail"):
if entry.get(field):
item.extras[field] = entry.get(field)
for key in entry:
if isinstance(key, str) and key.startswith("playlist") and entry.get(key):
item.extras[key] = entry.get(key)
dl = ItemDTO(
id=str(entry.get("id")),
title=str(entry.get("title")),
url=str(entry.get("webpage_url") or entry.get("url")),
preset=item.preset,
folder=item.folder,
download_dir=download_dir,
temp_dir=self.config.temp_path,
cookies=item.cookies,
template=item.template if item.template else self.config.output_template,
template_chapter=self.config.output_template_chapter,
datetime=formatdate(time.time()),
error=error,
is_live=is_live,
live_in=live_in if live_in else item.extras.get("live_in", None),
options=options,
cli=item.cli,
extras=item.extras,
)
try:
dlInfo: Download = Download(info=dl, info_dict=entry, logs=logs)
text_logs = ""
if filtered_logs := extract_ytdlp_logs(logs):
text_logs = f" Logs: {', '.join(filtered_logs)}"
if live_in or "is_upcoming" == entry.get("live_status"):
NotifyEvent = Events.COMPLETED
dlInfo.info.status = "not_live"
dlInfo.info.msg = "Stream is not live yet." + text_logs
itemDownload = self.done.put(dlInfo)
elif len(entry.get("formats", [])) < 1:
availability = entry.get("availability", "public")
msg = "No formats found."
if availability and availability not in ("public",):
msg += f" Availability is set for '{availability}'."
dlInfo.info.error = msg + text_logs
dlInfo.info.status = "error"
itemDownload = self.done.put(dlInfo)
NotifyEvent = Events.COMPLETED
await self._notify.emit(Events.LOG_WARNING, data=event_warning(msg))
elif self.config.allow_manifestless is False and is_manifestless is True:
dlInfo.info.status = "error"
dlInfo.info.error = "Video is in post-live manifestless mode." + text_logs
itemDownload = self.done.put(dlInfo)
NotifyEvent = Events.COMPLETED
else:
NotifyEvent = Events.ADDED
itemDownload = self.queue.put(dlInfo)
self.event.set()
await self._notify.emit(NotifyEvent, data=itemDownload.info.serialize())
return {"status": "ok"}
except Exception as e:
LOG.exception(e)
LOG.error(f"Failed to download item. '{e!s}'")
return {"status": "error", "msg": str(e)}
if eventType.startswith("url"):
if event_type.startswith("url"):
return await self.add(item=item.new_with(url=entry.get("url")), already=already)
return {"status": "error", "msg": f'Unsupported resource "{eventType}"'}
if not event_type.startswith("video"):
return {"status": "error", "msg": f'Unsupported event type "{event_type}".'}
return await self._add_video(entry=entry, item=item, logs=logs)
async def add(self, item: Item, already: set | None = None):
"""
@ -485,7 +476,7 @@ class DownloadQueue(metaclass=Singleton):
if condition is not None:
already.pop()
LOG.info(f"Condition '{condition}' matched for '{item.url}'.")
return await self.add(item=item.new_with(requeued=True, cli=condition.cli),already=already)
return await self.add(item=item.new_with(requeued=True, cli=condition.cli), already=already)
end_time = time.perf_counter() - started
LOG.debug(f"extract_info: for 'URL: {item.url}' is done in '{end_time:.3f}'. Length: '{len(entry)}/keys'.")
@ -504,12 +495,12 @@ class DownloadQueue(metaclass=Singleton):
finally:
if cookie_file and os.path.exists(cookie_file):
try:
os.remove(yt_conf["cookiefile"])
os.remove(cookie_file)
del yt_conf["cookiefile"]
except Exception as e:
LOG.error(f"Failed to remove cookie file '{yt_conf['cookiefile']}'. {e!s}")
return await self.__add_entry(entry=entry, item=item, already=already, logs=logs)
return await self._add_item(entry=entry, item=item, already=already, logs=logs)
async def cancel(self, ids: list[str]) -> dict[str, str]:
"""
@ -767,20 +758,6 @@ class DownloadQueue(metaclass=Singleton):
return is_downloaded(file, url)
def keep_extra_key(self, key: str) -> bool:
"""
Check if the extra key should be kept.
Args:
key (str): The extra key to check.
Returns:
bool: True if the extra key should be kept, False otherwise.
"""
keys = ("playlist", "external_downloader", "live_in")
return any(key == k or key.startswith(k) for k in keys)
async def monitor_stale(self):
"""
Monitor the queue and pool for stale downloads and cancel them if needed.

36
app/library/HttpAPI.py
View file

@ -268,6 +268,8 @@ class HttpAPI(Common):
HttpAPI: The instance of the HttpAPI.
"""
registered_options = []
for attr_name in dir(self):
method = getattr(self, attr_name)
if hasattr(method, "_http_method") and hasattr(method, "_http_path"):
@ -277,6 +279,15 @@ class HttpAPI(Common):
self.routes.route(method._http_method, f"/{http_path}")(method)
if http_path in registered_options:
continue
async def options_handler(_: Request) -> Response:
return web.Response(status=204)
self.routes.route("OPTIONS", f"/{http_path}")(options_handler)
registered_options.append(http_path)
self.routes.static("/api/download/", self.config.download_path)
self._preload_static(app)
@ -391,7 +402,16 @@ class HttpAPI(Common):
},
)
response = await handler(request)
try:
response = await handler(request)
except web.HTTPException as e:
return web.json_response(data={"error": str(e)}, status=e.status_code)
except Exception as e:
LOG.exception(e)
response = web.json_response(
data={"error": "Internal Server Error"},
status=web.HTTPInternalServerError.status_code,
)
if isinstance(response, web.FileResponse):
try:
@ -405,20 +425,6 @@ class HttpAPI(Common):
return middleware_handler
@route("OPTIONS", "/{path:.*}")
async def add_coors(self, _: Request) -> Response:
"""
Add CORS headers to the response.
Args:
_: The request object.
Returns:
Response: The response object.
"""
return web.json_response(data={"status": "ok"}, status=web.HTTPOk.status_code)
@route("GET", "api/ping")
async def ping(self, _: Request) -> Response:
"""

1
app/library/ItemDTO.py
View file

@ -68,6 +68,7 @@ class Item:
"""
return self.cli and len(self.cli) > 2
@staticmethod
def _default_preset() -> str:
from .config import Config

29
app/library/Utils.py
View file

@ -941,7 +941,7 @@ async def tail_log(file: str, emitter: callable, sleep_time: float = 0.5):
from anyio import open_file
if not pathlib(file).exists():
if not pathlib.Path(file).exists():
return
async with await open_file(file, "rb") as f:
@ -1020,20 +1020,23 @@ def get_archive_id(url: str) -> tuple[bool, dict[str | None, str | None, str | N
)
for key, ie in YTDLP_INFO_CLS._ies.items():
if not ie.suitable(url):
continue
try:
if not ie.suitable(url):
continue
if not ie.working():
if not ie.working():
break
temp_id = ie.get_temp_id(url)
if not temp_id:
break
idDict["id"] = temp_id
idDict["ie_key"] = key
idDict["archive_id"] = YTDLP_INFO_CLS._make_archive_id(idDict)
break
temp_id = ie.get_temp_id(url)
if not temp_id:
break
idDict["id"] = temp_id
idDict["ie_key"] = key
idDict["archive_id"] = YTDLP_INFO_CLS._make_archive_id(idDict)
break
except Exception as e:
LOG.error(f"Error getting archive ID: {e}")
return idDict

2
app/library/YTDLPOpts.py
View file

@ -131,7 +131,7 @@ class YTDLPOpts(metaclass=Singleton):
with open(file, "w") as f:
f.write(preset.cookies)
load_cookies(file)
load_cookies(str(file))
self._preset_opts["cookiefile"] = str(file)

4
app/library/config.py
View file

@ -43,9 +43,6 @@ class Config:
ytdl_debug: bool = False
"""Enable yt-dlp debugging."""
allow_manifestless: bool = False
"""Allow downloading videos without manifest."""
host: str = "0.0.0.0"
"""The host to bind the server to."""
@ -194,7 +191,6 @@ class Config:
"ytdl_debug",
"debug",
"temp_keep",
"allow_manifestless",
"access_log",
"remove_files",
"ignore_ui",

2
ui/components/History.vue
View file

@ -639,7 +639,7 @@ const reQueueItem = (item, event = null) => {
cookies: item.cookies,
template: item.template,
cli: item?.cli,
extras: item?.extras && Object.keys(item.extras) > 0 ? item.extras : {},
extras: toRaw(item.extras ?? {}) ?? {},
};
socket.emit('item_delete', { id: item._id, remove_file: false })

2
ui/pages/tasks.vue
View file

@ -81,7 +81,7 @@ div.is-centered {
<span class="icon"><i class="fa-solid fa-tv" /></span>
<span>{{ item.preset ?? config.app.default_preset }}</span>
</p>
<p class="is-text-overflow">
<p class="is-text-overflow" v-if="item.cli">
<span class="icon"><i class="fa-solid fa-terminal" /></span>
<span>{{ item.cli }}</span>
</p>