diff --git a/FAQ.md b/FAQ.md index 8db37eaa..5ba877b9 100644 --- a/FAQ.md +++ b/FAQ.md @@ -47,6 +47,7 @@ or the `environment:` section in `compose.yaml` file. | YTP_ALLOW_INTERNAL_URLS | Allow requests to internal URLs | `false` | | YTP_SIMPLE_MODE | Switch default interface to Simple mode. | `false` | | YTP_STATIC_UI_PATH | Path to custom static UI files. | `(not_set)` | +| YTP_AUTO_CLEAR_HISTORY_DAYS | Number of days after which completed download history is cleared. | `0` | > [!NOTE] > To raise the maximum workers for specific extractor, you need to add a ENV variable that follows the pattern `YTP_MAX_WORKERS_FOR_`. @@ -54,8 +55,12 @@ or the `environment:` section in `compose.yaml` file. > The limit should not exceed the `YTP_MAX_WORKERS` value as it will be ignored. > [!IMPORTANT] -> The env variable `YTP_SIMPLE_MODE` only control what being displayed for first time visitor, the users can still switch between the two modes via the WebUI settings page. +> The env variable `YTP_SIMPLE_MODE` only control what being displayed for first time visitor, the users can still switch between the two modes via the WebUI settings page. +## Notes about YTP_AUTO_CLEAR_HISTORY_DAYS + +- `0` days means no automatic clearing of the download history. lowest value that will trigger the clearing is `1` day. +- This setting will **NOT** delete the downloaded files, it will only clear the history from the database. # Browser extensions & bookmarklets @@ -507,3 +512,40 @@ Thats it, the `main.yml` will now disable the docker/github container registries naming, your container name will be named `REGISTRY/ytptube` and the tags will be the same as the ones used in the github registry. Unfortunately, the `native-builder.yml` workflow doesn't support self-hosted repositories at the moment. + +# Getting No space left on device error + +If you encounter this error: `OSError: [Errno 28] No space left on device` This indicates that either +the `/tmp` or `/downloads` directory has run out of available space. + +This issue commonly occurs when: + +- `/tmp` is mounted as `tmpfs` (memory-based storage) +- Your system has limited RAM +- You're downloading large video files + +Since videos are temporarily stored in `/tmp` before being moved to the final download location, memory-based storage +may be insufficient for large downloads. + +To fix the issue, modify your `compose.yaml` to use a disk-based directory for temporary files: + +```yaml +services: + ytptube: + user: "${UID:-1000}:${UID:-1000}" + image: ghcr.io/arabcoders/ytptube:latest + container_name: ytptube + restart: unless-stopped + ports: + - "8081:8081" + volumes: + - ./config:/config:rw + - ./downloads:/downloads/local:rw + - ./temp:/tmp:rw +``` + +> [!NOTE] +> Replace the `tmpfs` mount with a local directory volume (`./temp:/tmp:rw`). This allows temporary files to use disk space instead of RAM. + +After making the changes, restart your container. This should resolve the "No space left on device" +error during download. diff --git a/README.md b/README.md index e3c19ecc..30619b10 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,7 @@ services: > [!IMPORTANT] > Make sure to change the `user` line to match your user id and group id +> If you have low RAM, remove the `tmpfs` and mount a disk-based directory to `/tmp` instead. See [FAQ](FAQ.md#getting-no-space-left-on-device-error) for more information. ```bash $ mkdir -p ./{config,downloads} && docker compose -f compose.yaml up -d diff --git a/app/library/DownloadQueue.py b/app/library/DownloadQueue.py index b5c38786..68b02946 100644 --- a/app/library/DownloadQueue.py +++ b/app/library/DownloadQueue.py @@ -140,6 +140,15 @@ class DownloadQueue(metaclass=Singleton): func=self._check_live, id=f"{__class__.__name__}.{__class__._check_live.__name__}", ) + + if self.config.auto_clear_history_days > 0: + Scheduler.get_instance().add( + # timer="8 */1 * * *", + timer="* * * * *", + func=self._delete_old_history, + id=f"{__class__.__name__}.{__class__._delete_old_history.__name__}", + ) + # app.on_shutdown.append(self.on_shutdown) async def test(self) -> bool: @@ -1262,6 +1271,48 @@ class DownloadQueue(metaclass=Singleton): LOG.exception(e) LOG.error(f"Failed to retry item '{item_ref}'. {e!s}") + async def _delete_old_history(self) -> None: + """ + Automatically delete old download history based on user specified days. + 0 or negative value disables this feature. + """ + if self.config.auto_clear_history_days < 0 or self.is_paused() or self.done.empty(): + return + + cutoff_date: datetime = datetime.now(UTC) - timedelta(days=self.config.auto_clear_history_days) + items_to_delete: list[tuple[str, ItemDTO]] = [] + + for key, download in self.done.items(): + info: ItemDTO = download.info + if not info or not info.timestamp: + continue + + if "finished" != info.status or not info.filename: + continue + + try: + timestamp_seconds: float = info.timestamp / 1e9 + item_datetime: datetime = datetime.fromtimestamp(timestamp_seconds, tz=UTC) + if item_datetime < cutoff_date: + items_to_delete.append((key, info)) + except (OSError, ValueError, OverflowError) as e: + LOG.error(f"Failed to parse timestamp '{info.timestamp}' for item '{info.title}': {e}") + + titles: list[str] = [] + for key, info in items_to_delete: + item_name: str = info.title or info.id or info._id + self._notify.emit( + Events.ITEM_DELETED, + data=info, + title="Download Cleared", + message=f"'{item_name}' record removed from history.", + ) + titles.append(item_name) + self.done.delete(key) + + if titles: + LOG.info(f"Automatically cleared '{', '.join(titles)}' from download history due to age.") + def _handle_task_exception(self, task: asyncio.Task) -> None: if task.cancelled(): return diff --git a/app/library/Events.py b/app/library/Events.py index e9973182..f7528927 100644 --- a/app/library/Events.py +++ b/app/library/Events.py @@ -24,6 +24,8 @@ class Events: CONNECTED: str = "connected" + CONFIGURATION: str = "configuration" + LOG_INFO: str = "log_info" LOG_WARNING: str = "log_warning" LOG_ERROR: str = "log_error" @@ -90,6 +92,7 @@ class Events: """ return [ + Events.CONFIGURATION, Events.CONNECTED, Events.LOG_INFO, Events.LOG_WARNING, @@ -97,7 +100,6 @@ class Events: Events.LOG_SUCCESS, Events.ITEM_ADDED, Events.ITEM_UPDATED, - Events.ITEM_COMPLETED, Events.ITEM_CANCELLED, Events.ITEM_DELETED, Events.ITEM_MOVED, diff --git a/app/library/HttpSocket.py b/app/library/HttpSocket.py index 543ccf2a..16f3e4b2 100644 --- a/app/library/HttpSocket.py +++ b/app/library/HttpSocket.py @@ -44,7 +44,7 @@ class HttpSocket: self.sio = sio or socketio.AsyncServer( async_handlers=True, async_mode="aiohttp", - cors_allowed_origins=[], + cors_allowed_origins="*", transports=["websocket", "polling"], logger=self.config.debug, engineio_logger=self.config.debug, diff --git a/app/library/Utils.py b/app/library/Utils.py index efd77d19..358c26a2 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -482,6 +482,9 @@ def check_id(file: Path) -> bool | str: id: str | None = match.groupdict().get("id") try: + if not file.parent.exists(): + return False + for f in file.parent.iterdir(): if id not in f.stem: continue diff --git a/app/library/config.py b/app/library/config.py index 8825ecaf..05abc8f2 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -192,6 +192,9 @@ class Config(metaclass=Singleton): playlist_items_concurrency: int = 4 """The number of concurrent playlist items to be processed at same time.""" + auto_clear_history_days: int = 0 + """Number of days after which completed download history is automatically cleared. 0 to disable.""" + pictures_backends: list[str] = [ "https://unsplash.it/1920/1080?random", "https://picsum.photos/1920/1080", @@ -230,6 +233,7 @@ class Config(metaclass=Singleton): "playlist_items_concurrency", "download_path_depth", "download_info_expires", + "auto_clear_history_days", ) "The variables that are integers." diff --git a/app/routes/api/images.py b/app/routes/api/images.py index db7c92b3..c7e4e389 100644 --- a/app/routes/api/images.py +++ b/app/routes/api/images.py @@ -71,6 +71,12 @@ async def get_thumbnail(request: Request, config: Config) -> Response: LOG.debug(f"Fetching thumbnail from '{url}'.") response = await client.request(method="GET", url=url, follow_redirects=True) + if response.status_code != web.HTTPOk.status_code: + LOG.error(f"Failed to fetch thumbnail from '{url}'. Status code: {response.status_code}.") + return web.json_response( + data={"error": "failed to retrieve the thumbnail."}, status=response.status_code + ) + return web.Response( body=response.content, headers={ diff --git a/app/routes/socket/connection.py b/app/routes/socket/connection.py index 34c685d9..3364386d 100644 --- a/app/routes/socket/connection.py +++ b/app/routes/socket/connection.py @@ -23,25 +23,31 @@ class _Data: @route(RouteType.SOCKET, "connect", "socket_connect") async def connect(config: Config, queue: DownloadQueue, notify: EventBus, sid: str): - data = { - **queue.get(), - "config": config.frontend(), - "presets": Presets.get_instance().get_all(), - "dl_fields": DLFields.get_instance().get_all(), - "paused": queue.is_paused(), - } - - data["folders"] = list_folders( - path=Path(config.download_path), - base=Path(config.download_path), - depth_limit=config.download_path_depth-1, + notify.emit( + Events.CONFIGURATION, + data={ + "config": config.frontend(), + "presets": Presets.get_instance().get_all(), + "dl_fields": DLFields.get_instance().get_all(), + "paused": queue.is_paused(), + }, + title="Client connected", + message=f"Client '{sid}' connected.", + to=sid, ) notify.emit( Events.CONNECTED, - data=data, - title="Client connected", - message=f"Client '{sid}' connected.", + data={ + "folders": list_folders( + path=Path(config.download_path), + base=Path(config.download_path), + depth_limit=config.download_path_depth - 1, + ), + **queue.get(), + }, + title="Sending initial download data", + message=f"Sending initial download data to client '{sid}'.", to=sid, ) diff --git a/app/tests/test_events.py b/app/tests/test_events.py index a23ee1e1..2b3d2912 100644 --- a/app/tests/test_events.py +++ b/app/tests/test_events.py @@ -91,7 +91,6 @@ class TestEvents: Events.LOG_SUCCESS, Events.ITEM_ADDED, Events.ITEM_UPDATED, - Events.ITEM_COMPLETED, ] for expected in expected_frontend: diff --git a/pyproject.toml b/pyproject.toml index bf58a242..4f9812f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "aiocron>=1.8", "python-dotenv>=1.0.1", "debugpy>=1.8.1", - "httpx", + "httpx[brotli,http2,socks,zstd]", "async-timeout", "pyjson5", "curl_cffi>=0.13", diff --git a/ui/app/layouts/default.vue b/ui/app/layouts/default.vue index 8413251f..7242880e 100644 --- a/ui/app/layouts/default.vue +++ b/ui/app/layouts/default.vue @@ -103,7 +103,7 @@