diff --git a/README.md b/README.md index c4cba8f..0e7d27b 100644 --- a/README.md +++ b/README.md @@ -40,10 +40,9 @@ Certain values can be set via environment variables, using the `-e` parameter on * __SUBSCRIPTION_SCAN_PLAYLIST_END__: Maximum playlist/channel entries to fetch per subscription check (newest-first). Defaults to `50`. * __SUBSCRIPTION_MAX_SEEN_IDS__: Cap on stored video IDs per subscription to limit state file growth. Defaults to `50000`. * __CLEAR_COMPLETED_AFTER__: Number of seconds after which completed (and failed) downloads are automatically removed from the "Completed" list. Defaults to `0` (disabled). -* __HASHARR_ENABLED__: If `true`, call hasharr when a download completes. Defaults to `false`. -* __HASHARR_URL__: Base URL for hasharr service (for example `http://hasharr:9995`). Defaults to `http://hasharr:9995`. -* __HASHARR_SERVICE_ID__: Hash service profile ID to call at `POST /api/hash-service/{id}`. Defaults to `1`. -* __HASHARR_TIMEOUT_SEC__: Timeout (seconds) for hasharr callback requests. Defaults to `20`. +* __WEBSERVICE_ENABLED__: If `true`, call a webhook endpoint when a download completes. Defaults to `false`. +* __WEBSERVICE_ENDPOINT__: Full webhook URL to call (for example `"http://localhost:9876/api/web-service/"`). Defaults to `"http://localhost:9876/api/web-service/"`. +* __WEBSERVICE_TIMEOUT_SEC__: Timeout (seconds) for webhook requests. Defaults to `20`. ### 📁 Storage & Directories @@ -93,29 +92,39 @@ The project's Wiki contains examples of useful configurations contributed by use * [YTDL_OPTIONS Cookbook](https://github.com/alexta69/metube/wiki/YTDL_OPTIONS-Cookbook) * [OUTPUT_TEMPLATE Cookbook](https://github.com/alexta69/metube/wiki/OUTPUT_TEMPLATE-Cookbook) -## hasharr integration +## Webhook integration -MeTube can call hasharr after download completion to perform pHash matching and action policy. +MeTube can call a generic webhook endpoint after download completion. The payload is fixed JSON so any consumer can implement it. ### Runtime API endpoints -- `GET /hasharr-settings` -> current effective integration settings -- `POST /hasharr-settings` -> update runtime settings +- `GET /webhook-settings` -> current effective integration settings +- `POST /webhook-settings` -> update runtime settings +- `POST /webhook-settings/test` -> send test payload and return endpoint response + +Settings saved via `POST /webhook-settings` are written to `{STATE_DIR}/webhook_settings.json` and reloaded on startup, so they survive container restarts. If this file is present, it overrides the `WEBSERVICE_*` environment defaults for the running process. Example: ```json { "enabled": true, - "url": "http://hasharr:9995", - "service_id": 1, + "endpoint": "http://localhost:9876/api/web-service/", "timeout_sec": 20 } ``` -When enabled, MeTube posts each completed output file to: +When enabled, MeTube posts each completed output file to `POST {WEBSERVICE_ENDPOINT}` with: -`POST {HASHARR_URL}/api/hash-service/{HASHARR_SERVICE_ID}` +```json +{ + "filePath": "/downloads/example.mp4", + "source": "metube", + "jobId": "optional-job-id" +} +``` + +The test route posts the same shape with an empty `filePath`. ## 🍪 Using browser cookies diff --git a/app/main.py b/app/main.py index 487da82..2965972 100644 --- a/app/main.py +++ b/app/main.py @@ -20,6 +20,7 @@ from watchfiles import DefaultFilter, Change, awatch from ytdl import DownloadQueueNotifier, DownloadQueue, Download from subscriptions import SubscriptionManager, SubscriptionNotifier, SubscriptionInfo +from state_store import AtomicJsonStore from yt_dlp.version import __version__ as yt_dlp_version log = logging.getLogger('main') @@ -70,13 +71,12 @@ class Config: 'MAX_CONCURRENT_DOWNLOADS': '3', 'LOGLEVEL': 'INFO', 'ENABLE_ACCESSLOG': 'false', - 'HASHARR_ENABLED': 'false', - 'HASHARR_URL': 'http://hasharr:9995', - 'HASHARR_SERVICE_ID': '1', - 'HASHARR_TIMEOUT_SEC': '20', + 'WEBSERVICE_ENABLED': 'false', + 'WEBSERVICE_ENDPOINT': 'http://localhost:9876/api/web-service/', + 'WEBSERVICE_TIMEOUT_SEC': '20', } - _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG', 'HASHARR_ENABLED') + _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG', 'WEBSERVICE_ENABLED') def __init__(self): for k, v in self._DEFAULTS.items(): @@ -124,10 +124,9 @@ class Config: 'PUBLIC_HOST_URL', 'PUBLIC_HOST_AUDIO_URL', 'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT', - 'HASHARR_ENABLED', - 'HASHARR_URL', - 'HASHARR_SERVICE_ID', - 'HASHARR_TIMEOUT_SEC', + 'WEBSERVICE_ENABLED', + 'WEBSERVICE_ENDPOINT', + 'WEBSERVICE_TIMEOUT_SEC', 'SUBSCRIPTION_DEFAULT_CHECK_INTERVAL', ) @@ -171,6 +170,43 @@ class Config: return (True, '') config = Config() + +_webhook_settings_store = AtomicJsonStore( + os.path.join(config.STATE_DIR, 'webhook_settings.json'), + kind='webhook_settings', +) + + +def _load_persisted_webhook_settings() -> None: + """Apply webhook endpoint settings saved under STATE_DIR (survives container restarts).""" + payload = _webhook_settings_store.load() + if not payload: + return + try: + if 'endpoint' in payload: + ep = str(payload['endpoint'] or '').strip() + if ep: + config.WEBSERVICE_ENDPOINT = ep + if 'timeout_sec' in payload: + ts = int(payload['timeout_sec']) + if ts > 0: + config.WEBSERVICE_TIMEOUT_SEC = ts + if 'enabled' in payload: + config.WEBSERVICE_ENABLED = bool(payload['enabled']) + except (TypeError, ValueError) as exc: + log.warning('Ignoring invalid persisted webhook settings: %s', exc) + + +def _save_persisted_webhook_settings() -> None: + _webhook_settings_store.save({ + 'enabled': bool(config.WEBSERVICE_ENABLED), + 'endpoint': str(config.WEBSERVICE_ENDPOINT), + 'timeout_sec': int(config.WEBSERVICE_TIMEOUT_SEC), + }) + + +_load_persisted_webhook_settings() + # Align root logger level with Config (keeps a single source of truth). # This re-applies the log level after Config loads, in case LOGLEVEL was # overridden by config file settings or differs from the environment variable. @@ -693,92 +729,91 @@ async def history(request): log.info("Sending download history") return web.Response(text=serializer.encode(history)) -@routes.get(config.URL_PREFIX + 'hasharr-settings') -async def get_hasharr_settings(request): +@routes.get(config.URL_PREFIX + 'webhook-settings') +async def get_webhook_settings(request): return web.Response(text=serializer.encode({ - 'enabled': bool(config.HASHARR_ENABLED), - 'url': str(config.HASHARR_URL), - 'service_id': int(config.HASHARR_SERVICE_ID), - 'timeout_sec': int(config.HASHARR_TIMEOUT_SEC), + 'enabled': bool(config.WEBSERVICE_ENABLED), + 'endpoint': str(config.WEBSERVICE_ENDPOINT), + 'timeout_sec': int(config.WEBSERVICE_TIMEOUT_SEC), }), content_type='application/json') -@routes.post(config.URL_PREFIX + 'hasharr-settings') -async def set_hasharr_settings(request): +@routes.post(config.URL_PREFIX + 'webhook-settings') +async def set_webhook_settings(request): post = await _read_json_request(request) - enabled = bool(post.get('enabled', config.HASHARR_ENABLED)) - url = str(post.get('url', config.HASHARR_URL)).strip() - service_id = int(post.get('service_id', config.HASHARR_SERVICE_ID)) - timeout_sec = int(post.get('timeout_sec', config.HASHARR_TIMEOUT_SEC)) - if not url: - raise web.HTTPBadRequest(reason='url is required') - if service_id <= 0: - raise web.HTTPBadRequest(reason='service_id must be > 0') + enabled = bool(post.get('enabled', config.WEBSERVICE_ENABLED)) + endpoint = str(post.get('endpoint', config.WEBSERVICE_ENDPOINT)).strip() + timeout_sec = int(post.get('timeout_sec', config.WEBSERVICE_TIMEOUT_SEC)) + if not endpoint: + raise web.HTTPBadRequest(reason='endpoint is required') if timeout_sec <= 0: raise web.HTTPBadRequest(reason='timeout_sec must be > 0') - config.HASHARR_ENABLED = enabled - config.HASHARR_URL = url - config.HASHARR_SERVICE_ID = service_id - config.HASHARR_TIMEOUT_SEC = timeout_sec + config.WEBSERVICE_ENABLED = enabled + config.WEBSERVICE_ENDPOINT = endpoint + config.WEBSERVICE_TIMEOUT_SEC = timeout_sec + try: + _save_persisted_webhook_settings() + except OSError as exc: + log.exception('Failed to persist webhook settings to %s', _webhook_settings_store.path) + raise web.HTTPInternalServerError(reason=f'could not persist webhook settings: {exc}') from exc return web.Response(text=serializer.encode({'status': 'ok'}), content_type='application/json') -@routes.post(config.URL_PREFIX + 'hasharr-settings/test') -async def test_hasharr_settings(request): +@routes.post(config.URL_PREFIX + 'webhook-settings/test') +async def test_webhook_settings(request): post = await _read_json_request(request) - url = str(post.get('url', config.HASHARR_URL)).strip() - service_id = int(post.get('service_id', config.HASHARR_SERVICE_ID)) - timeout_sec = int(post.get('timeout_sec', config.HASHARR_TIMEOUT_SEC)) - if not url: - raise web.HTTPBadRequest(reason='url is required') - if service_id <= 0: - raise web.HTTPBadRequest(reason='service_id must be > 0') + endpoint = str(post.get('endpoint', config.WEBSERVICE_ENDPOINT)).strip() + timeout_sec = int(post.get('timeout_sec', config.WEBSERVICE_TIMEOUT_SEC)) + if not endpoint: + raise web.HTTPBadRequest(reason='endpoint is required') if timeout_sec <= 0: raise web.HTTPBadRequest(reason='timeout_sec must be > 0') - base_url = url.rstrip('/') - profile_url = f"{base_url}/v1/hash-service-profiles/{service_id}" + payload = { + 'filePath': '', + 'source': 'metube', + 'jobId': '', + } + body = json.dumps(payload).encode('utf-8') - def _fetch_profile(): - req = urlrequest.Request(profile_url, method='GET') + def _call_endpoint(): + req = urlrequest.Request(endpoint, data=body, headers={'Content-Type': 'application/json'}, method='POST') with urlrequest.urlopen(req, timeout=timeout_sec) as resp: - body = resp.read().decode('utf-8', errors='replace') - return resp.status, body + raw = resp.read().decode('utf-8', errors='replace') + return resp.status, raw try: - status, body = await asyncio.get_running_loop().run_in_executor(None, _fetch_profile) - if status != 200: - return web.json_response({ - 'status': 'error', - 'reachable': True, - 'valid_service_id': False, - 'message': f'hasharr responded with status {status}', - }, status=502) - profile = json.loads(body) + status, raw = await asyncio.get_running_loop().run_in_executor(None, _call_endpoint) + if not raw: + response_obj = {} + else: + try: + response_obj = json.loads(raw) + except json.JSONDecodeError: + response_obj = {'raw': raw} return web.json_response({ 'status': 'ok', - 'reachable': True, - 'valid_service_id': True, - 'profile': profile, + 'message': f'Endpoint returned HTTP {status}.', + 'status_code': status, + 'response': response_obj, }) except urlerror.HTTPError as exc: - if exc.code == 404: - return web.json_response({ - 'status': 'ok', - 'reachable': True, - 'valid_service_id': False, - 'message': f'Service ID {service_id} was not found in hasharr.', - }) + raw = exc.read().decode('utf-8', errors='replace') + if not raw: + response_obj = {} + else: + try: + response_obj = json.loads(raw) + except json.JSONDecodeError: + response_obj = {'raw': raw} return web.json_response({ 'status': 'error', - 'reachable': True, - 'valid_service_id': False, - 'message': f'hasharr returned HTTP {exc.code}', + 'message': f'Endpoint returned HTTP {exc.code}.', + 'status_code': int(exc.code), + 'response': response_obj, }, status=502) except Exception as exc: return web.json_response({ 'status': 'error', - 'reachable': False, - 'valid_service_id': False, - 'message': f'Could not reach hasharr: {exc}', + 'message': f'Could not reach endpoint: {exc}', }, status=502) @sio.event @@ -911,6 +946,8 @@ app.router.add_route('OPTIONS', config.URL_PREFIX + 'subscriptions/delete', add_ app.router.add_route('OPTIONS', config.URL_PREFIX + 'subscriptions/check', add_cors) app.router.add_route('OPTIONS', config.URL_PREFIX + 'upload-cookies', add_cors) app.router.add_route('OPTIONS', config.URL_PREFIX + 'delete-cookies', add_cors) +app.router.add_route('OPTIONS', config.URL_PREFIX + 'webhook-settings', add_cors) +app.router.add_route('OPTIONS', config.URL_PREFIX + 'webhook-settings/test', add_cors) async def on_prepare(request, response): if 'Origin' in request.headers: diff --git a/app/ytdl.py b/app/ytdl.py index 910e5aa..ef287ae 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -766,7 +766,7 @@ class DownloadQueue: else: self.done.put(download) asyncio.create_task(self.notifier.completed(download.info)) - asyncio.create_task(self._notify_hasharr(download.info)) + asyncio.create_task(self._notify_webhook(download.info)) try: clear_after = int(self.config.CLEAR_COMPLETED_AFTER) except ValueError: @@ -776,12 +776,13 @@ class DownloadQueue: task = asyncio.create_task(self.__auto_clear_after_delay(download.info.url, clear_after)) task.add_done_callback(lambda t: log.error(f'Auto-clear task failed: {t.exception()}') if not t.cancelled() and t.exception() else None) - async def _notify_hasharr(self, info): - if not getattr(self.config, 'HASHARR_ENABLED', False): + async def _notify_webhook(self, info): + if not getattr(self.config, 'WEBSERVICE_ENABLED', False): + return + endpoint = str(getattr(self.config, 'WEBSERVICE_ENDPOINT', '')).strip() + timeout_sec = int(getattr(self.config, 'WEBSERVICE_TIMEOUT_SEC', 20)) + if not endpoint: return - base_url = str(getattr(self.config, 'HASHARR_URL', 'http://hasharr:9995')).rstrip('/') - service_id = int(getattr(self.config, 'HASHARR_SERVICE_ID', 1)) - timeout_sec = int(getattr(self.config, 'HASHARR_TIMEOUT_SEC', 20)) files = [] if getattr(info, 'filename', None): files.append(info.filename) @@ -811,7 +812,7 @@ class DownloadQueue: } data = json.dumps(payload).encode('utf-8') req = urlrequest.Request( - f"{base_url}/api/hash-service/{service_id}", + endpoint, data=data, headers={"Content-Type": "application/json"}, method="POST", @@ -822,9 +823,9 @@ class DownloadQueue: for rel_name in dedup: try: status = await asyncio.get_running_loop().run_in_executor(None, _post_one, rel_name) - log.info(f"hasharr callback status={status} file={rel_name}") + log.info(f"webhook callback status={status} file={rel_name}") except Exception as exc: - log.warning(f"hasharr callback failed for {rel_name}: {exc}") + log.warning(f"webhook callback failed for {rel_name}: {exc}") async def __auto_clear_after_delay(self, url, delay_seconds): await asyncio.sleep(delay_seconds) diff --git a/ui/src/app/app.html b/ui/src/app/app.html index d04b225..ebb01fc 100644 --- a/ui/src/app/app.html +++ b/ui/src/app/app.html @@ -6,34 +6,34 @@
@if (activeDownloads > 0) { -
- - {{activeDownloads}} downloading -
+
+ + {{activeDownloads}} downloading +
} @if (queuedDownloads > 0) { -
- - {{queuedDownloads}} queued -
+
+ + {{queuedDownloads}} queued +
} @if (completedDownloads > 0) { -
- - {{completedDownloads}} completed -
+
+ + {{completedDownloads}} completed +
} @if (failedDownloads > 0) { -
- - {{failedDownloads}} failed -
+
+ + {{failedDownloads}} failed +
} @if ((totalSpeed | speed) !== '') { -
- - {{totalSpeed | speed }} -
+
+ + {{totalSpeed | speed }} +
}