From 7f7d9909f6b87cce19c32710a99381942f9183db Mon Sep 17 00:00:00 2001 From: KennyG Date: Wed, 1 Apr 2026 23:33:33 -0400 Subject: [PATCH 1/7] Refactor MeTube integration to generic webhook settings. Replace hasharr-specific settings/routes with webhook endpoint+timeout configuration, post fixed JSON payloads directly to endpoint, add generic JSON test response rendering, and add a gear modal for cookie-backed focused/background refresh cadence. Made-with: Cursor --- README.md | 31 +++-- app/main.py | 132 +++++++++++----------- app/ytdl.py | 19 ++-- ui/src/app/app.html | 115 +++++++++++-------- ui/src/app/app.sass | 10 +- ui/src/app/app.ts | 138 ++++++++++++++--------- ui/src/app/services/downloads.service.ts | 33 +++--- 7 files changed, 270 insertions(+), 208 deletions(-) diff --git a/README.md b/README.md index c4cba8f..68002ed 100644 --- a/README.md +++ b/README.md @@ -40,10 +40,9 @@ Certain values can be set via environment variables, using the `-e` parameter on * __SUBSCRIPTION_SCAN_PLAYLIST_END__: Maximum playlist/channel entries to fetch per subscription check (newest-first). Defaults to `50`. * __SUBSCRIPTION_MAX_SEEN_IDS__: Cap on stored video IDs per subscription to limit state file growth. Defaults to `50000`. * __CLEAR_COMPLETED_AFTER__: Number of seconds after which completed (and failed) downloads are automatically removed from the "Completed" list. Defaults to `0` (disabled). -* __HASHARR_ENABLED__: If `true`, call hasharr when a download completes. Defaults to `false`. -* __HASHARR_URL__: Base URL for hasharr service (for example `http://hasharr:9995`). Defaults to `http://hasharr:9995`. -* __HASHARR_SERVICE_ID__: Hash service profile ID to call at `POST /api/hash-service/{id}`. Defaults to `1`. -* __HASHARR_TIMEOUT_SEC__: Timeout (seconds) for hasharr callback requests. Defaults to `20`. +* __WEBSERVICE_ENABLED__: If `true`, call a webhook endpoint when a download completes. Defaults to `false`. +* __WEBSERVICE_ENDPOINT__: Full webhook URL to call (for example `http://localhost:9995/api/hash-service/1`). Defaults to `http://localhost:9995/api/hash-service/1`. +* __WEBSERVICE_TIMEOUT_SEC__: Timeout (seconds) for webhook requests. Defaults to `20`. ### 📁 Storage & Directories @@ -93,29 +92,37 @@ The project's Wiki contains examples of useful configurations contributed by use * [YTDL_OPTIONS Cookbook](https://github.com/alexta69/metube/wiki/YTDL_OPTIONS-Cookbook) * [OUTPUT_TEMPLATE Cookbook](https://github.com/alexta69/metube/wiki/OUTPUT_TEMPLATE-Cookbook) -## hasharr integration +## Webhook integration -MeTube can call hasharr after download completion to perform pHash matching and action policy. +MeTube can call a generic webhook endpoint after download completion. The payload is fixed JSON so any consumer can implement it. ### Runtime API endpoints -- `GET /hasharr-settings` -> current effective integration settings -- `POST /hasharr-settings` -> update runtime settings +- `GET /webhook-settings` -> current effective integration settings +- `POST /webhook-settings` -> update runtime settings +- `POST /webhook-settings/test` -> send test payload and return endpoint response Example: ```json { "enabled": true, - "url": "http://hasharr:9995", - "service_id": 1, + "endpoint": "http://localhost:9995/api/hash-service/1", "timeout_sec": 20 } ``` -When enabled, MeTube posts each completed output file to: +When enabled, MeTube posts each completed output file to `POST {WEBSERVICE_ENDPOINT}` with: -`POST {HASHARR_URL}/api/hash-service/{HASHARR_SERVICE_ID}` +```json +{ + "filePath": "/downloads/example.mp4", + "source": "metube", + "jobId": "optional-job-id" +} +``` + +The test route posts the same shape with an empty `filePath`. ## 🍪 Using browser cookies diff --git a/app/main.py b/app/main.py index 487da82..38910fc 100644 --- a/app/main.py +++ b/app/main.py @@ -70,13 +70,12 @@ class Config: 'MAX_CONCURRENT_DOWNLOADS': '3', 'LOGLEVEL': 'INFO', 'ENABLE_ACCESSLOG': 'false', - 'HASHARR_ENABLED': 'false', - 'HASHARR_URL': 'http://hasharr:9995', - 'HASHARR_SERVICE_ID': '1', - 'HASHARR_TIMEOUT_SEC': '20', + 'WEBSERVICE_ENABLED': 'false', + 'WEBSERVICE_ENDPOINT': 'http://localhost:9995/api/hash-service/1', + 'WEBSERVICE_TIMEOUT_SEC': '20', } - _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG', 'HASHARR_ENABLED') + _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG', 'WEBSERVICE_ENABLED') def __init__(self): for k, v in self._DEFAULTS.items(): @@ -124,10 +123,9 @@ class Config: 'PUBLIC_HOST_URL', 'PUBLIC_HOST_AUDIO_URL', 'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT', - 'HASHARR_ENABLED', - 'HASHARR_URL', - 'HASHARR_SERVICE_ID', - 'HASHARR_TIMEOUT_SEC', + 'WEBSERVICE_ENABLED', + 'WEBSERVICE_ENDPOINT', + 'WEBSERVICE_TIMEOUT_SEC', 'SUBSCRIPTION_DEFAULT_CHECK_INTERVAL', ) @@ -693,92 +691,86 @@ async def history(request): log.info("Sending download history") return web.Response(text=serializer.encode(history)) -@routes.get(config.URL_PREFIX + 'hasharr-settings') -async def get_hasharr_settings(request): +@routes.get(config.URL_PREFIX + 'webhook-settings') +async def get_webhook_settings(request): return web.Response(text=serializer.encode({ - 'enabled': bool(config.HASHARR_ENABLED), - 'url': str(config.HASHARR_URL), - 'service_id': int(config.HASHARR_SERVICE_ID), - 'timeout_sec': int(config.HASHARR_TIMEOUT_SEC), + 'enabled': bool(config.WEBSERVICE_ENABLED), + 'endpoint': str(config.WEBSERVICE_ENDPOINT), + 'timeout_sec': int(config.WEBSERVICE_TIMEOUT_SEC), }), content_type='application/json') -@routes.post(config.URL_PREFIX + 'hasharr-settings') -async def set_hasharr_settings(request): +@routes.post(config.URL_PREFIX + 'webhook-settings') +async def set_webhook_settings(request): post = await _read_json_request(request) - enabled = bool(post.get('enabled', config.HASHARR_ENABLED)) - url = str(post.get('url', config.HASHARR_URL)).strip() - service_id = int(post.get('service_id', config.HASHARR_SERVICE_ID)) - timeout_sec = int(post.get('timeout_sec', config.HASHARR_TIMEOUT_SEC)) - if not url: - raise web.HTTPBadRequest(reason='url is required') - if service_id <= 0: - raise web.HTTPBadRequest(reason='service_id must be > 0') + enabled = bool(post.get('enabled', config.WEBSERVICE_ENABLED)) + endpoint = str(post.get('endpoint', config.WEBSERVICE_ENDPOINT)).strip() + timeout_sec = int(post.get('timeout_sec', config.WEBSERVICE_TIMEOUT_SEC)) + if not endpoint: + raise web.HTTPBadRequest(reason='endpoint is required') if timeout_sec <= 0: raise web.HTTPBadRequest(reason='timeout_sec must be > 0') - config.HASHARR_ENABLED = enabled - config.HASHARR_URL = url - config.HASHARR_SERVICE_ID = service_id - config.HASHARR_TIMEOUT_SEC = timeout_sec + config.WEBSERVICE_ENABLED = enabled + config.WEBSERVICE_ENDPOINT = endpoint + config.WEBSERVICE_TIMEOUT_SEC = timeout_sec return web.Response(text=serializer.encode({'status': 'ok'}), content_type='application/json') -@routes.post(config.URL_PREFIX + 'hasharr-settings/test') -async def test_hasharr_settings(request): +@routes.post(config.URL_PREFIX + 'webhook-settings/test') +async def test_webhook_settings(request): post = await _read_json_request(request) - url = str(post.get('url', config.HASHARR_URL)).strip() - service_id = int(post.get('service_id', config.HASHARR_SERVICE_ID)) - timeout_sec = int(post.get('timeout_sec', config.HASHARR_TIMEOUT_SEC)) - if not url: - raise web.HTTPBadRequest(reason='url is required') - if service_id <= 0: - raise web.HTTPBadRequest(reason='service_id must be > 0') + endpoint = str(post.get('endpoint', config.WEBSERVICE_ENDPOINT)).strip() + timeout_sec = int(post.get('timeout_sec', config.WEBSERVICE_TIMEOUT_SEC)) + if not endpoint: + raise web.HTTPBadRequest(reason='endpoint is required') if timeout_sec <= 0: raise web.HTTPBadRequest(reason='timeout_sec must be > 0') - base_url = url.rstrip('/') - profile_url = f"{base_url}/v1/hash-service-profiles/{service_id}" + payload = { + 'filePath': '', + 'source': 'metube', + 'jobId': '', + } + body = json.dumps(payload).encode('utf-8') - def _fetch_profile(): - req = urlrequest.Request(profile_url, method='GET') + def _call_endpoint(): + req = urlrequest.Request(endpoint, data=body, headers={'Content-Type': 'application/json'}, method='POST') with urlrequest.urlopen(req, timeout=timeout_sec) as resp: - body = resp.read().decode('utf-8', errors='replace') - return resp.status, body + raw = resp.read().decode('utf-8', errors='replace') + return resp.status, raw try: - status, body = await asyncio.get_running_loop().run_in_executor(None, _fetch_profile) - if status != 200: - return web.json_response({ - 'status': 'error', - 'reachable': True, - 'valid_service_id': False, - 'message': f'hasharr responded with status {status}', - }, status=502) - profile = json.loads(body) + status, raw = await asyncio.get_running_loop().run_in_executor(None, _call_endpoint) + if not raw: + response_obj = {} + else: + try: + response_obj = json.loads(raw) + except json.JSONDecodeError: + response_obj = {'raw': raw} return web.json_response({ 'status': 'ok', - 'reachable': True, - 'valid_service_id': True, - 'profile': profile, + 'message': f'Endpoint returned HTTP {status}.', + 'status_code': status, + 'response': response_obj, }) except urlerror.HTTPError as exc: - if exc.code == 404: - return web.json_response({ - 'status': 'ok', - 'reachable': True, - 'valid_service_id': False, - 'message': f'Service ID {service_id} was not found in hasharr.', - }) + raw = exc.read().decode('utf-8', errors='replace') + if not raw: + response_obj = {} + else: + try: + response_obj = json.loads(raw) + except json.JSONDecodeError: + response_obj = {'raw': raw} return web.json_response({ 'status': 'error', - 'reachable': True, - 'valid_service_id': False, - 'message': f'hasharr returned HTTP {exc.code}', + 'message': f'Endpoint returned HTTP {exc.code}.', + 'status_code': int(exc.code), + 'response': response_obj, }, status=502) except Exception as exc: return web.json_response({ 'status': 'error', - 'reachable': False, - 'valid_service_id': False, - 'message': f'Could not reach hasharr: {exc}', + 'message': f'Could not reach endpoint: {exc}', }, status=502) @sio.event @@ -911,6 +903,8 @@ app.router.add_route('OPTIONS', config.URL_PREFIX + 'subscriptions/delete', add_ app.router.add_route('OPTIONS', config.URL_PREFIX + 'subscriptions/check', add_cors) app.router.add_route('OPTIONS', config.URL_PREFIX + 'upload-cookies', add_cors) app.router.add_route('OPTIONS', config.URL_PREFIX + 'delete-cookies', add_cors) +app.router.add_route('OPTIONS', config.URL_PREFIX + 'webhook-settings', add_cors) +app.router.add_route('OPTIONS', config.URL_PREFIX + 'webhook-settings/test', add_cors) async def on_prepare(request, response): if 'Origin' in request.headers: diff --git a/app/ytdl.py b/app/ytdl.py index 910e5aa..ef287ae 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -766,7 +766,7 @@ class DownloadQueue: else: self.done.put(download) asyncio.create_task(self.notifier.completed(download.info)) - asyncio.create_task(self._notify_hasharr(download.info)) + asyncio.create_task(self._notify_webhook(download.info)) try: clear_after = int(self.config.CLEAR_COMPLETED_AFTER) except ValueError: @@ -776,12 +776,13 @@ class DownloadQueue: task = asyncio.create_task(self.__auto_clear_after_delay(download.info.url, clear_after)) task.add_done_callback(lambda t: log.error(f'Auto-clear task failed: {t.exception()}') if not t.cancelled() and t.exception() else None) - async def _notify_hasharr(self, info): - if not getattr(self.config, 'HASHARR_ENABLED', False): + async def _notify_webhook(self, info): + if not getattr(self.config, 'WEBSERVICE_ENABLED', False): + return + endpoint = str(getattr(self.config, 'WEBSERVICE_ENDPOINT', '')).strip() + timeout_sec = int(getattr(self.config, 'WEBSERVICE_TIMEOUT_SEC', 20)) + if not endpoint: return - base_url = str(getattr(self.config, 'HASHARR_URL', 'http://hasharr:9995')).rstrip('/') - service_id = int(getattr(self.config, 'HASHARR_SERVICE_ID', 1)) - timeout_sec = int(getattr(self.config, 'HASHARR_TIMEOUT_SEC', 20)) files = [] if getattr(info, 'filename', None): files.append(info.filename) @@ -811,7 +812,7 @@ class DownloadQueue: } data = json.dumps(payload).encode('utf-8') req = urlrequest.Request( - f"{base_url}/api/hash-service/{service_id}", + endpoint, data=data, headers={"Content-Type": "application/json"}, method="POST", @@ -822,9 +823,9 @@ class DownloadQueue: for rel_name in dedup: try: status = await asyncio.get_running_loop().run_in_executor(None, _post_one, rel_name) - log.info(f"hasharr callback status={status} file={rel_name}") + log.info(f"webhook callback status={status} file={rel_name}") except Exception as exc: - log.warning(f"hasharr callback failed for {rel_name}: {exc}") + log.warning(f"webhook callback failed for {rel_name}: {exc}") async def __auto_clear_after_delay(self, url, delay_seconds): await asyncio.sleep(delay_seconds) diff --git a/ui/src/app/app.html b/ui/src/app/app.html index d04b225..4fa0f31 100644 --- a/ui/src/app/app.html +++ b/ui/src/app/app.html @@ -49,6 +49,12 @@ -->
+
-
-
- @if (hasharrSettingsStatus) { - {{ hasharrSettingsStatus }} +
+ @if (webhookSettingsStatus) { + {{ webhookSettingsStatus }} } - @if (hasharrTestStatus) { - {{ hasharrTestStatus }} + @if (webhookTestStatus) { + {{ webhookTestStatus }} }
- @if (hasharrTestProfile) { + @if (webhookTestResponsePretty) {
-
-
Profile Name: {{ hasharrTestProfile['name'] }}
-
Enabled: {{ hasharrTestProfile['enabled'] }}
-
Apply Actions: {{ hasharrTestProfile['applyActions'] }}
-
Stash Index: {{ hasharrTestProfile['stashIndex'] }}
-
Max Time Delta: {{ hasharrTestProfile['maxTimeDelta'] }}
-
Max Distance: {{ hasharrTestProfile['maxDistance'] }}
-
+ 
{{ webhookTestResponsePretty }}
}
@@ -666,6 +654,45 @@ + + @if (downloads.loading) {