diff --git a/README.md b/README.md index 16228af..bbdd969 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,10 @@ Certain values can be set via environment variables, using the `-e` parameter on * __DELETE_FILE_ON_TRASHCAN__: if `true`, downloaded files are deleted on the server, when they are trashed from the "Completed" section of the UI. Defaults to `false`. * __DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT__: Maximum number of playlist items that can be downloaded. Defaults to `0` (no limit). * __CLEAR_COMPLETED_AFTER__: Number of seconds after which completed (and failed) downloads are automatically removed from the "Completed" list. Defaults to `0` (disabled). +* __HASHARR_ENABLED__: If `true`, call hasharr when a download completes. Defaults to `false`. +* __HASHARR_URL__: Base URL for hasharr service (for example `http://hasharr:9995`). Defaults to `http://hasharr:9995`. +* __HASHARR_SERVICE_ID__: Hash service profile ID to call at `POST /api/hash-service/{id}`. Defaults to `1`. +* __HASHARR_TIMEOUT_SEC__: Timeout (seconds) for hasharr callback requests. Defaults to `20`. ### 📁 Storage & Directories @@ -86,6 +90,30 @@ The project's Wiki contains examples of useful configurations contributed by use * [YTDL_OPTIONS Cookbook](https://github.com/alexta69/metube/wiki/YTDL_OPTIONS-Cookbook) * [OUTPUT_TEMPLATE Cookbook](https://github.com/alexta69/metube/wiki/OUTPUT_TEMPLATE-Cookbook) +## hasharr integration + +MeTube can call hasharr after download completion to perform pHash matching and action policy. + +### Runtime API endpoints + +- `GET /hasharr-settings` -> current effective integration settings +- `POST /hasharr-settings` -> update runtime settings + +Example: + +```json +{ + "enabled": true, + "url": "http://hasharr:9995", + "service_id": 1, + "timeout_sec": 20 +} +``` + +When enabled, MeTube posts each completed output file to: + +`POST {HASHARR_URL}/api/hash-service/{HASHARR_SERVICE_ID}` + ## 🍪 Using browser cookies In case you need to use your browser's cookies with MeTube, for example to download restricted or private videos: diff --git a/app/main.py b/app/main.py index db7be18..d235cb3 100644 --- a/app/main.py +++ b/app/main.py @@ -14,6 +14,8 @@ import logging import json import pathlib import re +from urllib import request as urlrequest +from urllib import error as urlerror from watchfiles import DefaultFilter, Change, awatch from ytdl import DownloadQueueNotifier, DownloadQueue, Download @@ -64,9 +66,13 @@ class Config: 'MAX_CONCURRENT_DOWNLOADS': '3', 'LOGLEVEL': 'INFO', 'ENABLE_ACCESSLOG': 'false', + 'HASHARR_ENABLED': 'false', + 'HASHARR_URL': 'http://hasharr:9995', + 'HASHARR_SERVICE_ID': '1', + 'HASHARR_TIMEOUT_SEC': '20', } - _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG') + _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG', 'HASHARR_ENABLED') def __init__(self): for k, v in self._DEFAULTS.items(): @@ -114,6 +120,10 @@ class Config: 'PUBLIC_HOST_URL', 'PUBLIC_HOST_AUDIO_URL', 'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT', + 'HASHARR_ENABLED', + 'HASHARR_URL', + 'HASHARR_SERVICE_ID', + 'HASHARR_TIMEOUT_SEC', ) def frontend_safe(self) -> dict: @@ -550,6 +560,94 @@ async def history(request): log.info("Sending download history") return web.Response(text=serializer.encode(history)) +@routes.get(config.URL_PREFIX + 'hasharr-settings') +async def get_hasharr_settings(request): + return web.Response(text=serializer.encode({ + 'enabled': bool(config.HASHARR_ENABLED), + 'url': str(config.HASHARR_URL), + 'service_id': int(config.HASHARR_SERVICE_ID), + 'timeout_sec': int(config.HASHARR_TIMEOUT_SEC), + }), content_type='application/json') + +@routes.post(config.URL_PREFIX + 'hasharr-settings') +async def set_hasharr_settings(request): + post = await _read_json_request(request) + enabled = bool(post.get('enabled', config.HASHARR_ENABLED)) + url = str(post.get('url', config.HASHARR_URL)).strip() + service_id = int(post.get('service_id', config.HASHARR_SERVICE_ID)) + timeout_sec = int(post.get('timeout_sec', config.HASHARR_TIMEOUT_SEC)) + if not url: + raise web.HTTPBadRequest(reason='url is required') + if service_id <= 0: + raise web.HTTPBadRequest(reason='service_id must be > 0') + if timeout_sec <= 0: + raise web.HTTPBadRequest(reason='timeout_sec must be > 0') + config.HASHARR_ENABLED = enabled + config.HASHARR_URL = url + config.HASHARR_SERVICE_ID = service_id + config.HASHARR_TIMEOUT_SEC = timeout_sec + return web.Response(text=serializer.encode({'status': 'ok'}), content_type='application/json') + +@routes.post(config.URL_PREFIX + 'hasharr-settings/test') +async def test_hasharr_settings(request): + post = await _read_json_request(request) + url = str(post.get('url', config.HASHARR_URL)).strip() + service_id = int(post.get('service_id', config.HASHARR_SERVICE_ID)) + timeout_sec = int(post.get('timeout_sec', config.HASHARR_TIMEOUT_SEC)) + if not url: + raise web.HTTPBadRequest(reason='url is required') + if service_id <= 0: + raise web.HTTPBadRequest(reason='service_id must be > 0') + if timeout_sec <= 0: + raise web.HTTPBadRequest(reason='timeout_sec must be > 0') + + base_url = url.rstrip('/') + profile_url = f"{base_url}/v1/hash-service-profiles/{service_id}" + + def _fetch_profile(): + req = urlrequest.Request(profile_url, method='GET') + with urlrequest.urlopen(req, timeout=timeout_sec) as resp: + body = resp.read().decode('utf-8', errors='replace') + return resp.status, body + + try: + status, body = await asyncio.get_running_loop().run_in_executor(None, _fetch_profile) + if status != 200: + return web.json_response({ + 'status': 'error', + 'reachable': True, + 'valid_service_id': False, + 'message': f'hasharr responded with status {status}', + }, status=502) + profile = json.loads(body) + return web.json_response({ + 'status': 'ok', + 'reachable': True, + 'valid_service_id': True, + 'profile': profile, + }) + except urlerror.HTTPError as exc: + if exc.code == 404: + return web.json_response({ + 'status': 'ok', + 'reachable': True, + 'valid_service_id': False, + 'message': f'Service ID {service_id} was not found in hasharr.', + }) + return web.json_response({ + 'status': 'error', + 'reachable': True, + 'valid_service_id': False, + 'message': f'hasharr returned HTTP {exc.code}', + }, status=502) + except Exception as exc: + return web.json_response({ + 'status': 'error', + 'reachable': False, + 'valid_service_id': False, + 'message': f'Could not reach hasharr: {exc}', + }, status=502) + @sio.event async def connect(sid, environ): log.info(f"Client connected: {sid}") diff --git a/app/ytdl.py b/app/ytdl.py index ba5aada..77255dd 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -14,6 +14,8 @@ import re import types import dbm import subprocess +import json +from urllib import request as urlrequest from typing import Any from functools import lru_cache @@ -694,6 +696,7 @@ class DownloadQueue: else: self.done.put(download) asyncio.create_task(self.notifier.completed(download.info)) + asyncio.create_task(self._notify_hasharr(download.info)) try: clear_after = int(self.config.CLEAR_COMPLETED_AFTER) except ValueError: @@ -703,6 +706,56 @@ class DownloadQueue: task = asyncio.create_task(self.__auto_clear_after_delay(download.info.url, clear_after)) task.add_done_callback(lambda t: log.error(f'Auto-clear task failed: {t.exception()}') if not t.cancelled() and t.exception() else None) + async def _notify_hasharr(self, info): + if not getattr(self.config, 'HASHARR_ENABLED', False): + return + base_url = str(getattr(self.config, 'HASHARR_URL', 'http://hasharr:9995')).rstrip('/') + service_id = int(getattr(self.config, 'HASHARR_SERVICE_ID', 1)) + timeout_sec = int(getattr(self.config, 'HASHARR_TIMEOUT_SEC', 20)) + files = [] + if getattr(info, 'filename', None): + files.append(info.filename) + for cf in getattr(info, 'chapter_files', []) or []: + if isinstance(cf, dict) and cf.get('filename'): + files.append(cf['filename']) + for sf in getattr(info, 'subtitle_files', []) or []: + if isinstance(sf, dict) and sf.get('filename'): + files.append(sf['filename']) + dedup = [] + seen = set() + for f in files: + if f and f not in seen: + seen.add(f) + dedup.append(f) + if not dedup: + return + + def _post_one(rel_name): + download_type = getattr(info, 'download_type', 'video') + base_dir = self.config.AUDIO_DOWNLOAD_DIR if download_type == 'audio' else self.config.DOWNLOAD_DIR + full_path = os.path.join(base_dir, rel_name) + payload = { + "filePath": full_path, + "source": "metube", + "jobId": str(getattr(info, 'id', '')), + } + data = json.dumps(payload).encode('utf-8') + req = urlrequest.Request( + f"{base_url}/api/hash-service/{service_id}", + data=data, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlrequest.urlopen(req, timeout=timeout_sec) as resp: + return resp.status + + for rel_name in dedup: + try: + status = await asyncio.get_running_loop().run_in_executor(None, _post_one, rel_name) + log.info(f"hasharr callback status={status} file={rel_name}") + except Exception as exc: + log.warning(f"hasharr callback failed for {rel_name}: {exc}") + async def __auto_clear_after_delay(self, url, delay_seconds): await asyncio.sleep(delay_seconds) if self.done.exists(url): diff --git a/ui/src/app/app.html b/ui/src/app/app.html index 8ef50f8..f653ccd 100644 --- a/ui/src/app/app.html +++ b/ui/src/app/app.html @@ -428,6 +428,84 @@
+
+
+
Hasharr Integration
+
+
+
+ + +
+
+
+
+ Hasharr URL + +
+
+
+
+ Service ID + +
+
+
+
+ Timeout + +
+
+
+
+ + +
+
+
+
+ @if (hasharrSettingsStatus) { + {{ hasharrSettingsStatus }} + } + @if (hasharrTestStatus) { + {{ hasharrTestStatus }} + } +
+
+ @if (hasharrTestProfile) { +
+
+
Profile Name: {{ hasharrTestProfile['name'] }}
+
Enabled: {{ hasharrTestProfile['enabled'] }}
+
Apply Actions: {{ hasharrTestProfile['applyActions'] }}
+
Stash Index: {{ hasharrTestProfile['stashIndex'] }}
+
Max Time Delta: {{ hasharrTestProfile['maxTimeDelta'] }}
+
Max Distance: {{ hasharrTestProfile['maxDistance'] }}
+
+
+ } +
Cookies
diff --git a/ui/src/app/app.sass b/ui/src/app/app.sass index 9368c48..dd729c3 100644 --- a/ui/src/app/app.sass +++ b/ui/src/app/app.sass @@ -196,3 +196,26 @@ main &.active color: var(--bs-success-text-emphasis) + +.hasharr-integration + .input-group + margin-bottom: 0 + +.hasharr-action-row + display: flex + flex-wrap: wrap + gap: 0.5rem + +.hasharr-status-block + min-height: 2.5rem + display: flex + flex-direction: column + justify-content: center + text-align: left + +.hasharr-profile-result + margin-top: 0.25rem + padding: 0.5rem 0.75rem + border: 1px solid var(--bs-border-color) + border-radius: 0.375rem + background: var(--bs-tertiary-bg) diff --git a/ui/src/app/app.ts b/ui/src/app/app.ts index b1b8c1e..aac0ea6 100644 --- a/ui/src/app/app.ts +++ b/ui/src/app/app.ts @@ -10,7 +10,7 @@ import { NgSelectModule } from '@ng-select/ng-select'; import { faTrashAlt, faCheckCircle, faTimesCircle, faRedoAlt, faSun, faMoon, faCheck, faCircleHalfStroke, faDownload, faExternalLinkAlt, faFileImport, faFileExport, faCopy, faClock, faTachometerAlt, faSortAmountDown, faSortAmountUp, faChevronRight, faChevronDown, faUpload } from '@fortawesome/free-solid-svg-icons'; import { faGithub } from '@fortawesome/free-brands-svg-icons'; import { CookieService } from 'ngx-cookie-service'; -import { AddDownloadPayload, DownloadsService } from './services/downloads.service'; +import { AddDownloadPayload, DownloadsService, HasharrSettings, HasharrServiceTestResponse } from './services/downloads.service'; import { Themes } from './theme'; import { Download, @@ -96,6 +96,13 @@ export class App implements AfterViewInit, OnInit, OnDestroy { ytDlpVersion: string | null = null; metubeVersion: string | null = null; isAdvancedOpen = false; + hasharrEnabled = false; + hasharrUrl = 'http://hasharr:9995'; + hasharrServiceID = 1; + hasharrTimeoutSec = 20; + hasharrSettingsStatus = ''; + hasharrTestStatus = ''; + hasharrTestProfile: Record | null = null; sortAscending = false; expandedErrors: Set = new Set(); cachedSortedDone: [string, Download][] = []; @@ -268,6 +275,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy { this.cdr.markForCheck(); }); this.getConfiguration(); + this.loadHasharrSettings(); this.getYtdlOptionsUpdateTime(); this.customDirs$ = this.getMatchingCustomDir(); this.setTheme(this.activeTheme!); @@ -275,6 +283,94 @@ export class App implements AfterViewInit, OnInit, OnDestroy { this.colorSchemeMediaQuery.addEventListener('change', this.onColorSchemeChanged); } + loadHasharrSettings() { + this.downloads.getHasharrSettings().pipe(takeUntilDestroyed(this.destroyRef)).subscribe({ + next: (data) => { + if (!data || typeof data !== 'object' || ('status' in data && data.status === 'error')) { + this.hasharrSettingsStatus = 'Unable to load hasharr settings.'; + this.cdr.markForCheck(); + return; + } + const settings = data as HasharrSettings; + this.hasharrEnabled = !!settings.enabled; + this.hasharrUrl = String(settings.url || 'http://hasharr:9995'); + this.hasharrServiceID = Math.max(1, Number(settings.service_id || 1)); + this.hasharrTimeoutSec = Math.max(1, Number(settings.timeout_sec || 20)); + this.hasharrSettingsStatus = ''; + this.cdr.markForCheck(); + }, + error: () => { + this.hasharrSettingsStatus = 'Unable to load hasharr settings.'; + this.cdr.markForCheck(); + }, + }); + } + + saveHasharrSettings() { + const payload: HasharrSettings = { + enabled: !!this.hasharrEnabled, + url: String(this.hasharrUrl || '').trim(), + service_id: Math.max(1, Number(this.hasharrServiceID || 1)), + timeout_sec: Math.max(1, Number(this.hasharrTimeoutSec || 20)), + }; + if (!payload.url) { + this.hasharrSettingsStatus = 'Hasharr URL is required.'; + this.cdr.markForCheck(); + return; + } + this.downloads.saveHasharrSettings(payload).pipe(takeUntilDestroyed(this.destroyRef)).subscribe({ + next: (out) => { + if (out && typeof out === 'object' && 'status' in out && out.status === 'ok') { + this.hasharrSettingsStatus = 'Hasharr settings saved.'; + } else { + this.hasharrSettingsStatus = 'Failed to save hasharr settings.'; + } + this.cdr.markForCheck(); + }, + error: () => { + this.hasharrSettingsStatus = 'Failed to save hasharr settings.'; + this.cdr.markForCheck(); + }, + }); + } + + testHasharrService() { + this.hasharrTestStatus = 'Testing hasharr service...'; + this.hasharrTestProfile = null; + this.cdr.markForCheck(); + this.downloads.testHasharrSettings({ + url: String(this.hasharrUrl || '').trim(), + service_id: Math.max(1, Number(this.hasharrServiceID || 1)), + timeout_sec: Math.max(1, Number(this.hasharrTimeoutSec || 20)), + }).pipe(takeUntilDestroyed(this.destroyRef)).subscribe({ + next: (resp) => { + if (!resp || typeof resp !== 'object' || ('status' in resp && resp.status === 'error')) { + this.hasharrTestStatus = 'Hasharr test failed.'; + this.hasharrTestProfile = null; + this.cdr.markForCheck(); + return; + } + const result = resp as HasharrServiceTestResponse; + if (result.valid_service_id) { + this.hasharrTestStatus = 'Hasharr reachable. Service ID is valid.'; + this.hasharrTestProfile = result.profile || null; + } else if (result.reachable) { + this.hasharrTestStatus = result.message || 'Hasharr reachable, but service ID is not valid.'; + this.hasharrTestProfile = null; + } else { + this.hasharrTestStatus = result.message || 'Could not reach hasharr.'; + this.hasharrTestProfile = null; + } + this.cdr.markForCheck(); + }, + error: () => { + this.hasharrTestStatus = 'Hasharr test failed.'; + this.hasharrTestProfile = null; + this.cdr.markForCheck(); + }, + }); + } + ngAfterViewInit() { this.downloads.queueChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => { this.queueMasterCheckbox()?.selectionChanged(); diff --git a/ui/src/app/services/downloads.service.ts b/ui/src/app/services/downloads.service.ts index 94d43c8..e414921 100644 --- a/ui/src/app/services/downloads.service.ts +++ b/ui/src/app/services/downloads.service.ts @@ -6,6 +6,21 @@ import { MeTubeSocket } from './metube-socket.service'; import { Download, Status, State } from '../interfaces'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +export interface HasharrSettings { + enabled: boolean; + url: string; + service_id: number; + timeout_sec: number; +} + +export interface HasharrServiceTestResponse { + status: string; + reachable: boolean; + valid_service_id: boolean; + message?: string; + profile?: Record; +} + export interface AddDownloadPayload { url: string; downloadType: string; @@ -224,4 +239,22 @@ export class DownloadsService { catchError(this.handleHTTPError) ); } + + getHasharrSettings() { + return this.http.get('hasharr-settings').pipe( + catchError(this.handleHTTPError) + ); + } + + saveHasharrSettings(settings: HasharrSettings) { + return this.http.post<{ status: string; msg?: string }>('hasharr-settings', settings).pipe( + catchError(this.handleHTTPError) + ); + } + + testHasharrSettings(settings: Pick) { + return this.http.post('hasharr-settings/test', settings).pipe( + catchError(this.handleHTTPError) + ); + } }