Merge pull request #1 from Stash-KennyG/hash-mt-integration

Add native hasharr callback integration for completed downloads

README.md

@@ -37,6 +37,10 @@ Certain values can be set via environment variables, using the `-e` parameter on
 * __DELETE_FILE_ON_TRASHCAN__: if `true`, downloaded files are deleted on the server, when they are trashed from the "Completed" section of the UI. Defaults to `false`.
 * __DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT__: Maximum number of playlist items that can be downloaded. Defaults to `0` (no limit).
 * __CLEAR_COMPLETED_AFTER__: Number of seconds after which completed (and failed) downloads are automatically removed from the "Completed" list. Defaults to `0` (disabled).
+* __HASHARR_ENABLED__: If `true`, call hasharr when a download completes. Defaults to `false`.
+* __HASHARR_URL__: Base URL for hasharr service (for example `http://hasharr:9995`). Defaults to `http://hasharr:9995`.
+* __HASHARR_SERVICE_ID__: Hash service profile ID to call at `POST /api/hash-service/{id}`. Defaults to `1`.
+* __HASHARR_TIMEOUT_SEC__: Timeout (seconds) for hasharr callback requests. Defaults to `20`.
 
 ### 📁 Storage & Directories
 
@@ -86,6 +90,30 @@ The project's Wiki contains examples of useful configurations contributed by use
 * [YTDL_OPTIONS Cookbook](https://github.com/alexta69/metube/wiki/YTDL_OPTIONS-Cookbook)
 * [OUTPUT_TEMPLATE Cookbook](https://github.com/alexta69/metube/wiki/OUTPUT_TEMPLATE-Cookbook)
 
+## hasharr integration
+
+MeTube can call hasharr after download completion to perform pHash matching and action policy.
+
+### Runtime API endpoints
+
+- `GET /hasharr-settings` -> current effective integration settings
+- `POST /hasharr-settings` -> update runtime settings
+
+Example:
+
+```json
+{
+  "enabled": true,
+  "url": "http://hasharr:9995",
+  "service_id": 1,
+  "timeout_sec": 20
+}
+```
+
+When enabled, MeTube posts each completed output file to:
+
+`POST {HASHARR_URL}/api/hash-service/{HASHARR_SERVICE_ID}`
+
 ## 🍪 Using browser cookies
 
 In case you need to use your browser's cookies with MeTube, for example to download restricted or private videos:

app/main.py

@@ -14,6 +14,8 @@ import logging
 import json
 import pathlib
 import re
+from urllib import request as urlrequest
+from urllib import error as urlerror
 from watchfiles import DefaultFilter, Change, awatch
 
 from ytdl import DownloadQueueNotifier, DownloadQueue, Download
@@ -64,9 +66,13 @@ class Config:
         'MAX_CONCURRENT_DOWNLOADS': '3',
         'LOGLEVEL': 'INFO',
         'ENABLE_ACCESSLOG': 'false',
+        'HASHARR_ENABLED': 'false',
+        'HASHARR_URL': 'http://hasharr:9995',
+        'HASHARR_SERVICE_ID': '1',
+        'HASHARR_TIMEOUT_SEC': '20',
     }
 
-    _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG')
+    _BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG', 'HASHARR_ENABLED')
 
     def __init__(self):
         for k, v in self._DEFAULTS.items():
@@ -114,6 +120,10 @@ class Config:
         'PUBLIC_HOST_URL',
         'PUBLIC_HOST_AUDIO_URL',
         'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT',
+        'HASHARR_ENABLED',
+        'HASHARR_URL',
+        'HASHARR_SERVICE_ID',
+        'HASHARR_TIMEOUT_SEC',
     )
 
     def frontend_safe(self) -> dict:
@@ -550,6 +560,94 @@ async def history(request):
     log.info("Sending download history")
     return web.Response(text=serializer.encode(history))
 
+@routes.get(config.URL_PREFIX + 'hasharr-settings')
+async def get_hasharr_settings(request):
+    return web.Response(text=serializer.encode({
+        'enabled': bool(config.HASHARR_ENABLED),
+        'url': str(config.HASHARR_URL),
+        'service_id': int(config.HASHARR_SERVICE_ID),
+        'timeout_sec': int(config.HASHARR_TIMEOUT_SEC),
+    }), content_type='application/json')
+
+@routes.post(config.URL_PREFIX + 'hasharr-settings')
+async def set_hasharr_settings(request):
+    post = await _read_json_request(request)
+    enabled = bool(post.get('enabled', config.HASHARR_ENABLED))
+    url = str(post.get('url', config.HASHARR_URL)).strip()
+    service_id = int(post.get('service_id', config.HASHARR_SERVICE_ID))
+    timeout_sec = int(post.get('timeout_sec', config.HASHARR_TIMEOUT_SEC))
+    if not url:
+        raise web.HTTPBadRequest(reason='url is required')
+    if service_id <= 0:
+        raise web.HTTPBadRequest(reason='service_id must be > 0')
+    if timeout_sec <= 0:
+        raise web.HTTPBadRequest(reason='timeout_sec must be > 0')
+    config.HASHARR_ENABLED = enabled
+    config.HASHARR_URL = url
+    config.HASHARR_SERVICE_ID = service_id
+    config.HASHARR_TIMEOUT_SEC = timeout_sec
+    return web.Response(text=serializer.encode({'status': 'ok'}), content_type='application/json')
+
+@routes.post(config.URL_PREFIX + 'hasharr-settings/test')
+async def test_hasharr_settings(request):
+    post = await _read_json_request(request)
+    url = str(post.get('url', config.HASHARR_URL)).strip()
+    service_id = int(post.get('service_id', config.HASHARR_SERVICE_ID))
+    timeout_sec = int(post.get('timeout_sec', config.HASHARR_TIMEOUT_SEC))
+    if not url:
+        raise web.HTTPBadRequest(reason='url is required')
+    if service_id <= 0:
+        raise web.HTTPBadRequest(reason='service_id must be > 0')
+    if timeout_sec <= 0:
+        raise web.HTTPBadRequest(reason='timeout_sec must be > 0')
+
+    base_url = url.rstrip('/')
+    profile_url = f"{base_url}/v1/hash-service-profiles/{service_id}"
+
+    def _fetch_profile():
+        req = urlrequest.Request(profile_url, method='GET')
+        with urlrequest.urlopen(req, timeout=timeout_sec) as resp:
+            body = resp.read().decode('utf-8', errors='replace')
+            return resp.status, body
+
+    try:
+        status, body = await asyncio.get_running_loop().run_in_executor(None, _fetch_profile)
+        if status != 200:
+            return web.json_response({
+                'status': 'error',
+                'reachable': True,
+                'valid_service_id': False,
+                'message': f'hasharr responded with status {status}',
+            }, status=502)
+        profile = json.loads(body)
+        return web.json_response({
+            'status': 'ok',
+            'reachable': True,
+            'valid_service_id': True,
+            'profile': profile,
+        })
+    except urlerror.HTTPError as exc:
+        if exc.code == 404:
+            return web.json_response({
+                'status': 'ok',
+                'reachable': True,
+                'valid_service_id': False,
+                'message': f'Service ID {service_id} was not found in hasharr.',
+            })
+        return web.json_response({
+            'status': 'error',
+            'reachable': True,
+            'valid_service_id': False,
+            'message': f'hasharr returned HTTP {exc.code}',
+        }, status=502)
+    except Exception as exc:
+        return web.json_response({
+            'status': 'error',
+            'reachable': False,
+            'valid_service_id': False,
+            'message': f'Could not reach hasharr: {exc}',
+        }, status=502)
+
 @sio.event
 async def connect(sid, environ):
     log.info(f"Client connected: {sid}")

app/ytdl.py

@@ -14,6 +14,8 @@ import re
 import types
 import dbm
 import subprocess
+import json
+from urllib import request as urlrequest
 from typing import Any
 from functools import lru_cache
 
@@ -694,6 +696,7 @@ class DownloadQueue:
             else:
                 self.done.put(download)
                 asyncio.create_task(self.notifier.completed(download.info))
+                asyncio.create_task(self._notify_hasharr(download.info))
                 try:
                     clear_after = int(self.config.CLEAR_COMPLETED_AFTER)
                 except ValueError:
@@ -703,6 +706,56 @@ class DownloadQueue:
                     task = asyncio.create_task(self.__auto_clear_after_delay(download.info.url, clear_after))
                     task.add_done_callback(lambda t: log.error(f'Auto-clear task failed: {t.exception()}') if not t.cancelled() and t.exception() else None)
 
+    async def _notify_hasharr(self, info):
+        if not getattr(self.config, 'HASHARR_ENABLED', False):
+            return
+        base_url = str(getattr(self.config, 'HASHARR_URL', 'http://hasharr:9995')).rstrip('/')
+        service_id = int(getattr(self.config, 'HASHARR_SERVICE_ID', 1))
+        timeout_sec = int(getattr(self.config, 'HASHARR_TIMEOUT_SEC', 20))
+        files = []
+        if getattr(info, 'filename', None):
+            files.append(info.filename)
+        for cf in getattr(info, 'chapter_files', []) or []:
+            if isinstance(cf, dict) and cf.get('filename'):
+                files.append(cf['filename'])
+        for sf in getattr(info, 'subtitle_files', []) or []:
+            if isinstance(sf, dict) and sf.get('filename'):
+                files.append(sf['filename'])
+        dedup = []
+        seen = set()
+        for f in files:
+            if f and f not in seen:
+                seen.add(f)
+                dedup.append(f)
+        if not dedup:
+            return
+
+        def _post_one(rel_name):
+            download_type = getattr(info, 'download_type', 'video')
+            base_dir = self.config.AUDIO_DOWNLOAD_DIR if download_type == 'audio' else self.config.DOWNLOAD_DIR
+            full_path = os.path.join(base_dir, rel_name)
+            payload = {
+                "filePath": full_path,
+                "source": "metube",
+                "jobId": str(getattr(info, 'id', '')),
+            }
+            data = json.dumps(payload).encode('utf-8')
+            req = urlrequest.Request(
+                f"{base_url}/api/hash-service/{service_id}",
+                data=data,
+                headers={"Content-Type": "application/json"},
+                method="POST",
+            )
+            with urlrequest.urlopen(req, timeout=timeout_sec) as resp:
+                return resp.status
+
+        for rel_name in dedup:
+            try:
+                status = await asyncio.get_running_loop().run_in_executor(None, _post_one, rel_name)
+                log.info(f"hasharr callback status={status} file={rel_name}")
+            except Exception as exc:
+                log.warning(f"hasharr callback failed for {rel_name}: {exc}")
+
     async def __auto_clear_after_delay(self, url, delay_seconds):
         await asyncio.sleep(delay_seconds)
         if self.done.exists(url):

ui/src/app/app.html

@@ -428,6 +428,84 @@
               <div class="row">
                 <div class="col-12">
                   <hr class="my-3">
+                  <div class="row g-3 mb-3 hasharr-integration">
+                    <div class="col-12">
+                      <div class="action-group-label">Hasharr Integration</div>
+                    </div>
+                    <div class="col-12 col-md-3 d-flex align-items-center">
+                      <div class="form-check form-switch mb-0">
+                        <input class="form-check-input" type="checkbox" role="switch" id="hasharrEnabled"
+                          name="hasharrEnabled" [(ngModel)]="hasharrEnabled"
+                          [disabled]="addInProgress || downloads.loading">
+                        <label class="form-check-label" for="hasharrEnabled">Enabled</label>
+                      </div>
+                    </div>
+                    <div class="col-12 col-md-5">
+                      <div class="input-group">
+                        <span class="input-group-text">Hasharr URL</span>
+                        <input type="text"
+                          class="form-control"
+                          placeholder="http://hasharr:9995"
+                          name="hasharrUrl"
+                          [(ngModel)]="hasharrUrl"
+                          [disabled]="addInProgress || downloads.loading">
+                      </div>
+                    </div>
+                    <div class="col-6 col-md-2">
+                      <div class="input-group">
+                        <span class="input-group-text">Service ID</span>
+                        <input type="number"
+                          min="1"
+                          class="form-control"
+                          name="hasharrServiceID"
+                          [(ngModel)]="hasharrServiceID"
+                          [disabled]="addInProgress || downloads.loading">
+                      </div>
+                    </div>
+                    <div class="col-6 col-md-2">
+                      <div class="input-group">
+                        <span class="input-group-text">Timeout</span>
+                        <input type="number"
+                          min="1"
+                          class="form-control"
+                          name="hasharrTimeoutSec"
+                          [(ngModel)]="hasharrTimeoutSec"
+                          [disabled]="addInProgress || downloads.loading">
+                      </div>
+                    </div>
+                    <div class="col-12 col-md-7">
+                      <div class="hasharr-action-row">
+                        <button type="button" class="btn btn-secondary" (click)="saveHasharrSettings()">
+                          Save Hasharr Settings
+                        </button>
+                        <button type="button" class="btn btn-outline-secondary" (click)="testHasharrService()">
+                          Test Service
+                        </button>
+                      </div>
+                    </div>
+                    <div class="col-12 col-md-5">
+                      <div class="hasharr-status-block">
+                        @if (hasharrSettingsStatus) {
+                          <small class="text-muted d-block">{{ hasharrSettingsStatus }}</small>
+                        }
+                        @if (hasharrTestStatus) {
+                          <small class="text-muted d-block">{{ hasharrTestStatus }}</small>
+                        }
+                      </div>
+                    </div>
+                    @if (hasharrTestProfile) {
+                      <div class="col-12">
+                        <div class="small text-muted hasharr-profile-result">
+                          <div><strong>Profile Name:</strong> {{ hasharrTestProfile['name'] }}</div>
+                          <div><strong>Enabled:</strong> {{ hasharrTestProfile['enabled'] }}</div>
+                          <div><strong>Apply Actions:</strong> {{ hasharrTestProfile['applyActions'] }}</div>
+                          <div><strong>Stash Index:</strong> {{ hasharrTestProfile['stashIndex'] }}</div>
+                          <div><strong>Max Time Delta:</strong> {{ hasharrTestProfile['maxTimeDelta'] }}</div>
+                          <div><strong>Max Distance:</strong> {{ hasharrTestProfile['maxDistance'] }}</div>
+                        </div>
+                      </div>
+                    }
+                  </div>
                   <div class="row g-3">
                     <div class="col-md-4">
                       <div class="action-group-label">Cookies</div>

ui/src/app/app.sass

@@ -196,3 +196,26 @@ main
 
   &.active
     color: var(--bs-success-text-emphasis)
+
+.hasharr-integration
+  .input-group
+    margin-bottom: 0
+
+.hasharr-action-row
+  display: flex
+  flex-wrap: wrap
+  gap: 0.5rem
+
+.hasharr-status-block
+  min-height: 2.5rem
+  display: flex
+  flex-direction: column
+  justify-content: center
+  text-align: left
+
+.hasharr-profile-result
+  margin-top: 0.25rem
+  padding: 0.5rem 0.75rem
+  border: 1px solid var(--bs-border-color)
+  border-radius: 0.375rem
+  background: var(--bs-tertiary-bg)

ui/src/app/app.ts

@@ -10,7 +10,7 @@ import { NgSelectModule } from '@ng-select/ng-select';
 import { faTrashAlt, faCheckCircle, faTimesCircle, faRedoAlt, faSun, faMoon, faCheck, faCircleHalfStroke, faDownload, faExternalLinkAlt, faFileImport, faFileExport, faCopy, faClock, faTachometerAlt, faSortAmountDown, faSortAmountUp, faChevronRight, faChevronDown, faUpload } from '@fortawesome/free-solid-svg-icons';
 import { faGithub } from '@fortawesome/free-brands-svg-icons';
 import { CookieService } from 'ngx-cookie-service';
-import { AddDownloadPayload, DownloadsService } from './services/downloads.service';
+import { AddDownloadPayload, DownloadsService, HasharrSettings, HasharrServiceTestResponse } from './services/downloads.service';
 import { Themes } from './theme';
 import {
   Download,
@@ -96,6 +96,13 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
   ytDlpVersion: string | null = null;
   metubeVersion: string | null = null;
   isAdvancedOpen = false;
+  hasharrEnabled = false;
+  hasharrUrl = 'http://hasharr:9995';
+  hasharrServiceID = 1;
+  hasharrTimeoutSec = 20;
+  hasharrSettingsStatus = '';
+  hasharrTestStatus = '';
+  hasharrTestProfile: Record<string, unknown> | null = null;
   sortAscending = false;
   expandedErrors: Set<string> = new Set<string>();
   cachedSortedDone: [string, Download][] = [];
@@ -268,6 +275,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
       this.cdr.markForCheck();
     });
     this.getConfiguration();
+    this.loadHasharrSettings();
     this.getYtdlOptionsUpdateTime();
     this.customDirs$ = this.getMatchingCustomDir();
     this.setTheme(this.activeTheme!);
@@ -275,6 +283,94 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
     this.colorSchemeMediaQuery.addEventListener('change', this.onColorSchemeChanged);
   }
 
+  loadHasharrSettings() {
+    this.downloads.getHasharrSettings().pipe(takeUntilDestroyed(this.destroyRef)).subscribe({
+      next: (data) => {
+        if (!data || typeof data !== 'object' || ('status' in data && data.status === 'error')) {
+          this.hasharrSettingsStatus = 'Unable to load hasharr settings.';
+          this.cdr.markForCheck();
+          return;
+        }
+        const settings = data as HasharrSettings;
+        this.hasharrEnabled = !!settings.enabled;
+        this.hasharrUrl = String(settings.url || 'http://hasharr:9995');
+        this.hasharrServiceID = Math.max(1, Number(settings.service_id || 1));
+        this.hasharrTimeoutSec = Math.max(1, Number(settings.timeout_sec || 20));
+        this.hasharrSettingsStatus = '';
+        this.cdr.markForCheck();
+      },
+      error: () => {
+        this.hasharrSettingsStatus = 'Unable to load hasharr settings.';
+        this.cdr.markForCheck();
+      },
+    });
+  }
+
+  saveHasharrSettings() {
+    const payload: HasharrSettings = {
+      enabled: !!this.hasharrEnabled,
+      url: String(this.hasharrUrl || '').trim(),
+      service_id: Math.max(1, Number(this.hasharrServiceID || 1)),
+      timeout_sec: Math.max(1, Number(this.hasharrTimeoutSec || 20)),
+    };
+    if (!payload.url) {
+      this.hasharrSettingsStatus = 'Hasharr URL is required.';
+      this.cdr.markForCheck();
+      return;
+    }
+    this.downloads.saveHasharrSettings(payload).pipe(takeUntilDestroyed(this.destroyRef)).subscribe({
+      next: (out) => {
+        if (out && typeof out === 'object' && 'status' in out && out.status === 'ok') {
+          this.hasharrSettingsStatus = 'Hasharr settings saved.';
+        } else {
+          this.hasharrSettingsStatus = 'Failed to save hasharr settings.';
+        }
+        this.cdr.markForCheck();
+      },
+      error: () => {
+        this.hasharrSettingsStatus = 'Failed to save hasharr settings.';
+        this.cdr.markForCheck();
+      },
+    });
+  }
+
+  testHasharrService() {
+    this.hasharrTestStatus = 'Testing hasharr service...';
+    this.hasharrTestProfile = null;
+    this.cdr.markForCheck();
+    this.downloads.testHasharrSettings({
+      url: String(this.hasharrUrl || '').trim(),
+      service_id: Math.max(1, Number(this.hasharrServiceID || 1)),
+      timeout_sec: Math.max(1, Number(this.hasharrTimeoutSec || 20)),
+    }).pipe(takeUntilDestroyed(this.destroyRef)).subscribe({
+      next: (resp) => {
+        if (!resp || typeof resp !== 'object' || ('status' in resp && resp.status === 'error')) {
+          this.hasharrTestStatus = 'Hasharr test failed.';
+          this.hasharrTestProfile = null;
+          this.cdr.markForCheck();
+          return;
+        }
+        const result = resp as HasharrServiceTestResponse;
+        if (result.valid_service_id) {
+          this.hasharrTestStatus = 'Hasharr reachable. Service ID is valid.';
+          this.hasharrTestProfile = result.profile || null;
+        } else if (result.reachable) {
+          this.hasharrTestStatus = result.message || 'Hasharr reachable, but service ID is not valid.';
+          this.hasharrTestProfile = null;
+        } else {
+          this.hasharrTestStatus = result.message || 'Could not reach hasharr.';
+          this.hasharrTestProfile = null;
+        }
+        this.cdr.markForCheck();
+      },
+      error: () => {
+        this.hasharrTestStatus = 'Hasharr test failed.';
+        this.hasharrTestProfile = null;
+        this.cdr.markForCheck();
+      },
+    });
+  }
+
   ngAfterViewInit() {
     this.downloads.queueChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => {
       this.queueMasterCheckbox()?.selectionChanged();

ui/src/app/services/downloads.service.ts

@@ -6,6 +6,21 @@ import { MeTubeSocket } from './metube-socket.service';
 import { Download, Status, State } from '../interfaces';
 import { takeUntilDestroyed } from '@angular/core/rxjs-interop';
 
+export interface HasharrSettings {
+  enabled: boolean;
+  url: string;
+  service_id: number;
+  timeout_sec: number;
+}
+
+export interface HasharrServiceTestResponse {
+  status: string;
+  reachable: boolean;
+  valid_service_id: boolean;
+  message?: string;
+  profile?: Record<string, unknown>;
+}
+
 export interface AddDownloadPayload {
   url: string;
   downloadType: string;
@@ -224,4 +239,22 @@ export class DownloadsService {
       catchError(this.handleHTTPError)
     );
   }
+
+  getHasharrSettings() {
+    return this.http.get<HasharrSettings>('hasharr-settings').pipe(
+      catchError(this.handleHTTPError)
+    );
+  }
+
+  saveHasharrSettings(settings: HasharrSettings) {
+    return this.http.post<{ status: string; msg?: string }>('hasharr-settings', settings).pipe(
+      catchError(this.handleHTTPError)
+    );
+  }
+
+  testHasharrSettings(settings: Pick<HasharrSettings, 'url' | 'service_id' | 'timeout_sec'>) {
+    return this.http.post<HasharrServiceTestResponse>('hasharr-settings/test', settings).pipe(
+      catchError(this.handleHTTPError)
+    );
+  }
 }