User configurable youtube rate limiting and optional cookies for bot detection
This commit is contained in:
Broque Thomas
parent
aef056cfcb
commit
75f9b7364a
5 changed files with 118 additions and 30 deletions
|
|
@ -240,6 +240,10 @@ class ConfigManager:
|
|||
"m3u_export": {
|
||||
"enabled": False
|
||||
},
|
||||
"youtube": {
|
||||
"cookies_browser": "", # "", "chrome", "firefox", "edge", "brave", "opera", "safari"
|
||||
"download_delay": 3, # seconds between sequential downloads
|
||||
},
|
||||
"hydrabase": {
|
||||
"url": "",
|
||||
"api_key": "",
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ This client provides:
|
|||
import sys
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import platform
|
||||
import asyncio
|
||||
import uuid
|
||||
|
|
@ -111,6 +112,11 @@ class YouTubeClient:
|
|||
# Callback for shutdown check (avoids circular imports)
|
||||
self.shutdown_check = None
|
||||
|
||||
# Rate limiting — serialize YouTube downloads with delay
|
||||
self._download_semaphore = threading.Semaphore(1)
|
||||
self._download_delay = config_manager.get('youtube.download_delay', 3)
|
||||
self._last_download_time = 0
|
||||
|
||||
def set_shutdown_check(self, check_callable):
|
||||
"""Set a callback function to check for system shutdown"""
|
||||
self.shutdown_check = check_callable
|
||||
|
|
@ -153,6 +159,12 @@ class YouTubeClient:
|
|||
'age_limit': None, # Don't skip age-restricted
|
||||
}
|
||||
|
||||
# Cookie support — use browser cookies for YouTube auth
|
||||
from config.settings import config_manager
|
||||
cookies_browser = config_manager.get('youtube.cookies_browser', '')
|
||||
if cookies_browser:
|
||||
self.download_opts['cookiesfrombrowser'] = (cookies_browser,)
|
||||
|
||||
# Track current download progress (mirrors Soulseek transfer tracking)
|
||||
self.current_download_id: Optional[str] = None
|
||||
self.current_download_progress = {
|
||||
|
|
@ -187,6 +199,17 @@ class YouTubeClient:
|
|||
logger.error("yt-dlp is not installed")
|
||||
return False
|
||||
|
||||
def reload_settings(self):
|
||||
"""Reload YouTube settings from config (called when settings are saved)."""
|
||||
from config.settings import config_manager
|
||||
self._download_delay = config_manager.get('youtube.download_delay', 3)
|
||||
cookies_browser = config_manager.get('youtube.cookies_browser', '')
|
||||
if cookies_browser:
|
||||
self.download_opts['cookiesfrombrowser'] = (cookies_browser,)
|
||||
elif 'cookiesfrombrowser' in self.download_opts:
|
||||
del self.download_opts['cookiesfrombrowser']
|
||||
logger.info(f"🔄 YouTube settings reloaded (delay={self._download_delay}s, cookies={'enabled' if cookies_browser else 'disabled'})")
|
||||
|
||||
async def check_connection(self) -> bool:
|
||||
"""
|
||||
Test if YouTube is accessible by attempting a lightweight API call (async, Soulseek-compatible).
|
||||
|
|
@ -568,6 +591,7 @@ class YouTubeClient:
|
|||
loop = asyncio.get_event_loop()
|
||||
|
||||
def _search():
|
||||
from config.settings import config_manager
|
||||
ydl_opts = {
|
||||
'quiet': True,
|
||||
'no_warnings': True,
|
||||
|
|
@ -583,6 +607,11 @@ class YouTubeClient:
|
|||
'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
}
|
||||
|
||||
# Add cookie support for search (avoids bot detection)
|
||||
cookies_browser = config_manager.get('youtube.cookies_browser', '')
|
||||
if cookies_browser:
|
||||
ydl_opts['cookiesfrombrowser'] = (cookies_browser,)
|
||||
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
# Search YouTube (max 50 results)
|
||||
search_results = ydl.extract_info(f"ytsearch50:{query}", download=False)
|
||||
|
|
@ -837,42 +866,54 @@ class YouTubeClient:
|
|||
"""
|
||||
Background thread worker for downloading YouTube videos.
|
||||
Updates active_downloads dict with progress.
|
||||
Serialized via semaphore with configurable delay between downloads.
|
||||
"""
|
||||
try:
|
||||
# Update state to downloading
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'InProgress, Downloading' # Match Soulseek state
|
||||
with self._download_semaphore:
|
||||
# Enforce delay since last download completed
|
||||
elapsed = time.time() - self._last_download_time
|
||||
if self._last_download_time > 0 and elapsed < self._download_delay:
|
||||
wait_time = self._download_delay - elapsed
|
||||
logger.info(f"⏳ Rate limiting: waiting {wait_time:.1f}s before next YouTube download")
|
||||
time.sleep(wait_time)
|
||||
|
||||
# Set current download ID for progress hook
|
||||
self.current_download_id = download_id
|
||||
|
||||
# Perform actual download
|
||||
file_path = self._download_sync(youtube_url, title)
|
||||
|
||||
# Clear current download ID
|
||||
self.current_download_id = None
|
||||
|
||||
if file_path:
|
||||
# Mark as completed/succeeded (match Soulseek state)
|
||||
# Update state to downloading
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
# IMPORTANT: Keep original filename for context lookup!
|
||||
# The filename must match what was used to create the context entry
|
||||
# We store the actual file path separately
|
||||
self.active_downloads[download_id]['state'] = 'Completed, Succeeded' # Match Soulseek
|
||||
self.active_downloads[download_id]['progress'] = 100.0
|
||||
self.active_downloads[download_id]['file_path'] = file_path
|
||||
# DO NOT update filename - keep original_filename for context matching
|
||||
self.active_downloads[download_id]['state'] = 'InProgress, Downloading' # Match Soulseek state
|
||||
|
||||
logger.info(f"✅ YouTube download {download_id} completed: {file_path}")
|
||||
else:
|
||||
# Mark as errored
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Errored'
|
||||
# Set current download ID for progress hook
|
||||
self.current_download_id = download_id
|
||||
|
||||
logger.error(f"❌ YouTube download {download_id} failed")
|
||||
# Perform actual download
|
||||
file_path = self._download_sync(youtube_url, title)
|
||||
|
||||
# Clear current download ID
|
||||
self.current_download_id = None
|
||||
|
||||
# Record completion time for rate limiting
|
||||
self._last_download_time = time.time()
|
||||
|
||||
if file_path:
|
||||
# Mark as completed/succeeded (match Soulseek state)
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
# IMPORTANT: Keep original filename for context lookup!
|
||||
# The filename must match what was used to create the context entry
|
||||
# We store the actual file path separately
|
||||
self.active_downloads[download_id]['state'] = 'Completed, Succeeded' # Match Soulseek
|
||||
self.active_downloads[download_id]['progress'] = 100.0
|
||||
self.active_downloads[download_id]['file_path'] = file_path
|
||||
# DO NOT update filename - keep original_filename for context matching
|
||||
|
||||
logger.info(f"✅ YouTube download {download_id} completed: {file_path}")
|
||||
else:
|
||||
# Mark as errored
|
||||
with self._download_lock:
|
||||
if download_id in self.active_downloads:
|
||||
self.active_downloads[download_id]['state'] = 'Errored'
|
||||
|
||||
logger.error(f"❌ YouTube download {download_id} failed")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ YouTube download thread failed for {download_id}: {e}")
|
||||
|
|
|
|||
|
|
@ -2796,7 +2796,7 @@ def handle_settings():
|
|||
if 'active_media_server' in new_settings:
|
||||
config_manager.set_active_media_server(new_settings['active_media_server'])
|
||||
|
||||
for service in ['spotify', 'plex', 'jellyfin', 'navidrome', 'soulseek', 'download_source', 'settings', 'database', 'metadata_enhancement', 'file_organization', 'playlist_sync', 'tidal', 'tidal_download', 'listenbrainz', 'acoustid', 'import', 'lossy_copy', 'ui_appearance']:
|
||||
for service in ['spotify', 'plex', 'jellyfin', 'navidrome', 'soulseek', 'download_source', 'settings', 'database', 'metadata_enhancement', 'file_organization', 'playlist_sync', 'tidal', 'tidal_download', 'listenbrainz', 'acoustid', 'import', 'lossy_copy', 'ui_appearance', 'youtube']:
|
||||
if service in new_settings:
|
||||
for key, value in new_settings[service].items():
|
||||
config_manager.set(f'{service}.{key}', value)
|
||||
|
|
@ -2816,6 +2816,9 @@ def handle_settings():
|
|||
navidrome_client.reload_config()
|
||||
# Reload orchestrator settings (download source mode, hybrid_primary, etc.)
|
||||
soulseek_client.reload_settings()
|
||||
# Reload YouTube client settings (rate limiting, cookies)
|
||||
if hasattr(soulseek_client, 'youtube'):
|
||||
soulseek_client.youtube.reload_settings()
|
||||
# FIX: Re-instantiate the global tidal_client to pick up new settings
|
||||
tidal_client = TidalClient()
|
||||
print("✅ Service clients re-initialized with new settings.")
|
||||
|
|
|
|||
|
|
@ -3267,6 +3267,34 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<!-- YouTube Settings (shown when youtube or hybrid mode) -->
|
||||
<div id="youtube-settings-container" style="display: none;">
|
||||
<div class="form-group">
|
||||
<label>YouTube Browser Cookies:</label>
|
||||
<select id="youtube-cookies-browser" class="form-select">
|
||||
<option value="">None (No cookies)</option>
|
||||
<option value="chrome">Chrome</option>
|
||||
<option value="firefox">Firefox</option>
|
||||
<option value="edge">Edge</option>
|
||||
<option value="brave">Brave</option>
|
||||
<option value="opera">Opera</option>
|
||||
<option value="safari">Safari</option>
|
||||
</select>
|
||||
<div class="setting-help-text">
|
||||
If YouTube shows "Sign in to confirm you're not a bot", select your browser here.
|
||||
SoulSync will use your browser's YouTube cookies to authenticate.
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label>YouTube Download Delay:</label>
|
||||
<input type="number" id="youtube-download-delay" min="0" max="30" step="1"
|
||||
value="3" placeholder="3">
|
||||
<div class="setting-help-text">
|
||||
Seconds between YouTube downloads to avoid bot detection. Default: 3
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label>Log Level:</label>
|
||||
<select id="log-level-select" class="form-select" onchange="changeLogLevel()">
|
||||
|
|
|
|||
|
|
@ -2845,6 +2845,10 @@ async function loadSettingsData() {
|
|||
document.getElementById('youtube-min-confidence').value = settings.download_source?.youtube_min_confidence || 0.65;
|
||||
document.getElementById('tidal-download-quality').value = settings.tidal_download?.quality || 'lossless';
|
||||
|
||||
// Populate YouTube settings
|
||||
document.getElementById('youtube-cookies-browser').value = settings.youtube?.cookies_browser || '';
|
||||
document.getElementById('youtube-download-delay').value = settings.youtube?.download_delay ?? 3;
|
||||
|
||||
// Update UI based on download source mode
|
||||
updateDownloadSourceUI();
|
||||
|
||||
|
|
@ -3019,6 +3023,7 @@ function updateDownloadSourceUI() {
|
|||
const mode = document.getElementById('download-source-mode').value;
|
||||
const hybridContainer = document.getElementById('hybrid-settings-container');
|
||||
const tidalContainer = document.getElementById('tidal-download-settings-container');
|
||||
const youtubeContainer = document.getElementById('youtube-settings-container');
|
||||
|
||||
// Show hybrid settings only when hybrid mode is selected
|
||||
hybridContainer.style.display = mode === 'hybrid' ? 'block' : 'none';
|
||||
|
|
@ -3026,6 +3031,9 @@ function updateDownloadSourceUI() {
|
|||
// Show Tidal download settings when tidal mode is selected
|
||||
tidalContainer.style.display = mode === 'tidal' ? 'block' : 'none';
|
||||
|
||||
// Show YouTube settings when youtube or hybrid mode is selected
|
||||
youtubeContainer.style.display = (mode === 'youtube' || mode === 'hybrid') ? 'block' : 'none';
|
||||
|
||||
// Check Tidal download auth status when switching to tidal mode
|
||||
if (mode === 'tidal') {
|
||||
checkTidalDownloadAuthStatus();
|
||||
|
|
@ -3573,6 +3581,10 @@ async function saveSettings(quiet = false) {
|
|||
ui_appearance: {
|
||||
accent_preset: document.getElementById('accent-preset')?.value || '#1db954',
|
||||
accent_color: document.getElementById('accent-custom-color')?.value || '#1db954'
|
||||
},
|
||||
youtube: {
|
||||
cookies_browser: document.getElementById('youtube-cookies-browser').value,
|
||||
download_delay: parseInt(document.getElementById('youtube-download-delay').value) || 3,
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue