Spotify Public Profile scrapping (missing the sync button flow)
This commit is contained in:
Rafael Richard
parent
05c2cd7320
commit
5b5ff7443c
8 changed files with 746 additions and 7 deletions
|
|
@ -23,7 +23,8 @@ RUN useradd --create-home --shell /bin/bash --uid 1000 soulsync
|
|||
# Copy requirements and install Python dependencies
|
||||
COPY requirements-webui.txt .
|
||||
RUN pip install --no-cache-dir --upgrade pip && \
|
||||
pip install --no-cache-dir -r requirements-webui.txt
|
||||
pip install --no-cache-dir -r requirements-webui.txt && \
|
||||
playwright install --with-deps chromium
|
||||
|
||||
# Copy application code
|
||||
COPY . .
|
||||
|
|
|
|||
|
|
@ -3,7 +3,11 @@
|
|||
"spotify": {
|
||||
"client_id": "SpotifyClientID",
|
||||
"client_secret": "SpotifyClientSecret",
|
||||
"redirect_uri": "http://127.0.0.1:8888/callback"
|
||||
"redirect_uri": "http://127.0.0.1:8888/callback",
|
||||
"friend_profiles": [
|
||||
"userID1",
|
||||
"userID2"
|
||||
]
|
||||
},
|
||||
"tidal": {
|
||||
"client_id": "TidalClientID",
|
||||
|
|
|
|||
411
core/spotify_profile_scraper.py
Normal file
411
core/spotify_profile_scraper.py
Normal file
|
|
@ -0,0 +1,411 @@
|
|||
"""
|
||||
Spotify Profile Scraper - Fetches public playlists from Spotify user profiles.
|
||||
|
||||
This module scrapes the public Spotify profile page to extract playlist information,
|
||||
bypassing API restrictions for users who don't have API access or want to sync
|
||||
playlists from friends' profiles.
|
||||
|
||||
Requires: playwright (headless browser) for full JS-rendered page scraping.
|
||||
Install with: pip install playwright && python -m playwright install chromium
|
||||
"""
|
||||
|
||||
import requests
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from typing import List, Dict, Any, Optional, Tuple
|
||||
from utils.logging_config import get_logger
|
||||
from config.settings import config_manager
|
||||
|
||||
logger = get_logger("spotify_profile_scraper")
|
||||
|
||||
# Import playwright - required for this module
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
except ImportError:
|
||||
sync_playwright = None
|
||||
logger.warning("Playwright not installed - friend playlist scraping will not work")
|
||||
logger.warning("Install with: pip install playwright && python -m playwright install chromium")
|
||||
|
||||
# User agent for HTTP requests (fetching individual playlist details)
|
||||
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
|
||||
|
||||
|
||||
def is_playwright_available() -> bool:
|
||||
"""Check if playwright is installed and available."""
|
||||
return sync_playwright is not None
|
||||
|
||||
|
||||
def fetch_profile_playlists(user_id: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Fetch public playlists from a Spotify user's profile page using playwright.
|
||||
|
||||
Args:
|
||||
user_id: The Spotify user ID (e.g., "12166842163")
|
||||
|
||||
Returns:
|
||||
List of playlist dictionaries with id, name, owner, track_count, etc.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If playwright is not installed
|
||||
"""
|
||||
if not is_playwright_available():
|
||||
raise RuntimeError(
|
||||
"Playwright is required for friend playlist scraping. "
|
||||
"Install with: pip install playwright && python -m playwright install chromium"
|
||||
)
|
||||
|
||||
return _fetch_playlists_with_playwright(user_id)
|
||||
|
||||
|
||||
def _fetch_playlists_with_playwright(user_id: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Fetch all playlists using playwright headless browser.
|
||||
|
||||
This renders the /playlists page with JavaScript to get the complete list.
|
||||
"""
|
||||
playlists_url = f"https://open.spotify.com/user/{user_id}/playlists"
|
||||
logger.info(f"Playwright: Fetching playlists from {playlists_url}")
|
||||
|
||||
playlists = []
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True)
|
||||
# Use default Chrome user agent (custom UA causes Spotify to redirect)
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
|
||||
try:
|
||||
# Navigate to playlists page
|
||||
page.goto(playlists_url, wait_until="networkidle", timeout=30000)
|
||||
|
||||
# Wait for page to fully render (no specific selector needed)
|
||||
page.wait_for_timeout(5000)
|
||||
|
||||
# Extract playlist links from rendered page
|
||||
playlist_links = page.query_selector_all('a[href*="/playlist/"]')
|
||||
|
||||
seen_ids = set()
|
||||
for link in playlist_links:
|
||||
href = link.get_attribute('href')
|
||||
if href and '/playlist/' in href:
|
||||
# Extract playlist ID from href
|
||||
match = re.search(r'/playlist/([a-zA-Z0-9]+)', href)
|
||||
if match:
|
||||
playlist_id = match.group(1)
|
||||
if playlist_id not in seen_ids:
|
||||
seen_ids.add(playlist_id)
|
||||
|
||||
# Get playlist name from link text or parent
|
||||
name = link.inner_text().strip() or None
|
||||
|
||||
playlists.append({
|
||||
'id': playlist_id,
|
||||
'name': name,
|
||||
'owner': user_id,
|
||||
'source': 'friend_profile',
|
||||
'source_user_id': user_id
|
||||
})
|
||||
|
||||
logger.info(f"Playwright: Found {len(playlists)} playlist links")
|
||||
|
||||
finally:
|
||||
browser.close()
|
||||
|
||||
# Fetch details (name, track count) for each playlist
|
||||
for playlist in playlists:
|
||||
track_count, fetched_name = _fetch_playlist_details(playlist['id'])
|
||||
playlist['track_count'] = track_count
|
||||
if not playlist.get('name') and fetched_name:
|
||||
playlist['name'] = fetched_name
|
||||
|
||||
# Skip playlists with no name (likely deleted/inaccessible)
|
||||
if not playlist.get('name'):
|
||||
logger.debug(f"Skipping playlist with no name: {playlist['id']}")
|
||||
|
||||
# Filter out playlists with no name
|
||||
playlists = [p for p in playlists if p.get('name')]
|
||||
|
||||
return playlists
|
||||
|
||||
|
||||
def _fetch_playlists_with_http(user_id: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Fetch playlists using simple HTTP request (limited to ~10 playlists).
|
||||
|
||||
Fallback method when playwright is not available.
|
||||
"""
|
||||
profile_url = f"https://open.spotify.com/user/{user_id}"
|
||||
|
||||
logger.info(f"HTTP: Fetching playlists from {profile_url}")
|
||||
|
||||
try:
|
||||
headers = {"User-Agent": USER_AGENT}
|
||||
|
||||
response = requests.get(profile_url, headers=headers, timeout=15)
|
||||
response.raise_for_status()
|
||||
|
||||
html_content = response.text
|
||||
|
||||
# Try to extract from initialState JSON first (most reliable)
|
||||
playlists = _extract_from_initial_state(html_content, user_id)
|
||||
|
||||
if playlists:
|
||||
logger.info(f"HTTP: Extracted {len(playlists)} playlists from initialState")
|
||||
return playlists
|
||||
|
||||
# Fallback: Extract playlist IDs from href links
|
||||
playlists = _extract_from_html_links(html_content)
|
||||
|
||||
if playlists:
|
||||
logger.info(f"HTTP: Extracted {len(playlists)} playlists from HTML links")
|
||||
return playlists
|
||||
|
||||
logger.warning(f"HTTP: No playlists found for user {user_id}")
|
||||
return []
|
||||
|
||||
except requests.RequestException as e:
|
||||
logger.error(f"Failed to fetch profile page for {user_id}: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error fetching profile for {user_id}: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def _extract_from_initial_state(html_content: str, user_id: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Extract playlist data from the base64-encoded initialState script tag.
|
||||
|
||||
The initialState contains structured JSON with full playlist metadata.
|
||||
"""
|
||||
try:
|
||||
# Find the initialState script tag
|
||||
match = re.search(r'<script id="initialState" type="text/plain">([^<]+)</script>', html_content)
|
||||
|
||||
if not match:
|
||||
logger.debug("initialState script tag not found")
|
||||
return []
|
||||
|
||||
# Decode base64
|
||||
encoded_data = match.group(1)
|
||||
decoded_bytes = base64.b64decode(encoded_data)
|
||||
state_json = json.loads(decoded_bytes.decode('utf-8'))
|
||||
|
||||
# Navigate to the playlists data
|
||||
entities = state_json.get('entities', {})
|
||||
items = entities.get('items', {})
|
||||
|
||||
# The user data is stored with key like "spotify:user:12166842163"
|
||||
user_key = f"spotify:user:{user_id}"
|
||||
user_data = items.get(user_key, {})
|
||||
|
||||
if not user_data:
|
||||
logger.debug(f"No user data found for key {user_key}")
|
||||
return []
|
||||
|
||||
public_playlists = user_data.get('publicPlaylistsV2', {})
|
||||
playlist_items = public_playlists.get('items', [])
|
||||
|
||||
playlists = []
|
||||
for item in playlist_items:
|
||||
data = item.get('data', {})
|
||||
|
||||
# Skip playlists that are NotFound (deleted, private, or inaccessible)
|
||||
if data.get('__typename') == 'NotFound':
|
||||
logger.debug(f"Skipping NotFound playlist: {item.get('_uri')}")
|
||||
continue
|
||||
|
||||
uri = item.get('_uri', '') or data.get('uri', '')
|
||||
|
||||
# Extract playlist ID from URI (spotify:playlist:XXXXX)
|
||||
playlist_id = uri.split(':')[-1] if uri else None
|
||||
|
||||
if not playlist_id:
|
||||
continue
|
||||
|
||||
# Extract image URL from nested structure
|
||||
image_url = None
|
||||
images = data.get('images', {})
|
||||
image_items = images.get('items', [])
|
||||
if image_items:
|
||||
sources = image_items[0].get('sources', [])
|
||||
if sources:
|
||||
image_url = sources[0].get('url')
|
||||
|
||||
# Fetch track count (and name if missing) from playlist page
|
||||
playlist_name = data.get('name')
|
||||
track_count, fetched_name = _fetch_playlist_details(playlist_id)
|
||||
|
||||
# Use fetched name if original is missing
|
||||
if not playlist_name and fetched_name:
|
||||
playlist_name = fetched_name
|
||||
|
||||
# Skip if still no valid name (likely deleted/inaccessible)
|
||||
if not playlist_name:
|
||||
logger.debug(f"Skipping playlist with no name: {playlist_id}")
|
||||
continue
|
||||
|
||||
playlist_info = {
|
||||
'id': playlist_id,
|
||||
'name': playlist_name,
|
||||
'owner': user_id,
|
||||
'owner_display_name': user_data.get('name', user_id),
|
||||
'image_url': image_url,
|
||||
'followers': data.get('followers', 0),
|
||||
'track_count': track_count,
|
||||
'source': 'friend_profile',
|
||||
'source_user_id': user_id
|
||||
}
|
||||
|
||||
playlists.append(playlist_info)
|
||||
|
||||
return playlists
|
||||
|
||||
except (json.JSONDecodeError, ValueError) as e:
|
||||
logger.debug(f"Failed to parse initialState JSON: {e}")
|
||||
return []
|
||||
except Exception as e:
|
||||
logger.debug(f"Error extracting from initialState: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def _fetch_playlist_details(playlist_id: str) -> tuple:
|
||||
"""
|
||||
Fetch track count and name for a playlist by scraping its page.
|
||||
|
||||
Args:
|
||||
playlist_id: Spotify playlist ID
|
||||
|
||||
Returns:
|
||||
Tuple of (track_count, name) - defaults to (0, None) if unable to fetch
|
||||
"""
|
||||
try:
|
||||
playlist_url = f"https://open.spotify.com/playlist/{playlist_id}"
|
||||
headers = {"User-Agent": USER_AGENT}
|
||||
|
||||
response = requests.get(playlist_url, headers=headers, timeout=10)
|
||||
response.raise_for_status()
|
||||
|
||||
# Find initialState
|
||||
match = re.search(r'<script id="initialState" type="text/plain">([^<]+)</script>', response.text)
|
||||
if not match:
|
||||
return (0, None)
|
||||
|
||||
decoded = base64.b64decode(match.group(1)).decode('utf-8')
|
||||
data = json.loads(decoded)
|
||||
|
||||
# Get playlist data
|
||||
items = data.get('entities', {}).get('items', {})
|
||||
playlist_key = f"spotify:playlist:{playlist_id}"
|
||||
playlist_data = items.get(playlist_key, {})
|
||||
|
||||
# Extract totalCount from content
|
||||
content = playlist_data.get('content', {})
|
||||
track_count = content.get('totalCount', 0)
|
||||
|
||||
# Extract name
|
||||
name = playlist_data.get('name')
|
||||
|
||||
logger.debug(f"Playlist {playlist_id}: '{name}' with {track_count} tracks")
|
||||
return (track_count, name)
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not fetch details for {playlist_id}: {e}")
|
||||
return (0, None)
|
||||
|
||||
|
||||
# Keep the old function for backward compatibility with tests
|
||||
def _fetch_playlist_track_count(playlist_id: str) -> int:
|
||||
"""Fetch track count only (wrapper for backward compatibility)."""
|
||||
track_count, _ = _fetch_playlist_details(playlist_id)
|
||||
return track_count
|
||||
|
||||
|
||||
def _extract_from_html_links(html_content: str) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Fallback method: Extract playlist IDs from href links in the HTML.
|
||||
|
||||
Looks for patterns like href="/playlist/4xJiUcKvrFEhhfhthMeOx7"
|
||||
"""
|
||||
try:
|
||||
# Find all playlist links
|
||||
pattern = r'href="/playlist/([a-zA-Z0-9]+)"'
|
||||
matches = re.findall(pattern, html_content)
|
||||
|
||||
# Deduplicate while preserving order
|
||||
seen = set()
|
||||
unique_ids = []
|
||||
for playlist_id in matches:
|
||||
if playlist_id not in seen:
|
||||
seen.add(playlist_id)
|
||||
unique_ids.append(playlist_id)
|
||||
|
||||
playlists = []
|
||||
for playlist_id in unique_ids:
|
||||
playlist_info = {
|
||||
'id': playlist_id,
|
||||
'name': f'Playlist {playlist_id[:8]}...', # Placeholder name
|
||||
'owner': 'Unknown',
|
||||
'image_url': None,
|
||||
'followers': 0,
|
||||
'source': 'friend_profile'
|
||||
}
|
||||
playlists.append(playlist_info)
|
||||
|
||||
return playlists
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Error extracting from HTML links: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def get_all_friend_playlists() -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Fetch playlists from all configured friend profiles.
|
||||
|
||||
Reads the friend_profiles list from config and fetches playlists from each.
|
||||
|
||||
Returns:
|
||||
Combined list of playlists from all friend profiles
|
||||
"""
|
||||
spotify_config = config_manager.get_spotify_config()
|
||||
friend_profiles = spotify_config.get('friend_profiles', [])
|
||||
|
||||
if not friend_profiles:
|
||||
logger.debug("No friend profiles configured")
|
||||
return []
|
||||
|
||||
logger.info(f"Fetching playlists from {len(friend_profiles)} friend profile(s)")
|
||||
|
||||
all_playlists = []
|
||||
for user_id in friend_profiles:
|
||||
try:
|
||||
playlists = fetch_profile_playlists(user_id)
|
||||
all_playlists.extend(playlists)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch playlists for friend {user_id}: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"Total friend playlists fetched: {len(all_playlists)}")
|
||||
return all_playlists
|
||||
|
||||
|
||||
# For standalone testing
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
if len(sys.argv) > 1:
|
||||
test_user_id = sys.argv[1]
|
||||
else:
|
||||
test_user_id = "12166842163"
|
||||
|
||||
print(f"Testing profile scraper for user: {test_user_id}")
|
||||
print("-" * 50)
|
||||
|
||||
playlists = fetch_profile_playlists(test_user_id)
|
||||
|
||||
print(f"Found {len(playlists)} playlists:")
|
||||
for i, p in enumerate(playlists, 1):
|
||||
print(f"{i}. {p['name']} (ID: {p['id']})")
|
||||
if p.get('followers'):
|
||||
print(f" Followers: {p['followers']}")
|
||||
|
|
@ -11,4 +11,5 @@ aiohttp>=3.9.0
|
|||
unidecode>=1.3.8
|
||||
yt-dlp>=2024.12.13
|
||||
Flask>=3.0.0
|
||||
lrclibapi>=0.3.1
|
||||
lrclibapi>=0.3.1
|
||||
playwright>=1.40.0
|
||||
227
tools/test_spotify_profile_scraper.py
Normal file
227
tools/test_spotify_profile_scraper.py
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Spotify Profile Scraper Test Suite
|
||||
|
||||
Tests the spotify_profile_scraper module for fetching public playlists
|
||||
from Spotify user profiles without using the API.
|
||||
|
||||
All tests use MOCKED data - no network requests to Spotify.
|
||||
|
||||
Usage:
|
||||
python tools/test_spotify_profile_scraper.py
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import base64
|
||||
import json
|
||||
from unittest.mock import patch, Mock
|
||||
|
||||
# Add parent directory to path to import from core
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from utils.logging_config import get_logger
|
||||
from core.spotify_profile_scraper import (
|
||||
fetch_profile_playlists,
|
||||
_extract_from_initial_state,
|
||||
_extract_from_html_links,
|
||||
_fetch_playlist_track_count
|
||||
)
|
||||
|
||||
logger = get_logger("spotify_scraper_test")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MOCK DATA
|
||||
# =============================================================================
|
||||
|
||||
# Well-known Spotify editorial playlist IDs for realistic mock data
|
||||
MOCK_PLAYLISTS = [
|
||||
{"id": "37i9dQZF1DXcBWIGoYBM5M", "name": "Today's Top Hits", "followers": 35000000},
|
||||
{"id": "37i9dQZF1DX0XUsuxWHRQd", "name": "RapCaviar", "followers": 15000000},
|
||||
{"id": "37i9dQZF1DX4JAvHpjipBk", "name": "New Music Friday", "followers": 8000000},
|
||||
]
|
||||
|
||||
|
||||
def _create_mock_initial_state(user_id: str, playlists: list) -> str:
|
||||
"""Create a mock initialState JSON structure matching Spotify's format"""
|
||||
playlist_items = []
|
||||
for p in playlists:
|
||||
playlist_items.append({
|
||||
"__typename": "PlaylistResponseWrapper",
|
||||
"_uri": f"spotify:playlist:{p['id']}",
|
||||
"data": {
|
||||
"__typename": "Playlist",
|
||||
"followers": p.get("followers", 0),
|
||||
"name": p["name"],
|
||||
"uri": f"spotify:playlist:{p['id']}",
|
||||
"images": {"items": [{"sources": [{"url": f"https://example.com/{p['id']}.jpg"}]}]}
|
||||
}
|
||||
})
|
||||
|
||||
state = {
|
||||
"entities": {
|
||||
"items": {
|
||||
f"spotify:user:{user_id}": {
|
||||
"__typename": "User",
|
||||
"id": user_id,
|
||||
"name": f"Test User",
|
||||
"publicPlaylistsV2": {"items": playlist_items, "totalCount": len(playlist_items)}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return base64.b64encode(json.dumps(state).encode()).decode()
|
||||
|
||||
|
||||
def _create_mock_html(user_id: str, playlists: list) -> str:
|
||||
"""Create mock Spotify profile HTML page"""
|
||||
state = _create_mock_initial_state(user_id, playlists)
|
||||
return f'<html><body><script id="initialState" type="text/plain">{state}</script></body></html>'
|
||||
|
||||
|
||||
MOCK_USER = "testuser"
|
||||
MOCK_HTML = _create_mock_html(MOCK_USER, MOCK_PLAYLISTS)
|
||||
|
||||
|
||||
def _create_mock_playlist_page(playlist_id: str, track_count: int) -> str:
|
||||
"""Create mock Spotify playlist page HTML with track count in initialState"""
|
||||
state = {
|
||||
"entities": {
|
||||
"items": {
|
||||
f"spotify:playlist:{playlist_id}": {
|
||||
"__typename": "Playlist",
|
||||
"content": {
|
||||
"__typename": "PlaylistItemsPage",
|
||||
"items": [],
|
||||
"totalCount": track_count,
|
||||
"pagingInfo": {"nextOffset": None}
|
||||
},
|
||||
"name": "Test Playlist",
|
||||
"id": playlist_id
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
encoded = base64.b64encode(json.dumps(state).encode()).decode()
|
||||
return f'<html><body><script id="initialState" type="text/plain">{encoded}</script></body></html>'
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TESTS
|
||||
# =============================================================================
|
||||
|
||||
class TestSpotifyProfileScraper:
|
||||
"""All tests use mocked data"""
|
||||
|
||||
def __init__(self):
|
||||
self.passed = 0
|
||||
self.failed = 0
|
||||
|
||||
def run_all(self):
|
||||
print("\n" + "=" * 60)
|
||||
print("🧪 SPOTIFY PROFILE SCRAPER TESTS (mocked)")
|
||||
print("=" * 60 + "\n")
|
||||
|
||||
self.test_extract_initial_state()
|
||||
self.test_extract_html_links()
|
||||
self.test_playlist_fields()
|
||||
self.test_fetch_with_playwright_mock()
|
||||
self.test_playwright_not_installed()
|
||||
self.test_track_count_fetch()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print(f"📊 RESULTS: {self.passed}/{self.passed + self.failed} passed")
|
||||
print("=" * 60)
|
||||
if self.failed == 0:
|
||||
print("🎉 All tests passed!")
|
||||
return self.failed == 0
|
||||
|
||||
def _check(self, cond, name, detail=""):
|
||||
if cond:
|
||||
print(f" ✅ {name}")
|
||||
self.passed += 1
|
||||
else:
|
||||
print(f" ❌ {name}" + (f" - {detail}" if detail else ""))
|
||||
self.failed += 1
|
||||
|
||||
def test_extract_initial_state(self):
|
||||
"""Parse playlists from initialState JSON"""
|
||||
print("📦 Parse initialState JSON")
|
||||
result = _extract_from_initial_state(MOCK_HTML, MOCK_USER)
|
||||
self._check(len(result) == 3, "Extracts 3 playlists")
|
||||
self._check(result[0]['id'] == MOCK_PLAYLISTS[0]['id'], "Correct playlist ID")
|
||||
self._check(result[0]['name'] == "Today's Top Hits", "Correct playlist name")
|
||||
self._check(result[0]['source'] == 'friend_profile', "Source is friend_profile")
|
||||
|
||||
def test_extract_html_links(self):
|
||||
"""Fallback: parse playlist IDs from HTML links"""
|
||||
print("\n📦 Parse HTML links (fallback)")
|
||||
html = '<a href="/playlist/abc123">P1</a><a href="/playlist/def456">P2</a><a href="/playlist/abc123">P1</a>'
|
||||
result = _extract_from_html_links(html)
|
||||
self._check(len(result) == 2, "Deduplicates to 2 playlists")
|
||||
self._check(result[0]['id'] == 'abc123', "First ID correct")
|
||||
self._check(result[1]['id'] == 'def456', "Second ID correct")
|
||||
|
||||
def test_playlist_fields(self):
|
||||
"""Playlist dict has required fields"""
|
||||
print("\n📦 Playlist structure")
|
||||
result = _extract_from_initial_state(MOCK_HTML, MOCK_USER)
|
||||
p = result[0] if result else {}
|
||||
for f in ['id', 'name', 'owner', 'source']:
|
||||
self._check(f in p, f"Has '{f}' field")
|
||||
|
||||
def test_fetch_with_playwright_mock(self):
|
||||
"""fetch_profile_playlists with mocked playwright"""
|
||||
print("\n📦 fetch_profile_playlists (playwright mocked)")
|
||||
|
||||
# Create expected result from playwright
|
||||
mock_playlists = [
|
||||
{'id': MOCK_PLAYLISTS[0]['id'], 'name': MOCK_PLAYLISTS[0]['name'],
|
||||
'owner': MOCK_USER, 'track_count': 50, 'source': 'friend_profile'}
|
||||
]
|
||||
|
||||
with patch('core.spotify_profile_scraper._fetch_playlists_with_playwright', return_value=mock_playlists):
|
||||
result = fetch_profile_playlists(MOCK_USER)
|
||||
|
||||
self._check(len(result) == 1, "Returns mocked playlist")
|
||||
self._check(result[0]['name'] == MOCK_PLAYLISTS[0]['name'], "Correct playlist name")
|
||||
|
||||
def test_playwright_not_installed(self):
|
||||
"""Raises RuntimeError when playwright is not installed"""
|
||||
print("\n📦 Playwright not installed handling")
|
||||
|
||||
with patch('core.spotify_profile_scraper.sync_playwright', None):
|
||||
from core.spotify_profile_scraper import is_playwright_available
|
||||
# Need to reimport to pick up the patched value
|
||||
with patch('core.spotify_profile_scraper.is_playwright_available', return_value=False):
|
||||
try:
|
||||
fetch_profile_playlists("any_user")
|
||||
self._check(False, "Should raise RuntimeError")
|
||||
except RuntimeError as e:
|
||||
self._check("Playwright is required" in str(e), "Raises correct RuntimeError")
|
||||
|
||||
def test_track_count_fetch(self):
|
||||
"""Fetches track count from playlist page"""
|
||||
print("\n📦 Track count fetching")
|
||||
playlist_id = "test123"
|
||||
expected_count = 42
|
||||
|
||||
mock_html = _create_mock_playlist_page(playlist_id, expected_count)
|
||||
mock_resp = Mock(text=mock_html, raise_for_status=Mock())
|
||||
|
||||
with patch('core.spotify_profile_scraper.requests.get', return_value=mock_resp):
|
||||
result = _fetch_playlist_track_count(playlist_id)
|
||||
|
||||
self._check(result == expected_count, f"Returns correct track count ({expected_count})")
|
||||
|
||||
# Test error handling - returns 0 on failure
|
||||
import requests
|
||||
with patch('core.spotify_profile_scraper.requests.get', side_effect=requests.RequestException("fail")):
|
||||
result = _fetch_playlist_track_count(playlist_id)
|
||||
self._check(result == 0, "Returns 0 on request failure")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
suite = TestSpotifyProfileScraper()
|
||||
sys.exit(0 if suite.run_all() else 1)
|
||||
|
|
@ -33,6 +33,7 @@ from core.navidrome_client import NavidromeClient
|
|||
from core.soulseek_client import SoulseekClient
|
||||
from core.download_orchestrator import DownloadOrchestrator
|
||||
from core.tidal_client import TidalClient # Added import for Tidal
|
||||
from core.spotify_profile_scraper import get_all_friend_playlists, fetch_profile_playlists, is_playwright_available
|
||||
from core.matching_engine import MusicMatchingEngine
|
||||
from core.database_update_worker import DatabaseUpdateWorker, DatabaseStatsWorker
|
||||
from core.web_scan_manager import WebScanManager
|
||||
|
|
@ -13994,6 +13995,52 @@ def get_spotify_playlists():
|
|||
except Exception as e:
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@app.route('/api/spotify/friend_playlists', methods=['GET'])
|
||||
def get_friend_playlists():
|
||||
"""Fetches public playlists from configured friend Spotify profiles."""
|
||||
try:
|
||||
# Check if any friend profiles are configured
|
||||
spotify_config = config_manager.get_spotify_config()
|
||||
friend_profiles = spotify_config.get('friend_profiles', [])
|
||||
|
||||
if not friend_profiles:
|
||||
return jsonify([])
|
||||
|
||||
# Check if playwright is available (required for scraping)
|
||||
if not is_playwright_available():
|
||||
print("⚠️ Playwright not installed - friend playlist scraping disabled")
|
||||
print(" Install with: pip install playwright && python -m playwright install chromium")
|
||||
return jsonify([])
|
||||
|
||||
# Get playlists from all configured friend profiles
|
||||
friend_playlists = get_all_friend_playlists()
|
||||
|
||||
if not friend_playlists:
|
||||
print("📋 No friend playlists found from configured profiles")
|
||||
return jsonify([])
|
||||
|
||||
# Format response to match the standard playlist format
|
||||
playlist_data = []
|
||||
for p in friend_playlists:
|
||||
playlist_data.append({
|
||||
"id": p['id'],
|
||||
"name": p['name'],
|
||||
"owner": p.get('owner_display_name', p.get('owner', 'Unknown')),
|
||||
"track_count": p.get('track_count', 0),
|
||||
"image_url": p.get('image_url'),
|
||||
"sync_status": "Never Synced",
|
||||
"snapshot_id": "",
|
||||
"source": "friend_profile",
|
||||
"source_user_id": p.get('source_user_id', p.get('owner'))
|
||||
})
|
||||
|
||||
print(f"🎵 Returning {len(playlist_data)} playlists from friend profiles")
|
||||
return jsonify(playlist_data)
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error fetching friend playlists: {e}")
|
||||
return jsonify({"error": str(e)}), 500
|
||||
|
||||
@app.route('/api/spotify/playlist/<playlist_id>', methods=['GET'])
|
||||
def get_playlist_tracks(playlist_id):
|
||||
"""Fetches full track details for a specific playlist."""
|
||||
|
|
|
|||
|
|
@ -4876,12 +4876,28 @@ async function loadSpotifyPlaylists() {
|
|||
refreshBtn.textContent = '🔄 Loading...';
|
||||
|
||||
try {
|
||||
// Fetch user's own playlists
|
||||
const response = await fetch('/api/spotify/playlists');
|
||||
if (!response.ok) {
|
||||
const error = await response.json();
|
||||
throw new Error(error.error || 'Failed to fetch playlists');
|
||||
}
|
||||
spotifyPlaylists = await response.json();
|
||||
const userPlaylists = await response.json();
|
||||
|
||||
// Fetch friend playlists (non-blocking, don't fail if this errors)
|
||||
let friendPlaylists = [];
|
||||
try {
|
||||
const friendResponse = await fetch('/api/spotify/friend_playlists');
|
||||
if (friendResponse.ok) {
|
||||
friendPlaylists = await friendResponse.json();
|
||||
console.log(`📋 Loaded ${friendPlaylists.length} friend playlists`);
|
||||
}
|
||||
} catch (friendError) {
|
||||
console.warn('Could not load friend playlists:', friendError.message);
|
||||
}
|
||||
|
||||
// Combine: user playlists first, then friend playlists
|
||||
spotifyPlaylists = [...userPlaylists, ...friendPlaylists];
|
||||
renderSpotifyPlaylists();
|
||||
spotifyPlaylistsLoaded = true;
|
||||
|
||||
|
|
@ -4908,14 +4924,19 @@ function renderSpotifyPlaylists() {
|
|||
if (p.sync_status.startsWith('Synced')) statusClass = 'status-synced';
|
||||
if (p.sync_status === 'Needs Sync') statusClass = 'status-needs-sync';
|
||||
|
||||
// Check if this is a friend playlist
|
||||
const isFriendPlaylist = p.source === 'friend_profile';
|
||||
const friendBadge = isFriendPlaylist ? `<span class="friend-playlist-badge" title="From friend: ${escapeHtml(p.owner)}">👥</span>` : '';
|
||||
const ownerInfo = isFriendPlaylist ? ` • <span class="playlist-owner-name">from ${escapeHtml(p.owner)}</span>` : '';
|
||||
|
||||
// This HTML structure creates the interactive playlist cards
|
||||
return `
|
||||
<div class="playlist-card" data-playlist-id="${p.id}" onclick="togglePlaylistSelection(event)">
|
||||
<div class="playlist-card ${isFriendPlaylist ? 'friend-playlist' : ''}" data-playlist-id="${p.id}" onclick="togglePlaylistSelection(event)">
|
||||
<div class="playlist-card-main">
|
||||
<div class="playlist-card-content">
|
||||
<div class="playlist-card-name">${escapeHtml(p.name)}</div>
|
||||
<div class="playlist-card-name">${friendBadge}${escapeHtml(p.name)}</div>
|
||||
<div class="playlist-card-info">
|
||||
<span>${p.track_count} tracks</span> •
|
||||
<span>${p.track_count} tracks</span>${ownerInfo} •
|
||||
<span class="playlist-card-status ${statusClass}">${p.sync_status}</span>
|
||||
</div>
|
||||
<div class="sync-progress-indicator" id="progress-${p.id}"></div>
|
||||
|
|
|
|||
|
|
@ -6826,6 +6826,33 @@ body {
|
|||
inset 0 1px 0 rgba(255, 255, 255, 0.1);
|
||||
}
|
||||
|
||||
/* Friend Playlist Styles */
|
||||
.friend-playlist {
|
||||
border-color: rgba(100, 149, 237, 0.25);
|
||||
border-top-color: rgba(100, 149, 237, 0.35);
|
||||
}
|
||||
|
||||
.friend-playlist:hover {
|
||||
border-color: rgba(100, 149, 237, 0.4);
|
||||
box-shadow:
|
||||
0 16px 48px rgba(0, 0, 0, 0.5),
|
||||
0 8px 16px rgba(0, 0, 0, 0.3),
|
||||
0 0 20px rgba(100, 149, 237, 0.15),
|
||||
inset 0 1px 0 rgba(255, 255, 255, 0.15);
|
||||
}
|
||||
|
||||
.friend-playlist-badge {
|
||||
display: inline-block;
|
||||
margin-right: 8px;
|
||||
font-size: 14px;
|
||||
cursor: help;
|
||||
}
|
||||
|
||||
.playlist-owner-name {
|
||||
color: rgba(100, 149, 237, 0.9);
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.playlist-card-actions {
|
||||
flex-shrink: 0;
|
||||
margin-left: 20px;
|
||||
|
|
|
|||
Loading…
Reference in a new issue