Spotify Public Profile scrapping (missing the sync button flow)
This commit is contained in:
parent
05c2cd7320
commit
5b5ff7443c
8 changed files with 746 additions and 7 deletions
|
|
@ -23,7 +23,8 @@ RUN useradd --create-home --shell /bin/bash --uid 1000 soulsync
|
||||||
# Copy requirements and install Python dependencies
|
# Copy requirements and install Python dependencies
|
||||||
COPY requirements-webui.txt .
|
COPY requirements-webui.txt .
|
||||||
RUN pip install --no-cache-dir --upgrade pip && \
|
RUN pip install --no-cache-dir --upgrade pip && \
|
||||||
pip install --no-cache-dir -r requirements-webui.txt
|
pip install --no-cache-dir -r requirements-webui.txt && \
|
||||||
|
playwright install --with-deps chromium
|
||||||
|
|
||||||
# Copy application code
|
# Copy application code
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,11 @@
|
||||||
"spotify": {
|
"spotify": {
|
||||||
"client_id": "SpotifyClientID",
|
"client_id": "SpotifyClientID",
|
||||||
"client_secret": "SpotifyClientSecret",
|
"client_secret": "SpotifyClientSecret",
|
||||||
"redirect_uri": "http://127.0.0.1:8888/callback"
|
"redirect_uri": "http://127.0.0.1:8888/callback",
|
||||||
|
"friend_profiles": [
|
||||||
|
"userID1",
|
||||||
|
"userID2"
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"tidal": {
|
"tidal": {
|
||||||
"client_id": "TidalClientID",
|
"client_id": "TidalClientID",
|
||||||
|
|
|
||||||
411
core/spotify_profile_scraper.py
Normal file
411
core/spotify_profile_scraper.py
Normal file
|
|
@ -0,0 +1,411 @@
|
||||||
|
"""
|
||||||
|
Spotify Profile Scraper - Fetches public playlists from Spotify user profiles.
|
||||||
|
|
||||||
|
This module scrapes the public Spotify profile page to extract playlist information,
|
||||||
|
bypassing API restrictions for users who don't have API access or want to sync
|
||||||
|
playlists from friends' profiles.
|
||||||
|
|
||||||
|
Requires: playwright (headless browser) for full JS-rendered page scraping.
|
||||||
|
Install with: pip install playwright && python -m playwright install chromium
|
||||||
|
"""
|
||||||
|
|
||||||
|
import requests
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from typing import List, Dict, Any, Optional, Tuple
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
from config.settings import config_manager
|
||||||
|
|
||||||
|
logger = get_logger("spotify_profile_scraper")
|
||||||
|
|
||||||
|
# Import playwright - required for this module
|
||||||
|
try:
|
||||||
|
from playwright.sync_api import sync_playwright
|
||||||
|
except ImportError:
|
||||||
|
sync_playwright = None
|
||||||
|
logger.warning("Playwright not installed - friend playlist scraping will not work")
|
||||||
|
logger.warning("Install with: pip install playwright && python -m playwright install chromium")
|
||||||
|
|
||||||
|
# User agent for HTTP requests (fetching individual playlist details)
|
||||||
|
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
|
||||||
|
|
||||||
|
|
||||||
|
def is_playwright_available() -> bool:
|
||||||
|
"""Check if playwright is installed and available."""
|
||||||
|
return sync_playwright is not None
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_profile_playlists(user_id: str) -> List[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Fetch public playlists from a Spotify user's profile page using playwright.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
user_id: The Spotify user ID (e.g., "12166842163")
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of playlist dictionaries with id, name, owner, track_count, etc.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
RuntimeError: If playwright is not installed
|
||||||
|
"""
|
||||||
|
if not is_playwright_available():
|
||||||
|
raise RuntimeError(
|
||||||
|
"Playwright is required for friend playlist scraping. "
|
||||||
|
"Install with: pip install playwright && python -m playwright install chromium"
|
||||||
|
)
|
||||||
|
|
||||||
|
return _fetch_playlists_with_playwright(user_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_playlists_with_playwright(user_id: str) -> List[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Fetch all playlists using playwright headless browser.
|
||||||
|
|
||||||
|
This renders the /playlists page with JavaScript to get the complete list.
|
||||||
|
"""
|
||||||
|
playlists_url = f"https://open.spotify.com/user/{user_id}/playlists"
|
||||||
|
logger.info(f"Playwright: Fetching playlists from {playlists_url}")
|
||||||
|
|
||||||
|
playlists = []
|
||||||
|
|
||||||
|
with sync_playwright() as p:
|
||||||
|
browser = p.chromium.launch(headless=True)
|
||||||
|
# Use default Chrome user agent (custom UA causes Spotify to redirect)
|
||||||
|
context = browser.new_context()
|
||||||
|
page = context.new_page()
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Navigate to playlists page
|
||||||
|
page.goto(playlists_url, wait_until="networkidle", timeout=30000)
|
||||||
|
|
||||||
|
# Wait for page to fully render (no specific selector needed)
|
||||||
|
page.wait_for_timeout(5000)
|
||||||
|
|
||||||
|
# Extract playlist links from rendered page
|
||||||
|
playlist_links = page.query_selector_all('a[href*="/playlist/"]')
|
||||||
|
|
||||||
|
seen_ids = set()
|
||||||
|
for link in playlist_links:
|
||||||
|
href = link.get_attribute('href')
|
||||||
|
if href and '/playlist/' in href:
|
||||||
|
# Extract playlist ID from href
|
||||||
|
match = re.search(r'/playlist/([a-zA-Z0-9]+)', href)
|
||||||
|
if match:
|
||||||
|
playlist_id = match.group(1)
|
||||||
|
if playlist_id not in seen_ids:
|
||||||
|
seen_ids.add(playlist_id)
|
||||||
|
|
||||||
|
# Get playlist name from link text or parent
|
||||||
|
name = link.inner_text().strip() or None
|
||||||
|
|
||||||
|
playlists.append({
|
||||||
|
'id': playlist_id,
|
||||||
|
'name': name,
|
||||||
|
'owner': user_id,
|
||||||
|
'source': 'friend_profile',
|
||||||
|
'source_user_id': user_id
|
||||||
|
})
|
||||||
|
|
||||||
|
logger.info(f"Playwright: Found {len(playlists)} playlist links")
|
||||||
|
|
||||||
|
finally:
|
||||||
|
browser.close()
|
||||||
|
|
||||||
|
# Fetch details (name, track count) for each playlist
|
||||||
|
for playlist in playlists:
|
||||||
|
track_count, fetched_name = _fetch_playlist_details(playlist['id'])
|
||||||
|
playlist['track_count'] = track_count
|
||||||
|
if not playlist.get('name') and fetched_name:
|
||||||
|
playlist['name'] = fetched_name
|
||||||
|
|
||||||
|
# Skip playlists with no name (likely deleted/inaccessible)
|
||||||
|
if not playlist.get('name'):
|
||||||
|
logger.debug(f"Skipping playlist with no name: {playlist['id']}")
|
||||||
|
|
||||||
|
# Filter out playlists with no name
|
||||||
|
playlists = [p for p in playlists if p.get('name')]
|
||||||
|
|
||||||
|
return playlists
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_playlists_with_http(user_id: str) -> List[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Fetch playlists using simple HTTP request (limited to ~10 playlists).
|
||||||
|
|
||||||
|
Fallback method when playwright is not available.
|
||||||
|
"""
|
||||||
|
profile_url = f"https://open.spotify.com/user/{user_id}"
|
||||||
|
|
||||||
|
logger.info(f"HTTP: Fetching playlists from {profile_url}")
|
||||||
|
|
||||||
|
try:
|
||||||
|
headers = {"User-Agent": USER_AGENT}
|
||||||
|
|
||||||
|
response = requests.get(profile_url, headers=headers, timeout=15)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
html_content = response.text
|
||||||
|
|
||||||
|
# Try to extract from initialState JSON first (most reliable)
|
||||||
|
playlists = _extract_from_initial_state(html_content, user_id)
|
||||||
|
|
||||||
|
if playlists:
|
||||||
|
logger.info(f"HTTP: Extracted {len(playlists)} playlists from initialState")
|
||||||
|
return playlists
|
||||||
|
|
||||||
|
# Fallback: Extract playlist IDs from href links
|
||||||
|
playlists = _extract_from_html_links(html_content)
|
||||||
|
|
||||||
|
if playlists:
|
||||||
|
logger.info(f"HTTP: Extracted {len(playlists)} playlists from HTML links")
|
||||||
|
return playlists
|
||||||
|
|
||||||
|
logger.warning(f"HTTP: No playlists found for user {user_id}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
except requests.RequestException as e:
|
||||||
|
logger.error(f"Failed to fetch profile page for {user_id}: {e}")
|
||||||
|
return []
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Unexpected error fetching profile for {user_id}: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_from_initial_state(html_content: str, user_id: str) -> List[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Extract playlist data from the base64-encoded initialState script tag.
|
||||||
|
|
||||||
|
The initialState contains structured JSON with full playlist metadata.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Find the initialState script tag
|
||||||
|
match = re.search(r'<script id="initialState" type="text/plain">([^<]+)</script>', html_content)
|
||||||
|
|
||||||
|
if not match:
|
||||||
|
logger.debug("initialState script tag not found")
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Decode base64
|
||||||
|
encoded_data = match.group(1)
|
||||||
|
decoded_bytes = base64.b64decode(encoded_data)
|
||||||
|
state_json = json.loads(decoded_bytes.decode('utf-8'))
|
||||||
|
|
||||||
|
# Navigate to the playlists data
|
||||||
|
entities = state_json.get('entities', {})
|
||||||
|
items = entities.get('items', {})
|
||||||
|
|
||||||
|
# The user data is stored with key like "spotify:user:12166842163"
|
||||||
|
user_key = f"spotify:user:{user_id}"
|
||||||
|
user_data = items.get(user_key, {})
|
||||||
|
|
||||||
|
if not user_data:
|
||||||
|
logger.debug(f"No user data found for key {user_key}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
public_playlists = user_data.get('publicPlaylistsV2', {})
|
||||||
|
playlist_items = public_playlists.get('items', [])
|
||||||
|
|
||||||
|
playlists = []
|
||||||
|
for item in playlist_items:
|
||||||
|
data = item.get('data', {})
|
||||||
|
|
||||||
|
# Skip playlists that are NotFound (deleted, private, or inaccessible)
|
||||||
|
if data.get('__typename') == 'NotFound':
|
||||||
|
logger.debug(f"Skipping NotFound playlist: {item.get('_uri')}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
uri = item.get('_uri', '') or data.get('uri', '')
|
||||||
|
|
||||||
|
# Extract playlist ID from URI (spotify:playlist:XXXXX)
|
||||||
|
playlist_id = uri.split(':')[-1] if uri else None
|
||||||
|
|
||||||
|
if not playlist_id:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Extract image URL from nested structure
|
||||||
|
image_url = None
|
||||||
|
images = data.get('images', {})
|
||||||
|
image_items = images.get('items', [])
|
||||||
|
if image_items:
|
||||||
|
sources = image_items[0].get('sources', [])
|
||||||
|
if sources:
|
||||||
|
image_url = sources[0].get('url')
|
||||||
|
|
||||||
|
# Fetch track count (and name if missing) from playlist page
|
||||||
|
playlist_name = data.get('name')
|
||||||
|
track_count, fetched_name = _fetch_playlist_details(playlist_id)
|
||||||
|
|
||||||
|
# Use fetched name if original is missing
|
||||||
|
if not playlist_name and fetched_name:
|
||||||
|
playlist_name = fetched_name
|
||||||
|
|
||||||
|
# Skip if still no valid name (likely deleted/inaccessible)
|
||||||
|
if not playlist_name:
|
||||||
|
logger.debug(f"Skipping playlist with no name: {playlist_id}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
playlist_info = {
|
||||||
|
'id': playlist_id,
|
||||||
|
'name': playlist_name,
|
||||||
|
'owner': user_id,
|
||||||
|
'owner_display_name': user_data.get('name', user_id),
|
||||||
|
'image_url': image_url,
|
||||||
|
'followers': data.get('followers', 0),
|
||||||
|
'track_count': track_count,
|
||||||
|
'source': 'friend_profile',
|
||||||
|
'source_user_id': user_id
|
||||||
|
}
|
||||||
|
|
||||||
|
playlists.append(playlist_info)
|
||||||
|
|
||||||
|
return playlists
|
||||||
|
|
||||||
|
except (json.JSONDecodeError, ValueError) as e:
|
||||||
|
logger.debug(f"Failed to parse initialState JSON: {e}")
|
||||||
|
return []
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(f"Error extracting from initialState: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_playlist_details(playlist_id: str) -> tuple:
|
||||||
|
"""
|
||||||
|
Fetch track count and name for a playlist by scraping its page.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
playlist_id: Spotify playlist ID
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple of (track_count, name) - defaults to (0, None) if unable to fetch
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
playlist_url = f"https://open.spotify.com/playlist/{playlist_id}"
|
||||||
|
headers = {"User-Agent": USER_AGENT}
|
||||||
|
|
||||||
|
response = requests.get(playlist_url, headers=headers, timeout=10)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
# Find initialState
|
||||||
|
match = re.search(r'<script id="initialState" type="text/plain">([^<]+)</script>', response.text)
|
||||||
|
if not match:
|
||||||
|
return (0, None)
|
||||||
|
|
||||||
|
decoded = base64.b64decode(match.group(1)).decode('utf-8')
|
||||||
|
data = json.loads(decoded)
|
||||||
|
|
||||||
|
# Get playlist data
|
||||||
|
items = data.get('entities', {}).get('items', {})
|
||||||
|
playlist_key = f"spotify:playlist:{playlist_id}"
|
||||||
|
playlist_data = items.get(playlist_key, {})
|
||||||
|
|
||||||
|
# Extract totalCount from content
|
||||||
|
content = playlist_data.get('content', {})
|
||||||
|
track_count = content.get('totalCount', 0)
|
||||||
|
|
||||||
|
# Extract name
|
||||||
|
name = playlist_data.get('name')
|
||||||
|
|
||||||
|
logger.debug(f"Playlist {playlist_id}: '{name}' with {track_count} tracks")
|
||||||
|
return (track_count, name)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(f"Could not fetch details for {playlist_id}: {e}")
|
||||||
|
return (0, None)
|
||||||
|
|
||||||
|
|
||||||
|
# Keep the old function for backward compatibility with tests
|
||||||
|
def _fetch_playlist_track_count(playlist_id: str) -> int:
|
||||||
|
"""Fetch track count only (wrapper for backward compatibility)."""
|
||||||
|
track_count, _ = _fetch_playlist_details(playlist_id)
|
||||||
|
return track_count
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_from_html_links(html_content: str) -> List[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Fallback method: Extract playlist IDs from href links in the HTML.
|
||||||
|
|
||||||
|
Looks for patterns like href="/playlist/4xJiUcKvrFEhhfhthMeOx7"
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Find all playlist links
|
||||||
|
pattern = r'href="/playlist/([a-zA-Z0-9]+)"'
|
||||||
|
matches = re.findall(pattern, html_content)
|
||||||
|
|
||||||
|
# Deduplicate while preserving order
|
||||||
|
seen = set()
|
||||||
|
unique_ids = []
|
||||||
|
for playlist_id in matches:
|
||||||
|
if playlist_id not in seen:
|
||||||
|
seen.add(playlist_id)
|
||||||
|
unique_ids.append(playlist_id)
|
||||||
|
|
||||||
|
playlists = []
|
||||||
|
for playlist_id in unique_ids:
|
||||||
|
playlist_info = {
|
||||||
|
'id': playlist_id,
|
||||||
|
'name': f'Playlist {playlist_id[:8]}...', # Placeholder name
|
||||||
|
'owner': 'Unknown',
|
||||||
|
'image_url': None,
|
||||||
|
'followers': 0,
|
||||||
|
'source': 'friend_profile'
|
||||||
|
}
|
||||||
|
playlists.append(playlist_info)
|
||||||
|
|
||||||
|
return playlists
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug(f"Error extracting from HTML links: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def get_all_friend_playlists() -> List[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Fetch playlists from all configured friend profiles.
|
||||||
|
|
||||||
|
Reads the friend_profiles list from config and fetches playlists from each.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Combined list of playlists from all friend profiles
|
||||||
|
"""
|
||||||
|
spotify_config = config_manager.get_spotify_config()
|
||||||
|
friend_profiles = spotify_config.get('friend_profiles', [])
|
||||||
|
|
||||||
|
if not friend_profiles:
|
||||||
|
logger.debug("No friend profiles configured")
|
||||||
|
return []
|
||||||
|
|
||||||
|
logger.info(f"Fetching playlists from {len(friend_profiles)} friend profile(s)")
|
||||||
|
|
||||||
|
all_playlists = []
|
||||||
|
for user_id in friend_profiles:
|
||||||
|
try:
|
||||||
|
playlists = fetch_profile_playlists(user_id)
|
||||||
|
all_playlists.extend(playlists)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Failed to fetch playlists for friend {user_id}: {e}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
logger.info(f"Total friend playlists fetched: {len(all_playlists)}")
|
||||||
|
return all_playlists
|
||||||
|
|
||||||
|
|
||||||
|
# For standalone testing
|
||||||
|
if __name__ == "__main__":
|
||||||
|
import sys
|
||||||
|
|
||||||
|
if len(sys.argv) > 1:
|
||||||
|
test_user_id = sys.argv[1]
|
||||||
|
else:
|
||||||
|
test_user_id = "12166842163"
|
||||||
|
|
||||||
|
print(f"Testing profile scraper for user: {test_user_id}")
|
||||||
|
print("-" * 50)
|
||||||
|
|
||||||
|
playlists = fetch_profile_playlists(test_user_id)
|
||||||
|
|
||||||
|
print(f"Found {len(playlists)} playlists:")
|
||||||
|
for i, p in enumerate(playlists, 1):
|
||||||
|
print(f"{i}. {p['name']} (ID: {p['id']})")
|
||||||
|
if p.get('followers'):
|
||||||
|
print(f" Followers: {p['followers']}")
|
||||||
|
|
@ -11,4 +11,5 @@ aiohttp>=3.9.0
|
||||||
unidecode>=1.3.8
|
unidecode>=1.3.8
|
||||||
yt-dlp>=2024.12.13
|
yt-dlp>=2024.12.13
|
||||||
Flask>=3.0.0
|
Flask>=3.0.0
|
||||||
lrclibapi>=0.3.1
|
lrclibapi>=0.3.1
|
||||||
|
playwright>=1.40.0
|
||||||
227
tools/test_spotify_profile_scraper.py
Normal file
227
tools/test_spotify_profile_scraper.py
Normal file
|
|
@ -0,0 +1,227 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Spotify Profile Scraper Test Suite
|
||||||
|
|
||||||
|
Tests the spotify_profile_scraper module for fetching public playlists
|
||||||
|
from Spotify user profiles without using the API.
|
||||||
|
|
||||||
|
All tests use MOCKED data - no network requests to Spotify.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python tools/test_spotify_profile_scraper.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
from unittest.mock import patch, Mock
|
||||||
|
|
||||||
|
# Add parent directory to path to import from core
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
from core.spotify_profile_scraper import (
|
||||||
|
fetch_profile_playlists,
|
||||||
|
_extract_from_initial_state,
|
||||||
|
_extract_from_html_links,
|
||||||
|
_fetch_playlist_track_count
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = get_logger("spotify_scraper_test")
|
||||||
|
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# MOCK DATA
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
# Well-known Spotify editorial playlist IDs for realistic mock data
|
||||||
|
MOCK_PLAYLISTS = [
|
||||||
|
{"id": "37i9dQZF1DXcBWIGoYBM5M", "name": "Today's Top Hits", "followers": 35000000},
|
||||||
|
{"id": "37i9dQZF1DX0XUsuxWHRQd", "name": "RapCaviar", "followers": 15000000},
|
||||||
|
{"id": "37i9dQZF1DX4JAvHpjipBk", "name": "New Music Friday", "followers": 8000000},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _create_mock_initial_state(user_id: str, playlists: list) -> str:
|
||||||
|
"""Create a mock initialState JSON structure matching Spotify's format"""
|
||||||
|
playlist_items = []
|
||||||
|
for p in playlists:
|
||||||
|
playlist_items.append({
|
||||||
|
"__typename": "PlaylistResponseWrapper",
|
||||||
|
"_uri": f"spotify:playlist:{p['id']}",
|
||||||
|
"data": {
|
||||||
|
"__typename": "Playlist",
|
||||||
|
"followers": p.get("followers", 0),
|
||||||
|
"name": p["name"],
|
||||||
|
"uri": f"spotify:playlist:{p['id']}",
|
||||||
|
"images": {"items": [{"sources": [{"url": f"https://example.com/{p['id']}.jpg"}]}]}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
state = {
|
||||||
|
"entities": {
|
||||||
|
"items": {
|
||||||
|
f"spotify:user:{user_id}": {
|
||||||
|
"__typename": "User",
|
||||||
|
"id": user_id,
|
||||||
|
"name": f"Test User",
|
||||||
|
"publicPlaylistsV2": {"items": playlist_items, "totalCount": len(playlist_items)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return base64.b64encode(json.dumps(state).encode()).decode()
|
||||||
|
|
||||||
|
|
||||||
|
def _create_mock_html(user_id: str, playlists: list) -> str:
|
||||||
|
"""Create mock Spotify profile HTML page"""
|
||||||
|
state = _create_mock_initial_state(user_id, playlists)
|
||||||
|
return f'<html><body><script id="initialState" type="text/plain">{state}</script></body></html>'
|
||||||
|
|
||||||
|
|
||||||
|
MOCK_USER = "testuser"
|
||||||
|
MOCK_HTML = _create_mock_html(MOCK_USER, MOCK_PLAYLISTS)
|
||||||
|
|
||||||
|
|
||||||
|
def _create_mock_playlist_page(playlist_id: str, track_count: int) -> str:
|
||||||
|
"""Create mock Spotify playlist page HTML with track count in initialState"""
|
||||||
|
state = {
|
||||||
|
"entities": {
|
||||||
|
"items": {
|
||||||
|
f"spotify:playlist:{playlist_id}": {
|
||||||
|
"__typename": "Playlist",
|
||||||
|
"content": {
|
||||||
|
"__typename": "PlaylistItemsPage",
|
||||||
|
"items": [],
|
||||||
|
"totalCount": track_count,
|
||||||
|
"pagingInfo": {"nextOffset": None}
|
||||||
|
},
|
||||||
|
"name": "Test Playlist",
|
||||||
|
"id": playlist_id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
encoded = base64.b64encode(json.dumps(state).encode()).decode()
|
||||||
|
return f'<html><body><script id="initialState" type="text/plain">{encoded}</script></body></html>'
|
||||||
|
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# TESTS
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
class TestSpotifyProfileScraper:
|
||||||
|
"""All tests use mocked data"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.passed = 0
|
||||||
|
self.failed = 0
|
||||||
|
|
||||||
|
def run_all(self):
|
||||||
|
print("\n" + "=" * 60)
|
||||||
|
print("🧪 SPOTIFY PROFILE SCRAPER TESTS (mocked)")
|
||||||
|
print("=" * 60 + "\n")
|
||||||
|
|
||||||
|
self.test_extract_initial_state()
|
||||||
|
self.test_extract_html_links()
|
||||||
|
self.test_playlist_fields()
|
||||||
|
self.test_fetch_with_playwright_mock()
|
||||||
|
self.test_playwright_not_installed()
|
||||||
|
self.test_track_count_fetch()
|
||||||
|
|
||||||
|
print("\n" + "=" * 60)
|
||||||
|
print(f"📊 RESULTS: {self.passed}/{self.passed + self.failed} passed")
|
||||||
|
print("=" * 60)
|
||||||
|
if self.failed == 0:
|
||||||
|
print("🎉 All tests passed!")
|
||||||
|
return self.failed == 0
|
||||||
|
|
||||||
|
def _check(self, cond, name, detail=""):
|
||||||
|
if cond:
|
||||||
|
print(f" ✅ {name}")
|
||||||
|
self.passed += 1
|
||||||
|
else:
|
||||||
|
print(f" ❌ {name}" + (f" - {detail}" if detail else ""))
|
||||||
|
self.failed += 1
|
||||||
|
|
||||||
|
def test_extract_initial_state(self):
|
||||||
|
"""Parse playlists from initialState JSON"""
|
||||||
|
print("📦 Parse initialState JSON")
|
||||||
|
result = _extract_from_initial_state(MOCK_HTML, MOCK_USER)
|
||||||
|
self._check(len(result) == 3, "Extracts 3 playlists")
|
||||||
|
self._check(result[0]['id'] == MOCK_PLAYLISTS[0]['id'], "Correct playlist ID")
|
||||||
|
self._check(result[0]['name'] == "Today's Top Hits", "Correct playlist name")
|
||||||
|
self._check(result[0]['source'] == 'friend_profile', "Source is friend_profile")
|
||||||
|
|
||||||
|
def test_extract_html_links(self):
|
||||||
|
"""Fallback: parse playlist IDs from HTML links"""
|
||||||
|
print("\n📦 Parse HTML links (fallback)")
|
||||||
|
html = '<a href="/playlist/abc123">P1</a><a href="/playlist/def456">P2</a><a href="/playlist/abc123">P1</a>'
|
||||||
|
result = _extract_from_html_links(html)
|
||||||
|
self._check(len(result) == 2, "Deduplicates to 2 playlists")
|
||||||
|
self._check(result[0]['id'] == 'abc123', "First ID correct")
|
||||||
|
self._check(result[1]['id'] == 'def456', "Second ID correct")
|
||||||
|
|
||||||
|
def test_playlist_fields(self):
|
||||||
|
"""Playlist dict has required fields"""
|
||||||
|
print("\n📦 Playlist structure")
|
||||||
|
result = _extract_from_initial_state(MOCK_HTML, MOCK_USER)
|
||||||
|
p = result[0] if result else {}
|
||||||
|
for f in ['id', 'name', 'owner', 'source']:
|
||||||
|
self._check(f in p, f"Has '{f}' field")
|
||||||
|
|
||||||
|
def test_fetch_with_playwright_mock(self):
|
||||||
|
"""fetch_profile_playlists with mocked playwright"""
|
||||||
|
print("\n📦 fetch_profile_playlists (playwright mocked)")
|
||||||
|
|
||||||
|
# Create expected result from playwright
|
||||||
|
mock_playlists = [
|
||||||
|
{'id': MOCK_PLAYLISTS[0]['id'], 'name': MOCK_PLAYLISTS[0]['name'],
|
||||||
|
'owner': MOCK_USER, 'track_count': 50, 'source': 'friend_profile'}
|
||||||
|
]
|
||||||
|
|
||||||
|
with patch('core.spotify_profile_scraper._fetch_playlists_with_playwright', return_value=mock_playlists):
|
||||||
|
result = fetch_profile_playlists(MOCK_USER)
|
||||||
|
|
||||||
|
self._check(len(result) == 1, "Returns mocked playlist")
|
||||||
|
self._check(result[0]['name'] == MOCK_PLAYLISTS[0]['name'], "Correct playlist name")
|
||||||
|
|
||||||
|
def test_playwright_not_installed(self):
|
||||||
|
"""Raises RuntimeError when playwright is not installed"""
|
||||||
|
print("\n📦 Playwright not installed handling")
|
||||||
|
|
||||||
|
with patch('core.spotify_profile_scraper.sync_playwright', None):
|
||||||
|
from core.spotify_profile_scraper import is_playwright_available
|
||||||
|
# Need to reimport to pick up the patched value
|
||||||
|
with patch('core.spotify_profile_scraper.is_playwright_available', return_value=False):
|
||||||
|
try:
|
||||||
|
fetch_profile_playlists("any_user")
|
||||||
|
self._check(False, "Should raise RuntimeError")
|
||||||
|
except RuntimeError as e:
|
||||||
|
self._check("Playwright is required" in str(e), "Raises correct RuntimeError")
|
||||||
|
|
||||||
|
def test_track_count_fetch(self):
|
||||||
|
"""Fetches track count from playlist page"""
|
||||||
|
print("\n📦 Track count fetching")
|
||||||
|
playlist_id = "test123"
|
||||||
|
expected_count = 42
|
||||||
|
|
||||||
|
mock_html = _create_mock_playlist_page(playlist_id, expected_count)
|
||||||
|
mock_resp = Mock(text=mock_html, raise_for_status=Mock())
|
||||||
|
|
||||||
|
with patch('core.spotify_profile_scraper.requests.get', return_value=mock_resp):
|
||||||
|
result = _fetch_playlist_track_count(playlist_id)
|
||||||
|
|
||||||
|
self._check(result == expected_count, f"Returns correct track count ({expected_count})")
|
||||||
|
|
||||||
|
# Test error handling - returns 0 on failure
|
||||||
|
import requests
|
||||||
|
with patch('core.spotify_profile_scraper.requests.get', side_effect=requests.RequestException("fail")):
|
||||||
|
result = _fetch_playlist_track_count(playlist_id)
|
||||||
|
self._check(result == 0, "Returns 0 on request failure")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
suite = TestSpotifyProfileScraper()
|
||||||
|
sys.exit(0 if suite.run_all() else 1)
|
||||||
|
|
@ -33,6 +33,7 @@ from core.navidrome_client import NavidromeClient
|
||||||
from core.soulseek_client import SoulseekClient
|
from core.soulseek_client import SoulseekClient
|
||||||
from core.download_orchestrator import DownloadOrchestrator
|
from core.download_orchestrator import DownloadOrchestrator
|
||||||
from core.tidal_client import TidalClient # Added import for Tidal
|
from core.tidal_client import TidalClient # Added import for Tidal
|
||||||
|
from core.spotify_profile_scraper import get_all_friend_playlists, fetch_profile_playlists, is_playwright_available
|
||||||
from core.matching_engine import MusicMatchingEngine
|
from core.matching_engine import MusicMatchingEngine
|
||||||
from core.database_update_worker import DatabaseUpdateWorker, DatabaseStatsWorker
|
from core.database_update_worker import DatabaseUpdateWorker, DatabaseStatsWorker
|
||||||
from core.web_scan_manager import WebScanManager
|
from core.web_scan_manager import WebScanManager
|
||||||
|
|
@ -13994,6 +13995,52 @@ def get_spotify_playlists():
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return jsonify({"error": str(e)}), 500
|
return jsonify({"error": str(e)}), 500
|
||||||
|
|
||||||
|
@app.route('/api/spotify/friend_playlists', methods=['GET'])
|
||||||
|
def get_friend_playlists():
|
||||||
|
"""Fetches public playlists from configured friend Spotify profiles."""
|
||||||
|
try:
|
||||||
|
# Check if any friend profiles are configured
|
||||||
|
spotify_config = config_manager.get_spotify_config()
|
||||||
|
friend_profiles = spotify_config.get('friend_profiles', [])
|
||||||
|
|
||||||
|
if not friend_profiles:
|
||||||
|
return jsonify([])
|
||||||
|
|
||||||
|
# Check if playwright is available (required for scraping)
|
||||||
|
if not is_playwright_available():
|
||||||
|
print("⚠️ Playwright not installed - friend playlist scraping disabled")
|
||||||
|
print(" Install with: pip install playwright && python -m playwright install chromium")
|
||||||
|
return jsonify([])
|
||||||
|
|
||||||
|
# Get playlists from all configured friend profiles
|
||||||
|
friend_playlists = get_all_friend_playlists()
|
||||||
|
|
||||||
|
if not friend_playlists:
|
||||||
|
print("📋 No friend playlists found from configured profiles")
|
||||||
|
return jsonify([])
|
||||||
|
|
||||||
|
# Format response to match the standard playlist format
|
||||||
|
playlist_data = []
|
||||||
|
for p in friend_playlists:
|
||||||
|
playlist_data.append({
|
||||||
|
"id": p['id'],
|
||||||
|
"name": p['name'],
|
||||||
|
"owner": p.get('owner_display_name', p.get('owner', 'Unknown')),
|
||||||
|
"track_count": p.get('track_count', 0),
|
||||||
|
"image_url": p.get('image_url'),
|
||||||
|
"sync_status": "Never Synced",
|
||||||
|
"snapshot_id": "",
|
||||||
|
"source": "friend_profile",
|
||||||
|
"source_user_id": p.get('source_user_id', p.get('owner'))
|
||||||
|
})
|
||||||
|
|
||||||
|
print(f"🎵 Returning {len(playlist_data)} playlists from friend profiles")
|
||||||
|
return jsonify(playlist_data)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"❌ Error fetching friend playlists: {e}")
|
||||||
|
return jsonify({"error": str(e)}), 500
|
||||||
|
|
||||||
@app.route('/api/spotify/playlist/<playlist_id>', methods=['GET'])
|
@app.route('/api/spotify/playlist/<playlist_id>', methods=['GET'])
|
||||||
def get_playlist_tracks(playlist_id):
|
def get_playlist_tracks(playlist_id):
|
||||||
"""Fetches full track details for a specific playlist."""
|
"""Fetches full track details for a specific playlist."""
|
||||||
|
|
|
||||||
|
|
@ -4876,12 +4876,28 @@ async function loadSpotifyPlaylists() {
|
||||||
refreshBtn.textContent = '🔄 Loading...';
|
refreshBtn.textContent = '🔄 Loading...';
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
// Fetch user's own playlists
|
||||||
const response = await fetch('/api/spotify/playlists');
|
const response = await fetch('/api/spotify/playlists');
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const error = await response.json();
|
const error = await response.json();
|
||||||
throw new Error(error.error || 'Failed to fetch playlists');
|
throw new Error(error.error || 'Failed to fetch playlists');
|
||||||
}
|
}
|
||||||
spotifyPlaylists = await response.json();
|
const userPlaylists = await response.json();
|
||||||
|
|
||||||
|
// Fetch friend playlists (non-blocking, don't fail if this errors)
|
||||||
|
let friendPlaylists = [];
|
||||||
|
try {
|
||||||
|
const friendResponse = await fetch('/api/spotify/friend_playlists');
|
||||||
|
if (friendResponse.ok) {
|
||||||
|
friendPlaylists = await friendResponse.json();
|
||||||
|
console.log(`📋 Loaded ${friendPlaylists.length} friend playlists`);
|
||||||
|
}
|
||||||
|
} catch (friendError) {
|
||||||
|
console.warn('Could not load friend playlists:', friendError.message);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Combine: user playlists first, then friend playlists
|
||||||
|
spotifyPlaylists = [...userPlaylists, ...friendPlaylists];
|
||||||
renderSpotifyPlaylists();
|
renderSpotifyPlaylists();
|
||||||
spotifyPlaylistsLoaded = true;
|
spotifyPlaylistsLoaded = true;
|
||||||
|
|
||||||
|
|
@ -4908,14 +4924,19 @@ function renderSpotifyPlaylists() {
|
||||||
if (p.sync_status.startsWith('Synced')) statusClass = 'status-synced';
|
if (p.sync_status.startsWith('Synced')) statusClass = 'status-synced';
|
||||||
if (p.sync_status === 'Needs Sync') statusClass = 'status-needs-sync';
|
if (p.sync_status === 'Needs Sync') statusClass = 'status-needs-sync';
|
||||||
|
|
||||||
|
// Check if this is a friend playlist
|
||||||
|
const isFriendPlaylist = p.source === 'friend_profile';
|
||||||
|
const friendBadge = isFriendPlaylist ? `<span class="friend-playlist-badge" title="From friend: ${escapeHtml(p.owner)}">👥</span>` : '';
|
||||||
|
const ownerInfo = isFriendPlaylist ? ` • <span class="playlist-owner-name">from ${escapeHtml(p.owner)}</span>` : '';
|
||||||
|
|
||||||
// This HTML structure creates the interactive playlist cards
|
// This HTML structure creates the interactive playlist cards
|
||||||
return `
|
return `
|
||||||
<div class="playlist-card" data-playlist-id="${p.id}" onclick="togglePlaylistSelection(event)">
|
<div class="playlist-card ${isFriendPlaylist ? 'friend-playlist' : ''}" data-playlist-id="${p.id}" onclick="togglePlaylistSelection(event)">
|
||||||
<div class="playlist-card-main">
|
<div class="playlist-card-main">
|
||||||
<div class="playlist-card-content">
|
<div class="playlist-card-content">
|
||||||
<div class="playlist-card-name">${escapeHtml(p.name)}</div>
|
<div class="playlist-card-name">${friendBadge}${escapeHtml(p.name)}</div>
|
||||||
<div class="playlist-card-info">
|
<div class="playlist-card-info">
|
||||||
<span>${p.track_count} tracks</span> •
|
<span>${p.track_count} tracks</span>${ownerInfo} •
|
||||||
<span class="playlist-card-status ${statusClass}">${p.sync_status}</span>
|
<span class="playlist-card-status ${statusClass}">${p.sync_status}</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="sync-progress-indicator" id="progress-${p.id}"></div>
|
<div class="sync-progress-indicator" id="progress-${p.id}"></div>
|
||||||
|
|
|
||||||
|
|
@ -6826,6 +6826,33 @@ body {
|
||||||
inset 0 1px 0 rgba(255, 255, 255, 0.1);
|
inset 0 1px 0 rgba(255, 255, 255, 0.1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Friend Playlist Styles */
|
||||||
|
.friend-playlist {
|
||||||
|
border-color: rgba(100, 149, 237, 0.25);
|
||||||
|
border-top-color: rgba(100, 149, 237, 0.35);
|
||||||
|
}
|
||||||
|
|
||||||
|
.friend-playlist:hover {
|
||||||
|
border-color: rgba(100, 149, 237, 0.4);
|
||||||
|
box-shadow:
|
||||||
|
0 16px 48px rgba(0, 0, 0, 0.5),
|
||||||
|
0 8px 16px rgba(0, 0, 0, 0.3),
|
||||||
|
0 0 20px rgba(100, 149, 237, 0.15),
|
||||||
|
inset 0 1px 0 rgba(255, 255, 255, 0.15);
|
||||||
|
}
|
||||||
|
|
||||||
|
.friend-playlist-badge {
|
||||||
|
display: inline-block;
|
||||||
|
margin-right: 8px;
|
||||||
|
font-size: 14px;
|
||||||
|
cursor: help;
|
||||||
|
}
|
||||||
|
|
||||||
|
.playlist-owner-name {
|
||||||
|
color: rgba(100, 149, 237, 0.9);
|
||||||
|
font-weight: 500;
|
||||||
|
}
|
||||||
|
|
||||||
.playlist-card-actions {
|
.playlist-card-actions {
|
||||||
flex-shrink: 0;
|
flex-shrink: 0;
|
||||||
margin-left: 20px;
|
margin-left: 20px;
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue