diff --git a/beatport_unified_scraper.py b/beatport_unified_scraper.py index 3a1a2f42..9e92186b 100644 --- a/beatport_unified_scraper.py +++ b/beatport_unified_scraper.py @@ -798,13 +798,17 @@ class BeatportUnifiedScraper: return tracks - def scrape_top_100(self, limit: int = 100) -> List[Dict]: + def scrape_top_100(self, limit: int = 100, enrich: bool = True) -> List[Dict]: """Scrape Beatport Top 100""" print("\nš„ Scraping Beatport Top 100...") soup = self.get_page(f"{self.base_url}/top-100") tracks = self.extract_tracks_from_page(soup, "Top 100", limit) + # Enrich with per-track release metadata + if tracks and enrich: + tracks = self.enrich_chart_tracks(tracks) + print(f"ā Extracted {len(tracks)} tracks from Top 100") return tracks @@ -909,6 +913,247 @@ class BeatportUnifiedScraper: return None + def _try_json_chart_extraction(self, soup, chart_name: str, limit: int) -> List[Dict]: + """Try to extract rich track data from page JSON. Returns list of tracks or empty list.""" + json_obj = self.extract_json_object_from_release_page(soup) + if not json_obj: + return [] + + json_tracks = self.extract_all_tracks_from_json(json_obj) + if not json_tracks: + return [] + + converted = [] + for i, td in enumerate(json_tracks[:limit]): + track = self.convert_chart_json_to_rich_track_format(td, i + 1, chart_name) + if track: + converted.append(track) + + if len(converted) >= 5: + print(f" ā JSON extraction found {len(converted)} tracks with rich metadata") + return converted + + return [] + + def extract_all_tracks_from_json(self, json_obj: Dict) -> List[Dict]: + """Extract ALL track objects from page JSON (no release filtering). Works for charts, top-100, track pages, etc.""" + try: + queries = json_obj.get('props', {}).get('pageProps', {}).get('dehydratedState', {}).get('queries', []) + + # Try multiple query indices ā charts may use index 0 or 1 + for idx in range(min(len(queries), 5)): + query = queries[idx] + if not isinstance(query, dict): + continue + + data = query.get('state', {}).get('data', {}) + + # Pattern 1: data.results[] (release pages, chart pages) + if isinstance(data, dict): + results = data.get('results', []) + if results and isinstance(results, list) and len(results) > 0: + first = results[0] if isinstance(results[0], dict) else {} + if first.get('title') or first.get('name'): + print(f" ā Found {len(results)} tracks in queries[{idx}].data.results") + return results + + # Pattern 2: data itself is a single track object (track pages) + if isinstance(data, dict) and (data.get('title') or data.get('name')) and data.get('id'): + print(f" ā Found single track in queries[{idx}].data") + return [data] + + # Pattern 3: data.tracks[] + if isinstance(data, dict): + tracks = data.get('tracks', []) + if tracks and isinstance(tracks, list) and len(tracks) > 0: + first = tracks[0] if isinstance(tracks[0], dict) else {} + if first.get('title') or first.get('name'): + print(f" ā Found {len(tracks)} tracks in queries[{idx}].data.tracks") + return tracks + + except Exception as e: + print(f" ā Error extracting tracks from JSON: {e}") + + return [] + + def convert_chart_json_to_rich_track_format(self, track_data: Dict, position: int, chart_name: str) -> Dict: + """Convert a JSON track object from a chart page to a rich format with per-track release metadata.""" + try: + if not isinstance(track_data, dict): + return None + + title = track_data.get('title') or track_data.get('name', 'Unknown Title') + mix_name = track_data.get('mix_name') or track_data.get('mix', '') + + # Artists + artist = 'Unknown Artist' + if 'artists' in track_data and isinstance(track_data['artists'], list): + artist_names = [] + for a in track_data['artists']: + if isinstance(a, dict) and 'name' in a: + artist_names.append(a['name']) + elif isinstance(a, str): + artist_names.append(a) + if artist_names: + artist = ', '.join(artist_names) + + # Duration ā may be int seconds, float, or "m:ss" string + raw_duration = track_data.get('duration') or track_data.get('length') or 0 + if isinstance(raw_duration, str) and ':' in raw_duration: + parts = raw_duration.split(':') + try: + duration = (int(parts[0]) * 60 + int(parts[1])) + except (ValueError, IndexError): + duration = 0 + else: + try: + duration = int(raw_duration) if raw_duration else 0 + except (ValueError, TypeError): + duration = 0 + + # BPM, key, genre + bpm = track_data.get('bpm') + key_data = track_data.get('key') + key = key_data.get('name') if isinstance(key_data, dict) else None + genre_data = track_data.get('genre') + genre = genre_data.get('name') if isinstance(genre_data, dict) else None + + # Per-track release metadata + release_data = track_data.get('release', {}) if isinstance(track_data.get('release'), dict) else {} + release_name = release_data.get('name', '') + release_id = str(release_data.get('id', '')) + + release_image = '' + image_data = release_data.get('image') + if isinstance(image_data, dict): + release_image = image_data.get('uri', image_data.get('url', '')) + elif isinstance(image_data, str): + release_image = image_data + + label = 'Unknown Label' + label_data = release_data.get('label') + if isinstance(label_data, dict): + label = label_data.get('name', 'Unknown Label') + + publish_date = release_data.get('publish_date', release_data.get('date', '')) + + # Track URL + track_url = '' + if 'slug' in track_data and 'id' in track_data: + track_url = f"{self.base_url}/track/{track_data['slug']}/{track_data['id']}" + + track = { + 'position': position, + 'title': title, + 'mix_name': mix_name, + 'artist': artist, + 'list_name': chart_name, + 'url': track_url, + 'label': label, + 'bpm': bpm, + 'key': key, + 'genre': genre, + 'duration': duration, + 'type': 'track', + # Per-track release context + 'release_name': release_name, + 'release_id': release_id, + 'release_image': release_image, + 'release_date': publish_date + } + + return track + + except Exception as e: + print(f" ā Error converting chart JSON track: {e}") + return None + + def enrich_chart_tracks(self, tracks: List[Dict], progress_callback=None) -> List[Dict]: + """ + Enrich basic chart tracks by visiting each track's page to get release metadata. + Each track must have a 'url' field pointing to /track/slug/id. + Returns tracks with added: release_name, release_id, release_image, release_date, + duration, bpm, key, genre, mix_name, label. + progress_callback: optional callable(completed, total, track_name) for progress updates. + """ + import time + + enriched = [] + total = len(tracks) + print(f" š Enriching {total} chart tracks with per-track metadata...") + + for i, track in enumerate(tracks): + track_url = track.get('url', '') + if not track_url or '/track/' not in track_url: + enriched.append(track) + continue + + try: + soup = self.get_page(track_url) + if not soup: + enriched.append(track) + continue + + json_obj = self.extract_json_object_from_release_page(soup) + if not json_obj: + enriched.append(track) + continue + + # Get all tracks from the JSON ā the track page JSON contains the track itself + json_tracks = self.extract_all_tracks_from_json(json_obj) + + # Find the matching track by URL id + track_id_from_url = track_url.rstrip('/').split('/')[-1] + matched = None + for jt in json_tracks: + if str(jt.get('id', '')) == track_id_from_url: + matched = jt + break + + if not matched and json_tracks: + # Debug: show what IDs we have vs what we're looking for + sample_ids = [str(jt.get('id', '')) for jt in json_tracks[:5]] + print(f" ā ļø [{i+1}] No ID match for '{track_id_from_url}' in {sample_ids}... trying title match") + # Fallback: match by title similarity + track_title = track.get('title', '').lower().strip() + for jt in json_tracks: + jt_title = (jt.get('title') or jt.get('name', '')).lower().strip() + if track_title and jt_title and (track_title in jt_title or jt_title in track_title): + matched = jt + print(f" ā [{i+1}] Title matched: '{jt_title}'") + break + + # Fallback: use first track if only one result + if not matched and len(json_tracks) == 1: + matched = json_tracks[0] + + if matched: + rich = self.convert_chart_json_to_rich_track_format(matched, i + 1, track.get('list_name', '')) + if rich: + enriched.append(rich) + if (i + 1) <= 3 or (i + 1) % 25 == 0: + print(f" ā [{i+1}/{total}] {rich.get('artist', '?')} - {rich.get('title', '?')} | {rich.get('release_name', 'no release')}") + continue + + enriched.append(track) + + except Exception as e: + print(f" ā ļø [{i+1}/{total}] Error enriching track: {e}") + enriched.append(track) + + # Report progress + if progress_callback: + track_name = track.get('title', 'Unknown') + progress_callback(i + 1, total, track_name) + + # Small delay between requests + if i < total - 1: + time.sleep(0.3) + + enriched_count = sum(1 for t in enriched if t.get('release_name')) + print(f" ā Enrichment complete: {enriched_count}/{total} tracks have release metadata") + return enriched + def filter_tracks_for_specific_release(self, json_obj: Dict, release_url: str) -> List[Dict]: """Filter tracks to only include those from the specific release""" # Extract release ID from URL (e.g., /release/capoeira-feat-jessica-gaspar/5361445) @@ -1003,6 +1248,164 @@ class BeatportUnifiedScraper: print(f" ā Error converting track data: {e}") return None + def get_release_metadata(self, release_url: str) -> Dict: + """ + Fetch a release page and return structured metadata suitable for the download modal. + Returns album/artist/tracks in a Spotify-compatible format so the frontend can + open openDownloadMissingModalForArtistAlbum() directly. + """ + try: + soup = self.get_page(release_url) + if not soup: + return {'success': False, 'error': 'Failed to fetch release page'} + + json_obj = self.extract_json_object_from_release_page(soup) + if not json_obj: + return {'success': False, 'error': 'Could not extract JSON data from release page'} + + release_tracks = self.filter_tracks_for_specific_release(json_obj, release_url) + if not release_tracks: + return {'success': False, 'error': 'No tracks found for this release'} + + # Extract release-level info from the first track's release object + first_track = release_tracks[0] + release_data = first_track.get('release', {}) if isinstance(first_track, dict) else {} + + release_name = release_data.get('name', 'Unknown Release') + release_id = str(release_data.get('id', '')) + publish_date = release_data.get('publish_date', release_data.get('date', '')) + catalog_number = release_data.get('catalog_number', '') + + # Release image + image_url = '' + image_data = release_data.get('image') + if isinstance(image_data, dict): + image_url = image_data.get('uri', image_data.get('url', '')) + elif isinstance(image_data, str): + image_url = image_data + + # Label + label_name = 'Unknown Label' + label_data = release_data.get('label') + if isinstance(label_data, dict): + label_name = label_data.get('name', 'Unknown Label') + + # Primary artist from first track + primary_artist = 'Unknown Artist' + primary_artist_id = '' + if 'artists' in first_track and isinstance(first_track['artists'], list) and first_track['artists']: + artist_obj = first_track['artists'][0] + if isinstance(artist_obj, dict): + primary_artist = artist_obj.get('name', 'Unknown Artist') + primary_artist_id = str(artist_obj.get('id', '')) + + # Infer album type from track count + track_count = len(release_tracks) + if track_count == 1: + album_type = 'single' + elif track_count <= 6: + album_type = 'ep' + else: + album_type = 'album' + + # Build album object (same on every track) + album_obj = { + 'id': f'beatport_release_{release_id}', + 'name': release_name, + 'album_type': album_type, + 'images': [{'url': image_url}] if image_url else [], + 'release_date': publish_date, + 'total_tracks': track_count, + 'label': label_name, + 'catalog_number': catalog_number + } + + # Build artist object + artist_obj = { + 'id': f'beatport_artist_{primary_artist_id}', + 'name': primary_artist + } + + # Build track list + tracks = [] + for i, track_data in enumerate(release_tracks): + if not isinstance(track_data, dict): + continue + + title = track_data.get('title') or track_data.get('name', 'Unknown Title') + mix_name = track_data.get('mix_name') or track_data.get('mix', '') + if mix_name and mix_name.lower() != 'original mix': + name = f"{title} ({mix_name})" + else: + name = title + + # Artists list + artists = [] + if 'artists' in track_data and isinstance(track_data['artists'], list): + for a in track_data['artists']: + if isinstance(a, dict) and 'name' in a: + artists.append({'name': a['name']}) + elif isinstance(a, str): + artists.append({'name': a}) + if not artists: + artists = [{'name': 'Unknown Artist'}] + + raw_duration = track_data.get('duration') or track_data.get('length') or 0 + # Duration may be seconds (int/float) or "m:ss" / "mm:ss" string + if isinstance(raw_duration, str) and ':' in raw_duration: + parts = raw_duration.split(':') + try: + duration_ms = (int(parts[0]) * 60 + int(parts[1])) * 1000 + except (ValueError, IndexError): + duration_ms = 0 + else: + try: + duration_ms = int(raw_duration) * 1000 if raw_duration else 0 + except (ValueError, TypeError): + duration_ms = 0 + + # Extra Beatport metadata + bpm = track_data.get('bpm') + key_data = track_data.get('key') + key_name = key_data.get('name') if isinstance(key_data, dict) else None + genre_data = track_data.get('genre') + genre_name = genre_data.get('name') if isinstance(genre_data, dict) else None + isrc = track_data.get('isrc', '') + + track_obj = { + 'id': f"beatport_{track_data.get('id', i)}", + 'name': name, + 'artists': artists, + 'duration_ms': duration_ms, + 'track_number': i + 1, + 'disc_number': 1, + 'album': album_obj + } + + # Store extra metadata that may be useful + if bpm or key_name or genre_name or isrc: + track_obj['beatport_extra'] = { + 'bpm': bpm, + 'key': key_name, + 'genre': genre_name, + 'isrc': isrc + } + + tracks.append(track_obj) + + return { + 'success': True, + 'tracks': tracks, + 'album': album_obj, + 'artist': artist_obj + } + + except Exception as e: + print(f"ā Error getting release metadata from {release_url}: {e}") + import traceback + traceback.print_exc() + return {'success': False, 'error': str(e)} + def extract_individual_tracks_from_release_url(self, release_url: str, source_name: str) -> List[Dict]: """Extract individual tracks from a release URL using JSON method - used for Top 10/100""" try: @@ -1096,7 +1499,7 @@ class BeatportUnifiedScraper: return all_tracks - def scrape_hype_top_100(self, limit: int = 100) -> List[Dict]: + def scrape_hype_top_100(self, limit: int = 100, enrich: bool = True) -> List[Dict]: """Scrape Beatport Hype Top 100 - Fixed URL based on parser discovery""" print("\nš„ Scraping Beatport Hype Top 100...") @@ -1104,6 +1507,9 @@ class BeatportUnifiedScraper: soup = self.get_page(f"{self.base_url}/hype-100") if soup: tracks = self.extract_tracks_from_page(soup, "Hype Top 100", limit) + if tracks and enrich: + print(f" Enriching {len(tracks)} Hype Top 100 tracks with per-track metadata...") + tracks = self.enrich_chart_tracks(tracks) print(f"ā Extracted {len(tracks)} tracks from Hype Top 100") return tracks else: @@ -2428,7 +2834,7 @@ class BeatportUnifiedScraper: print(f"ā Extracted {len(all_individual_tracks)} individual tracks from {len(release_urls)} Top 10 releases") return all_individual_tracks - def scrape_genre_charts(self, genre: Dict, limit: int = 100) -> List[Dict]: + def scrape_genre_charts(self, genre: Dict, limit: int = 100, enrich: bool = True) -> List[Dict]: """Scrape charts for a specific genre (default: top tracks)""" tracks = [] @@ -2449,7 +2855,7 @@ class BeatportUnifiedScraper: soup = self.get_page(chart_url) if soup: tracks = self.extract_tracks_from_page(soup, f"{genre['name']} Top 100", limit) - if tracks and len(tracks) >= min(limit, 50): # If we got a decent number of tracks + if tracks and len(tracks) >= min(limit, 50): print(f" ā Successfully extracted {len(tracks)} tracks from {chart_url}") break elif tracks: @@ -2457,6 +2863,10 @@ class BeatportUnifiedScraper: else: print(f" ā No tracks found at {chart_url}") + if tracks and enrich: + print(f" Enriching {len(tracks)} {genre['name']} chart tracks with per-track metadata...") + tracks = self.enrich_chart_tracks(tracks) + return tracks def scrape_genre_top_10(self, genre: Dict) -> List[Dict]: @@ -3237,7 +3647,7 @@ class BeatportUnifiedScraper: return charts[:limit] def extract_tracks_from_chart(self, chart_url: str, chart_name: str, limit: int) -> List[Dict]: - """Extract individual tracks from a chart page - OPTIMIZED FOR CHART PAGES""" + """Extract tracks from a chart page, then enrich each with per-track release metadata""" tracks = [] try: @@ -3248,42 +3658,26 @@ class BeatportUnifiedScraper: print(f" š Extracting tracks from chart page: {chart_url}") print(f" š Chart name: {chart_name}") - # DEBUG: Check page title to confirm we're on the right page - page_title = soup.find('title') - if page_title: - print(f" š Page title: {page_title.get_text(strip=True)}") - - # DEBUG: Look for the chart title on the page - chart_title_elem = soup.find(['h1', 'h2'], string=re.compile(chart_name.split(':')[0], re.I)) - if chart_title_elem: - print(f" ā Found chart title on page: {chart_title_elem.get_text(strip=True)}") - else: - print(f" ā ļø Chart title '{chart_name}' not found on page") - - # Method 1: Try chart-specific table extraction first (most reliable for chart pages) + # Step 1: Get basic track list from HTML tracks = self.extract_tracks_from_chart_table(soup, chart_name, limit) - if len(tracks) >= 10: - print(f" ā Chart table extraction found {len(tracks)} tracks") - return tracks - - # Method 2: Fallback to general page extraction - print(f" ā ļø Chart table extraction found {len(tracks)} tracks, trying general extraction...") - general_tracks = self.extract_tracks_from_page(soup, f"New Chart: {chart_name}", limit) - - if len(general_tracks) > len(tracks): - tracks = general_tracks - print(f" ā General extraction found {len(tracks)} tracks") - - # Method 3: Last resort - generic table extraction if len(tracks) < 10: - print(f" ā ļø Still low track count, trying generic table extraction...") + print(f" ā ļø Chart table extraction found {len(tracks)} tracks, trying general extraction...") + general_tracks = self.extract_tracks_from_page(soup, f"New Chart: {chart_name}", limit) + if len(general_tracks) > len(tracks): + tracks = general_tracks + + if len(tracks) < 10: table_tracks = self.extract_tracks_from_table_format(soup, chart_name, limit) if len(table_tracks) > len(tracks): tracks = table_tracks - print(f" ā Generic table extraction found {len(tracks)} tracks") - print(f" š Final result: {len(tracks)} tracks extracted from {chart_name}") + print(f" š Found {len(tracks)} tracks, enriching with per-track metadata...") + + # Step 2: Enrich each track by visiting its individual page + if tracks: + tracks = self.enrich_chart_tracks(tracks) + return tracks except Exception as e: diff --git a/web_server.py b/web_server.py index b81b96f6..45e88dd5 100644 --- a/web_server.py +++ b/web_server.py @@ -14883,22 +14883,23 @@ def _extract_spotify_metadata(context: dict, artist: dict, album_info: dict) -> track_info = context.get("track_info", {}) if track_info and track_info.get('id'): # Spotify track IDs are alphanumeric strings; iTunes IDs are numeric + # Beatport IDs (beatport_*) are neither ā skip them for external ID tagging track_id = str(track_info['id']) if track_id.isdigit(): metadata['itunes_track_id'] = track_id - else: + elif not track_id.startswith('beatport_'): metadata['spotify_track_id'] = track_id if artist.get('id'): artist_id = str(artist['id']) if artist_id.isdigit(): metadata['itunes_artist_id'] = artist_id - else: + elif not artist_id.startswith('beatport_'): metadata['spotify_artist_id'] = artist_id if spotify_album and spotify_album.get('id'): album_id = str(spotify_album['id']) if album_id.isdigit(): metadata['itunes_album_id'] = album_id - else: + elif not album_id.startswith('beatport_'): metadata['spotify_album_id'] = album_id # Summary log for debugging metadata issues (e.g. wrong album_artist / track_number) @@ -16684,14 +16685,14 @@ def _record_retag_download(context, spotify_artist, album_info, final_path): album_name = album_info.get('album_name', '') if album_info else ( original_search.get('spotify_clean_title', 'Unknown')) - # Determine album IDs (Spotify vs iTunes) + # Determine album IDs (Spotify vs iTunes) ā skip beatport_ prefixed IDs spotify_album_id = None itunes_album_id = None if spotify_album: album_id_raw = str(spotify_album.get('id', '')) if album_id_raw and album_id_raw.isdigit(): itunes_album_id = album_id_raw - elif album_id_raw: + elif album_id_raw and not album_id_raw.startswith('beatport_'): spotify_album_id = album_id_raw image_url = album_info.get('album_image_url') if album_info else None @@ -16718,14 +16719,14 @@ def _record_retag_download(context, spotify_artist, album_info, final_path): album_info.get('clean_track_name', 'Unknown Track') if album_info else 'Unknown Track') file_format = os.path.splitext(str(final_path))[1].lstrip('.').lower() - # Track IDs (Spotify vs iTunes) + # Track IDs (Spotify vs iTunes) ā skip beatport_ prefixed IDs spotify_track_id = None itunes_track_id = None if track_info and track_info.get('id'): tid = str(track_info['id']) if tid.isdigit(): itunes_track_id = tid - else: + elif not tid.startswith('beatport_'): spotify_track_id = tid # Avoid duplicate track entries @@ -34931,6 +34932,7 @@ def get_beatport_genre_tracks(genre_slug, genre_id): # Get query parameters limit = int(request.args.get('limit', '100')) + enrich = request.args.get('enrich', 'true').lower() != 'false' # Create genre dict for scraper genre = { @@ -34940,7 +34942,7 @@ def get_beatport_genre_tracks(genre_slug, genre_id): } # Scrape tracks for this genre - tracks = scraper.scrape_genre_charts(genre, limit=limit) + tracks = scraper.scrape_genre_charts(genre, limit=limit, enrich=enrich) logger.info(f"ā Successfully scraped {len(tracks)} tracks for {genre_slug}") @@ -35596,9 +35598,10 @@ def get_beatport_top_100(): # Get query parameters limit = int(request.args.get('limit', '100')) + enrich = request.args.get('enrich', 'true').lower() != 'false' # Scrape Top 100 - tracks = scraper.scrape_top_100(limit=limit) + tracks = scraper.scrape_top_100(limit=limit, enrich=enrich) logger.info(f"ā Successfully scraped {len(tracks)} tracks from Beatport Top 100") @@ -35669,9 +35672,10 @@ def get_beatport_hype_top_100(): # Get query parameters limit = int(request.args.get('limit', '100')) + enrich = request.args.get('enrich', 'true').lower() != 'false' # Scrape Hype Top 100 using improved method - tracks = scraper.scrape_hype_top_100(limit=limit) + tracks = scraper.scrape_hype_top_100(limit=limit, enrich=enrich) logger.info(f"ā Successfully scraped {len(tracks)} tracks from Beatport Hype Top 100") @@ -35988,6 +35992,97 @@ def scrape_beatport_releases(): "track_count": 0 }), 500 +@app.route('/api/beatport/release-metadata', methods=['POST']) +def get_beatport_release_metadata(): + """Fetch structured release metadata for direct download modal (skip discovery)""" + try: + data = request.get_json() + if not data: + return jsonify({"success": False, "error": "No JSON data provided"}), 400 + + release_url = data.get('release_url', '') + if not release_url: + return jsonify({"success": False, "error": "No release_url provided"}), 400 + + logger.info(f"šÆ API request for release metadata: {release_url}") + + scraper = BeatportUnifiedScraper() + result = scraper.get_release_metadata(release_url) + + if not result.get('success'): + return jsonify(result), 404 + + # Apply text cleaning + album = result['album'] + artist = result['artist'] + album['name'] = clean_beatport_text(album['name']) + artist['name'] = clean_beatport_text(artist['name']) + + for track in result['tracks']: + track['name'] = clean_beatport_text(track['name']) + for a in track.get('artists', []): + a['name'] = clean_beatport_text(a['name']) + # Update the embedded album name too + track['album']['name'] = album['name'] + + logger.info(f"ā Release metadata: {album['name']} by {artist['name']} ({len(result['tracks'])} tracks)") + + return jsonify(result) + + except Exception as e: + logger.error(f"ā Error getting release metadata: {e}") + import traceback + logger.error(f"Full traceback: {traceback.format_exc()}") + return jsonify({"success": False, "error": str(e)}), 500 + +@app.route('/api/beatport/enrich-tracks', methods=['POST']) +def enrich_beatport_tracks(): + """Enrich basic track data (from DOM extraction) by visiting each track's page""" + try: + data = request.get_json() + if not data: + return jsonify({"success": False, "error": "No JSON data provided"}), 400 + + tracks = data.get('tracks', []) + if not tracks: + return jsonify({"success": False, "error": "No tracks provided"}), 400 + + enrichment_id = data.get('enrichment_id', str(uuid.uuid4())) + + logger.info(f"šÆ Enriching {len(tracks)} Beatport tracks with per-track metadata (id: {enrichment_id})") + + def on_progress(completed, total, track_name): + socketio.emit('beatport:enrich_progress', { + 'enrichment_id': enrichment_id, + 'completed': completed, + 'total': total, + 'current_track': track_name + }) + + scraper = BeatportUnifiedScraper() + enriched = scraper.enrich_chart_tracks(tracks, progress_callback=on_progress) + + # Apply text cleaning + for track in enriched: + if track.get('title'): + track['title'] = clean_beatport_text(track['title']) + if track.get('artist'): + track['artist'] = clean_beatport_text(track['artist']) + if track.get('release_name'): + track['release_name'] = clean_beatport_text(track['release_name']) + if track.get('label'): + track['label'] = clean_beatport_text(track['label']) + + logger.info(f"ā Enriched {len(enriched)} tracks") + + return jsonify({"success": True, "tracks": enriched}) + + except Exception as e: + logger.error(f"ā Error enriching tracks: {e}") + import traceback + logger.error(f"Full traceback: {traceback.format_exc()}") + return jsonify({"success": False, "error": str(e)}), 500 + @app.route('/api/beatport/homepage/featured-charts', methods=['GET']) def get_beatport_homepage_featured_charts(): """Get Beatport Featured Charts from homepage section""" diff --git a/webui/static/script.js b/webui/static/script.js index df16ec9c..3b40a7e2 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -12090,7 +12090,7 @@ async function startMissingTracksProcess(playlistId) { // If this is an artist album download, use album name and include full context // Match 'artist_album_', 'enhanced_search_album_', 'enhanced_search_track_', 'discover_album_', and 'seasonal_album_' prefixes - if (playlistId.startsWith('artist_album_') || playlistId.startsWith('enhanced_search_album_') || playlistId.startsWith('enhanced_search_track_') || playlistId.startsWith('discover_album_') || playlistId.startsWith('seasonal_album_') || playlistId.startsWith('spotify_library_') || playlistId.startsWith('issue_download_') || playlistId.startsWith('library_redownload_')) { + if (playlistId.startsWith('artist_album_') || playlistId.startsWith('enhanced_search_album_') || playlistId.startsWith('enhanced_search_track_') || playlistId.startsWith('discover_album_') || playlistId.startsWith('seasonal_album_') || playlistId.startsWith('spotify_library_') || playlistId.startsWith('issue_download_') || playlistId.startsWith('library_redownload_') || playlistId.startsWith('beatport_release_')) { requestBody.playlist_name = process.album?.name || process.playlist.name; requestBody.is_album_download = true; requestBody.album_context = process.album; // Full Spotify album object @@ -24221,24 +24221,9 @@ async function handleHomepageChartTypeClick(chartType, chartEndpoint, chartName) } try { - // Check if we already have a card for this specific chart type (following genre page pattern) - const existingState = Object.values(beatportChartStates).find(state => - state.chart && state.chart.name === chartConfig.name && state.chart.chart_type === `homepage_${chartType}` - ); - - if (existingState) { - console.log(`š Found existing Beatport card for ${chartConfig.name}, opening existing modal`); - handleBeatportCardClick(existingState.chart.hash); - return; - } - - // Create a chart hash for state management (following genre page pattern) - const chartHash = `homepage_${chartType}_${Date.now()}`; - showToast(`Loading ${chartConfig.name}...`, 'info'); showLoadingOverlay(`Loading ${chartConfig.name}...`); - // Fetch tracks from the specific endpoint (following genre page pattern) const response = await fetch(`${chartConfig.endpoint}?limit=${chartConfig.limit}`); if (!response.ok) { throw new Error(`Failed to fetch ${chartConfig.name}: ${response.status}`); @@ -24249,31 +24234,9 @@ async function handleHomepageChartTypeClick(chartType, chartEndpoint, chartName) throw new Error(`No tracks found in ${chartConfig.name}`); } - // Create chart data object for playlist card (following genre page pattern) - const chartData = { - hash: chartHash, - name: chartConfig.name, - chart_type: `homepage_${chartType}`, - track_count: data.tracks.length, - tracks: data.tracks.map(track => ({ - name: track.title || 'Unknown Title', - artists: [track.artist || 'Unknown Artist'], - album: chartConfig.name, - duration_ms: 0, - external_urls: { beatport: track.url || '' }, - source: 'beatport' - })) - }; - - // Add card to container (in background, like YouTube does - following genre page pattern) - console.log(`š Creating Beatport playlist card for: ${chartConfig.name}`); - addBeatportCardToContainer(chartData); - - // Automatically open discovery modal (like when you click a YouTube or Tidal card in fresh state - following genre page pattern) + console.log(`ā Fetched ${data.tracks.length} tracks from ${chartConfig.name}`); hideLoadingOverlay(); - handleBeatportCardClick(chartHash); - - console.log(`ā Created Beatport card and opened discovery modal for ${chartConfig.name}`); + openBeatportChartAsDownloadModal(data.tracks, chartConfig.name, null); } catch (error) { console.error(`ā Error loading ${chartConfig.name}:`, error); @@ -24547,55 +24510,28 @@ async function handleRebuildHypeTop10Click() { // The old handleRebuildHeroSliderClick function has been removed in favor of individual release discovery async function handleRebuildChartClick(trackDataKey, chartName, chartType) { + if (_beatportModalOpening) return; + _beatportModalOpening = true; + setTimeout(() => { _beatportModalOpening = false; }, 2000); + try { - // Create chart hash (following Browse Charts pattern) - const chartHash = `${chartType}_${Date.now()}`; - - // Check if we already have an existing state (following Browse Charts pattern) - const existingState = Object.values(beatportChartStates).find(state => - state.chart && state.chart.name === chartName && state.chart.chart_type === chartType - ); - - if (existingState) { - console.log(`š Found existing ${chartName} card, opening existing modal`); - // Use existing card click handler (following Browse Charts pattern) - handleBeatportCardClick(existingState.chart.hash); - return; - } - - // Get track data from rebuild page data (instead of API scraping) + // Get basic track data from DOM const trackData = await getRebuildPageTrackData(trackDataKey); if (!trackData || trackData.length === 0) { throw new Error(`No track data found for ${chartName}`); } - // Transform rebuild data to Browse Charts format EXACTLY - const chartData = { - hash: chartHash, - name: chartName, - chart_type: chartType, - track_count: trackData.length, - tracks: trackData.map(track => ({ - name: cleanTrackText(track.title || 'Unknown Title'), - artists: [cleanTrackText(track.artist || 'Unknown Artist')], - album: chartName, - duration_ms: 0, - external_urls: { beatport: track.url || '' }, - source: 'beatport' - })) - }; + console.log(`ā Got ${trackData.length} tracks from ${chartName}, enriching one-by-one...`); + showLoadingOverlay(`Fetching track metadata... (0/${trackData.length})`); - // Follow Browse Charts pattern EXACTLY: - // 1. Add card to container (creates playlist card) - console.log(`š Creating Beatport playlist card for: ${chartData.name}`); - addBeatportCardToContainer(chartData); + const enrichedTracks = await _enrichTracksWithProgress(trackData, chartName); - // 2. Automatically open discovery modal (like when you click a card in fresh state) - handleBeatportCardClick(chartHash); - - console.log(`ā Created ${chartName} card and opened discovery modal`); + console.log(`ā Enriched ${enrichedTracks.length} tracks`); + hideLoadingOverlay(); + openBeatportChartAsDownloadModal(enrichedTracks, chartName, null); } catch (error) { + hideLoadingOverlay(); console.error(`ā Error handling ${chartName} click:`, error); showToast(`Error loading ${chartName}: ${error.message}`, 'error'); } @@ -25398,23 +25334,8 @@ async function handleBeatportChartClick(chartType, chartId, chartName, chartEndp console.log(`šµ Beatport chart clicked: ${chartType} - ${chartId} - ${chartName}`); try { - // Check if we already have a card for this chart - const existingState = Object.values(beatportChartStates).find(state => - state.chart && state.chart.name === chartName && state.chart.chart_type === chartType - ); - - if (existingState) { - console.log(`š Found existing Beatport card for ${chartName}, opening existing modal`); - handleBeatportCardClick(existingState.chart.hash); - return; - } - - // First, create a chart hash for state management - const chartHash = `${chartType}_${chartId}_${Date.now()}`; - - // Load chart data from backend using the specific endpoint - console.log(`š Loading ${chartName} tracks from ${chartEndpoint}...`); showToast(`Loading ${chartName}...`, 'info'); + showLoadingOverlay(`Loading ${chartName}...`); const response = await fetch(`${chartEndpoint}?limit=100`); if (!response.ok) { @@ -25426,32 +25347,13 @@ async function handleBeatportChartClick(chartType, chartId, chartName, chartEndp throw new Error(`No tracks found in ${chartName}`); } - // Create chart data object - const chartData = { - hash: chartHash, - name: chartName, - chart_type: chartType, - track_count: data.tracks.length, - tracks: data.tracks.map(track => ({ - name: track.title || 'Unknown Title', - artists: [track.artist || 'Unknown Artist'], - album: chartName, - duration_ms: 0, - external_urls: { beatport: track.url || '' }, - source: 'beatport' - })) - }; - - // Add card to container (in background, like YouTube does) - addBeatportCardToContainer(chartData); - - // Automatically open discovery modal (like when you click a YouTube or Tidal card in fresh state) - handleBeatportCardClick(chartHash); - - console.log(`ā Created Beatport card and opened discovery modal for ${chartName}`); + console.log(`ā Fetched ${data.tracks.length} tracks from ${chartName}`); + hideLoadingOverlay(); + openBeatportChartAsDownloadModal(data.tracks, chartName, null); } catch (error) { console.error(`ā Error handling Beatport chart click:`, error); + hideLoadingOverlay(); showToast(`Error loading ${chartName || chartId}: ${error.message}`, 'error'); } } @@ -25900,32 +25802,14 @@ function setupDJChartItemHandlers() { console.log(`š§ DJ Chart clicked: ${chartName}`); - // Check if state already exists by name and type (follow same pattern as homepage Beatport cards) - const existingState = Object.values(beatportChartStates).find(state => - state.chart && state.chart.name === chartName && state.chart.chart_type === 'dj-chart' - ); - - if (existingState) { - console.log(`š Found existing DJ chart state for ${chartName}, opening existing modal`); - handleBeatportCardClick(existingState.chart.hash); - return; - } - try { showToast(`Loading ${chartName}...`, 'info'); showLoadingOverlay(`Loading ${chartName}...`); - // Extract tracks from the DJ chart const response = await fetch('/api/beatport/chart/extract', { method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - chart_url: chartUrl, - chart_name: chartName, - limit: 100 - }) + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ chart_url: chartUrl, chart_name: chartName, limit: 100 }) }); if (!response.ok) { @@ -25933,54 +25817,14 @@ function setupDJChartItemHandlers() { } const data = await response.json(); - if (data.success && data.tracks && data.tracks.length > 0) { - console.log(`ā Extracted ${data.tracks.length} tracks from DJ chart: ${chartName}`); - - // Generate a unique hash for state management (following homepage pattern) - const chartHash = `dj_chart_${Date.now()}`; - - // Create chart data in the format expected by the state system - const chartData = { - hash: chartHash, - name: chartName, - chart_type: 'dj-chart', - track_count: data.tracks.length, - tracks: data.tracks.map(track => ({ - name: track.title || 'Unknown Title', - artists: [track.artist || 'Unknown Artist'], - album: chartName, - duration_ms: 0, - external_urls: { spotify: null }, - preview_url: null, - popularity: 0, - explicit: false, - track_number: track.position || 1, - disc_number: 1, - id: `dj_chart_${chartHash}_${track.position || Math.random()}`, - uri: null, - type: 'track', - is_local: false, - source: 'beatport_dj_chart' - })) - }; - - // Create state in beatportChartStates (follow same pattern as other Beatport cards) - beatportChartStates[chartHash] = { - chart: chartData, - phase: 'fresh', - cardElement: null, // Will be set when actual card is created - discovery_results: [], - discoveryProgress: 0 - }; - - // Use the same click handler as other Beatport cards - hideLoadingOverlay(); - handleBeatportCardClick(chartHash); - - } else { + if (!data.success || !data.tracks || data.tracks.length === 0) { throw new Error('No tracks found in chart'); } + console.log(`ā Extracted ${data.tracks.length} tracks from DJ chart: ${chartName}`); + hideLoadingOverlay(); + openBeatportChartAsDownloadModal(data.tracks, chartName, null); + } catch (error) { console.error('ā Error extracting DJ chart tracks:', error); hideLoadingOverlay(); @@ -26000,32 +25844,14 @@ function setupFeaturedChartItemHandlers() { console.log(`ā Featured Chart clicked: ${chartName}`); - // Check if state already exists by name and type (follow same pattern as homepage Beatport cards) - const existingState = Object.values(beatportChartStates).find(state => - state.chart && state.chart.name === chartName && state.chart.chart_type === 'featured-chart' - ); - - if (existingState) { - console.log(`š Found existing Featured chart state for ${chartName}, opening existing modal`); - handleBeatportCardClick(existingState.chart.hash); - return; - } - try { showToast(`Loading ${chartName}...`, 'info'); showLoadingOverlay(`Loading ${chartName}...`); - // Extract tracks from the Featured chart const response = await fetch('/api/beatport/chart/extract', { method: 'POST', - headers: { - 'Content-Type': 'application/json' - }, - body: JSON.stringify({ - chart_url: chartUrl, - chart_name: chartName, - limit: 100 - }) + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ chart_url: chartUrl, chart_name: chartName, limit: 100 }) }); if (!response.ok) { @@ -26033,54 +25859,14 @@ function setupFeaturedChartItemHandlers() { } const data = await response.json(); - if (data.success && data.tracks && data.tracks.length > 0) { - console.log(`ā Extracted ${data.tracks.length} tracks from Featured chart: ${chartName}`); - - // Generate a unique hash for state management (following homepage pattern) - const chartHash = `featured_chart_${Date.now()}`; - - // Create chart data in the format expected by the state system - const chartData = { - hash: chartHash, - name: chartName, - chart_type: 'featured-chart', - track_count: data.tracks.length, - tracks: data.tracks.map(track => ({ - name: track.title || 'Unknown Title', - artists: [track.artist || 'Unknown Artist'], - album: chartName, - duration_ms: 0, - external_urls: { spotify: null }, - preview_url: null, - popularity: 0, - explicit: false, - track_number: track.position || 1, - disc_number: 1, - id: `featured_chart_${chartHash}_${track.position || Math.random()}`, - uri: null, - type: 'track', - is_local: false, - source: 'beatport_featured_chart' - })) - }; - - // Create state in beatportChartStates (follow same pattern as other Beatport cards) - beatportChartStates[chartHash] = { - chart: chartData, - phase: 'fresh', - cardElement: null, // Will be set when actual card is created - discovery_results: [], - discoveryProgress: 0 - }; - - // Use the same click handler as other Beatport cards - hideLoadingOverlay(); - handleBeatportCardClick(chartHash); - - } else { + if (!data.success || !data.tracks || data.tracks.length === 0) { throw new Error('No tracks found in chart'); } + console.log(`ā Extracted ${data.tracks.length} tracks from Featured chart: ${chartName}`); + hideLoadingOverlay(); + openBeatportChartAsDownloadModal(data.tracks, chartName, null); + } catch (error) { console.error('ā Error extracting Featured chart tracks:', error); hideLoadingOverlay(); @@ -26100,74 +25886,31 @@ function setupNewChartItemHandlers(genreSlug, genreId, genreName) { const chartUrl = item.dataset.chartUrl; console.log(`šµ Chart clicked: ${chartName} by ${chartArtist}`); - console.log(`š Chart URL: ${chartUrl}`); const fullChartName = `${chartName} (${genreName})`; - // Check if state already exists by name and type (follow same pattern as homepage Beatport cards) - const existingState = Object.values(beatportChartStates).find(state => - state.chart && state.chart.name === fullChartName && state.chart.chart_type === 'individual_chart' - ); - - if (existingState) { - console.log(`š Found existing individual chart state for ${fullChartName}, opening existing modal`); - handleBeatportCardClick(existingState.chart.hash); - return; - } - try { showToast(`Loading ${chartName}...`, 'info'); showLoadingOverlay(`Loading ${chartName}...`); - // Use the new chart extraction endpoint with the actual chart URL const response = await fetch('/api/beatport/chart/extract', { method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - chart_url: chartUrl, - chart_name: chartName, - limit: 100 - }) + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ chart_url: chartUrl, chart_name: chartName, limit: 100 }) }); + if (!response.ok) { throw new Error(`Failed to fetch chart content: ${response.status}`); } const data = await response.json(); if (!data.success || !data.tracks || data.tracks.length === 0) { - throw new Error(`No tracks found in chart`); + throw new Error('No tracks found in chart'); } - // Generate a unique hash for state management (following homepage pattern) - const chartHash = `individual_chart_${Date.now()}`; - - // Create chart data object for playlist card - const chartData = { - hash: chartHash, - name: fullChartName, - chart_type: 'individual_chart', - track_count: data.tracks.length, - tracks: data.tracks.map(track => ({ - name: track.title || 'Unknown Title', - artists: [track.artist || 'Unknown Artist'], - album: fullChartName, - duration_ms: 0, - external_urls: { beatport: track.url || chartUrl }, - source: 'beatport' - })) - }; - - // Add card to container (in background, like YouTube does) - console.log(`š Creating Beatport playlist card for: ${fullChartName}`); - addBeatportCardToContainer(chartData); - - // Automatically open discovery modal + console.log(`ā Extracted ${data.tracks.length} tracks from ${fullChartName}`); hideLoadingOverlay(); - handleBeatportCardClick(chartHash); - - console.log(`ā Created Beatport card and opened discovery modal for ${fullChartName}`); + openBeatportChartAsDownloadModal(data.tracks, fullChartName, null); } catch (error) { console.error(`ā Error loading chart: ${error.message}`); @@ -26328,62 +26071,31 @@ function setupGenreChartItemHandlers(genreSlug, genreId, genreName) { const chartUrl = item.dataset.chartUrl; console.log(`šµ Chart clicked: ${chartName} by ${chartArtist}`); - console.log(`š Chart URL: ${chartUrl}`); + + const fullChartName = `${chartName} (${genreName})`; try { - // Create a virtual chart data object - const chartHash = `individual_chart_${genreSlug}_${Date.now()}`; - const fullChartName = `${chartName} (${genreName})`; - showToast(`Loading ${chartName}...`, 'info'); showLoadingOverlay(`Loading ${chartName}...`); - // Use the new chart extraction endpoint with the actual chart URL const response = await fetch('/api/beatport/chart/extract', { method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - chart_url: chartUrl, - chart_name: chartName, - limit: 100 - }) + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ chart_url: chartUrl, chart_name: chartName, limit: 100 }) }); + if (!response.ok) { throw new Error(`Failed to fetch chart content: ${response.status}`); } const data = await response.json(); if (!data.success || !data.tracks || data.tracks.length === 0) { - throw new Error(`No tracks found in chart`); + throw new Error('No tracks found in chart'); } - // Create chart data object for playlist card - const chartData = { - hash: chartHash, - name: fullChartName, - chart_type: 'individual_chart', - track_count: data.tracks.length, - tracks: data.tracks.map(track => ({ - name: track.title || 'Unknown Title', - artists: [track.artist || 'Unknown Artist'], - album: fullChartName, - duration_ms: 0, - external_urls: { beatport: track.url || chartUrl }, - source: 'beatport' - })) - }; - - // Add card to container (in background, like YouTube does) - console.log(`š Creating Beatport playlist card for: ${fullChartName}`); - addBeatportCardToContainer(chartData); - - // Automatically open discovery modal + console.log(`ā Extracted ${data.tracks.length} tracks from ${fullChartName}`); hideLoadingOverlay(); - handleBeatportCardClick(chartHash); - - console.log(`ā Created Beatport card and opened discovery modal for ${fullChartName}`); + openBeatportChartAsDownloadModal(data.tracks, fullChartName, null); } catch (error) { console.error(`ā Error loading chart: ${error.message}`); @@ -26459,24 +26171,9 @@ async function handleGenreChartTypeClick(genreSlug, genreId, genreName, chartTyp } try { - // Check if we already have a card for this specific chart type - const existingState = Object.values(beatportChartStates).find(state => - state.chart && state.chart.name === chartConfig.name && state.chart.chart_type === `genre_${chartType}` - ); - - if (existingState) { - console.log(`š Found existing Beatport card for ${chartConfig.name}, opening existing modal`); - handleBeatportCardClick(existingState.chart.hash); - return; - } - - // Create a chart hash for state management - const chartHash = `genre_${chartType}_${genreSlug}_${genreId}_${Date.now()}`; - showToast(`Loading ${chartConfig.name}...`, 'info'); showLoadingOverlay(`Loading ${chartConfig.name}...`); - // Fetch tracks from the specific endpoint const response = await fetch(`${chartConfig.endpoint}?limit=${chartConfig.limit}`); if (!response.ok) { throw new Error(`Failed to fetch ${chartConfig.name}: ${response.status}`); @@ -26487,31 +26184,9 @@ async function handleGenreChartTypeClick(genreSlug, genreId, genreName, chartTyp throw new Error(`No tracks found in ${chartConfig.name}`); } - // Create chart data object for playlist card - const chartData = { - hash: chartHash, - name: chartConfig.name, - chart_type: `genre_${chartType}`, - track_count: data.tracks.length, - tracks: data.tracks.map(track => ({ - name: track.title || 'Unknown Title', - artists: [track.artist || 'Unknown Artist'], - album: chartConfig.name, - duration_ms: 0, - external_urls: { beatport: track.url || '' }, - source: 'beatport' - })) - }; - - // Add card to container (in background, like YouTube does) - console.log(`š Creating Beatport playlist card for: ${chartConfig.name}`); - addBeatportCardToContainer(chartData); - - // Automatically open discovery modal (like when you click a YouTube or Tidal card in fresh state) + console.log(`ā Fetched ${data.tracks.length} tracks from ${chartConfig.name}`); hideLoadingOverlay(); - handleBeatportCardClick(chartHash); - - console.log(`ā Created Beatport card and opened discovery modal for ${chartConfig.name}`); + openBeatportChartAsDownloadModal(data.tracks, chartConfig.name, null); } catch (error) { console.error(`ā Error loading ${chartConfig.name}:`, error); @@ -31879,7 +31554,7 @@ async function createArtistAlbumVirtualPlaylist(album, albumType) { * Open download missing tracks modal specifically for artist albums * Similar to openDownloadMissingModalForYouTube but for artist albums */ -async function openDownloadMissingModalForArtistAlbum(virtualPlaylistId, playlistName, spotifyTracks, album, artist, showLoadingOverlayParam = true) { +async function openDownloadMissingModalForArtistAlbum(virtualPlaylistId, playlistName, spotifyTracks, album, artist, showLoadingOverlayParam = true, contextType = 'artist_album') { if (showLoadingOverlayParam) { showLoadingOverlay('Loading album...'); } @@ -31933,8 +31608,13 @@ async function openDownloadMissingModalForArtistAlbum(virtualPlaylistId, playlis albumType: album.album_type }; - // Generate hero section for artist album context - const heroContext = { + // Generate hero section ā 'artist_album' for releases, 'playlist' for charts/compilations + const heroContext = contextType === 'playlist' ? { + type: 'playlist', + playlist: { name: playlistName, owner: 'Beatport' }, + trackCount: spotifyTracks.length, + playlistId: virtualPlaylistId + } : { type: 'artist_album', artist: artist, album: album, @@ -31944,7 +31624,7 @@ async function openDownloadMissingModalForArtistAlbum(virtualPlaylistId, playlis // Use the exact same modal HTML structure as the existing modals modal.innerHTML = ` -