diff --git a/beatport_unified_scraper.py b/beatport_unified_scraper.py index eced8d24..3983a891 100644 --- a/beatport_unified_scraper.py +++ b/beatport_unified_scraper.py @@ -7,6 +7,8 @@ Focused on extracting clean artist and track names for virtual playlists import requests from bs4 import BeautifulSoup import json +import logging +import os import time import re from urllib.parse import urljoin @@ -14,6 +16,39 @@ from typing import Dict, List, Optional import concurrent.futures from threading import Lock +from utils.logging_config import get_logger + + +logger = get_logger("beatport_scraper") +_BEATPORT_LOGGING_ENABLED = os.environ.get("SOULSYNC_BEATPORT_SCRAPER_LOGS", "").lower() in ("1", "true", "yes", "on") +if _BEATPORT_LOGGING_ENABLED: + logger.setLevel(logging.DEBUG) +else: + logger.setLevel(logging.CRITICAL + 1) + +def _beatport_log(message: str): + """Route scraper output through logging when explicitly enabled.""" + if not _BEATPORT_LOGGING_ENABLED: + return + + text = str(message) + stripped = text.strip() + + if not stripped: + return + + lowered = stripped.lower() + level = logging.DEBUG + + if lowered.startswith("error") or " exception" in lowered or lowered.startswith("failed") or " failed" in lowered: + level = logging.ERROR + elif lowered.startswith("could not") or lowered.startswith("couldn't"): + level = logging.WARNING + elif lowered.startswith("no ") and "found" not in lowered: + level = logging.WARNING + + logger.log(level, text) + class BeatportUnifiedScraper: def __init__(self): self.base_url = "https://beatport.com" @@ -127,7 +162,7 @@ class BeatportUnifiedScraper: response.raise_for_status() return BeautifulSoup(response.content, 'html.parser') except requests.RequestException as e: - print(f"Error fetching {url}: {e}") + _beatport_log(f"Error fetching {url}: {e}") return None def clean_artist_track_data(self, raw_artist: str, raw_title: str) -> Dict[str, str]: @@ -171,12 +206,12 @@ class BeatportUnifiedScraper: def discover_genres_from_homepage(self) -> List[Dict]: """Dynamically discover all genres from Beatport homepage dropdown""" - print("Discovering genres from Beatport homepage...") + _beatport_log("Discovering genres from Beatport homepage...") try: soup = self.get_page(self.base_url) if not soup: - print("Could not fetch homepage") + _beatport_log("Could not fetch homepage") return self.fallback_genres genres = [] @@ -185,14 +220,14 @@ class BeatportUnifiedScraper: genres_dropdown = soup.find('div', {'id': 'genres-dropdown-menu'}) if genres_dropdown: - print("Found genres-dropdown-menu") + _beatport_log("Found genres-dropdown-menu") # Look for the two main div containers as described genre_containers = genres_dropdown.find_all('div', recursive=False) - print(f"Found {len(genre_containers)} top-level containers in dropdown") + _beatport_log(f"Found {len(genre_containers)} top-level containers in dropdown") for container_idx, container in enumerate(genre_containers): - print(f"Processing container {container_idx + 1}") + _beatport_log(f"Processing container {container_idx + 1}") # Look specifically for .dropdown_menu classes dropdown_menus = container.find_all(class_='dropdown_menu') @@ -202,17 +237,17 @@ class BeatportUnifiedScraper: dropdown_menus = container.find_all(class_=re.compile(r'dropdown.*menu', re.I)) if not dropdown_menus: - print(f"No .dropdown_menu found in container {container_idx + 1}") + _beatport_log(f"No .dropdown_menu found in container {container_idx + 1}") continue for menu_idx, menu in enumerate(dropdown_menus): - print(f"Processing dropdown_menu {menu_idx + 1} in container {container_idx + 1}") + _beatport_log(f"Processing dropdown_menu {menu_idx + 1} in container {container_idx + 1}") # Look for
  • elements first, then elements within them list_items = menu.find_all('li') if list_items: - print(f"Found {len(list_items)} list items in menu") + _beatport_log(f"Found {len(list_items)} list items in menu") for li in list_items: # Find anchor tag within the list item @@ -239,16 +274,16 @@ class BeatportUnifiedScraper: 'id': genre_id, 'url': urljoin(self.base_url, href) }) - print(f" Added: {name} ({slug}/{genre_id})") + _beatport_log(f" Added: {name} ({slug}/{genre_id})") else: - print(f" Filtered out: '{name}' (appears to be a section title)") + _beatport_log(f" Filtered out: '{name}' (appears to be a section title)") else: # Fallback: try the old method if no
  • elements found - print(f"No
  • elements found, trying direct search...") + _beatport_log(f"No
  • elements found, trying direct search...") genre_links = menu.find_all('a', href=re.compile(r'/genre/[^/]+/\d+')) if genre_links: - print(f"Found {len(genre_links)} genre links in menu (fallback method)") + _beatport_log(f"Found {len(genre_links)} genre links in menu (fallback method)") for link in genre_links: href = link.get('href', '') name_text = link.get_text(strip=True) @@ -266,16 +301,16 @@ class BeatportUnifiedScraper: 'id': genre_id, 'url': urljoin(self.base_url, href) }) - print(f" Added: {name} ({slug}/{genre_id})") + _beatport_log(f" Added: {name} ({slug}/{genre_id})") else: - print(f"No genre links found in dropdown_menu {menu_idx + 1}") + _beatport_log(f"No genre links found in dropdown_menu {menu_idx + 1}") if genres: - print(f"Successfully extracted {len(genres)} genres from dropdown menu") + _beatport_log(f"Successfully extracted {len(genres)} genres from dropdown menu") else: - print("No genre links found in dropdown menu structure") + _beatport_log("No genre links found in dropdown menu structure") else: - print("Could not find genres-dropdown-menu, trying fallback methods...") + _beatport_log("Could not find genres-dropdown-menu, trying fallback methods...") # Fallback: Look for other potential dropdown structures potential_dropdowns = [ @@ -289,11 +324,11 @@ class BeatportUnifiedScraper: for dropdown in potential_dropdowns: if dropdown: - print(f"Found fallback dropdown: {dropdown.name} with class {dropdown.get('class')}") + _beatport_log(f"Found fallback dropdown: {dropdown.name} with class {dropdown.get('class')}") genre_links = dropdown.find_all('a', href=re.compile(r'/genre/[^/]+/\d+')) if genre_links: - print(f"Found {len(genre_links)} genre links in fallback dropdown") + _beatport_log(f"Found {len(genre_links)} genre links in fallback dropdown") for link in genre_links: href = link.get('href', '') name_text = link.get_text(strip=True) @@ -313,14 +348,14 @@ class BeatportUnifiedScraper: }) if genres: - print(f"Successfully extracted {len(genres)} genres from fallback dropdown") + _beatport_log(f"Successfully extracted {len(genres)} genres from fallback dropdown") break # Method 2: Look for any genre links on the page if not genres: - print("Dropdown not found, searching for genre links...") + _beatport_log("Dropdown not found, searching for genre links...") all_genre_links = soup.find_all('a', href=re.compile(r'/genre/[^/]+/\d+')) - print(f"Found {len(all_genre_links)} potential genre links on page") + _beatport_log(f"Found {len(all_genre_links)} potential genre links on page") seen_genres = set() for link in all_genre_links: @@ -343,18 +378,18 @@ class BeatportUnifiedScraper: # Method 3: Try to find a genres page link and scrape from there if not genres: - print("Searching for genres page...") + _beatport_log("Searching for genres page...") genres_page_link = soup.find('a', href=re.compile(r'/genres$')) or \ soup.find('a', href=re.compile(r'/browse.*genre', re.I)) if genres_page_link: genres_page_url = urljoin(self.base_url, genres_page_link['href']) - print(f"Found genres page: {genres_page_url}") + _beatport_log(f"Found genres page: {genres_page_url}") genres_soup = self.get_page(genres_page_url) if genres_soup: genre_links = genres_soup.find_all('a', href=re.compile(r'/genre/[^/]+/\d+')) - print(f"Found {len(genre_links)} genre links on genres page") + _beatport_log(f"Found {len(genre_links)} genre links on genres page") seen_genres = set() for link in genre_links: @@ -386,19 +421,19 @@ class BeatportUnifiedScraper: final_genres = list(unique_genres.values()) final_genres.sort(key=lambda x: x['name']) - print(f"Discovered {len(final_genres)} unique genres from homepage") + _beatport_log(f"Discovered {len(final_genres)} unique genres from homepage") return final_genres else: - print("No genres found, using fallback list") + _beatport_log("No genres found, using fallback list") return self.fallback_genres except Exception as e: - print(f"Error discovering genres: {e}") + _beatport_log(f"Error discovering genres: {e}") return self.fallback_genres def discover_chart_sections(self) -> Dict[str, List[Dict]]: """Dynamically discover chart sections from homepage""" - print("Discovering chart sections from Beatport homepage...") + _beatport_log("Discovering chart sections from Beatport homepage...") soup = self.get_page(self.base_url) if not soup: @@ -411,7 +446,7 @@ class BeatportUnifiedScraper: } # Method 1: Find H2 section headings - print(" Finding H2 section headings...") + _beatport_log(" Finding H2 section headings...") h2_headings = soup.find_all('h2') for heading in h2_headings: @@ -426,10 +461,10 @@ class BeatportUnifiedScraper: # Categorize into our three main groups category = self._categorize_chart_section(text) chart_sections[category].append(section_info) - print(f" Found: '{text}' -> {category}") + _beatport_log(f" Found: '{text}' -> {category}") # Method 2: Find specific chart links - print(" Finding chart page links...") + _beatport_log(" Finding chart page links...") chart_links = [] # Look for the specific links we discovered @@ -450,10 +485,10 @@ class BeatportUnifiedScraper: 'expected': link_info['expected_href'], 'matches_expected': href == link_info['expected_href'] }) - print(f" Found: '{link.get_text(strip=True)}' -> {href}") + _beatport_log(f" Found: '{link.get_text(strip=True)}' -> {href}") # Method 3: Count individual DJ charts - print(" Counting individual DJ charts...") + _beatport_log(" Counting individual DJ charts...") dj_chart_links = soup.find_all('a', href=re.compile(r'/chart/')) individual_dj_charts = [] @@ -467,7 +502,7 @@ class BeatportUnifiedScraper: 'full_url': urljoin(self.base_url, href) }) - print(f" Found {len(dj_chart_links)} individual DJ charts") + _beatport_log(f" Found {len(dj_chart_links)} individual DJ charts") return { 'sections': chart_sections, @@ -529,20 +564,20 @@ class BeatportUnifiedScraper: for img in artwork_imgs: src = img.get('src', '') if 'geo-media' in src and ('1050x508' in src or '500x500' in src): - print(f" Found high-quality artwork image: {src}") + _beatport_log(f" Found high-quality artwork image: {src}") return src # Second, try any geo-media images in artwork containers for img in artwork_imgs: src = img.get('src', '') if 'geo-media' in src: - print(f" Found geo-media artwork image: {src}") + _beatport_log(f" Found geo-media artwork image: {src}") return src # Third, use any artwork image as fallback first_artwork_src = artwork_imgs[0].get('src', '') if first_artwork_src: - print(f" Found artwork image (fallback): {first_artwork_src}") + _beatport_log(f" Found artwork image (fallback): {first_artwork_src}") return first_artwork_src # Priority 2: Original method - Look for hero release slideshow images @@ -553,19 +588,19 @@ class BeatportUnifiedScraper: for img in hero_images: src = img.get('src', '') if '1050x508' in src or '500x500' in src: - print(f" Found high-quality hero image: {src}") + _beatport_log(f" Found high-quality hero image: {src}") return src # Fallback to any geo-media image fallback_src = hero_images[0].get('src', '') - print(f" Found hero image (fallback): {fallback_src}") + _beatport_log(f" Found hero image (fallback): {fallback_src}") return fallback_src - print(f" No suitable images found on page") + _beatport_log(f" No suitable images found on page") return None except Exception as e: - print(f"Could not get image for {genre_url}: {e}") + _beatport_log(f"Could not get image for {genre_url}: {e}") return None def discover_genres_with_images(self, include_images: bool = False) -> List[Dict]: @@ -573,16 +608,16 @@ class BeatportUnifiedScraper: genres = self.discover_genres_from_homepage() if include_images: - print("Fetching genre images...") + _beatport_log("Fetching genre images...") for i, genre in enumerate(genres[:10]): # Limit to first 10 for demo - print(f"Getting image for {genre['name']} ({i+1}/{min(10, len(genres))})") + _beatport_log(f"Getting image for {genre['name']} ({i+1}/{min(10, len(genres))})") # Check if genre has URL if 'url' in genre and genre['url']: image_url = self.get_genre_image(genre['url']) genre['image_url'] = image_url else: - print(f" No URL available for {genre['name']}, skipping image") + _beatport_log(f" No URL available for {genre['name']}, skipping image") genre['image_url'] = None # Small delay to be respectful @@ -649,7 +684,7 @@ class BeatportUnifiedScraper: } except Exception as e: - print(f"Error extracting release data: {e}") + _beatport_log(f"Error extracting release data: {e}") return None def extract_chart_data_from_card(self, chart_card) -> Optional[Dict]: @@ -695,7 +730,7 @@ class BeatportUnifiedScraper: } except Exception as e: - print(f"Error extracting chart data: {e}") + _beatport_log(f"Error extracting chart data: {e}") return None def extract_tracks_from_page(self, soup: BeautifulSoup, list_name: str, limit: int = 100) -> List[Dict]: @@ -708,7 +743,7 @@ class BeatportUnifiedScraper: # Find all track links on the page track_links = soup.find_all('a', href=re.compile(r'/track/')) - print(f" Found {len(track_links)} track links on {list_name}") + _beatport_log(f" Found {len(track_links)} track links on {list_name}") for i, link in enumerate(track_links[:limit]): if len(tracks) >= limit: @@ -800,7 +835,7 @@ class BeatportUnifiedScraper: def scrape_top_100(self, limit: int = 100, enrich: bool = True) -> List[Dict]: """Scrape Beatport Top 100""" - print("\nScraping Beatport Top 100...") + _beatport_log("\nScraping Beatport Top 100...") soup = self.get_page(f"{self.base_url}/top-100") tracks = self.extract_tracks_from_page(soup, "Top 100", limit) @@ -809,12 +844,12 @@ class BeatportUnifiedScraper: if tracks and enrich: tracks = self.enrich_chart_tracks(tracks) - print(f"Extracted {len(tracks)} tracks from Top 100") + _beatport_log(f"Extracted {len(tracks)} tracks from Top 100") return tracks def scrape_new_releases(self, limit: int = 40) -> List[Dict]: """Scrape individual tracks from Beatport New Releases using JSON extraction - ENHANCED""" - print("\nšŸ†• Scraping Beatport New Releases (individual tracks)...") + _beatport_log("\nšŸ†• Scraping Beatport New Releases (individual tracks)...") # Step 1: Get release URLs from homepage cards release_urls = self.extract_new_releases_urls(limit) @@ -824,7 +859,7 @@ class BeatportUnifiedScraper: # Step 2: Extract individual tracks from each release all_tracks = [] for i, release_url in enumerate(release_urls): - print(f"\nProcessing release {i+1}/{len(release_urls)}") + _beatport_log(f"\nProcessing release {i+1}/{len(release_urls)}") tracks = self.extract_tracks_from_release_json(release_url) if tracks: all_tracks.extend(tracks) @@ -833,7 +868,7 @@ class BeatportUnifiedScraper: import time time.sleep(0.5) - print(f"Extracted {len(all_tracks)} individual tracks from {len(release_urls)} releases") + _beatport_log(f"Extracted {len(all_tracks)} individual tracks from {len(release_urls)} releases") return all_tracks def extract_new_releases_urls(self, limit: int) -> List[str]: @@ -844,7 +879,7 @@ class BeatportUnifiedScraper: # Find New Releases section using data-testid release_cards = soup.select('[data-testid="new-releases"]') - print(f" Found {len(release_cards)} release cards in New Releases section") + _beatport_log(f" Found {len(release_cards)} release cards in New Releases section") release_urls = [] for i, card in enumerate(release_cards[:limit]): @@ -860,13 +895,13 @@ class BeatportUnifiedScraper: if href.startswith('/'): href = self.base_url + href release_urls.append(href) - print(f" {i+1}. Found release URL: {href}") + _beatport_log(f" {i+1}. Found release URL: {href}") return release_urls def extract_tracks_from_release_json(self, release_url: str) -> List[Dict]: """Extract individual tracks from a release page using JSON data""" - print(f"Extracting tracks from: {release_url}") + _beatport_log(f"Extracting tracks from: {release_url}") soup = self.get_page(release_url) if not soup: @@ -875,13 +910,13 @@ class BeatportUnifiedScraper: # Extract JSON object from page json_obj = self.extract_json_object_from_release_page(soup) if not json_obj: - print(" No JSON data found") + _beatport_log(" No JSON data found") return [] # Filter tracks for this specific release release_tracks = self.filter_tracks_for_specific_release(json_obj, release_url) if not release_tracks: - print(" No matching tracks found") + _beatport_log(" No matching tracks found") return [] # Convert to our standard format @@ -891,7 +926,7 @@ class BeatportUnifiedScraper: if track: converted_tracks.append(track) - print(f" Extracted {len(converted_tracks)} tracks") + _beatport_log(f" Extracted {len(converted_tracks)} tracks") return converted_tracks def extract_json_object_from_release_page(self, soup): @@ -930,7 +965,7 @@ class BeatportUnifiedScraper: converted.append(track) if len(converted) >= 5: - print(f" JSON extraction found {len(converted)} tracks with rich metadata") + _beatport_log(f" JSON extraction found {len(converted)} tracks with rich metadata") return converted return [] @@ -954,12 +989,12 @@ class BeatportUnifiedScraper: if results and isinstance(results, list) and len(results) > 0: first = results[0] if isinstance(results[0], dict) else {} if first.get('title') or first.get('name'): - print(f" Found {len(results)} tracks in queries[{idx}].data.results") + _beatport_log(f" Found {len(results)} tracks in queries[{idx}].data.results") return results # Pattern 2: data itself is a single track object (track pages) if isinstance(data, dict) and (data.get('title') or data.get('name')) and data.get('id'): - print(f" Found single track in queries[{idx}].data") + _beatport_log(f" Found single track in queries[{idx}].data") return [data] # Pattern 3: data.tracks[] @@ -968,11 +1003,11 @@ class BeatportUnifiedScraper: if tracks and isinstance(tracks, list) and len(tracks) > 0: first = tracks[0] if isinstance(tracks[0], dict) else {} if first.get('title') or first.get('name'): - print(f" Found {len(tracks)} tracks in queries[{idx}].data.tracks") + _beatport_log(f" Found {len(tracks)} tracks in queries[{idx}].data.tracks") return tracks except Exception as e: - print(f" Error extracting tracks from JSON: {e}") + _beatport_log(f" Error extracting tracks from JSON: {e}") return [] @@ -1065,7 +1100,7 @@ class BeatportUnifiedScraper: return track except Exception as e: - print(f" Error converting chart JSON track: {e}") + _beatport_log(f" Error converting chart JSON track: {e}") return None def enrich_chart_tracks(self, tracks: List[Dict], progress_callback=None) -> List[Dict]: @@ -1080,7 +1115,7 @@ class BeatportUnifiedScraper: enriched = [] total = len(tracks) - print(f" Enriching {total} chart tracks with per-track metadata...") + _beatport_log(f" Enriching {total} chart tracks with per-track metadata...") for i, track in enumerate(tracks): track_url = track.get('url', '') @@ -1113,14 +1148,14 @@ class BeatportUnifiedScraper: if not matched and json_tracks: # Debug: show what IDs we have vs what we're looking for sample_ids = [str(jt.get('id', '')) for jt in json_tracks[:5]] - print(f" [{i+1}] No ID match for '{track_id_from_url}' in {sample_ids}... trying title match") + _beatport_log(f" [{i+1}] No ID match for '{track_id_from_url}' in {sample_ids}... trying title match") # Fallback: match by title similarity track_title = track.get('title', '').lower().strip() for jt in json_tracks: jt_title = (jt.get('title') or jt.get('name', '')).lower().strip() if track_title and jt_title and (track_title in jt_title or jt_title in track_title): matched = jt - print(f" [{i+1}] Title matched: '{jt_title}'") + _beatport_log(f" [{i+1}] Title matched: '{jt_title}'") break # Fallback: use first track if only one result @@ -1132,14 +1167,14 @@ class BeatportUnifiedScraper: if rich: enriched.append(rich) if (i + 1) <= 3 or (i + 1) % 25 == 0: - print(f" [{i+1}/{total}] {rich.get('artist', '?')} - {rich.get('title', '?')} | {rich.get('release_name', 'no release')}") + _beatport_log(f" [{i+1}/{total}] {rich.get('artist', '?')} - {rich.get('title', '?')} | {rich.get('release_name', 'no release')}") else: enriched.append(track) else: enriched.append(track) except Exception as e: - print(f" [{i+1}/{total}] Error enriching track: {e}") + _beatport_log(f" [{i+1}/{total}] Error enriching track: {e}") enriched.append(track) # Report progress (always runs ā€” success, failure, or exception) @@ -1152,7 +1187,7 @@ class BeatportUnifiedScraper: time.sleep(0.3) enriched_count = sum(1 for t in enriched if t.get('release_name')) - print(f" Enrichment complete: {enriched_count}/{total} tracks have release metadata") + _beatport_log(f" Enrichment complete: {enriched_count}/{total} tracks have release metadata") return enriched def filter_tracks_for_specific_release(self, json_obj: Dict, release_url: str) -> List[Dict]: @@ -1182,7 +1217,7 @@ class BeatportUnifiedScraper: return matching_tracks except Exception as e: - print(f" Error filtering tracks: {e}") + _beatport_log(f" Error filtering tracks: {e}") return [] @@ -1246,7 +1281,7 @@ class BeatportUnifiedScraper: return track except Exception as e: - print(f" Error converting track data: {e}") + _beatport_log(f" Error converting track data: {e}") return None def get_release_metadata(self, release_url: str) -> Dict: @@ -1402,7 +1437,7 @@ class BeatportUnifiedScraper: } except Exception as e: - print(f"Error getting release metadata from {release_url}: {e}") + _beatport_log(f"Error getting release metadata from {release_url}: {e}") import traceback traceback.print_exc() return {'success': False, 'error': str(e)} @@ -1436,7 +1471,7 @@ class BeatportUnifiedScraper: return tracks except Exception as e: - print(f" Error extracting tracks from {release_url}: {e}") + _beatport_log(f" Error extracting tracks from {release_url}: {e}") return [] def scrape_multiple_releases(self, release_urls, source_name: str = "General Release Scraper") -> List[Dict]: @@ -1456,65 +1491,65 @@ class BeatportUnifiedScraper: # Validate input if not release_urls or len(release_urls) == 0: - print("No release URLs provided") + _beatport_log("No release URLs provided") return [] - print(f"\nSCRAPING {len(release_urls)} RELEASE URL{'S' if len(release_urls) > 1 else ''}") - print("=" * 60) + _beatport_log(f"\nSCRAPING {len(release_urls)} RELEASE URL{'S' if len(release_urls) > 1 else ''}") + _beatport_log("=" * 60) all_tracks = [] for i, release_url in enumerate(release_urls, 1): - print(f"\nProcessing release {i}/{len(release_urls)}: {release_url}") + _beatport_log(f"\nProcessing release {i}/{len(release_urls)}: {release_url}") try: tracks = self.extract_individual_tracks_from_release_url(release_url, source_name) if tracks: all_tracks.extend(tracks) - print(f" Found {len(tracks)} tracks") + _beatport_log(f" Found {len(tracks)} tracks") # Show first few tracks for verification for j, track in enumerate(tracks[:3], 1): title = track.get('title', 'Unknown') artist = track.get('artist', 'Unknown') label = track.get('label', 'Unknown') - print(f" Track {j}: '{title}' by '{artist}' [{label}]") + _beatport_log(f" Track {j}: '{title}' by '{artist}' [{label}]") if len(tracks) > 3: - print(f" ... and {len(tracks) - 3} more tracks") + _beatport_log(f" ... and {len(tracks) - 3} more tracks") else: - print(f" No tracks found") + _beatport_log(f" No tracks found") except Exception as e: - print(f" Error processing release: {e}") + _beatport_log(f" Error processing release: {e}") continue # Small delay between requests to be respectful if i < len(release_urls): time.sleep(0.5) - print(f"\n" + "=" * 60) - print(f"SCRAPING COMPLETE") - print(f" Total releases processed: {len(release_urls)}") - print(f" Total tracks extracted: {len(all_tracks)}") + _beatport_log(f"\n" + "=" * 60) + _beatport_log(f"SCRAPING COMPLETE") + _beatport_log(f" Total releases processed: {len(release_urls)}") + _beatport_log(f" Total tracks extracted: {len(all_tracks)}") return all_tracks def scrape_hype_top_100(self, limit: int = 100, enrich: bool = True) -> List[Dict]: """Scrape Beatport Hype Top 100 - Fixed URL based on parser discovery""" - print("\nScraping Beatport Hype Top 100...") + _beatport_log("\nScraping Beatport Hype Top 100...") # Use the correct URL discovered by parser soup = self.get_page(f"{self.base_url}/hype-100") if soup: tracks = self.extract_tracks_from_page(soup, "Hype Top 100", limit) if tracks and enrich: - print(f" Enriching {len(tracks)} Hype Top 100 tracks with per-track metadata...") + _beatport_log(f" Enriching {len(tracks)} Hype Top 100 tracks with per-track metadata...") tracks = self.enrich_chart_tracks(tracks) - print(f"Extracted {len(tracks)} tracks from Hype Top 100") + _beatport_log(f"Extracted {len(tracks)} tracks from Hype Top 100") return tracks else: - print("Could not access /hype-100, trying homepage Hype Picks section...") + _beatport_log("Could not access /hype-100, trying homepage Hype Picks section...") # Fallback to homepage section soup = self.get_page(self.base_url) if soup: @@ -1534,7 +1569,7 @@ class BeatportUnifiedScraper: else: tracks = [] - print(f"Extracted {len(tracks)} tracks from Hype Top 100 (fallback)") + _beatport_log(f"Extracted {len(tracks)} tracks from Hype Top 100 (fallback)") return tracks def extract_releases_from_page(self, soup: BeautifulSoup, list_name: str, limit: int = 100) -> List[Dict]: @@ -1546,7 +1581,7 @@ class BeatportUnifiedScraper: # Find table rows - each track/release is in a table row table_rows = soup.find_all('div', class_=re.compile(r'Table-style__TableRow')) - print(f" Found {len(table_rows)} table rows on {list_name}") + _beatport_log(f" Found {len(table_rows)} table rows on {list_name}") for i, row in enumerate(table_rows[:limit]): if len(releases) >= limit: @@ -1557,13 +1592,13 @@ class BeatportUnifiedScraper: title_element = row.find('span', class_=re.compile(r'Tables-shared-style__ReleaseName')) if not title_element: if len(releases) < 5: - print(f" Row {i+1}: No release title found") + _beatport_log(f" Row {i+1}: No release title found") continue release_title = title_element.get_text(strip=True) if not release_title: if len(releases) < 5: - print(f" Row {i+1}: Empty release title") + _beatport_log(f" Row {i+1}: Empty release title") continue # Find the release URL from the title link @@ -1601,23 +1636,23 @@ class BeatportUnifiedScraper: # Debug print for first few items if len(releases) <= 5: - print(f" Release {len(releases)}: '{release_title}' by '{artist_text}' (found {len(artists)} artists)") + _beatport_log(f" Release {len(releases)}: '{release_title}' by '{artist_text}' (found {len(artists)} artists)") except Exception as e: - print(f" Error extracting row {i+1}: {e}") + _beatport_log(f" Error extracting row {i+1}: {e}") continue - print(f" Successfully extracted {len(releases)} releases from {len(table_rows)} rows") + _beatport_log(f" Successfully extracted {len(releases)} releases from {len(table_rows)} rows") return releases def scrape_top_100_releases(self, limit: int = 100) -> List[Dict]: """Scrape Beatport Top 100 Releases - Extract individual tracks using URL crawling""" - print("\nScraping Beatport Top 100 Releases...") + _beatport_log("\nScraping Beatport Top 100 Releases...") # Step 1: Extract release URLs from Top 100 page soup = self.get_page(f"{self.base_url}/top-100-releases") if not soup: - print(" Could not access /top-100-releases page") + _beatport_log(" Could not access /top-100-releases page") return [] # Look for rows with release links (Top 100 uses [class*="row"] elements, not tables) @@ -1626,7 +1661,7 @@ class BeatportUnifiedScraper: # Top 100 page uses row-based layout, not table structure table_rows = soup.select('[class*="row"]') - print(f" Found {len(table_rows)} rows on Top 100 page") + _beatport_log(f" Found {len(table_rows)} rows on Top 100 page") release_urls = [] urls_found = 0 @@ -1638,39 +1673,39 @@ class BeatportUnifiedScraper: release_url = urljoin(self.base_url, link_elem.get('href')) release_urls.append(release_url) urls_found += 1 - print(f" {urls_found}. Found Top 100 release URL: {release_url}") + _beatport_log(f" {urls_found}. Found Top 100 release URL: {release_url}") # Stop when we've found enough URLs if urls_found >= limit: break if not release_urls: - print(" No Top 100 release URLs found") + _beatport_log(" No Top 100 release URLs found") return [] # Step 2: Crawl each release URL to extract individual tracks all_individual_tracks = [] for i, release_url in enumerate(release_urls): - print(f" Processing Top 100 release {i+1}/{len(release_urls)}: {release_url}") + _beatport_log(f" Processing Top 100 release {i+1}/{len(release_urls)}: {release_url}") # Extract individual tracks from this release tracks = self.extract_individual_tracks_from_release_url(release_url, "Top 100 Releases") if tracks: - print(f" Found {len(tracks)} individual tracks") + _beatport_log(f" Found {len(tracks)} individual tracks") all_individual_tracks.extend(tracks) else: - print(f" No tracks found") + _beatport_log(f" No tracks found") # Add delay between requests to be respectful if i < len(release_urls) - 1: time.sleep(0.5) - print(f"Extracted {len(all_individual_tracks)} individual tracks from {len(release_urls)} Top 100 releases") + _beatport_log(f"Extracted {len(all_individual_tracks)} individual tracks from {len(release_urls)} Top 100 releases") return all_individual_tracks def scrape_dj_charts(self, limit: int = 20) -> List[Dict]: """Scrape Beatport DJ Charts from homepage section - Improved reliability""" - print("\nScraping Beatport DJ Charts...") + _beatport_log("\nScraping Beatport DJ Charts...") soup = self.get_page(self.base_url) if not soup: @@ -1681,7 +1716,7 @@ class BeatportUnifiedScraper: # Method 1: Find DJ Charts H2 section on homepage dj_charts_heading = soup.find(['h1', 'h2', 'h3'], string=re.compile(r'DJ Charts', re.I)) if dj_charts_heading: - print(" Found DJ Charts section heading") + _beatport_log(" Found DJ Charts section heading") # Get the section content after the heading section_container = dj_charts_heading.find_parent() if section_container: @@ -1689,7 +1724,7 @@ class BeatportUnifiedScraper: if content_area: # Look for individual chart links within this section chart_links = content_area.find_all('a', href=re.compile(r'/chart/')) - print(f" Found {len(chart_links)} individual DJ chart links") + _beatport_log(f" Found {len(chart_links)} individual DJ chart links") for chart_link in chart_links[:limit]: chart_name = chart_link.get_text(strip=True) @@ -1710,9 +1745,9 @@ class BeatportUnifiedScraper: # Method 2: If no section found, look for chart links across entire homepage if not charts: - print(" DJ Charts section not found, scanning entire homepage...") + _beatport_log(" DJ Charts section not found, scanning entire homepage...") all_chart_links = soup.find_all('a', href=re.compile(r'/chart/')) - print(f" Found {len(all_chart_links)} total chart links on homepage") + _beatport_log(f" Found {len(all_chart_links)} total chart links on homepage") for chart_link in all_chart_links[:limit]: chart_name = chart_link.get_text(strip=True) @@ -1730,12 +1765,12 @@ class BeatportUnifiedScraper: } charts.append(chart_info) - print(f"Extracted {len(charts)} DJ charts") + _beatport_log(f"Extracted {len(charts)} DJ charts") return charts def scrape_featured_charts(self, limit: int = 20) -> List[Dict]: """Scrape Beatport Featured Charts from homepage section - FIXED""" - print("\nScraping Beatport Featured Charts...") + _beatport_log("\nScraping Beatport Featured Charts...") soup = self.get_page(self.base_url) if not soup: @@ -1743,7 +1778,7 @@ class BeatportUnifiedScraper: # Find Featured Charts section using data-testid chart_cards = soup.select('[data-testid="featured-charts"]') - print(f" Found {len(chart_cards)} chart cards in Featured Charts section") + _beatport_log(f" Found {len(chart_cards)} chart cards in Featured Charts section") charts = [] for i, card in enumerate(chart_cards[:limit]): @@ -1765,12 +1800,12 @@ class BeatportUnifiedScraper: } charts.append(track_data) - print(f"Extracted {len(charts)} charts from Featured Charts") + _beatport_log(f"Extracted {len(charts)} charts from Featured Charts") return charts def scrape_hype_picks_homepage(self, limit: int = 40) -> List[Dict]: """Scrape individual tracks from Beatport Hype Picks using JSON extraction - ENHANCED""" - print("\nScraping Beatport Hype Picks (individual tracks)...") + _beatport_log("\nScraping Beatport Hype Picks (individual tracks)...") # Step 1: Get release URLs from homepage cards release_urls = self.extract_hype_picks_urls(limit) @@ -1780,7 +1815,7 @@ class BeatportUnifiedScraper: # Step 2: Extract individual tracks from each release all_tracks = [] for i, release_url in enumerate(release_urls): - print(f"\nProcessing release {i+1}/{len(release_urls)}") + _beatport_log(f"\nProcessing release {i+1}/{len(release_urls)}") tracks = self.extract_tracks_from_hype_picks_release_json(release_url) if tracks: all_tracks.extend(tracks) @@ -1789,7 +1824,7 @@ class BeatportUnifiedScraper: import time time.sleep(0.5) - print(f"Extracted {len(all_tracks)} individual tracks from {len(release_urls)} hype picks releases") + _beatport_log(f"Extracted {len(all_tracks)} individual tracks from {len(release_urls)} hype picks releases") return all_tracks def extract_hype_picks_urls(self, limit: int) -> List[str]: @@ -1800,7 +1835,7 @@ class BeatportUnifiedScraper: # Find Hype Picks section using data-testid hype_cards = soup.select('[data-testid="hype-picks"]') - print(f" Found {len(hype_cards)} hype picks cards in section") + _beatport_log(f" Found {len(hype_cards)} hype picks cards in section") release_urls = [] for i, card in enumerate(hype_cards[:limit]): @@ -1816,13 +1851,13 @@ class BeatportUnifiedScraper: if href.startswith('/'): href = self.base_url + href release_urls.append(href) - print(f" {i+1}. Found release URL: {href}") + _beatport_log(f" {i+1}. Found release URL: {href}") return release_urls def extract_tracks_from_hype_picks_release_json(self, release_url: str) -> List[Dict]: """Extract individual tracks from a hype picks release page using JSON data""" - print(f"Extracting tracks from: {release_url}") + _beatport_log(f"Extracting tracks from: {release_url}") soup = self.get_page(release_url) if not soup: @@ -1831,13 +1866,13 @@ class BeatportUnifiedScraper: # Extract JSON object from page (same method as New Releases) json_obj = self.extract_json_object_from_release_page(soup) if not json_obj: - print(" No JSON data found") + _beatport_log(" No JSON data found") return [] # Filter tracks for this specific release (same method as New Releases) release_tracks = self.filter_tracks_for_specific_release(json_obj, release_url) if not release_tracks: - print(" No matching tracks found") + _beatport_log(" No matching tracks found") return [] # Convert to our standard format (with Hype Picks branding) @@ -1847,7 +1882,7 @@ class BeatportUnifiedScraper: if track: converted_tracks.append(track) - print(f" Extracted {len(converted_tracks)} tracks") + _beatport_log(f" Extracted {len(converted_tracks)} tracks") return converted_tracks def convert_hype_picks_json_to_track_format(self, track_data: Dict, release_url: str, position: int): @@ -1912,12 +1947,12 @@ class BeatportUnifiedScraper: return track except Exception as e: - print(f" Error converting track data: {e}") + _beatport_log(f" Error converting track data: {e}") return None def scrape_homepage_top10_lists(self) -> Dict[str, List[Dict]]: """Scrape Top 10 Lists from homepage - Beatport Top 10 and Hype Top 10""" - print("\nScraping Top 10 Lists from homepage...") + _beatport_log("\nScraping Top 10 Lists from homepage...") soup = self.get_page(self.base_url) if not soup: @@ -1925,7 +1960,7 @@ class BeatportUnifiedScraper: # Extract Beatport Top 10 tracks beatport_top10_items = soup.select('[data-testid="top-10-item"]') - print(f" Found {len(beatport_top10_items)} Beatport Top 10 items") + _beatport_log(f" Found {len(beatport_top10_items)} Beatport Top 10 items") beatport_tracks = [] for i, item in enumerate(beatport_top10_items, 1): @@ -1934,11 +1969,11 @@ class BeatportUnifiedScraper: if track_data: beatport_tracks.append(track_data) except Exception as e: - print(f" Error extracting Beatport track {i}: {e}") + _beatport_log(f" Error extracting Beatport track {i}: {e}") # Extract Hype Top 10 tracks hype_top10_items = soup.select('[data-testid="hype-top-10-item"]') - print(f" Found {len(hype_top10_items)} Hype Top 10 items") + _beatport_log(f" Found {len(hype_top10_items)} Hype Top 10 items") hype_tracks = [] for i, item in enumerate(hype_top10_items, 1): @@ -1947,9 +1982,9 @@ class BeatportUnifiedScraper: if track_data: hype_tracks.append(track_data) except Exception as e: - print(f" Error extracting Hype track {i}: {e}") + _beatport_log(f" Error extracting Hype track {i}: {e}") - print(f"Extracted {len(beatport_tracks)} Beatport Top 10 + {len(hype_tracks)} Hype Top 10 tracks") + _beatport_log(f"Extracted {len(beatport_tracks)} Beatport Top 10 + {len(hype_tracks)} Hype Top 10 tracks") return { "beatport_top10": beatport_tracks, @@ -2031,24 +2066,24 @@ class BeatportUnifiedScraper: } except Exception as e: - print(f"Error extracting track data: {e}") + _beatport_log(f"Error extracting track data: {e}") return None def scrape_homepage_top10_releases(self) -> List[Dict]: """Scrape Top 10 Releases from homepage - FIXED VERSION""" - print("\nFIXED: Scraping Top 10 Releases from homepage...") + _beatport_log("\nFIXED: Scraping Top 10 Releases from homepage...") soup = self.get_page(self.base_url) if not soup: - print(" Could not get homepage") + _beatport_log(" Could not get homepage") return [] # Extract Top 10 Releases items - EXACT same as test script top10_releases_items = soup.select('[data-testid="top-10-releases-item"]') - print(f" FOUND {len(top10_releases_items)} Top 10 Releases items") + _beatport_log(f" FOUND {len(top10_releases_items)} Top 10 Releases items") if len(top10_releases_items) == 0: - print(" No items found - trying alternatives") + _beatport_log(" No items found - trying alternatives") return [] releases = [] @@ -2058,13 +2093,13 @@ class BeatportUnifiedScraper: release_data = self.extract_release_from_item_FIXED(item, i) if release_data: releases.append(release_data) - print(f" {i}. {release_data['artist']} - {release_data['title']}") + _beatport_log(f" {i}. {release_data['artist']} - {release_data['title']}") else: - print(f" {i}. No data extracted") + _beatport_log(f" {i}. No data extracted") except Exception as e: - print(f" Error extracting release {i}: {e}") + _beatport_log(f" Error extracting release {i}: {e}") - print(f"FINAL: Extracted {len(releases)} Top 10 Releases") + _beatport_log(f"FINAL: Extracted {len(releases)} Top 10 Releases") return releases def extract_release_from_item_FIXED(self, item, rank): @@ -2177,7 +2212,7 @@ class BeatportUnifiedScraper: } except Exception as e: - print(f"Error extracting release data: {e}") + _beatport_log(f"Error extracting release data: {e}") return None def extract_release_from_top10_item(self, item, rank): @@ -2288,12 +2323,12 @@ class BeatportUnifiedScraper: } except Exception as e: - print(f"Error extracting release data: {e}") + _beatport_log(f"Error extracting release data: {e}") return None def scrape_new_on_beatport_hero(self, limit: int = 10) -> List[Dict]: """Scrape the 'New on Beatport' hero slideshow from homepage using data-testid standard""" - print("\nScraping 'New on Beatport' hero slideshow...") + _beatport_log("\nScraping 'New on Beatport' hero slideshow...") soup = self.get_page(self.base_url) if not soup: @@ -2304,7 +2339,7 @@ class BeatportUnifiedScraper: # Method 1 (PRIMARY): Use data-testid standard like all other rebuild functions hero_items = soup.select('[data-testid="new-on-beatport"]') if hero_items: - print(f" Found {len(hero_items)} items using data-testid='new-on-beatport'") + _beatport_log(f" Found {len(hero_items)} items using data-testid='new-on-beatport'") for i, item in enumerate(hero_items[:limit]): track_data = self._extract_track_from_slide(item, f"Hero Item {i+1}") if track_data and track_data.get('url'): @@ -2314,14 +2349,14 @@ class BeatportUnifiedScraper: if len(tracks) < 5: hero_wrapper = soup.select_one('[class*="Homepage-style__NewOnBeatportWrapper"]') if hero_wrapper: - print(" Found Homepage NewOnBeatportWrapper (fallback)") + _beatport_log(" Found Homepage NewOnBeatportWrapper (fallback)") tracks.extend(self._extract_from_hero_wrapper(hero_wrapper, limit)) # Method 3 (FALLBACK): Look for carousel with aria attributes if len(tracks) < 5: carousel = soup.find('div', {'aria-roledescription': 'carousel', 'aria-label': 'Carousel'}) if carousel: - print(" Found carousel with aria-roledescription and aria-label (fallback)") + _beatport_log(" Found carousel with aria-roledescription and aria-label (fallback)") additional_tracks = self._extract_from_carousel(carousel, limit) # Merge without duplicates existing_urls = {track.get('url') for track in tracks} @@ -2331,9 +2366,9 @@ class BeatportUnifiedScraper: # Method 4 (LAST RESORT): Look for individual slide items more broadly if len(tracks) < 5: - print(" Looking for individual carousel items (last resort)...") + _beatport_log(" Looking for individual carousel items (last resort)...") carousel_items = soup.find_all(['div', 'article'], class_=re.compile(r'carousel.*item|item.*carousel|slide', re.I)) - print(f" Found {len(carousel_items)} potential carousel items") + _beatport_log(f" Found {len(carousel_items)} potential carousel items") for i, item in enumerate(carousel_items[:limit * 2]): # Check more items track_data = self._extract_track_from_slide(item, f"Carousel Item {i+1}") @@ -2343,7 +2378,7 @@ class BeatportUnifiedScraper: if track_data['url'] not in existing_urls: tracks.append(track_data) - print(f" Extracted {len(tracks)} tracks from New on Beatport hero") + _beatport_log(f" Extracted {len(tracks)} tracks from New on Beatport hero") return tracks[:limit] def _extract_from_hero_wrapper(self, wrapper, limit: int) -> List[Dict]: @@ -2529,21 +2564,21 @@ class BeatportUnifiedScraper: if (not title or not artist or title.lower() in ['no title', 'unknown title', 'unknown', ''] or artist.lower() in ['no artist', 'unknown artist', 'unknown', 'various artists', '']): - print(f" {context}: Filtered out invalid track - '{title}' by '{artist}'") + _beatport_log(f" {context}: Filtered out invalid track - '{title}' by '{artist}'") return None # Only return if we found meaningful data if track_data.get('url') or track_data.get('image_url'): track_data['source'] = f"New on Beatport Hero - {context}" track_data['scraped_at'] = time.time() - print(f" {context}: {title} - {artist}") + _beatport_log(f" {context}: {title} - {artist}") return track_data else: - print(f" {context}: No usable data found") + _beatport_log(f" {context}: No usable data found") return None except Exception as e: - print(f" Error extracting from {context}: {e}") + _beatport_log(f" Error extracting from {context}: {e}") return None def _extract_title_artist_from_url(self, url: str) -> Dict[str, str]: @@ -2792,7 +2827,7 @@ class BeatportUnifiedScraper: def scrape_top_10_releases_homepage(self, limit: int = 10) -> List[Dict]: """Scrape Top 10 Releases from homepage - Extract individual tracks using URL crawling""" - print("\nScraping Top 10 Releases from homepage...") + _beatport_log("\nScraping Top 10 Releases from homepage...") soup = self.get_page(self.base_url) if not soup: @@ -2800,7 +2835,7 @@ class BeatportUnifiedScraper: # Step 1: Extract release URLs from Top 10 section release_items = soup.select('[data-testid="top-10-releases-item"]') - print(f" Found {len(release_items)} release items in Top 10 Releases section") + _beatport_log(f" Found {len(release_items)} release items in Top 10 Releases section") release_urls = [] for i, item in enumerate(release_items[:limit]): @@ -2809,30 +2844,30 @@ class BeatportUnifiedScraper: if link_elem and link_elem.get('href'): release_url = urljoin(self.base_url, link_elem.get('href')) release_urls.append(release_url) - print(f" {i+1}. Found Top 10 release URL: {release_url}") + _beatport_log(f" {i+1}. Found Top 10 release URL: {release_url}") if not release_urls: - print(" No Top 10 release URLs found") + _beatport_log(" No Top 10 release URLs found") return [] # Step 2: Crawl each release URL to extract individual tracks all_individual_tracks = [] for i, release_url in enumerate(release_urls): - print(f" Processing Top 10 release {i+1}/{len(release_urls)}: {release_url}") + _beatport_log(f" Processing Top 10 release {i+1}/{len(release_urls)}: {release_url}") # Extract individual tracks from this release tracks = self.extract_individual_tracks_from_release_url(release_url, "Top 10 Releases") if tracks: - print(f" Found {len(tracks)} individual tracks") + _beatport_log(f" Found {len(tracks)} individual tracks") all_individual_tracks.extend(tracks) else: - print(f" No tracks found") + _beatport_log(f" No tracks found") # Add delay between requests to be respectful if i < len(release_urls) - 1: time.sleep(0.5) - print(f"Extracted {len(all_individual_tracks)} individual tracks from {len(release_urls)} Top 10 releases") + _beatport_log(f"Extracted {len(all_individual_tracks)} individual tracks from {len(release_urls)} Top 10 releases") return all_individual_tracks def scrape_genre_charts(self, genre: Dict, limit: int = 100, enrich: bool = True) -> List[Dict]: @@ -2852,20 +2887,20 @@ class BeatportUnifiedScraper: ] for chart_url in chart_urls_to_try: - print(f" Trying chart URL: {chart_url}") + _beatport_log(f" Trying chart URL: {chart_url}") soup = self.get_page(chart_url) if soup: tracks = self.extract_tracks_from_page(soup, f"{genre['name']} Top 100", limit) if tracks and len(tracks) >= min(limit, 50): - print(f" Successfully extracted {len(tracks)} tracks from {chart_url}") + _beatport_log(f" Successfully extracted {len(tracks)} tracks from {chart_url}") break elif tracks: - print(f" Only found {len(tracks)} tracks at {chart_url}, trying next URL...") + _beatport_log(f" Only found {len(tracks)} tracks at {chart_url}, trying next URL...") else: - print(f" No tracks found at {chart_url}") + _beatport_log(f" No tracks found at {chart_url}") if tracks and enrich: - print(f" Enriching {len(tracks)} {genre['name']} chart tracks with per-track metadata...") + _beatport_log(f" Enriching {len(tracks)} {genre['name']} chart tracks with per-track metadata...") tracks = self.enrich_chart_tracks(tracks) return tracks @@ -2892,7 +2927,7 @@ class BeatportUnifiedScraper: ] for release_url in release_urls_to_try: - print(f" Trying release URL: {release_url}") + _beatport_log(f" Trying release URL: {release_url}") soup = self.get_page(release_url) if soup: # Try to find releases section on the page @@ -2900,19 +2935,19 @@ class BeatportUnifiedScraper: # If no releases found with release extraction, try track extraction if not releases: - print(f" No releases found with release method, trying track method for {genre['name']}") + _beatport_log(f" No releases found with release method, trying track method for {genre['name']}") releases = self.extract_tracks_from_page(soup, f"{genre['name']} Top Releases", limit) # Mark these as releases for release in releases: release['type'] = 'release' if releases and len(releases) >= min(limit, 30): # If we got a decent number of releases - print(f" Successfully extracted {len(releases)} releases from {release_url}") + _beatport_log(f" Successfully extracted {len(releases)} releases from {release_url}") break elif releases: - print(f" Only found {len(releases)} releases at {release_url}, trying next URL...") + _beatport_log(f" Only found {len(releases)} releases at {release_url}, trying next URL...") else: - print(f" No releases found at {release_url}") + _beatport_log(f" No releases found at {release_url}") return releases @@ -2933,22 +2968,22 @@ class BeatportUnifiedScraper: ] for hype_url in hype_urls_to_try: - print(f" Trying hype URL: {hype_url}") + _beatport_log(f" Trying hype URL: {hype_url}") soup = self.get_page(hype_url) if soup: # Use the new dedicated hype extraction method tracks = self.extract_hype_tracks_from_beatport_page(soup, f"{genre['name']} Hype Charts", limit) if tracks and len(tracks) >= min(limit, 10): # If we got a decent number of tracks - print(f" Successfully extracted {len(tracks)} hype tracks from {hype_url}") + _beatport_log(f" Successfully extracted {len(tracks)} hype tracks from {hype_url}") break elif tracks: - print(f" Only found {len(tracks)} hype tracks at {hype_url}, trying next URL...") + _beatport_log(f" Only found {len(tracks)} hype tracks at {hype_url}, trying next URL...") else: - print(f" No hype tracks found at {hype_url}") + _beatport_log(f" No hype tracks found at {hype_url}") # If no dedicated hype page found, try main genre page for hype content if not tracks: - print(f" No dedicated hype page found, looking for hype content on main page...") + _beatport_log(f" No dedicated hype page found, looking for hype content on main page...") genre_url = f"{self.base_url}/genre/{genre['slug']}/{genre['id']}" soup = self.get_page(genre_url) if soup: @@ -2958,7 +2993,7 @@ class BeatportUnifiedScraper: def scrape_genre_hype_picks(self, genre: Dict, limit: int = 100) -> List[Dict]: """Scrape individual tracks from Genre Hype Picks using JSON extraction - ENHANCED (same pattern as Latest Releases)""" - print(f"\nScraping {genre['name']} Hype Picks (individual tracks)...") + _beatport_log(f"\nScraping {genre['name']} Hype Picks (individual tracks)...") # Step 1: Get release URLs from genre Hype Picks carousel (same logic as Latest Releases) release_urls = self.extract_genre_hype_picks_urls(genre, limit) @@ -2968,7 +3003,7 @@ class BeatportUnifiedScraper: # Step 2: Extract individual tracks from each release (same method as Latest Releases) all_tracks = [] for i, release_url in enumerate(release_urls): - print(f"\nProcessing {genre['name']} hype pick {i+1}/{len(release_urls)}") + _beatport_log(f"\nProcessing {genre['name']} hype pick {i+1}/{len(release_urls)}") tracks = self.extract_tracks_from_release_json(release_url) if tracks: # Update list_name to match genre context @@ -2980,7 +3015,7 @@ class BeatportUnifiedScraper: import time time.sleep(0.5) - print(f"Extracted {len(all_tracks)} individual tracks from {len(release_urls)} {genre['name']} hype picks") + _beatport_log(f"Extracted {len(all_tracks)} individual tracks from {len(release_urls)} {genre['name']} hype picks") return all_tracks def extract_genre_hype_picks_urls(self, genre: Dict, limit: int) -> List[str]: @@ -2998,16 +3033,16 @@ class BeatportUnifiedScraper: h2 = container.select_one('h2') if h2 and 'hype' in h2.get_text().lower() and 'pick' in h2.get_text().lower(): hype_container = container - print(f" Found Hype Picks section: '{h2.get_text().strip()}'") + _beatport_log(f" Found Hype Picks section: '{h2.get_text().strip()}'") break if not hype_container: - print(f" Could not find Hype Picks section for {genre['name']}") + _beatport_log(f" Could not find Hype Picks section for {genre['name']}") return [] # Extract release URLs from ALL releases in Hype Picks section (same as Latest Releases) release_links = hype_container.select('a[href*="/release/"]') - print(f" Found {len(release_links)} release links in Hype Picks section") + _beatport_log(f" Found {len(release_links)} release links in Hype Picks section") release_urls = [] seen_urls = set() @@ -3024,7 +3059,7 @@ class BeatportUnifiedScraper: if href not in seen_urls: release_urls.append(href) seen_urls.add(href) - print(f" {len(release_urls)}. Found hype pick URL: {href}") + _beatport_log(f" {len(release_urls)}. Found hype pick URL: {href}") # Stop when we reach the desired number of unique releases if len(release_urls) >= limit: @@ -3041,7 +3076,7 @@ class BeatportUnifiedScraper: string=re.compile(r'hype', re.I)) for heading in hype_headings: - print(f" Found hype heading: {heading.get_text(strip=True)}") + _beatport_log(f" Found hype heading: {heading.get_text(strip=True)}") # Get the section after this heading section_container = heading.find_parent() @@ -3146,7 +3181,7 @@ class BeatportUnifiedScraper: } tracks.append(track_data) - print(f" Release Track: {artist_text} - {track_title}") + _beatport_log(f" Release Track: {artist_text} - {track_title}") except Exception: continue @@ -3171,7 +3206,7 @@ class BeatportUnifiedScraper: string=re.compile(rf'{section_name}', re.I)) if section_heading: - print(f" Found hype picks section: {section_heading.get_text(strip=True)}") + _beatport_log(f" Found hype picks section: {section_heading.get_text(strip=True)}") section_container = section_heading.find_parent() if section_container: content_area = section_container.find_next_sibling() @@ -3193,7 +3228,7 @@ class BeatportUnifiedScraper: if not soup: return tracks - print(f" Looking for HYPE labeled tracks on page...") + _beatport_log(f" Looking for HYPE labeled tracks on page...") # Look for elements containing "HYPE" text hype_elements = soup.find_all(text=re.compile(r'HYPE', re.I)) @@ -3261,7 +3296,7 @@ class BeatportUnifiedScraper: # Avoid duplicates if not any(existing['url'] == track_data['url'] for existing in tracks): tracks.append(track_data) - print(f" Found HYPE track: {track_data['artist']} - {track_data['title']}") + _beatport_log(f" Found HYPE track: {track_data['artist']} - {track_data['title']}") except Exception as e: continue @@ -3269,7 +3304,7 @@ class BeatportUnifiedScraper: except Exception as e: continue - print(f" Extracted {len(tracks)} HYPE labeled tracks") + _beatport_log(f" Extracted {len(tracks)} HYPE labeled tracks") return tracks def extract_hype_tracks_from_beatport_page(self, soup: BeautifulSoup, list_name: str, limit: int = 100) -> List[Dict]: @@ -3279,7 +3314,7 @@ class BeatportUnifiedScraper: if not soup: return tracks - print(f" Extracting hype tracks from Beatport page...") + _beatport_log(f" Extracting hype tracks from Beatport page...") # Method 1: Extract from Hype Picks carousel (release cards with HYPE badges) hype_picks_tracks = self.extract_hype_picks_from_carousel(soup, list_name, limit) @@ -3295,7 +3330,7 @@ class BeatportUnifiedScraper: hype_table_tracks = self.extract_hype_from_track_table(soup, list_name, limit - len(tracks)) tracks.extend(hype_table_tracks) - print(f" Extracted {len(tracks)} hype tracks using actual Beatport structure") + _beatport_log(f" Extracted {len(tracks)} hype tracks using actual Beatport structure") return tracks[:limit] def extract_hype_picks_from_carousel(self, soup: BeautifulSoup, list_name: str, limit: int) -> List[Dict]: @@ -3342,7 +3377,7 @@ class BeatportUnifiedScraper: } tracks.append(track_data) - print(f" Hype Pick: {artist_text} - {release_title}") + _beatport_log(f" Hype Pick: {artist_text} - {release_title}") except Exception as e: continue @@ -3395,7 +3430,7 @@ class BeatportUnifiedScraper: } tracks.append(track_data) - print(f" Hype Track {position}: {artist_text} - {track_title}") + _beatport_log(f" Hype Track {position}: {artist_text} - {track_title}") except Exception as e: continue @@ -3452,7 +3487,7 @@ class BeatportUnifiedScraper: } tracks.append(track_data) - print(f" Hype Track {position}: {artist_text} - {track_title}") + _beatport_log(f" Hype Track {position}: {artist_text} - {track_title}") except Exception as e: continue @@ -3461,7 +3496,7 @@ class BeatportUnifiedScraper: def scrape_genre_staff_picks(self, genre: Dict, limit: int = 100) -> List[Dict]: """Scrape individual tracks from Genre Staff Picks using JSON extraction - ENHANCED (same pattern as Latest Releases)""" - print(f"\nScraping {genre['name']} Staff Picks (individual tracks)...") + _beatport_log(f"\nScraping {genre['name']} Staff Picks (individual tracks)...") # Step 1: Get release URLs from genre Staff Picks carousel (same logic as Latest Releases) release_urls = self.extract_genre_staff_picks_urls(genre, limit) @@ -3471,7 +3506,7 @@ class BeatportUnifiedScraper: # Step 2: Extract individual tracks from each release (same method as Latest Releases) all_tracks = [] for i, release_url in enumerate(release_urls): - print(f"\nProcessing {genre['name']} staff pick {i+1}/{len(release_urls)}") + _beatport_log(f"\nProcessing {genre['name']} staff pick {i+1}/{len(release_urls)}") tracks = self.extract_tracks_from_release_json(release_url) if tracks: # Update list_name to match genre context @@ -3483,7 +3518,7 @@ class BeatportUnifiedScraper: import time time.sleep(0.5) - print(f"Extracted {len(all_tracks)} individual tracks from {len(release_urls)} {genre['name']} staff picks") + _beatport_log(f"Extracted {len(all_tracks)} individual tracks from {len(release_urls)} {genre['name']} staff picks") return all_tracks def extract_genre_staff_picks_urls(self, genre: Dict, limit: int) -> List[str]: @@ -3501,16 +3536,16 @@ class BeatportUnifiedScraper: h2 = container.select_one('h2') if h2 and 'staff' in h2.get_text().lower() and 'pick' in h2.get_text().lower(): staff_container = container - print(f" Found Staff Picks section: '{h2.get_text().strip()}'") + _beatport_log(f" Found Staff Picks section: '{h2.get_text().strip()}'") break if not staff_container: - print(f" Could not find Staff Picks section for {genre['name']}") + _beatport_log(f" Could not find Staff Picks section for {genre['name']}") return [] # Extract release URLs from ALL releases in Staff Picks section (same as Latest Releases) release_links = staff_container.select('a[href*="/release/"]') - print(f" Found {len(release_links)} release links in Staff Picks section") + _beatport_log(f" Found {len(release_links)} release links in Staff Picks section") release_urls = [] seen_urls = set() @@ -3527,7 +3562,7 @@ class BeatportUnifiedScraper: if href not in seen_urls: release_urls.append(href) seen_urls.add(href) - print(f" {len(release_urls)}. Found staff pick URL: {href}") + _beatport_log(f" {len(release_urls)}. Found staff pick URL: {href}") # Stop when we reach the desired number of unique releases if len(release_urls) >= limit: @@ -3537,7 +3572,7 @@ class BeatportUnifiedScraper: def scrape_genre_latest_releases(self, genre: Dict, limit: int = 100) -> List[Dict]: """Scrape individual tracks from Genre Latest Releases using JSON extraction - ENHANCED (same pattern as homepage)""" - print(f"\nšŸ†• Scraping {genre['name']} Latest Releases (individual tracks)...") + _beatport_log(f"\nšŸ†• Scraping {genre['name']} Latest Releases (individual tracks)...") # Step 1: Get release URLs from genre Latest Releases carousel (same logic as homepage) release_urls = self.extract_genre_latest_releases_urls(genre, limit) @@ -3547,7 +3582,7 @@ class BeatportUnifiedScraper: # Step 2: Extract individual tracks from each release (same method as homepage) all_tracks = [] for i, release_url in enumerate(release_urls): - print(f"\nProcessing {genre['name']} latest release {i+1}/{len(release_urls)}") + _beatport_log(f"\nProcessing {genre['name']} latest release {i+1}/{len(release_urls)}") tracks = self.extract_tracks_from_release_json(release_url) if tracks: # Update list_name to match genre context @@ -3559,7 +3594,7 @@ class BeatportUnifiedScraper: import time time.sleep(0.5) - print(f"Extracted {len(all_tracks)} individual tracks from {len(release_urls)} latest {genre['name']} releases") + _beatport_log(f"Extracted {len(all_tracks)} individual tracks from {len(release_urls)} latest {genre['name']} releases") return all_tracks def extract_genre_latest_releases_urls(self, genre: Dict, limit: int) -> List[str]: @@ -3577,16 +3612,16 @@ class BeatportUnifiedScraper: h2 = container.select_one('h2') if h2 and 'latest' in h2.get_text().lower() and 'release' in h2.get_text().lower(): latest_container = container - print(f" Found Latest Releases section: '{h2.get_text().strip()}'") + _beatport_log(f" Found Latest Releases section: '{h2.get_text().strip()}'") break if not latest_container: - print(f" Could not find Latest Releases section for {genre['name']}") + _beatport_log(f" Could not find Latest Releases section for {genre['name']}") return [] # Extract release URLs from ALL releases in Latest Releases section (same as homepage gets all cards) release_links = latest_container.select('a[href*="/release/"]') - print(f" Found {len(release_links)} release links in Latest Releases section") + _beatport_log(f" Found {len(release_links)} release links in Latest Releases section") release_urls = [] seen_urls = set() @@ -3603,7 +3638,7 @@ class BeatportUnifiedScraper: if href not in seen_urls: release_urls.append(href) seen_urls.add(href) - print(f" {len(release_urls)}. Found latest release URL: {href}") + _beatport_log(f" {len(release_urls)}. Found latest release URL: {href}") # Stop when we reach the desired number of unique releases if len(release_urls) >= limit: @@ -3622,7 +3657,7 @@ class BeatportUnifiedScraper: charts = [] chart_links = soup.find_all('a', href=re.compile(r'/chart/')) - print(f" Found {len(chart_links)} chart links on genre page") + _beatport_log(f" Found {len(chart_links)} chart links on genre page") for chart_link in chart_links[:limit]: chart_name = chart_link.get_text(strip=True) @@ -3642,9 +3677,9 @@ class BeatportUnifiedScraper: } charts.append(chart_info) - print(f" Chart {len(charts)}: {chart_name}") + _beatport_log(f" Chart {len(charts)}: {chart_name}") - print(f" Found {len(charts)} charts in New Charts Collection") + _beatport_log(f" Found {len(charts)} charts in New Charts Collection") return charts[:limit] def extract_tracks_from_chart(self, chart_url: str, chart_name: str, limit: int) -> List[Dict]: @@ -3656,14 +3691,14 @@ class BeatportUnifiedScraper: if not soup: return tracks - print(f" Extracting tracks from chart page: {chart_url}") - print(f" Chart name: {chart_name}") + _beatport_log(f" Extracting tracks from chart page: {chart_url}") + _beatport_log(f" Chart name: {chart_name}") # Step 1: Get basic track list from HTML tracks = self.extract_tracks_from_chart_table(soup, chart_name, limit) if len(tracks) < 10: - print(f" Chart table extraction found {len(tracks)} tracks, trying general extraction...") + _beatport_log(f" Chart table extraction found {len(tracks)} tracks, trying general extraction...") general_tracks = self.extract_tracks_from_page(soup, f"New Chart: {chart_name}", limit) if len(general_tracks) > len(tracks): tracks = general_tracks @@ -3673,7 +3708,7 @@ class BeatportUnifiedScraper: if len(table_tracks) > len(tracks): tracks = table_tracks - print(f" Found {len(tracks)} tracks, enriching with per-track metadata...") + _beatport_log(f" Found {len(tracks)} tracks, enriching with per-track metadata...") # Step 2: Enrich each track by visiting its individual page if tracks: @@ -3682,47 +3717,47 @@ class BeatportUnifiedScraper: return tracks except Exception as e: - print(f" Error extracting tracks from chart {chart_name}: {e}") + _beatport_log(f" Error extracting tracks from chart {chart_name}: {e}") return [] def extract_tracks_from_chart_table(self, soup, chart_name: str, limit: int) -> List[Dict]: """Extract tracks from Beatport chart table structure (tracks-table class)""" tracks = [] - print(f" DEBUG: Looking for tracks-table container...") + _beatport_log(f" DEBUG: Looking for tracks-table container...") # Look for the tracks table container tracks_table = soup.find(class_=re.compile(r'tracks-table')) if not tracks_table: - print(f" No tracks-table container found") + _beatport_log(f" No tracks-table container found") # Debug: Let's see what table classes ARE available all_tables = soup.find_all(['table', 'div'], class_=re.compile(r'table|Table', re.I)) - print(f" DEBUG: Found {len(all_tables)} table-like elements") + _beatport_log(f" DEBUG: Found {len(all_tables)} table-like elements") for i, table in enumerate(all_tables[:5]): classes = table.get('class', []) - print(f" Table {i+1}: {' '.join(classes)}") + _beatport_log(f" Table {i+1}: {' '.join(classes)}") return tracks - print(f" Found tracks-table container with classes: {tracks_table.get('class', [])}") + _beatport_log(f" Found tracks-table container with classes: {tracks_table.get('class', [])}") # Find all track rows using data-testid or table row classes track_rows_testid = tracks_table.find_all(['div', 'tr'], attrs={'data-testid': 'tracks-table-row'}) track_rows_class = tracks_table.find_all(class_=re.compile(r'Table.*Row.*tracks-table')) track_rows_generic = tracks_table.find_all(class_=re.compile(r'Table.*Row')) - print(f" DEBUG: Track rows found:") - print(f" - By data-testid='tracks-table-row': {len(track_rows_testid)}") - print(f" - By class pattern 'Table.*Row.*tracks-table': {len(track_rows_class)}") - print(f" - By generic 'Table.*Row': {len(track_rows_generic)}") + _beatport_log(f" DEBUG: Track rows found:") + _beatport_log(f" - By data-testid='tracks-table-row': {len(track_rows_testid)}") + _beatport_log(f" - By class pattern 'Table.*Row.*tracks-table': {len(track_rows_class)}") + _beatport_log(f" - By generic 'Table.*Row': {len(track_rows_generic)}") # Use the best available option track_rows = track_rows_testid or track_rows_class or track_rows_generic if not track_rows: - print(f" No track rows found in any format") + _beatport_log(f" No track rows found in any format") return tracks - print(f" Using {len(track_rows)} track rows for extraction") + _beatport_log(f" Using {len(track_rows)} track rows for extraction") for i, row in enumerate(track_rows[:limit]): try: @@ -3760,11 +3795,11 @@ class BeatportUnifiedScraper: # DEBUG: Print track details for first few if len(tracks) < 3: - print(f" DEBUG Track {len(tracks)+1}:") - print(f" Title: '{track_title}'") - print(f" Artist: '{artist_text}'") - print(f" URL: {track_url}") - print(f" Track link href: {track_link.get('href', 'NO HREF')}") + _beatport_log(f" DEBUG Track {len(tracks)+1}:") + _beatport_log(f" Title: '{track_title}'") + _beatport_log(f" Artist: '{artist_text}'") + _beatport_log(f" URL: {track_url}") + _beatport_log(f" Track link href: {track_link.get('href', 'NO HREF')}") # Extract track number if available track_no_elem = row.find(class_=re.compile(r'TrackNo')) @@ -3783,13 +3818,13 @@ class BeatportUnifiedScraper: # Debug output for first few tracks if len(tracks) <= 5: - print(f" Track {len(tracks)}: {artist_text} - {track_title}") + _beatport_log(f" Track {len(tracks)}: {artist_text} - {track_title}") except Exception as e: - print(f" Error parsing track row {i+1}: {e}") + _beatport_log(f" Error parsing track row {i+1}: {e}") continue - print(f" Chart table extraction completed: {len(tracks)} tracks found") + _beatport_log(f" Chart table extraction completed: {len(tracks)} tracks found") return tracks def extract_tracks_from_table_format(self, soup, chart_name: str, limit: int) -> List[Dict]: @@ -3799,7 +3834,7 @@ class BeatportUnifiedScraper: # Look for table rows containing track data table_rows = soup.find_all('tr') + soup.find_all('div', class_=re.compile(r'Table.*Row|track.*row', re.I)) - print(f" Found {len(table_rows)} potential table rows") + _beatport_log(f" Found {len(table_rows)} potential table rows") for i, row in enumerate(table_rows[:limit]): try: @@ -3837,7 +3872,7 @@ class BeatportUnifiedScraper: tracks.append(track_data) if len(tracks) <= 3: # Debug first few - print(f" Track {len(tracks)}: {artist_text} - {track_title}") + _beatport_log(f" Track {len(tracks)}: {artist_text} - {track_title}") except Exception as e: continue @@ -3848,7 +3883,7 @@ class BeatportUnifiedScraper: """Analyze a genre page to discover all available sections""" genre_url = f"{self.base_url}/genre/{genre['slug']}/{genre['id']}" - print(f"Discovering sections for {genre['name']} genre page...") + _beatport_log(f"Discovering sections for {genre['name']} genre page...") soup = self.get_page(genre_url) if not soup: @@ -3886,17 +3921,17 @@ class BeatportUnifiedScraper: chart_links = soup.find_all('a', href=re.compile(r'/chart/')) sections['chart_count'] = len(chart_links) - print(f"Discovered sections for {genre['name']}:") + _beatport_log(f"Discovered sections for {genre['name']}:") for section_type, items in sections.items(): if items and section_type != 'chart_count': - print(f" ā€¢ {section_type}: {len(items)} sections") - print(f" ā€¢ Individual charts found: {sections['chart_count']}") + _beatport_log(f" ā€¢ {section_type}: {len(items)} sections") + _beatport_log(f" ā€¢ Individual charts found: {sections['chart_count']}") return sections def scrape_genre_hero_slider(self, genre_slug: str, genre_id: str) -> List[Dict]: """Scrape hero slider data from a genre page""" - print(f"\nScraping hero slider for {genre_slug}...") + _beatport_log(f"\nScraping hero slider for {genre_slug}...") genre_url = f"{self.base_url}/genre/{genre_slug}/{genre_id}" soup = self.get_page(genre_url) @@ -3906,18 +3941,18 @@ class BeatportUnifiedScraper: # Find the main section container main_section = soup.find('div', class_=re.compile(r'Genre-style__MainSection')) if not main_section: - print(f" Main section not found for {genre_slug}") + _beatport_log(f" Main section not found for {genre_slug}") return [] # Find the hero slider hero_slider = main_section.find('div', class_='hero-slider') if not hero_slider: - print(f" Hero slider not found for {genre_slug}") + _beatport_log(f" Hero slider not found for {genre_slug}") return [] # Extract all hero releases hero_releases = hero_slider.find_all(class_='hero-release') - print(f" Found {len(hero_releases)} hero releases") + _beatport_log(f" Found {len(hero_releases)} hero releases") releases_data = [] for i, release in enumerate(hero_releases): @@ -3925,18 +3960,18 @@ class BeatportUnifiedScraper: release_data = self.extract_hero_release_data(release) if release_data and release_data.get('url'): releases_data.append(release_data) - print(f" Extracted: {release_data.get('title', 'Unknown')} by {release_data.get('artists_string', 'Unknown')}") + _beatport_log(f" Extracted: {release_data.get('title', 'Unknown')} by {release_data.get('artists_string', 'Unknown')}") else: - print(f" Skipped release {i+1} - incomplete data") + _beatport_log(f" Skipped release {i+1} - incomplete data") except Exception as e: - print(f" Error extracting release {i+1}: {e}") + _beatport_log(f" Error extracting release {i+1}: {e}") - print(f" Successfully extracted {len(releases_data)} hero releases") + _beatport_log(f" Successfully extracted {len(releases_data)} hero releases") return releases_data def scrape_genre_top10_tracks(self, genre_slug, genre_id): """Scrape Top 10 tracks lists from genre page (Beatport Top 10 + Hype Top 10 if available)""" - print(f"Scraping Top 10 tracks for {genre_slug} (ID: {genre_id})") + _beatport_log(f"Scraping Top 10 tracks for {genre_slug} (ID: {genre_id})") genre_url = f"https://www.beatport.com/genre/{genre_slug}/{genre_id}" @@ -3948,7 +3983,7 @@ class BeatportUnifiedScraper: track_items = soup.find_all(attrs={'data-testid': 'tracks-list-item'}) if not track_items: - print(f"No tracks-list-item elements found on {genre_url}") + _beatport_log(f"No tracks-list-item elements found on {genre_url}") return { 'beatport_top10': [], 'hype_top10': [], @@ -3956,7 +3991,7 @@ class BeatportUnifiedScraper: 'has_hype_section': False } - print(f"Found {len(track_items)} total track items") + _beatport_log(f"Found {len(track_items)} total track items") # Extract track data from all items all_tracks = [] @@ -3983,7 +4018,7 @@ class BeatportUnifiedScraper: has_hype_section = len(all_tracks) > 10 - print(f"Extracted {len(beatport_top10)} Beatport Top 10 + {len(hype_top10)} Hype Top 10 tracks") + _beatport_log(f"Extracted {len(beatport_top10)} Beatport Top 10 + {len(hype_top10)} Hype Top 10 tracks") return { 'beatport_top10': beatport_top10, @@ -4059,12 +4094,12 @@ class BeatportUnifiedScraper: } except Exception as e: - print(f"Error extracting track data: {e}") + _beatport_log(f"Error extracting track data: {e}") return None def scrape_genre_top10_releases(self, genre_slug, genre_id): """Scrape Top 10 releases from genre page using .partial-artwork elements""" - print(f"Scraping Top 10 releases for {genre_slug} (ID: {genre_id})") + _beatport_log(f"Scraping Top 10 releases for {genre_slug} (ID: {genre_id})") genre_url = f"https://www.beatport.com/genre/{genre_slug}/{genre_id}" @@ -4076,10 +4111,10 @@ class BeatportUnifiedScraper: partial_artwork_elements = soup.find_all(class_='partial-artwork') if not partial_artwork_elements: - print(f"No .partial-artwork elements found on {genre_url}") + _beatport_log(f"No .partial-artwork elements found on {genre_url}") return [] - print(f"Found {len(partial_artwork_elements)} .partial-artwork elements") + _beatport_log(f"Found {len(partial_artwork_elements)} .partial-artwork elements") # Extract release data from each element releases = [] @@ -4088,7 +4123,7 @@ class BeatportUnifiedScraper: if release_data: releases.append(release_data) - print(f"Extracted {len(releases)} Top 10 releases") + _beatport_log(f"Extracted {len(releases)} Top 10 releases") return releases def extract_release_data_from_partial_artwork(self, artwork_element, rank): @@ -4144,7 +4179,7 @@ class BeatportUnifiedScraper: artist = self.clean_beatport_text(artist) if artist != "Unknown Artist" else artist label = self.clean_beatport_text(label) if label != "Unknown Label" else label - print(f" Release #{rank}: '{title}' by '{artist}' [{label}]") + _beatport_log(f" Release #{rank}: '{title}' by '{artist}' [{label}]") return { 'title': title, @@ -4158,7 +4193,7 @@ class BeatportUnifiedScraper: } except Exception as e: - print(f"Error extracting release data from .partial-artwork: {e}") + _beatport_log(f"Error extracting release data from .partial-artwork: {e}") return None def extract_hero_release_data(self, release_element) -> Dict: @@ -4234,7 +4269,7 @@ class BeatportUnifiedScraper: return data except Exception as e: - print(f"Error extracting hero release data: {e}") + _beatport_log(f"Error extracting hero release data: {e}") return {} def scrape_all_genres(self, tracks_per_genre: int = 100, max_workers: int = 5, include_images: bool = False) -> Dict[str, List[Dict]]: @@ -4243,7 +4278,7 @@ class BeatportUnifiedScraper: if not self.all_genres: self.all_genres = self.discover_genres_with_images(include_images=include_images) - print(f"\nScraping {len(self.all_genres)} genres...") + _beatport_log(f"\nScraping {len(self.all_genres)} genres...") all_results = {} completed = 0 @@ -4251,14 +4286,14 @@ class BeatportUnifiedScraper: def scrape_single_genre(genre): nonlocal completed - print(f"Scraping {genre['name']}...") + _beatport_log(f"Scraping {genre['name']}...") tracks = self.scrape_genre_charts(genre, tracks_per_genre) with self.results_lock: if tracks: # Only store genres that have tracks all_results[genre['name']] = tracks completed += 1 - print(f"{genre['name']}: {len(tracks)} tracks ({completed}/{len(self.all_genres)} complete)") + _beatport_log(f"{genre['name']}: {len(tracks)} tracks ({completed}/{len(self.all_genres)} complete)") return genre['name'], tracks @@ -4273,7 +4308,7 @@ class BeatportUnifiedScraper: try: future.result() except Exception as e: - print(f"Error processing {genre['name']}: {e}") + _beatport_log(f"Error processing {genre['name']}: {e}") return all_results @@ -4304,119 +4339,119 @@ class BeatportUnifiedScraper: def test_dynamic_genre_discovery(): """Test the dynamic genre discovery functionality""" - print("Dynamic Genre Discovery Test") - print("=" * 80) + _beatport_log("Dynamic Genre Discovery Test") + _beatport_log("=" * 80) scraper = BeatportUnifiedScraper() # Test genre discovery - print("\nTEST 1: Genre Discovery") + _beatport_log("\nTEST 1: Genre Discovery") genres = scraper.discover_genres_from_homepage() - print(f"\nDiscovered {len(genres)} genres:") + _beatport_log(f"\nDiscovered {len(genres)} genres:") for i, genre in enumerate(genres[:10]): # Show first 10 - print(f" {i+1:2}. {genre['name']} -> {genre['slug']} (ID: {genre['id']})") + _beatport_log(f" {i+1:2}. {genre['name']} -> {genre['slug']} (ID: {genre['id']})") if 'url' in genre: - print(f" URL: {genre['url']}") + _beatport_log(f" URL: {genre['url']}") if len(genres) > 10: - print(f" ... and {len(genres) - 10} more genres") + _beatport_log(f" ... and {len(genres) - 10} more genres") # Test with images (limit to 3 for demo) - print("\nTEST 2: Genre Discovery with Images (Sample)") + _beatport_log("\nTEST 2: Genre Discovery with Images (Sample)") genres_with_images = scraper.discover_genres_with_images(include_images=True) - print(f"\nSample genres with images:") + _beatport_log(f"\nSample genres with images:") for genre in genres_with_images[:3]: - print(f" ā€¢ {genre['name']}: {genre.get('image_url', 'No image')}") + _beatport_log(f" ā€¢ {genre['name']}: {genre.get('image_url', 'No image')}") # Test a few genre scrapes - print("\nTEST 3: Sample Genre Chart Scraping") + _beatport_log("\nTEST 3: Sample Genre Chart Scraping") sample_genres = genres[:3] for genre in sample_genres: - print(f"\nTesting {genre['name']}...") + _beatport_log(f"\nTesting {genre['name']}...") tracks = scraper.scrape_genre_charts(genre, limit=3) if tracks: - print(f" Found {len(tracks)} tracks:") + _beatport_log(f" Found {len(tracks)} tracks:") for track in tracks: - print(f" ā€¢ {track['artist']} - {track['title']}") + _beatport_log(f" ā€¢ {track['artist']} - {track['title']}") else: - print(f" No tracks found") + _beatport_log(f" No tracks found") return genres def test_improved_chart_sections(): """Test the improved chart section discovery and scraping""" - print("Testing Improved Chart Section Discovery & Scraping") - print("=" * 80) + _beatport_log("Testing Improved Chart Section Discovery & Scraping") + _beatport_log("=" * 80) scraper = BeatportUnifiedScraper() # Test 1: Chart Section Discovery - print("\nTEST 1: Chart Section Discovery") + _beatport_log("\nTEST 1: Chart Section Discovery") chart_discovery = scraper.discover_chart_sections() - print(f"\nDiscovery Results:") + _beatport_log(f"\nDiscovery Results:") summary = chart_discovery.get('summary', {}) - print(f" ā€¢ Top Charts sections: {summary.get('top_charts_sections', 0)}") - print(f" ā€¢ Staff Picks sections: {summary.get('staff_picks_sections', 0)}") - print(f" ā€¢ Other sections: {summary.get('other_sections', 0)}") - print(f" ā€¢ Main chart links: {summary.get('main_chart_links', 0)}") - print(f" ā€¢ Individual DJ charts: {summary.get('individual_dj_charts', 0)}") + _beatport_log(f" ā€¢ Top Charts sections: {summary.get('top_charts_sections', 0)}") + _beatport_log(f" ā€¢ Staff Picks sections: {summary.get('staff_picks_sections', 0)}") + _beatport_log(f" ā€¢ Other sections: {summary.get('other_sections', 0)}") + _beatport_log(f" ā€¢ Main chart links: {summary.get('main_chart_links', 0)}") + _beatport_log(f" ā€¢ Individual DJ charts: {summary.get('individual_dj_charts', 0)}") # Test 2: New/Improved Scraping Methods - print("\nTEST 2: Improved Chart Scraping Methods") + _beatport_log("\nTEST 2: Improved Chart Scraping Methods") # Test Hype Top 100 (fixed URL) - print("\n2a. Testing Hype Top 100 (fixed URL)...") + _beatport_log("\n2a. Testing Hype Top 100 (fixed URL)...") hype_tracks = scraper.scrape_hype_top_100(limit=5) if hype_tracks: - print(f" Found {len(hype_tracks)} tracks:") + _beatport_log(f" Found {len(hype_tracks)} tracks:") for track in hype_tracks[:3]: - print(f" ā€¢ {track['artist']} - {track['title']}") + _beatport_log(f" ā€¢ {track['artist']} - {track['title']}") else: - print(" No tracks found") + _beatport_log(" No tracks found") # Test Top 100 Releases (new method) - print("\n2b. Testing Top 100 Releases (new method)...") + _beatport_log("\n2b. Testing Top 100 Releases (new method)...") releases_tracks = scraper.scrape_top_100_releases(limit=5) if releases_tracks: - print(f" Found {len(releases_tracks)} tracks:") + _beatport_log(f" Found {len(releases_tracks)} tracks:") for track in releases_tracks[:3]: - print(f" ā€¢ {track['artist']} - {track['title']}") + _beatport_log(f" ā€¢ {track['artist']} - {track['title']}") else: - print(" No tracks found") + _beatport_log(" No tracks found") # Test Improved New Releases - print("\n2c. Testing Improved New Releases...") + _beatport_log("\n2c. Testing Improved New Releases...") new_releases = scraper.scrape_new_releases(limit=5) if new_releases: - print(f" Found {len(new_releases)} tracks:") + _beatport_log(f" Found {len(new_releases)} tracks:") for track in new_releases[:3]: - print(f" ā€¢ {track['artist']} - {track['title']}") + _beatport_log(f" ā€¢ {track['artist']} - {track['title']}") else: - print(" No tracks found") + _beatport_log(" No tracks found") # Test Improved DJ Charts - print("\n2d. Testing Improved DJ Charts...") + _beatport_log("\n2d. Testing Improved DJ Charts...") dj_charts = scraper.scrape_dj_charts(limit=5) if dj_charts: - print(f" Found {len(dj_charts)} charts:") + _beatport_log(f" Found {len(dj_charts)} charts:") for chart in dj_charts[:3]: - print(f" ā€¢ {chart['title']} by {chart['artist']}") + _beatport_log(f" ā€¢ {chart['title']} by {chart['artist']}") else: - print(" No charts found") + _beatport_log(" No charts found") # Test Improved Featured Charts - print("\n2e. Testing Improved Featured Charts...") + _beatport_log("\n2e. Testing Improved Featured Charts...") featured_charts = scraper.scrape_featured_charts(limit=5) if featured_charts: - print(f" Found {len(featured_charts)} items:") + _beatport_log(f" Found {len(featured_charts)} items:") for item in featured_charts[:3]: - print(f" ā€¢ {item['title']} by {item['artist']}") + _beatport_log(f" ā€¢ {item['title']} by {item['artist']}") else: - print(" No items found") + _beatport_log(" No items found") return { 'chart_discovery': chart_discovery, @@ -4429,66 +4464,66 @@ def test_improved_chart_sections(): def main(): """Test the unified Beatport scraper""" - print("Beatport Unified Scraper - Improved Chart Discovery") - print("=" * 80) + _beatport_log("Beatport Unified Scraper - Improved Chart Discovery") + _beatport_log("=" * 80) scraper = BeatportUnifiedScraper() # Test New on Beatport Hero first - print("\nNEW ON BEATPORT HERO TEST") + _beatport_log("\nNEW ON BEATPORT HERO TEST") hero_tracks = scraper.scrape_new_on_beatport_hero(limit=10) if hero_tracks: - print(f"Successfully extracted {len(hero_tracks)} tracks from hero slideshow") + _beatport_log(f"Successfully extracted {len(hero_tracks)} tracks from hero slideshow") for i, track in enumerate(hero_tracks[:3]): # Show first 3 - print(f" {i+1}. {track.get('title', 'No title')} - {track.get('artist', 'No artist')}") - print(f" URL: {track.get('url', 'No URL')}") - print(f" Classes: {track.get('element_classes', 'No classes')}") + _beatport_log(f" {i+1}. {track.get('title', 'No title')} - {track.get('artist', 'No artist')}") + _beatport_log(f" URL: {track.get('url', 'No URL')}") + _beatport_log(f" Classes: {track.get('element_classes', 'No classes')}") else: - print("No tracks found in hero slideshow") + _beatport_log("No tracks found in hero slideshow") # Test improved chart sections - print("\nšŸ†• IMPROVED CHART SECTIONS TEST") + _beatport_log("\nšŸ†• IMPROVED CHART SECTIONS TEST") improved_results = test_improved_chart_sections() # Test dynamic genre discovery (existing) - print("\n\nšŸ†• DYNAMIC GENRE DISCOVERY TEST") + _beatport_log("\n\nšŸ†• DYNAMIC GENRE DISCOVERY TEST") discovered_genres = test_dynamic_genre_discovery() # Update scraper with discovered genres scraper.all_genres = discovered_genres # Test 1: Top 100 - print("\nTEST 1: Top 100 Chart") + _beatport_log("\nTEST 1: Top 100 Chart") top_100 = scraper.scrape_top_100(limit=10) # Test with 10 for now if top_100: - print(f"\nTop 100 Sample (showing first 5):") + _beatport_log(f"\nTop 100 Sample (showing first 5):") for track in top_100[:5]: - print(f" {track['position']}. {track['artist']} - {track['title']}") + _beatport_log(f" {track['position']}. {track['artist']} - {track['title']}") quality = scraper.test_data_quality(top_100) - print(f"\nData Quality: {quality['quality_score']:.1f}% ({quality['valid_tracks']}/{quality['total_tracks']} tracks)") + _beatport_log(f"\nData Quality: {quality['quality_score']:.1f}% ({quality['valid_tracks']}/{quality['total_tracks']} tracks)") else: - print("Failed to extract Top 100") + _beatport_log("Failed to extract Top 100") # Test 2: Sample of discovered genres - print("\nTEST 2: Dynamic Genre Charts Sample") + _beatport_log("\nTEST 2: Dynamic Genre Charts Sample") test_genres = scraper.all_genres[:5] # Test first 5 discovered genres - print(f"Testing {len(test_genres)} dynamically discovered genres...") + _beatport_log(f"Testing {len(test_genres)} dynamically discovered genres...") genre_results = {} for genre in test_genres: tracks = scraper.scrape_genre_charts(genre, limit=5) # 5 tracks per genre for testing if tracks: genre_results[genre['name']] = tracks - print(f"\n{genre['name']} Top 5:") + _beatport_log(f"\n{genre['name']} Top 5:") for track in tracks[:3]: - print(f" ā€¢ {track['artist']} - {track['title']}") + _beatport_log(f" ā€¢ {track['artist']} - {track['title']}") # Test 3: Full genre scraping (smaller sample) - print("\nTEST 3: Full Multi-Genre Scraping") - print("Testing parallel scraping of 10 genres...") + _beatport_log("\nTEST 3: Full Multi-Genre Scraping") + _beatport_log("Testing parallel scraping of 10 genres...") sample_genres = scraper.all_genres[:10] scraper.all_genres = sample_genres # Temporarily limit for testing @@ -4496,29 +4531,29 @@ def main(): all_genre_results = scraper.scrape_all_genres(tracks_per_genre=5, max_workers=3) # Results summary - print("\n" + "=" * 80) - print("FINAL RESULTS SUMMARY") - print("=" * 80) + _beatport_log("\n" + "=" * 80) + _beatport_log("FINAL RESULTS SUMMARY") + _beatport_log("=" * 80) total_tracks = len(top_100) if top_100 else 0 total_genres = len(all_genre_results) total_genre_tracks = sum(len(tracks) for tracks in all_genre_results.values()) - print(f"ā€¢ Top 100 tracks extracted: {total_tracks}") - print(f"ā€¢ Genres successfully scraped: {total_genres}") - print(f"ā€¢ Total genre tracks: {total_genre_tracks}") - print(f"ā€¢ Grand total tracks: {total_tracks + total_genre_tracks}") + _beatport_log(f"ā€¢ Top 100 tracks extracted: {total_tracks}") + _beatport_log(f"ā€¢ Genres successfully scraped: {total_genres}") + _beatport_log(f"ā€¢ Total genre tracks: {total_genre_tracks}") + _beatport_log(f"ā€¢ Grand total tracks: {total_tracks + total_genre_tracks}") # Data quality assessment all_tracks = (top_100 or []) + [track for tracks in all_genre_results.values() for track in tracks] if all_tracks: overall_quality = scraper.test_data_quality(all_tracks) - print(f"\nOVERALL DATA QUALITY") - print(f"ā€¢ Quality Score: {overall_quality['quality_score']:.1f}%") - print(f"ā€¢ Valid Tracks: {overall_quality['valid_tracks']}/{overall_quality['total_tracks']}") + _beatport_log(f"\nOVERALL DATA QUALITY") + _beatport_log(f"ā€¢ Quality Score: {overall_quality['quality_score']:.1f}%") + _beatport_log(f"ā€¢ Valid Tracks: {overall_quality['valid_tracks']}/{overall_quality['total_tracks']}") if overall_quality['issues']: - print(f"ā€¢ Issues Found: {len(overall_quality['issues'])}") + _beatport_log(f"ā€¢ Issues Found: {len(overall_quality['issues'])}") # Save results results = { @@ -4536,28 +4571,28 @@ def main(): try: with open('beatport_unified_results.json', 'w', encoding='utf-8') as f: json.dump(results, f, indent=2, ensure_ascii=False) - print(f"\nResults saved to beatport_unified_results.json") + _beatport_log(f"\nResults saved to beatport_unified_results.json") except Exception as e: - print(f"Failed to save results: {e}") + _beatport_log(f"Failed to save results: {e}") # Virtual playlist possibilities if overall_quality['quality_score'] > 70: - print(f"\nSUCCESS! Ready for virtual playlist creation") - print(f"You can now create playlists for:") - print(f" ā€¢ Beatport Top 100") + _beatport_log(f"\nSUCCESS! Ready for virtual playlist creation") + _beatport_log(f"You can now create playlists for:") + _beatport_log(f" ā€¢ Beatport Top 100") for genre_name in list(all_genre_results.keys())[:5]: - print(f" ā€¢ {genre_name} Top 100") + _beatport_log(f" ā€¢ {genre_name} Top 100") if len(all_genre_results) > 5: - print(f" ā€¢ ...and {len(all_genre_results) - 5} more genres!") + _beatport_log(f" ā€¢ ...and {len(all_genre_results) - 5} more genres!") - print(f"\nIntegration Notes:") - print(f" ā€¢ Artist and title data is clean and ready") - print(f" ā€¢ {total_genres} genres confirmed working") - print(f" ā€¢ Data quality: {overall_quality['quality_score']:.1f}%") + _beatport_log(f"\nIntegration Notes:") + _beatport_log(f" ā€¢ Artist and title data is clean and ready") + _beatport_log(f" ā€¢ {total_genres} genres confirmed working") + _beatport_log(f" ā€¢ Data quality: {overall_quality['quality_score']:.1f}%") else: - print(f"\nData quality needs improvement ({overall_quality['quality_score']:.1f}%)") - print(f"Consider refining extraction methods") + _beatport_log(f"\nData quality needs improvement ({overall_quality['quality_score']:.1f}%)") + _beatport_log(f"Consider refining extraction methods") if __name__ == "__main__": - main() \ No newline at end of file + main()