diff --git a/beatport_unified_scraper.py b/beatport_unified_scraper.py index 01204489..4061581e 100644 --- a/beatport_unified_scraper.py +++ b/beatport_unified_scraper.py @@ -26,28 +26,8 @@ class BeatportUnifiedScraper: # Dynamic genres - will be populated by scraping homepage self.all_genres = [] - def clean_text(self, text): - """Clean and normalize text from HTML elements""" - if not text: - return text - - # Fix common spacing issues - text = re.sub(r'([a-z$!@#%&*])([A-Z])', r'\1 \2', text) # Add space between lowercase/symbols and uppercase - text = re.sub(r'([a-zA-Z])(\d)', r'\1 \2', text) # Add space between letter and number - text = re.sub(r'(\d)([a-zA-Z])', r'\1 \2', text) # Add space between number and letter - text = re.sub(r'([a-zA-Z]),([a-zA-Z])', r'\1, \2', text) # Add space after comma - text = re.sub(r'([a-zA-Z])Mix\b', r'\1 Mix', text) # Fix "hitMix" -> "hit Mix" - text = re.sub(r'([a-zA-Z])Remix\b', r'\1 Remix', text) # Fix "hitRemix" -> "hit Remix" - text = re.sub(r'([a-zA-Z])Extended\b', r'\1 Extended', text) # Fix "hitExtended" -> "hit Extended" - text = re.sub(r'([a-zA-Z])Version\b', r'\1 Version', text) # Fix "hitVersion" -> "hit Version" - text = re.sub(r'\s+', ' ', text) # Collapse multiple spaces - text = text.strip() - - return text - - # Comprehensive fallback genres based on current Beatport dropdown (39 genres) + # Current Beatport genres with correct URLs and IDs (updated from live site) self.fallback_genres = [ - # Electronic genres {'name': '140 / Deep Dubstep / Grime', 'slug': '140-deep-dubstep-grime', 'id': '95', 'url': f'{self.base_url}/genre/140-deep-dubstep-grime/95'}, {'name': 'Afro House', 'slug': 'afro-house', 'id': '89', 'url': f'{self.base_url}/genre/afro-house/89'}, {'name': 'Amapiano', 'slug': 'amapiano', 'id': '98', 'url': f'{self.base_url}/genre/amapiano/98'}, @@ -84,7 +64,7 @@ class BeatportUnifiedScraper: {'name': 'Trance (Raw / Deep / Hypnotic)', 'slug': 'trance-raw-deep-hypnotic', 'id': '99', 'url': f'{self.base_url}/genre/trance-raw-deep-hypnotic/99'}, {'name': 'Trap / Future Bass', 'slug': 'trap-future-bass', 'id': '38', 'url': f'{self.base_url}/genre/trap-future-bass/38'}, {'name': 'UK Garage / Bassline', 'slug': 'uk-garage-bassline', 'id': '86', 'url': f'{self.base_url}/genre/uk-garage-bassline/86'}, - # Open Format genres + # Additional genres from current Beatport {'name': 'African', 'slug': 'african', 'id': '102', 'url': f'{self.base_url}/genre/african/102'}, {'name': 'Caribbean', 'slug': 'caribbean', 'id': '103', 'url': f'{self.base_url}/genre/caribbean/103'}, {'name': 'Hip-Hop', 'slug': 'hip-hop', 'id': '105', 'url': f'{self.base_url}/genre/hip-hop/105'}, @@ -93,6 +73,53 @@ class BeatportUnifiedScraper: {'name': 'R&B', 'slug': 'rb', 'id': '108', 'url': f'{self.base_url}/genre/rb/108'} ] + def clean_text(self, text): + """Clean and normalize text from HTML elements""" + if not text: + return text + + # Fix common spacing issues + text = re.sub(r'([a-z$!@#%&*])([A-Z])', r'\1 \2', text) # Add space between lowercase/symbols and uppercase + text = re.sub(r'([a-zA-Z])(\d)', r'\1 \2', text) # Add space between letter and number + text = re.sub(r'(\d)([a-zA-Z])', r'\1 \2', text) # Add space between number and letter + text = re.sub(r'([a-zA-Z]),([a-zA-Z])', r'\1, \2', text) # Add space after comma + text = re.sub(r'([a-zA-Z])Mix\b', r'\1 Mix', text) # Fix "hitMix" -> "hit Mix" + text = re.sub(r'([a-zA-Z])Remix\b', r'\1 Remix', text) # Fix "hitRemix" -> "hit Remix" + text = re.sub(r'([a-zA-Z])Extended\b', r'\1 Extended', text) # Fix "hitExtended" -> "hit Extended" + text = re.sub(r'([a-zA-Z])Version\b', r'\1 Version', text) # Fix "hitVersion" -> "hit Version" + text = re.sub(r'\s+', ' ', text) # Collapse multiple spaces + text = text.strip() + + return text + + def _is_valid_genre_name(self, name: str) -> bool: + """Check if a name is a valid genre name and not a section title""" + # Filter out common section titles + section_titles = { + 'open format', 'electronic', 'genres', 'browse', 'charts', + 'new releases', 'trending', 'featured', 'popular', 'top', + 'main', 'explore', 'discover', 'all genres' + } + + name_lower = name.lower().strip() + + # Reject if it's a section title + if name_lower in section_titles: + return False + + # Reject if it's too short or too generic + if len(name_lower) < 3: + return False + + # Reject if it contains only common words + common_words = {'the', 'and', 'or', 'of', 'in', 'on', 'at', 'to', 'for'} + words = name_lower.split() + if len(words) == 1 and words[0] in common_words: + return False + + # Accept everything else + return True + def get_page(self, url: str) -> Optional[BeautifulSoup]: """Fetch and parse a page with error handling""" try: @@ -154,50 +181,140 @@ class BeatportUnifiedScraper: genres = [] - # Method 1: Look for genres dropdown menu (multiple selectors) - potential_dropdowns = [ - soup.find('div', {'id': 'genres-dropdown-menu'}), - soup.find('div', class_=re.compile(r'genres.*dropdown', re.I)), - soup.find('nav', class_=re.compile(r'genres', re.I)), - soup.find('div', class_=re.compile(r'dropdown.*genres', re.I)), - soup.find('ul', class_=re.compile(r'genres', re.I)), - soup.find('div', {'data-testid': 'genres-dropdown'}), - soup.find('div', {'aria-label': re.compile(r'genres', re.I)}) - ] + # Method 1: Look for the specific genres dropdown menu structure + genres_dropdown = soup.find('div', {'id': 'genres-dropdown-menu'}) - for dropdown in potential_dropdowns: - if dropdown: - print(f"β Found potential genres dropdown: {dropdown.name} with class {dropdown.get('class')}") - # Extract genre links from dropdown - look for the specific pattern - genre_links = dropdown.find_all('a', href=re.compile(r'/genre/[^/]+/\d+')) + if genres_dropdown: + print("β Found genres-dropdown-menu") - if genre_links: - print(f"π Found {len(genre_links)} genre links in dropdown") - for link in genre_links: - href = link.get('href', '') - # Get text content, handling nested elements - name_text = link.get_text(strip=True) + # Look for the two main div containers as described + genre_containers = genres_dropdown.find_all('div', recursive=False) + print(f"π Found {len(genre_containers)} top-level containers in dropdown") - # Clean up the name - remove "New" tags and extra whitespace - name = re.sub(r'\s*New\s*', '', name_text).strip() + for container_idx, container in enumerate(genre_containers): + print(f"π¦ Processing container {container_idx + 1}") - if href and name and len(name) > 1: # Filter out empty or single char names - # Parse URL: /genre/house/5 -> slug='house', id='5' - url_parts = href.strip('/').split('/') - if len(url_parts) >= 3 and url_parts[0] == 'genre': - slug = url_parts[1] - genre_id = url_parts[2] + # Look specifically for .dropdown_menu classes + dropdown_menus = container.find_all(class_='dropdown_menu') - genres.append({ - 'name': name, - 'slug': slug, - 'id': genre_id, - 'url': urljoin(self.base_url, href) - }) + if not dropdown_menus: + # Fallback: Look for any element with class containing 'dropdown' and 'menu' + dropdown_menus = container.find_all(class_=re.compile(r'dropdown.*menu', re.I)) - if genres: - print(f"π― Successfully extracted {len(genres)} genres from dropdown") - break # Stop after first successful dropdown + if not dropdown_menus: + print(f"β οΈ No .dropdown_menu found in container {container_idx + 1}") + continue + + for menu_idx, menu in enumerate(dropdown_menus): + print(f"π Processing dropdown_menu {menu_idx + 1} in container {container_idx + 1}") + + # Look for