Fix Beatport new releases scraper broken by hashed CSS class changes
Replace all hardcoded styled-components class selectors (with build hashes that break on every Beatport deploy) with partial class name matches using [class*="prefix"] pattern. Applies to new releases endpoint and hero release scraper.
This commit is contained in:
parent
a31dd9b81e
commit
b186fed05b
2 changed files with 32 additions and 11 deletions
|
|
@ -1905,7 +1905,7 @@ class BeatportUnifiedScraper:
|
||||||
|
|
||||||
# Method 2 (FALLBACK): Look for the specific wrapper class (legacy support)
|
# Method 2 (FALLBACK): Look for the specific wrapper class (legacy support)
|
||||||
if len(tracks) < 5:
|
if len(tracks) < 5:
|
||||||
hero_wrapper = soup.find('div', class_='Homepage-style__NewOnBeatportWrapper-sc-deeb4244-2 iyIchZ')
|
hero_wrapper = soup.select_one('[class*="Homepage-style__NewOnBeatportWrapper"]')
|
||||||
if hero_wrapper:
|
if hero_wrapper:
|
||||||
print(" ✅ Found Homepage NewOnBeatportWrapper (fallback)")
|
print(" ✅ Found Homepage NewOnBeatportWrapper (fallback)")
|
||||||
tracks.extend(self._extract_from_hero_wrapper(hero_wrapper, limit))
|
tracks.extend(self._extract_from_hero_wrapper(hero_wrapper, limit))
|
||||||
|
|
@ -3788,7 +3788,7 @@ class BeatportUnifiedScraper:
|
||||||
data['release_slug'] = url_parts[1]
|
data['release_slug'] = url_parts[1]
|
||||||
|
|
||||||
# Extract release title
|
# Extract release title
|
||||||
title_elem = release_element.select_one('.HeroRelease-style__ReleaseName-sc-aeec852a-3')
|
title_elem = release_element.select_one('[class*="HeroRelease-style__ReleaseName"]')
|
||||||
if title_elem:
|
if title_elem:
|
||||||
data['title'] = self.clean_text(title_elem.get_text(strip=True))
|
data['title'] = self.clean_text(title_elem.get_text(strip=True))
|
||||||
|
|
||||||
|
|
@ -3799,7 +3799,7 @@ class BeatportUnifiedScraper:
|
||||||
data['alt_text'] = img_elem.get('alt', '')
|
data['alt_text'] = img_elem.get('alt', '')
|
||||||
|
|
||||||
# Extract artists
|
# Extract artists
|
||||||
artists_container = release_element.select_one('.HeroRelease-style__Artists-sc-aeec852a-1')
|
artists_container = release_element.select_one('[class*="HeroRelease-style__Artists"]')
|
||||||
if artists_container:
|
if artists_container:
|
||||||
artist_links = artists_container.find_all('a')
|
artist_links = artists_container.find_all('a')
|
||||||
artists = []
|
artists = []
|
||||||
|
|
@ -3817,7 +3817,7 @@ class BeatportUnifiedScraper:
|
||||||
data['artists_string'] = ', '.join([a['name'] for a in artists])
|
data['artists_string'] = ', '.join([a['name'] for a in artists])
|
||||||
|
|
||||||
# Extract label
|
# Extract label
|
||||||
label_elem = release_element.select_one('.HeroRelease-style__Label-sc-aeec852a-0')
|
label_elem = release_element.select_one('[class*="HeroRelease-style__Label"]')
|
||||||
if label_elem:
|
if label_elem:
|
||||||
label_link = label_elem.find('a')
|
label_link = label_elem.find('a')
|
||||||
if label_link:
|
if label_link:
|
||||||
|
|
@ -3826,7 +3826,7 @@ class BeatportUnifiedScraper:
|
||||||
data['label_beatport_url'] = urljoin(self.base_url, data['label_url']) if data['label_url'] else None
|
data['label_beatport_url'] = urljoin(self.base_url, data['label_url']) if data['label_url'] else None
|
||||||
|
|
||||||
# Extract any badges (like EXCLUSIVE)
|
# Extract any badges (like EXCLUSIVE)
|
||||||
badges_elem = release_element.select_one('.HeroRelease-style__Badges-sc-aeec852a-8')
|
badges_elem = release_element.select_one('[class*="HeroRelease-style__Badges"]')
|
||||||
if badges_elem:
|
if badges_elem:
|
||||||
badge_text = self.clean_text(badges_elem.get_text(strip=True))
|
badge_text = self.clean_text(badges_elem.get_text(strip=True))
|
||||||
if badge_text:
|
if badge_text:
|
||||||
|
|
|
||||||
|
|
@ -6251,8 +6251,27 @@ def get_beatport_new_releases():
|
||||||
if not soup:
|
if not soup:
|
||||||
raise Exception("Could not fetch Beatport homepage")
|
raise Exception("Could not fetch Beatport homepage")
|
||||||
|
|
||||||
# Extract release cards using the working CSS selector
|
# Find New Releases GridSlider container by section heading
|
||||||
release_cards = soup.select('.ReleaseCard-style__Wrapper-sc-7c61989b-12.duhBUN')
|
# Use partial class match to avoid brittle hashed class names
|
||||||
|
gridsliders = soup.select('[class*="GridSlider-style__Wrapper"]')
|
||||||
|
releases_container = None
|
||||||
|
|
||||||
|
for container in gridsliders:
|
||||||
|
h2 = container.select_one('h2')
|
||||||
|
if h2:
|
||||||
|
title = h2.get_text(strip=True).lower()
|
||||||
|
if 'new release' in title:
|
||||||
|
releases_container = container
|
||||||
|
logger.info(f"🆕 FOUND NEW RELEASES: '{h2.get_text(strip=True)}'")
|
||||||
|
break
|
||||||
|
|
||||||
|
# Fallback: try ReleaseCard partial class match on whole page
|
||||||
|
if releases_container:
|
||||||
|
release_cards = releases_container.select('[class*="ReleaseCard-style__Wrapper"]')
|
||||||
|
else:
|
||||||
|
logger.warning("⚠️ No New Releases GridSlider found, trying page-wide ReleaseCard search")
|
||||||
|
release_cards = soup.select('[class*="ReleaseCard-style__Wrapper"]')
|
||||||
|
|
||||||
releases = []
|
releases = []
|
||||||
|
|
||||||
logger.info(f"🔍 Found {len(release_cards)} release cards")
|
logger.info(f"🔍 Found {len(release_cards)} release cards")
|
||||||
|
|
@ -6260,22 +6279,24 @@ def get_beatport_new_releases():
|
||||||
for i, card in enumerate(release_cards[:100]): # Limit to 100 for 10 slides
|
for i, card in enumerate(release_cards[:100]): # Limit to 100 for 10 slides
|
||||||
release_data = {}
|
release_data = {}
|
||||||
|
|
||||||
# Extract title
|
# Extract title from Meta section
|
||||||
title_elem = card.select_one('[class*="title"], [class*="Title"], h1, h2, h3, h4, h5, h6')
|
title_elem = card.select_one('[class*="ReleaseCard-style__Meta"] a[href*="/release/"]')
|
||||||
|
if not title_elem:
|
||||||
|
title_elem = card.select_one('[class*="title"], [class*="Title"], h3, h4, h5, h6')
|
||||||
if title_elem:
|
if title_elem:
|
||||||
title_text = title_elem.get_text(strip=True)
|
title_text = title_elem.get_text(strip=True)
|
||||||
if title_text and len(title_text) > 2 and title_text not in ['New Releases', 'Buy', 'Play']:
|
if title_text and len(title_text) > 2 and title_text not in ['New Releases', 'Buy', 'Play']:
|
||||||
release_data['title'] = title_text
|
release_data['title'] = title_text
|
||||||
|
|
||||||
# Extract artist
|
# Extract artist
|
||||||
artist_elem = card.select_one('[class*="artist"], [class*="Artist"], a[href*="/artist/"]')
|
artist_elem = card.select_one('a[href*="/artist/"]')
|
||||||
if artist_elem:
|
if artist_elem:
|
||||||
artist_text = artist_elem.get_text(strip=True)
|
artist_text = artist_elem.get_text(strip=True)
|
||||||
if artist_text and len(artist_text) > 1:
|
if artist_text and len(artist_text) > 1:
|
||||||
release_data['artist'] = artist_text
|
release_data['artist'] = artist_text
|
||||||
|
|
||||||
# Extract label
|
# Extract label
|
||||||
label_elem = card.select_one('[class*="label"], [class*="Label"], a[href*="/label/"]')
|
label_elem = card.select_one('a[href*="/label/"]')
|
||||||
if label_elem:
|
if label_elem:
|
||||||
label_text = label_elem.get_text(strip=True)
|
label_text = label_elem.get_text(strip=True)
|
||||||
if label_text and len(label_text) > 1:
|
if label_text and len(label_text) > 1:
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue