diff --git a/beatport_unified_scraper.py b/beatport_unified_scraper.py index ed039009..3c9e5aa7 100644 --- a/beatport_unified_scraper.py +++ b/beatport_unified_scraper.py @@ -1403,6 +1403,263 @@ class BeatportUnifiedScraper: print(f"Error extracting track data: {e}") return None + def scrape_homepage_top10_releases(self) -> List[Dict]: + """Scrape Top 10 Releases from homepage - FIXED VERSION""" + print("\nšæ FIXED: Scraping Top 10 Releases from homepage...") + + soup = self.get_page(self.base_url) + if not soup: + print(" ā Could not get homepage") + return [] + + # Extract Top 10 Releases items - EXACT same as test script + top10_releases_items = soup.select('[data-testid="top-10-releases-item"]') + print(f" FOUND {len(top10_releases_items)} Top 10 Releases items") + + if len(top10_releases_items) == 0: + print(" ā No items found - trying alternatives") + return [] + + releases = [] + for i, item in enumerate(top10_releases_items, 1): + try: + # Use the SAME function name as the test script + release_data = self.extract_release_from_item_FIXED(item, i) + if release_data: + releases.append(release_data) + print(f" ā {i}. {release_data['artist']} - {release_data['title']}") + else: + print(f" ā {i}. No data extracted") + except Exception as e: + print(f" ā Error extracting release {i}: {e}") + + print(f"ā FINAL: Extracted {len(releases)} Top 10 Releases") + return releases + + def extract_release_from_item_FIXED(self, item, rank): + """Extract release data from a list item - EXACT COPY FROM WORKING TEST SCRIPT""" + try: + # Get the release URL + link_elem = item.select_one('a[href*="/release/"]') + release_url = "" + if link_elem and link_elem.get('href'): + release_url = f"https://www.beatport.com{link_elem.get('href')}" + + # Extract release title + title = "Unknown Title" + # Try multiple selectors for title + title_selectors = [ + '[class*="ItemName"]', + '[class*="ReleaseName"]', + '[class*="release-name"]', + '[class*="TrackName"]', + '[class*="track-name"]', + 'a[href*="/release/"]', + 'h3', 'h4', 'h5', + '[class*="title"]', + '[class*="Title"]' + ] + + for selector in title_selectors: + title_elem = item.select_one(selector) + if title_elem: + title = title_elem.get_text(strip=True) + if title and title != "Unknown Title" and len(title) > 2: + break + + # Extract artist name - try multiple approaches + artist = "Unknown Artist" + artist_selectors = [ + '[class*="Artists"]', + '[class*="artist"]', + '[class*="Artist"]', + '[class*="ItemArtist"]', + 'a[href*="/artist/"]', + '[class*="by"]', + '[class*="By"]' + ] + + for selector in artist_selectors: + artist_elem = item.select_one(selector) + if artist_elem: + artist = artist_elem.get_text(strip=True) + if artist and artist != "Unknown Artist" and len(artist) > 1: + break + + # Extract label name - try multiple approaches + label = "Unknown Label" + label_selectors = [ + '[class*="Label"]', + '[class*="label"]', + '[class*="ItemLabel"]', + 'a[href*="/label/"]', + '[class*="imprint"]', + '[class*="Imprint"]' + ] + + for selector in label_selectors: + label_elem = item.select_one(selector) + if label_elem: + label = label_elem.get_text(strip=True) + if label and label != "Unknown Label" and len(label) > 2: + break + + # Extract image URL - this is important for releases + image_url = "" + image_selectors = [ + 'img[src]', + 'img[data-src]', + 'img[data-lazy]', + '[style*="background-image"]', + 'picture img', + '.artwork img', + '[class*="artwork"] img', + '[class*="Artwork"] img', + '[class*="image"] img', + '[class*="Image"] img' + ] + + for selector in image_selectors: + img_elem = item.select_one(selector) + if img_elem: + # Try different image source attributes + img_src = (img_elem.get('src') or + img_elem.get('data-src') or + img_elem.get('data-lazy') or + img_elem.get('data-original')) + + if img_src and img_src.startswith(('http', '//')): + image_url = img_src + break + elif img_src and img_src.startswith('/'): + image_url = f"https://www.beatport.com{img_src}" + break + + return { + "rank": rank, + "title": title, + "artist": artist, + "label": label, + "url": release_url, + "image_url": image_url, + "list_name": "Top 10 Releases" + } + + except Exception as e: + print(f"Error extracting release data: {e}") + return None + + def extract_release_from_top10_item(self, item, rank): + """Extract release data from a top 10 releases item""" + try: + # Get the release URL + link_elem = item.select_one('a[href*="/release/"]') + release_url = "" + if link_elem and link_elem.get('href'): + release_url = f"https://www.beatport.com{link_elem.get('href')}" + + # Extract release title + title = "Unknown Title" + title_selectors = [ + '[class*="ItemName"]', + '[class*="ReleaseName"]', + '[class*="release-name"]', + '[class*="TrackName"]', + '[class*="track-name"]', + 'a[href*="/release/"]', + 'h3', 'h4', 'h5', + '[class*="title"]', + '[class*="Title"]' + ] + + for selector in title_selectors: + title_elem = item.select_one(selector) + if title_elem: + title = title_elem.get_text(strip=True) + if title and title != "Unknown Title" and len(title) > 2: + break + + # Extract artist name + artist = "Unknown Artist" + artist_selectors = [ + '[class*="Artists"]', + '[class*="artist"]', + '[class*="Artist"]', + '[class*="ItemArtist"]', + 'a[href*="/artist/"]', + '[class*="by"]', + '[class*="By"]' + ] + + for selector in artist_selectors: + artist_elem = item.select_one(selector) + if artist_elem: + artist = artist_elem.get_text(strip=True) + if artist and artist != "Unknown Artist" and len(artist) > 1: + break + + # Extract label name + label = "Unknown Label" + label_selectors = [ + '[class*="Label"]', + '[class*="label"]', + '[class*="ItemLabel"]', + 'a[href*="/label/"]', + '[class*="imprint"]', + '[class*="Imprint"]' + ] + + for selector in label_selectors: + label_elem = item.select_one(selector) + if label_elem: + label = label_elem.get_text(strip=True) + if label and label != "Unknown Label" and len(label) > 2: + break + + # Extract image URL - important for releases + image_url = "" + image_selectors = [ + 'img[src]', + 'img[data-src]', + 'img[data-lazy]', + '[style*="background-image"]', + 'picture img', + '.artwork img', + '[class*="artwork"] img', + '[class*="Artwork"] img', + '[class*="image"] img', + '[class*="Image"] img' + ] + + for selector in image_selectors: + img_elem = item.select_one(selector) + if img_elem: + img_src = (img_elem.get('src') or + img_elem.get('data-src') or + img_elem.get('data-lazy') or + img_elem.get('data-original')) + + if img_src and img_src.startswith(('http', '//')): + image_url = img_src + break + elif img_src and img_src.startswith('/'): + image_url = f"https://www.beatport.com{img_src}" + break + + return { + "rank": rank, + "title": title, + "artist": artist, + "label": label, + "url": release_url, + "image_url": image_url, + "list_name": "Top 10 Releases" + } + + except Exception as e: + print(f"Error extracting release data: {e}") + return None + def scrape_new_on_beatport_hero(self, limit: int = 10) -> List[Dict]: """Scrape the 'New on Beatport' hero slideshow from homepage""" print("\nšÆ Scraping 'New on Beatport' hero slideshow...") diff --git a/web_server.py b/web_server.py index 2a97b8aa..8b3f6d97 100644 --- a/web_server.py +++ b/web_server.py @@ -13262,6 +13262,44 @@ def get_beatport_homepage_top10_lists(): "hype_count": 0 }), 500 +@app.route('/api/beatport/homepage/top-10-releases-cards', methods=['GET']) +def get_beatport_homepage_top10_releases_cards(): + """Get Beatport Top 10 Releases CARDS from homepage (not individual tracks)""" + try: + logger.info("šæ API request for Beatport homepage Top 10 Releases CARDS") + + # Initialize the Beatport scraper + scraper = BeatportUnifiedScraper() + + # Get top 10 releases from homepage + top10_releases = scraper.scrape_homepage_top10_releases() + + logger.info(f"ā API extracted {len(top10_releases)} Top 10 Release Cards") + + # Debug: Log first release if any + if top10_releases: + logger.info(f"First release: {top10_releases[0].get('title', 'No title')} by {top10_releases[0].get('artist', 'No artist')}") + else: + logger.warning("ā No releases found by scraper") + + return jsonify({ + "success": True, + "releases": top10_releases, + "releases_count": len(top10_releases), + "source": "beatport_homepage_top10_releases_cards" + }) + + except Exception as e: + logger.error(f"ā Error getting Beatport homepage Top 10 Releases cards: {e}") + import traceback + logger.error(f"Full traceback: {traceback.format_exc()}") + return jsonify({ + "success": False, + "error": str(e), + "releases": [], + "releases_count": 0 + }), 500 + @app.route('/api/beatport/homepage/featured-charts', methods=['GET']) def get_beatport_homepage_featured_charts(): """Get Beatport Featured Charts from homepage section""" diff --git a/webui/index.html b/webui/index.html index 3e66cf4e..64e28b04 100644 --- a/webui/index.html +++ b/webui/index.html @@ -850,6 +850,26 @@ + +
Most popular albums and EPs on Beatport
+Fetching trending albums and EPs
+${release.artist || 'Unknown Artist'}
+${release.label || 'Unknown Label'}
+${errorMessage}
+