feat: Async scraper, cloud upload, subreddit stats

analytics/subreddit_stats.py

@@ -0,0 +1,243 @@
+"""
+Subreddit Statistics - Subscribers, rules, mods, and metadata
+"""
+import requests
+from datetime import datetime
+import json
+
+USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+
+def get_subreddit_about(subreddit):
+    """
+    Fetch subreddit metadata (subscribers, description, rules, etc.)
+    
+    Args:
+        subreddit: Subreddit name (without r/)
+    
+    Returns:
+        Dictionary with subreddit info
+    """
+    url = f"https://old.reddit.com/r/{subreddit}/about.json"
+    
+    try:
+        response = requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=15)
+        
+        if response.status_code != 200:
+            print(f"❌ Failed to fetch r/{subreddit} info: {response.status_code}")
+            return None
+        
+        data = response.json()['data']
+        
+        return {
+            "name": data.get('display_name'),
+            "title": data.get('title'),
+            "description": data.get('public_description'),
+            "subscribers": data.get('subscribers', 0),
+            "active_users": data.get('accounts_active', 0),
+            "created_utc": datetime.fromtimestamp(data.get('created_utc', 0)).isoformat(),
+            "over_18": data.get('over18', False),
+            "subreddit_type": data.get('subreddit_type'),  # public, private, restricted
+            "lang": data.get('lang'),
+            "icon_url": data.get('icon_img', '').split('?')[0] if data.get('icon_img') else None,
+            "banner_url": data.get('banner_img', '').split('?')[0] if data.get('banner_img') else None,
+            "header_url": data.get('header_img'),
+            "community_icon": data.get('community_icon', '').split('?')[0] if data.get('community_icon') else None,
+            "wiki_enabled": data.get('wiki_enabled', False),
+            "spoilers_enabled": data.get('spoilers_enabled', False),
+            "allow_videos": data.get('allow_videos', False),
+            "allow_images": data.get('allow_images', False),
+            "allow_polls": data.get('allow_polls', False),
+        }
+    except Exception as e:
+        print(f"❌ Error fetching subreddit info: {e}")
+        return None
+
+def get_subreddit_rules(subreddit):
+    """
+    Fetch subreddit rules.
+    
+    Args:
+        subreddit: Subreddit name
+    
+    Returns:
+        List of rule dictionaries
+    """
+    url = f"https://old.reddit.com/r/{subreddit}/about/rules.json"
+    
+    try:
+        response = requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=15)
+        
+        if response.status_code != 200:
+            return []
+        
+        data = response.json()
+        rules = []
+        
+        for rule in data.get('rules', []):
+            rules.append({
+                "short_name": rule.get('short_name'),
+                "description": rule.get('description'),
+                "priority": rule.get('priority'),
+                "kind": rule.get('kind'),  # link, comment, all
+                "created_utc": datetime.fromtimestamp(rule.get('created_utc', 0)).isoformat()
+            })
+        
+        return rules
+    except Exception as e:
+        print(f"❌ Error fetching rules: {e}")
+        return []
+
+def get_subreddit_mods(subreddit):
+    """
+    Fetch subreddit moderators.
+    
+    Args:
+        subreddit: Subreddit name
+    
+    Returns:
+        List of moderator usernames
+    """
+    url = f"https://old.reddit.com/r/{subreddit}/about/moderators.json"
+    
+    try:
+        response = requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=15)
+        
+        if response.status_code != 200:
+            return []
+        
+        data = response.json()
+        mods = []
+        
+        for mod in data.get('data', {}).get('children', []):
+            mods.append({
+                "name": mod.get('name'),
+                "permissions": mod.get('mod_permissions', []),
+                "added_utc": datetime.fromtimestamp(mod.get('date', 0)).isoformat() if mod.get('date') else None
+            })
+        
+        return mods
+    except Exception as e:
+        print(f"❌ Error fetching mods: {e}")
+        return []
+
+def get_subreddit_flairs(subreddit):
+    """
+    Fetch available post flairs.
+    
+    Args:
+        subreddit: Subreddit name
+    
+    Returns:
+        List of flair options
+    """
+    url = f"https://old.reddit.com/r/{subreddit}/api/link_flair_v2.json"
+    
+    try:
+        response = requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=15)
+        
+        if response.status_code != 200:
+            return []
+        
+        flairs = []
+        for flair in response.json():
+            flairs.append({
+                "text": flair.get('text'),
+                "id": flair.get('id'),
+                "background_color": flair.get('background_color'),
+                "text_color": flair.get('text_color'),
+                "type": flair.get('type')
+            })
+        
+        return flairs
+    except Exception as e:
+        return []
+
+def get_full_subreddit_stats(subreddit):
+    """
+    Get comprehensive subreddit statistics.
+    
+    Args:
+        subreddit: Subreddit name
+    
+    Returns:
+        Dictionary with all stats
+    """
+    print(f"📊 Fetching stats for r/{subreddit}...")
+    
+    about = get_subreddit_about(subreddit)
+    
+    if not about:
+        return None
+    
+    rules = get_subreddit_rules(subreddit)
+    mods = get_subreddit_mods(subreddit)
+    flairs = get_subreddit_flairs(subreddit)
+    
+    stats = {
+        **about,
+        "rules": rules,
+        "rules_count": len(rules),
+        "moderators": mods,
+        "moderator_count": len(mods),
+        "flairs": flairs,
+        "flair_count": len(flairs),
+        "fetched_at": datetime.now().isoformat()
+    }
+    
+    # Print summary
+    print(f"\n📊 r/{subreddit} Statistics:")
+    print(f"   👥 Subscribers: {stats['subscribers']:,}")
+    print(f"   🟢 Active Users: {stats['active_users']:,}")
+    print(f"   📜 Rules: {stats['rules_count']}")
+    print(f"   👮 Moderators: {stats['moderator_count']}")
+    print(f"   🏷️ Flairs: {stats['flair_count']}")
+    print(f"   📅 Created: {stats['created_utc'][:10]}")
+    print(f"   🔞 NSFW: {stats['over_18']}")
+    
+    return stats
+
+def save_subreddit_stats(subreddit, output_dir="data"):
+    """
+    Fetch and save subreddit stats to JSON.
+    
+    Args:
+        subreddit: Subreddit name
+        output_dir: Output directory
+    
+    Returns:
+        Path to saved file
+    """
+    import os
+    
+    stats = get_full_subreddit_stats(subreddit)
+    
+    if not stats:
+        return None
+    
+    save_dir = f"{output_dir}/r_{subreddit}"
+    os.makedirs(save_dir, exist_ok=True)
+    
+    filepath = f"{save_dir}/subreddit_stats.json"
+    
+    with open(filepath, 'w', encoding='utf-8') as f:
+        json.dump(stats, f, indent=2, ensure_ascii=False)
+    
+    print(f"\n💾 Saved to {filepath}")
+    return filepath
+
+# CLI for testing
+if __name__ == "__main__":
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Subreddit Statistics")
+    parser.add_argument("subreddit", help="Subreddit name")
+    parser.add_argument("--save", action="store_true", help="Save to JSON")
+    
+    args = parser.parse_args()
+    
+    if args.save:
+        save_subreddit_stats(args.subreddit)
+    else:
+        stats = get_full_subreddit_stats(args.subreddit)
+        if stats:
+            print(f"\n📝 Description: {stats['description'][:200]}..." if stats['description'] else "")

export/cloud.py

@@ -0,0 +1,313 @@
+"""
+Cloud Upload Module - S3 and Google Drive integration
+"""
+import os
+import json
+from pathlib import Path
+from datetime import datetime
+
+# Try importing boto3 for S3
+try:
+    import boto3
+    from botocore.exceptions import ClientError
+    HAS_BOTO3 = True
+except ImportError:
+    HAS_BOTO3 = False
+
+# Try importing Google Drive API
+try:
+    from google.oauth2.credentials import Credentials
+    from googleapiclient.discovery import build
+    from googleapiclient.http import MediaFileUpload
+    HAS_GDRIVE = True
+except ImportError:
+    HAS_GDRIVE = False
+
+
+class S3Uploader:
+    """Upload scraped data to AWS S3."""
+    
+    def __init__(self, bucket_name, aws_access_key=None, aws_secret_key=None, region='us-east-1'):
+        """
+        Initialize S3 uploader.
+        
+        Args:
+            bucket_name: S3 bucket name
+            aws_access_key: Optional, uses env/config if not provided
+            aws_secret_key: Optional, uses env/config if not provided
+            region: AWS region
+        """
+        if not HAS_BOTO3:
+            raise ImportError("boto3 not installed. Run: pip install boto3")
+        
+        self.bucket_name = bucket_name
+        self.region = region
+        
+        # Use provided credentials or fall back to env vars
+        self.s3 = boto3.client(
+            's3',
+            aws_access_key_id=aws_access_key or os.getenv('AWS_ACCESS_KEY_ID'),
+            aws_secret_access_key=aws_secret_key or os.getenv('AWS_SECRET_ACCESS_KEY'),
+            region_name=region
+        )
+    
+    def upload_file(self, local_path, s3_key=None):
+        """
+        Upload a single file to S3.
+        
+        Args:
+            local_path: Local file path
+            s3_key: S3 object key (default: same as filename)
+        
+        Returns:
+            S3 URL or None on failure
+        """
+        local_path = Path(local_path)
+        
+        if not local_path.exists():
+            print(f"❌ File not found: {local_path}")
+            return None
+        
+        s3_key = s3_key or local_path.name
+        
+        try:
+            self.s3.upload_file(str(local_path), self.bucket_name, s3_key)
+            url = f"https://{self.bucket_name}.s3.{self.region}.amazonaws.com/{s3_key}"
+            print(f"✅ Uploaded: {s3_key}")
+            return url
+        except ClientError as e:
+            print(f"❌ S3 upload failed: {e}")
+            return None
+    
+    def upload_directory(self, local_dir, s3_prefix=""):
+        """
+        Upload entire directory to S3.
+        
+        Args:
+            local_dir: Local directory path
+            s3_prefix: Prefix for S3 keys
+        
+        Returns:
+            Dictionary of uploaded files
+        """
+        local_dir = Path(local_dir)
+        
+        if not local_dir.exists():
+            print(f"❌ Directory not found: {local_dir}")
+            return {}
+        
+        uploaded = {}
+        
+        for file_path in local_dir.rglob('*'):
+            if file_path.is_file():
+                relative_path = file_path.relative_to(local_dir)
+                s3_key = f"{s3_prefix}/{relative_path}" if s3_prefix else str(relative_path)
+                s3_key = s3_key.replace('\\', '/')  # Windows path fix
+                
+                url = self.upload_file(file_path, s3_key)
+                if url:
+                    uploaded[str(relative_path)] = url
+        
+        print(f"\n📤 Uploaded {len(uploaded)} files to S3")
+        return uploaded
+    
+    def upload_subreddit_data(self, subreddit, prefix="u"):
+        """
+        Upload all data for a subreddit.
+        
+        Args:
+            subreddit: Subreddit name
+            prefix: "r" for subreddit, "u" for user
+        
+        Returns:
+            Upload results
+        """
+        data_dir = Path(f"data/{prefix}_{subreddit}")
+        
+        if not data_dir.exists():
+            print(f"❌ Data not found for {prefix}/{subreddit}")
+            return {}
+        
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        s3_prefix = f"reddit/{prefix}_{subreddit}/{timestamp}"
+        
+        return self.upload_directory(data_dir, s3_prefix)
+    
+    def list_uploads(self, prefix="reddit/"):
+        """List all uploaded data in S3."""
+        try:
+            response = self.s3.list_objects_v2(
+                Bucket=self.bucket_name,
+                Prefix=prefix
+            )
+            
+            objects = response.get('Contents', [])
+            
+            print(f"\n📁 S3 Contents ({self.bucket_name}/{prefix}):")
+            for obj in objects[:50]:  # Limit to 50
+                size_kb = obj['Size'] / 1024
+                print(f"   {obj['Key']} ({size_kb:.1f} KB)")
+            
+            if len(objects) > 50:
+                print(f"   ... and {len(objects) - 50} more")
+            
+            return objects
+        except ClientError as e:
+            print(f"❌ S3 list failed: {e}")
+            return []
+
+
+class GDriveUploader:
+    """Upload scraped data to Google Drive."""
+    
+    def __init__(self, credentials_file='credentials.json', token_file='token.json'):
+        """
+        Initialize Google Drive uploader.
+        
+        Args:
+            credentials_file: Path to OAuth credentials JSON
+            token_file: Path to token JSON
+        """
+        if not HAS_GDRIVE:
+            raise ImportError("Google API client not installed. Run: pip install google-api-python-client google-auth-oauthlib")
+        
+        self.credentials_file = credentials_file
+        self.token_file = token_file
+        self.service = None
+        self._authenticate()
+    
+    def _authenticate(self):
+        """Authenticate with Google Drive API."""
+        creds = None
+        
+        if os.path.exists(self.token_file):
+            creds = Credentials.from_authorized_user_file(self.token_file)
+        
+        if not creds or not creds.valid:
+            if creds and creds.expired and creds.refresh_token:
+                creds.refresh(Request())
+            else:
+                from google_auth_oauthlib.flow import InstalledAppFlow
+                SCOPES = ['https://www.googleapis.com/auth/drive.file']
+                flow = InstalledAppFlow.from_client_secrets_file(self.credentials_file, SCOPES)
+                creds = flow.run_local_server(port=0)
+            
+            with open(self.token_file, 'w') as token:
+                token.write(creds.to_json())
+        
+        self.service = build('drive', 'v3', credentials=creds)
+        print("✅ Google Drive authenticated")
+    
+    def create_folder(self, name, parent_id=None):
+        """Create a folder in Google Drive."""
+        metadata = {
+            'name': name,
+            'mimeType': 'application/vnd.google-apps.folder'
+        }
+        
+        if parent_id:
+            metadata['parents'] = [parent_id]
+        
+        folder = self.service.files().create(body=metadata, fields='id').execute()
+        return folder.get('id')
+    
+    def upload_file(self, local_path, folder_id=None):
+        """Upload a file to Google Drive."""
+        local_path = Path(local_path)
+        
+        if not local_path.exists():
+            print(f"❌ File not found: {local_path}")
+            return None
+        
+        metadata = {'name': local_path.name}
+        if folder_id:
+            metadata['parents'] = [folder_id]
+        
+        media = MediaFileUpload(str(local_path), resumable=True)
+        
+        try:
+            file = self.service.files().create(
+                body=metadata,
+                media_body=media,
+                fields='id,webViewLink'
+            ).execute()
+            
+            print(f"✅ Uploaded: {local_path.name}")
+            return file.get('webViewLink')
+        except Exception as e:
+            print(f"❌ Upload failed: {e}")
+            return None
+    
+    def upload_subreddit_data(self, subreddit, prefix="r"):
+        """Upload all data for a subreddit."""
+        data_dir = Path(f"data/{prefix}_{subreddit}")
+        
+        if not data_dir.exists():
+            print(f"❌ Data not found for {prefix}/{subreddit}")
+            return {}
+        
+        # Create folder structure
+        root_folder = self.create_folder(f"reddit_{prefix}_{subreddit}_{datetime.now().strftime('%Y%m%d')}")
+        
+        uploaded = {}
+        
+        for file_path in data_dir.rglob('*'):
+            if file_path.is_file():
+                url = self.upload_file(file_path, root_folder)
+                if url:
+                    uploaded[str(file_path.name)] = url
+        
+        print(f"\n📤 Uploaded {len(uploaded)} files to Google Drive")
+        return uploaded
+
+
+def upload_to_s3(subreddit, bucket_name, prefix="r"):
+    """
+    Convenience function to upload subreddit data to S3.
+    
+    Args:
+        subreddit: Subreddit name
+        bucket_name: S3 bucket name
+        prefix: "r" or "u"
+    
+    Returns:
+        Upload results
+    """
+    uploader = S3Uploader(bucket_name)
+    return uploader.upload_subreddit_data(subreddit, prefix)
+
+
+def upload_to_gdrive(subreddit, prefix="r"):
+    """
+    Convenience function to upload subreddit data to Google Drive.
+    
+    Args:
+        subreddit: Subreddit name
+        prefix: "r" or "u"
+    
+    Returns:
+        Upload results
+    """
+    uploader = GDriveUploader()
+    return uploader.upload_subreddit_data(subreddit, prefix)
+
+
+# CLI for testing
+if __name__ == "__main__":
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Cloud Upload")
+    parser.add_argument("subreddit", help="Subreddit to upload")
+    parser.add_argument("--s3-bucket", help="S3 bucket name")
+    parser.add_argument("--gdrive", action="store_true", help="Upload to Google Drive")
+    parser.add_argument("--user", action="store_true", help="Is a user profile")
+    
+    args = parser.parse_args()
+    prefix = "u" if args.user else "r"
+    
+    if args.s3_bucket:
+        upload_to_s3(args.subreddit, args.s3_bucket, prefix)
+    elif args.gdrive:
+        upload_to_gdrive(args.subreddit, prefix)
+    else:
+        print("Please specify --s3-bucket or --gdrive")

requirements.txt

@@ -2,6 +2,10 @@
 pandas
 requests
 
+# Async
+aiohttp
+aiofiles
+
 # Dashboard
 streamlit
 

scraper/__init__.py

@@ -0,0 +1,2 @@
+# Scraper module
+from .async_scraper import run_async_scraper, scrape_async

scraper/async_scraper.py

@@ -0,0 +1,381 @@
+"""
+Async Reddit Scraper - 10x Speed Boost with aiohttp
+"""
+import asyncio
+import aiohttp
+import aiofiles
+import pandas as pd
+import datetime
+import time
+import os
+import random
+from pathlib import Path
+from urllib.parse import urlparse
+import sys
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+from config import USER_AGENT, MIRRORS, ASYNC_MAX_CONCURRENT, ASYNC_BATCH_SIZE
+
+# Semaphore to limit concurrent requests
+semaphore = None
+
+async def fetch_json(session, url, retries=3):
+    """Fetch JSON with retry logic."""
+    for attempt in range(retries):
+        try:
+            async with session.get(url, timeout=aiohttp.ClientTimeout(total=15)) as response:
+                if response.status == 200:
+                    return await response.json()
+                elif response.status == 429:  # Rate limited
+                    await asyncio.sleep(5 * (attempt + 1))
+        except Exception as e:
+            if attempt < retries - 1:
+                await asyncio.sleep(2)
+    return None
+
+async def fetch_posts_page(session, base_url, target, after=None, is_user=False):
+    """Fetch a single page of posts."""
+    if is_user:
+        path = f"/user/{target}/submitted.json"
+    else:
+        path = f"/r/{target}/new.json"
+    
+    url = f"{base_url}{path}?limit=100&raw_json=1"
+    if after:
+        url += f"&after={after}"
+    
+    return await fetch_json(session, url)
+
+async def download_media_async(session, url, save_path):
+    """Download media file asynchronously."""
+    global semaphore
+    
+    if os.path.exists(save_path):
+        return True
+    
+    async with semaphore:
+        try:
+            async with session.get(url, timeout=aiohttp.ClientTimeout(total=60)) as response:
+                if response.status == 200:
+                    async with aiofiles.open(save_path, 'wb') as f:
+                        async for chunk in response.content.iter_chunked(8192):
+                            await f.write(chunk)
+                    return True
+        except:
+            pass
+    return False
+
+async def fetch_comments_async(session, permalink):
+    """Fetch comments asynchronously."""
+    global semaphore
+    
+    async with semaphore:
+        url = f"https://old.reddit.com{permalink}.json?limit=100"
+        data = await fetch_json(session, url)
+        
+        if data and len(data) > 1:
+            return parse_comments_sync(data[1]['data']['children'], permalink)
+    return []
+
+def parse_comments_sync(comment_list, post_permalink, depth=0, max_depth=3):
+    """Parse comments (sync helper)."""
+    comments = []
+    
+    if depth > max_depth:
+        return comments
+    
+    for item in comment_list:
+        if item['kind'] != 't1':
+            continue
+        
+        c = item['data']
+        comments.append({
+            "post_permalink": post_permalink,
+            "comment_id": c.get('id'),
+            "parent_id": c.get('parent_id'),
+            "author": c.get('author'),
+            "body": c.get('body', ''),
+            "score": c.get('score', 0),
+            "created_utc": datetime.datetime.fromtimestamp(c.get('created_utc', 0)).isoformat(),
+            "depth": depth,
+            "is_submitter": c.get('is_submitter', False),
+        })
+        
+        replies = c.get('replies')
+        if replies and isinstance(replies, dict):
+            comments.extend(parse_comments_sync(
+                replies.get('data', {}).get('children', []),
+                post_permalink, depth + 1, max_depth
+            ))
+    
+    return comments
+
+def extract_media_urls(post_data):
+    """Extract all media URLs from a post."""
+    media = {"images": [], "videos": [], "galleries": []}
+    
+    url = post_data.get('url', '')
+    
+    if any(ext in url.lower() for ext in ['.jpg', '.jpeg', '.png', '.gif', '.webp']):
+        media["images"].append(url)
+    
+    if 'i.redd.it' in url:
+        media["images"].append(url)
+    
+    if post_data.get('is_video'):
+        reddit_video = post_data.get('media', {})
+        if reddit_video and 'reddit_video' in reddit_video:
+            video_url = reddit_video['reddit_video'].get('fallback_url', '')
+            if video_url:
+                media["videos"].append(video_url.split('?')[0])
+    
+    preview = post_data.get('preview', {})
+    if preview and 'images' in preview:
+        for img in preview['images']:
+            source = img.get('source', {})
+            if source.get('url'):
+                media["images"].append(source['url'].replace('&amp;', '&'))
+    
+    if post_data.get('is_gallery'):
+        gallery_data = post_data.get('gallery_data', {})
+        media_metadata = post_data.get('media_metadata', {})
+        
+        if gallery_data and media_metadata:
+            for item in gallery_data.get('items', []):
+                media_id = item.get('media_id')
+                if media_id and media_id in media_metadata:
+                    meta = media_metadata[media_id]
+                    if meta.get('s', {}).get('u'):
+                        media["galleries"].append(meta['s']['u'].replace('&amp;', '&'))
+    
+    return media
+
+def extract_post_data(p):
+    """Extract post data from JSON."""
+    post_type = "text"
+    if p.get('is_video'):
+        post_type = "video"
+    elif p.get('is_gallery'):
+        post_type = "gallery"
+    elif any(ext in p.get('url', '').lower() for ext in ['.jpg', '.jpeg', '.png', '.gif', '.webp']) or 'i.redd.it' in p.get('url', ''):
+        post_type = "image"
+    elif p.get('is_self'):
+        post_type = "text"
+    else:
+        post_type = "link"
+    
+    return {
+        "id": p.get('id'),
+        "title": p.get('title'),
+        "author": p.get('author'),
+        "created_utc": datetime.datetime.fromtimestamp(p.get('created_utc', 0)).isoformat(),
+        "permalink": p.get('permalink'),
+        "url": p.get('url_overridden_by_dest', p.get('url')),
+        "score": p.get('score', 0),
+        "upvote_ratio": p.get('upvote_ratio', 0),
+        "num_comments": p.get('num_comments', 0),
+        "num_crossposts": p.get('num_crossposts', 0),
+        "selftext": p.get('selftext', ''),
+        "post_type": post_type,
+        "is_nsfw": p.get('over_18', False),
+        "is_spoiler": p.get('spoiler', False),
+        "flair": p.get('link_flair_text', ''),
+        "total_awards": p.get('total_awards_received', 0),
+        "has_media": p.get('is_video', False) or p.get('is_gallery', False) or 'i.redd.it' in p.get('url', ''),
+        "media_downloaded": False,
+        "source": "Async-Scraper"
+    }
+
+async def scrape_async(target, limit=100, is_user=False, download_media=True, scrape_comments=True):
+    """
+    Main async scraping function.
+    
+    Args:
+        target: Subreddit or username
+        limit: Max posts to scrape
+        is_user: True if scraping a user
+        download_media: Download images/videos
+        scrape_comments: Scrape comments
+    """
+    global semaphore
+    semaphore = asyncio.Semaphore(ASYNC_MAX_CONCURRENT)
+    
+    prefix = "u" if is_user else "r"
+    print(f"🚀 ASYNC Scraper starting for {prefix}/{target}")
+    print(f"   Target: {limit} posts | Media: {download_media} | Comments: {scrape_comments}")
+    print(f"   Concurrency: {ASYNC_MAX_CONCURRENT} simultaneous requests")
+    print("-" * 50)
+    
+    # Setup directories
+    base_dir = f"data/{prefix}_{target}"
+    media_dir = f"{base_dir}/media"
+    images_dir = f"{media_dir}/images"
+    videos_dir = f"{media_dir}/videos"
+    
+    for d in [base_dir, media_dir, images_dir, videos_dir]:
+        os.makedirs(d, exist_ok=True)
+    
+    start_time = time.time()
+    all_posts = []
+    all_comments = []
+    media_tasks = []
+    seen_permalinks = set()
+    
+    # Load existing data
+    posts_file = f"{base_dir}/posts.csv"
+    if os.path.exists(posts_file):
+        try:
+            df = pd.read_csv(posts_file)
+            seen_permalinks = set(df['permalink'].astype(str).tolist())
+            print(f"📚 Loaded {len(seen_permalinks)} existing posts")
+        except:
+            pass
+    
+    async with aiohttp.ClientSession(headers={"User-Agent": USER_AGENT}) as session:
+        after = None
+        total_fetched = 0
+        
+        while total_fetched < limit:
+            # Try mirrors
+            mirrors = MIRRORS.copy()
+            random.shuffle(mirrors)
+            
+            data = None
+            for mirror in mirrors:
+                data = await fetch_posts_page(session, mirror, target, after, is_user)
+                if data:
+                    print(f"✅ Fetched from {mirror}")
+                    break
+            
+            if not data:
+                print("❌ All mirrors failed")
+                break
+            
+            children = data.get('data', {}).get('children', [])
+            if not children:
+                print("🏁 No more posts")
+                break
+            
+            print(f"   Processing {len(children)} posts...")
+            
+            # Process posts
+            batch_posts = []
+            comment_tasks = []
+            
+            for child in children:
+                p = child['data']
+                post = extract_post_data(p)
+                
+                if post['permalink'] in seen_permalinks:
+                    continue
+                
+                seen_permalinks.add(post['permalink'])
+                batch_posts.append(post)
+                
+                # Queue media downloads
+                if download_media:
+                    media = extract_media_urls(p)
+                    
+                    for i, img_url in enumerate(media['images'][:5]):
+                        ext = os.path.splitext(urlparse(img_url).path)[1] or '.jpg'
+                        save_path = f"{images_dir}/{post['id']}_{i}{ext}"
+                        media_tasks.append(download_media_async(session, img_url, save_path))
+                    
+                    for i, img_url in enumerate(media['galleries'][:10]):
+                        save_path = f"{images_dir}/{post['id']}_gallery_{i}.jpg"
+                        media_tasks.append(download_media_async(session, img_url, save_path))
+                    
+                    for i, vid_url in enumerate(media['videos'][:2]):
+                        if 'youtube' not in vid_url:
+                            save_path = f"{videos_dir}/{post['id']}_{i}.mp4"
+                            media_tasks.append(download_media_async(session, vid_url, save_path))
+                
+                # Queue comment fetching
+                if scrape_comments and post['num_comments'] > 0:
+                    comment_tasks.append(fetch_comments_async(session, post['permalink']))
+            
+            all_posts.extend(batch_posts)
+            total_fetched += len(batch_posts)
+            
+            # Fetch comments in parallel
+            if comment_tasks:
+                print(f"   💬 Fetching comments for {len(comment_tasks)} posts...")
+                comment_results = await asyncio.gather(*comment_tasks, return_exceptions=True)
+                for result in comment_results:
+                    if isinstance(result, list):
+                        all_comments.extend(result)
+            
+            print(f"   📊 Progress: {total_fetched}/{limit} posts | {len(all_comments)} comments")
+            
+            after = data.get('data', {}).get('after')
+            if not after:
+                print("🏁 Reached end of available posts")
+                break
+            
+            await asyncio.sleep(1)  # Small delay between pages
+        
+        # Download all media in parallel
+        if media_tasks:
+            print(f"\n🖼️ Downloading {len(media_tasks)} media files in parallel...")
+            media_results = await asyncio.gather(*media_tasks, return_exceptions=True)
+            downloaded = sum(1 for r in media_results if r is True)
+            print(f"   ✅ Downloaded {downloaded}/{len(media_tasks)} files")
+    
+    # Save data
+    if all_posts:
+        df = pd.DataFrame(all_posts)
+        if os.path.exists(posts_file):
+            df.to_csv(posts_file, mode='a', header=False, index=False)
+        else:
+            df.to_csv(posts_file, index=False)
+        print(f"\n💾 Saved {len(all_posts)} posts to {posts_file}")
+    
+    if all_comments:
+        comments_file = f"{base_dir}/comments.csv"
+        df = pd.DataFrame(all_comments)
+        if os.path.exists(comments_file):
+            df.to_csv(comments_file, mode='a', header=False, index=False)
+        else:
+            df.to_csv(comments_file, index=False)
+        print(f"💾 Saved {len(all_comments)} comments")
+    
+    duration = time.time() - start_time
+    
+    print("\n" + "=" * 50)
+    print("✅ ASYNC SCRAPE COMPLETE!")
+    print(f"   📊 Posts: {len(all_posts)}")
+    print(f"   💬 Comments: {len(all_comments)}")
+    print(f"   🖼️ Media: {len(media_tasks)} queued")
+    print(f"   ⏱️ Duration: {duration:.1f}s")
+    print(f"   ⚡ Speed: {len(all_posts) / duration:.1f} posts/sec")
+    
+    return {
+        'posts': len(all_posts),
+        'comments': len(all_comments),
+        'duration': duration
+    }
+
+def run_async_scraper(target, limit=100, is_user=False, download_media=True, scrape_comments=True):
+    """Wrapper to run async scraper from sync code."""
+    return asyncio.run(scrape_async(target, limit, is_user, download_media, scrape_comments))
+
+# CLI for testing
+if __name__ == "__main__":
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Async Reddit Scraper")
+    parser.add_argument("target", help="Subreddit or username")
+    parser.add_argument("--limit", type=int, default=100)
+    parser.add_argument("--user", action="store_true")
+    parser.add_argument("--no-media", action="store_true")
+    parser.add_argument("--no-comments", action="store_true")
+    
+    args = parser.parse_args()
+    
+    run_async_scraper(
+        args.target,
+        args.limit,
+        args.user,
+        not args.no_media,
+        not args.no_comments
+    )