Initial release of Universal Scraper

.gitignore

@@ -0,0 +1,3 @@
+data/
+__pycache__/
+.env

Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.9-slim
+ENV PYTHONUNBUFFERED=1
+WORKDIR /app
+COPY requirements.txt .
+COPY main.py .
+RUN pip install --no-cache-dir -r requirements.txt
+RUN mkdir data
+ENTRYPOINT ["python", "main.py"]

README.md

@@ -0,0 +1,75 @@
+# 🤖 Universal Reddit Scraper
+
+A robust, dual-mode Reddit scraper designed to run on low-resource servers (like AWS Free Tier). 
+
+## Features
+- **Zero API Keys Needed:** Uses RSS feeds and Public Mirrors (Redlib) to bypass API limits.
+- **Dual Modes:**
+  - `monitor`: Runs 24/7 to catch new posts via RSS.
+  - `history`: Digs into the past using mirror rotation to bypass IP blocks.
+- **Universal:** Works on any Subreddit (`r/name`) or User (`u/name`).
+- **Dockerized:** Ready to deploy in an isolated container.
+
+## Usage
+
+### 1. Run via Docker (Recommended)
+```bash
+# Build
+docker build -t reddit-scraper .
+
+# Monitor a Subreddit (e.g., r/CreditCardsIndia)
+docker run -d -v $(pwd)/data:/app/data reddit-scraper python main.py CreditCardsIndia --mode monitor
+
+# Monitor a User (e.g., u/spez)
+docker run -d -v $(pwd)/data:/app/data reddit-scraper python main.py spez --user --mode monitor
+
+# Scrape History (Last 1000 posts)
+docker run --rm -v $(pwd)/data:/app/data reddit-scraper python main.py CreditCardsIndia --mode history --limit 1000
+```
+
+### 2. Run Locally (Without Docker)
+```bash
+# Install dependencies
+pip install -r requirements.txt
+
+# Monitor a Subreddit
+python main.py python --mode monitor
+
+# Monitor a User
+python main.py spez --user --mode monitor
+
+# Scrape History
+python main.py python --mode history --limit 500
+```
+
+## Output
+Data is saved to the `/data` folder in CSV format:
+- `data/r_CreditCardsIndia.csv`
+- `data/u_spez.csv`
+
+## Command Line Options
+| Option | Description | Default |
+|--------|-------------|---------|
+| `target` | Name of Subreddit or User to scrape | Required |
+| `--mode` | `monitor` or `history` | `monitor` |
+| `--user` | Flag if target is a User, not Subreddit | `false` |
+| `--limit` | Max posts to scrape (History mode only) | `500` |
+
+## How It Works
+
+### Monitor Mode (RSS)
+- Polls Reddit's public RSS feed every 5 minutes
+- Catches new posts in real-time
+- Uses official Reddit RSS endpoints (no rate limits)
+
+### History Mode (Mirrors)
+- Uses public Redlib mirrors to access historical data
+- Rotates between multiple mirrors to avoid IP blocks
+- Implements cooldown periods to respect rate limits
+- Automatically resumes from where it left off
+
+## License
+MIT License - Feel free to use, modify, and distribute.
+
+## Contributing
+Pull requests are welcome! For major changes, please open an issue first to discuss what you would like to change.

main.py

@@ -0,0 +1,188 @@
+import requests
+import pandas as pd
+import datetime
+import time
+import os
+import xml.etree.ElementTree as ET
+import argparse
+import random
+import sys
+
+# --- CONFIGURATION ---
+USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+
+# Public Mirrors to bypass AWS/Data Center IP Blocks
+MIRRORS = [
+    "https://redlib.catsarch.com",
+    "https://redlib.vsls.cz",
+    "https://r.nf",
+    "https://libreddit.northboot.xyz",
+    "https://redlib.tux.pizza"
+]
+
+SEEN_URLS = set()
+
+def get_file_path(target, type_prefix):
+    """Generates a dynamic filename (e.g., data/r_python.csv or data/u_elonmusk.csv)"""
+    # Ensure data directory exists
+    if not os.path.exists("data"):
+        os.makedirs("data")
+    
+    sanitized_target = target.replace("/", "_")
+    return f"data/{type_prefix}_{sanitized_target}.csv"
+
+def load_history(filepath):
+    """Loads existing CSV history to prevent duplicates."""
+    SEEN_URLS.clear() # Reset for new target
+    if os.path.exists(filepath):
+        try:
+            df = pd.read_csv(filepath)
+            for url in df['URL']:
+                SEEN_URLS.add(str(url))
+            print(f"📚 Loaded {len(SEEN_URLS)} existing items from {filepath}")
+        except:
+            pass
+
+def save_to_csv(items, filepath):
+    """Appends new unique items to the specific CSV file."""
+    if not items:
+        return 0
+    
+    new_items = [i for i in items if i['URL'] not in SEEN_URLS]
+    
+    if new_items:
+        df = pd.DataFrame(new_items)
+        if os.path.exists(filepath):
+            df.to_csv(filepath, mode='a', header=False, index=False)
+        else:
+            df.to_csv(filepath, index=False)
+        
+        for i in new_items:
+            SEEN_URLS.add(i['URL'])
+            
+        print(f"✅ Saved {len(new_items)} new items to {filepath}")
+        return len(new_items)
+    else:
+        print("💤 No new unique data found.")
+        return 0
+
+# --- MODE 1: LIVE MONITOR (RSS) ---
+def run_monitor(target, is_user=False):
+    prefix = "u" if is_user else "r"
+    # Reddit RSS format: /r/sub/new.rss OR /user/name/submitted.rss
+    if is_user:
+        rss_url = f"https://www.reddit.com/user/{target}/submitted.rss?limit=100"
+    else:
+        rss_url = f"https://www.reddit.com/r/{target}/new.rss?limit=100"
+
+    print(f"[{datetime.datetime.now()}] 📡 Checking RSS Feed for {prefix}/{target}...")
+    
+    try:
+        headers = {"User-Agent": USER_AGENT}
+        response = requests.get(rss_url, headers=headers, timeout=15)
+        
+        if response.status_code != 200:
+            print(f"❌ Blocked/Error (Status {response.status_code})")
+            return
+
+        root = ET.fromstring(response.content)
+        namespace = {'atom': 'http://www.w3.org/2005/Atom'}
+        posts = []
+        
+        for entry in root.findall('atom:entry', namespace):
+            posts.append({
+                "Date": entry.find('atom:published', namespace).text,
+                "Title": entry.find('atom:title', namespace).text,
+                "URL": entry.find('atom:link', namespace).attrib['href'],
+                "Source": "Monitor-RSS"
+            })
+            
+        save_to_csv(posts, get_file_path(target, prefix))
+
+    except Exception as e:
+        print(f"❌ Monitor Error: {e}")
+
+# --- MODE 2: HISTORY SCRAPE (Mirrors) ---
+def run_history(target, limit, is_user=False):
+    prefix = "u" if is_user else "r"
+    print(f"🚀 Starting HISTORY scrape for {prefix}/{target} (Target: {limit})...")
+    
+    filepath = get_file_path(target, prefix)
+    load_history(filepath)
+    
+    after = None
+    count = 0
+    
+    while count < limit:
+        random.shuffle(MIRRORS)
+        success = False
+        
+        for base_url in MIRRORS:
+            try:
+                # Mirror URL format differs slightly for users
+                if is_user:
+                    path = f"/user/{target}/submitted.json"
+                else:
+                    path = f"/r/{target}/new.json"
+                
+                target_url = f"{base_url}{path}?limit=100"
+                if after:
+                    target_url += f"&after={after}"
+                
+                print(f"📡 Requesting: {base_url} (Page Token: {after})")
+                response = requests.get(target_url, headers={"User-Agent": USER_AGENT}, timeout=10)
+                
+                if response.status_code == 200:
+                    data = response.json()
+                    posts = []
+                    
+                    for child in data['data']['children']:
+                        p = child['data']
+                        date_str = datetime.datetime.fromtimestamp(p['created_utc']).isoformat()
+                        posts.append({
+                            "Date": date_str,
+                            "Title": p['title'],
+                            "URL": p.get('url_overridden_by_dest', p.get('url')),
+                            "Source": "History-Mirror"
+                        })
+                    
+                    saved = save_to_csv(posts, filepath)
+                    count += saved
+                    
+                    after = data['data'].get('after')
+                    if not after:
+                        print("🏁 Reached end of history.")
+                        return
+                    
+                    success = True
+                    break 
+            except:
+                continue 
+        
+        if not success:
+            print("❌ All mirrors failed. Waiting 30s...")
+            time.sleep(30)
+        else:
+            print(f"⏸️ Cooling down (5s)... Total: {count}")
+            time.sleep(5)
+
+# --- CLI ARGS ---
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Universal Reddit Scraper (Open Source)")
+    parser.add_argument("target", help="Name of Subreddit (e.g. 'python') or User (e.g. 'spez')")
+    parser.add_argument("--mode", choices=["monitor", "history"], default="monitor", help="Run mode")
+    parser.add_argument("--user", action="store_true", help="Flag if the target is a User, not a Subreddit")
+    parser.add_argument("--limit", type=int, default=500, help="Max posts to scrape (History mode only)")
+    
+    args = parser.parse_args()
+    
+    # Pre-load history for monitor mode
+    if args.mode == "monitor":
+        prefix = "u" if args.user else "r"
+        load_history(get_file_path(args.target, prefix))
+        print(f"🤖 Monitoring {prefix}/{args.target} every 5 mins...")
+        while True:
+            run_monitor(args.target, args.user)
+            time.sleep(300)
+    else:
+        run_history(args.target, args.limit, args.user)

requirements.txt

@@ -0,0 +1,2 @@
+pandas
+requests