From db4e363130ae8f78a20315583f3c76847a7ad78d Mon Sep 17 00:00:00 2001 From: Sanjeev Kumar Date: Sat, 13 Dec 2025 21:51:02 +0530 Subject: [PATCH] Initial release of Universal Scraper --- .gitignore | 3 + Dockerfile | 8 ++ README.md | 75 +++++++++++++++++++ main.py | 188 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 + 5 files changed, 276 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 main.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..81f405e --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +data/ +__pycache__/ +.env diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..39a0129 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.9-slim +ENV PYTHONUNBUFFERED=1 +WORKDIR /app +COPY requirements.txt . +COPY main.py . +RUN pip install --no-cache-dir -r requirements.txt +RUN mkdir data +ENTRYPOINT ["python", "main.py"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..a75cd32 --- /dev/null +++ b/README.md @@ -0,0 +1,75 @@ +# πŸ€– Universal Reddit Scraper + +A robust, dual-mode Reddit scraper designed to run on low-resource servers (like AWS Free Tier). + +## Features +- **Zero API Keys Needed:** Uses RSS feeds and Public Mirrors (Redlib) to bypass API limits. +- **Dual Modes:** + - `monitor`: Runs 24/7 to catch new posts via RSS. + - `history`: Digs into the past using mirror rotation to bypass IP blocks. +- **Universal:** Works on any Subreddit (`r/name`) or User (`u/name`). +- **Dockerized:** Ready to deploy in an isolated container. + +## Usage + +### 1. Run via Docker (Recommended) +```bash +# Build +docker build -t reddit-scraper . + +# Monitor a Subreddit (e.g., r/CreditCardsIndia) +docker run -d -v $(pwd)/data:/app/data reddit-scraper python main.py CreditCardsIndia --mode monitor + +# Monitor a User (e.g., u/spez) +docker run -d -v $(pwd)/data:/app/data reddit-scraper python main.py spez --user --mode monitor + +# Scrape History (Last 1000 posts) +docker run --rm -v $(pwd)/data:/app/data reddit-scraper python main.py CreditCardsIndia --mode history --limit 1000 +``` + +### 2. Run Locally (Without Docker) +```bash +# Install dependencies +pip install -r requirements.txt + +# Monitor a Subreddit +python main.py python --mode monitor + +# Monitor a User +python main.py spez --user --mode monitor + +# Scrape History +python main.py python --mode history --limit 500 +``` + +## Output +Data is saved to the `/data` folder in CSV format: +- `data/r_CreditCardsIndia.csv` +- `data/u_spez.csv` + +## Command Line Options +| Option | Description | Default | +|--------|-------------|---------| +| `target` | Name of Subreddit or User to scrape | Required | +| `--mode` | `monitor` or `history` | `monitor` | +| `--user` | Flag if target is a User, not Subreddit | `false` | +| `--limit` | Max posts to scrape (History mode only) | `500` | + +## How It Works + +### Monitor Mode (RSS) +- Polls Reddit's public RSS feed every 5 minutes +- Catches new posts in real-time +- Uses official Reddit RSS endpoints (no rate limits) + +### History Mode (Mirrors) +- Uses public Redlib mirrors to access historical data +- Rotates between multiple mirrors to avoid IP blocks +- Implements cooldown periods to respect rate limits +- Automatically resumes from where it left off + +## License +MIT License - Feel free to use, modify, and distribute. + +## Contributing +Pull requests are welcome! For major changes, please open an issue first to discuss what you would like to change. diff --git a/main.py b/main.py new file mode 100644 index 0000000..9c62489 --- /dev/null +++ b/main.py @@ -0,0 +1,188 @@ +import requests +import pandas as pd +import datetime +import time +import os +import xml.etree.ElementTree as ET +import argparse +import random +import sys + +# --- CONFIGURATION --- +USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + +# Public Mirrors to bypass AWS/Data Center IP Blocks +MIRRORS = [ + "https://redlib.catsarch.com", + "https://redlib.vsls.cz", + "https://r.nf", + "https://libreddit.northboot.xyz", + "https://redlib.tux.pizza" +] + +SEEN_URLS = set() + +def get_file_path(target, type_prefix): + """Generates a dynamic filename (e.g., data/r_python.csv or data/u_elonmusk.csv)""" + # Ensure data directory exists + if not os.path.exists("data"): + os.makedirs("data") + + sanitized_target = target.replace("/", "_") + return f"data/{type_prefix}_{sanitized_target}.csv" + +def load_history(filepath): + """Loads existing CSV history to prevent duplicates.""" + SEEN_URLS.clear() # Reset for new target + if os.path.exists(filepath): + try: + df = pd.read_csv(filepath) + for url in df['URL']: + SEEN_URLS.add(str(url)) + print(f"πŸ“š Loaded {len(SEEN_URLS)} existing items from {filepath}") + except: + pass + +def save_to_csv(items, filepath): + """Appends new unique items to the specific CSV file.""" + if not items: + return 0 + + new_items = [i for i in items if i['URL'] not in SEEN_URLS] + + if new_items: + df = pd.DataFrame(new_items) + if os.path.exists(filepath): + df.to_csv(filepath, mode='a', header=False, index=False) + else: + df.to_csv(filepath, index=False) + + for i in new_items: + SEEN_URLS.add(i['URL']) + + print(f"βœ… Saved {len(new_items)} new items to {filepath}") + return len(new_items) + else: + print("πŸ’€ No new unique data found.") + return 0 + +# --- MODE 1: LIVE MONITOR (RSS) --- +def run_monitor(target, is_user=False): + prefix = "u" if is_user else "r" + # Reddit RSS format: /r/sub/new.rss OR /user/name/submitted.rss + if is_user: + rss_url = f"https://www.reddit.com/user/{target}/submitted.rss?limit=100" + else: + rss_url = f"https://www.reddit.com/r/{target}/new.rss?limit=100" + + print(f"[{datetime.datetime.now()}] πŸ“‘ Checking RSS Feed for {prefix}/{target}...") + + try: + headers = {"User-Agent": USER_AGENT} + response = requests.get(rss_url, headers=headers, timeout=15) + + if response.status_code != 200: + print(f"❌ Blocked/Error (Status {response.status_code})") + return + + root = ET.fromstring(response.content) + namespace = {'atom': 'http://www.w3.org/2005/Atom'} + posts = [] + + for entry in root.findall('atom:entry', namespace): + posts.append({ + "Date": entry.find('atom:published', namespace).text, + "Title": entry.find('atom:title', namespace).text, + "URL": entry.find('atom:link', namespace).attrib['href'], + "Source": "Monitor-RSS" + }) + + save_to_csv(posts, get_file_path(target, prefix)) + + except Exception as e: + print(f"❌ Monitor Error: {e}") + +# --- MODE 2: HISTORY SCRAPE (Mirrors) --- +def run_history(target, limit, is_user=False): + prefix = "u" if is_user else "r" + print(f"πŸš€ Starting HISTORY scrape for {prefix}/{target} (Target: {limit})...") + + filepath = get_file_path(target, prefix) + load_history(filepath) + + after = None + count = 0 + + while count < limit: + random.shuffle(MIRRORS) + success = False + + for base_url in MIRRORS: + try: + # Mirror URL format differs slightly for users + if is_user: + path = f"/user/{target}/submitted.json" + else: + path = f"/r/{target}/new.json" + + target_url = f"{base_url}{path}?limit=100" + if after: + target_url += f"&after={after}" + + print(f"πŸ“‘ Requesting: {base_url} (Page Token: {after})") + response = requests.get(target_url, headers={"User-Agent": USER_AGENT}, timeout=10) + + if response.status_code == 200: + data = response.json() + posts = [] + + for child in data['data']['children']: + p = child['data'] + date_str = datetime.datetime.fromtimestamp(p['created_utc']).isoformat() + posts.append({ + "Date": date_str, + "Title": p['title'], + "URL": p.get('url_overridden_by_dest', p.get('url')), + "Source": "History-Mirror" + }) + + saved = save_to_csv(posts, filepath) + count += saved + + after = data['data'].get('after') + if not after: + print("🏁 Reached end of history.") + return + + success = True + break + except: + continue + + if not success: + print("❌ All mirrors failed. Waiting 30s...") + time.sleep(30) + else: + print(f"⏸️ Cooling down (5s)... Total: {count}") + time.sleep(5) + +# --- CLI ARGS --- +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Universal Reddit Scraper (Open Source)") + parser.add_argument("target", help="Name of Subreddit (e.g. 'python') or User (e.g. 'spez')") + parser.add_argument("--mode", choices=["monitor", "history"], default="monitor", help="Run mode") + parser.add_argument("--user", action="store_true", help="Flag if the target is a User, not a Subreddit") + parser.add_argument("--limit", type=int, default=500, help="Max posts to scrape (History mode only)") + + args = parser.parse_args() + + # Pre-load history for monitor mode + if args.mode == "monitor": + prefix = "u" if args.user else "r" + load_history(get_file_path(args.target, prefix)) + print(f"πŸ€– Monitoring {prefix}/{args.target} every 5 mins...") + while True: + run_monitor(args.target, args.user) + time.sleep(300) + else: + run_history(args.target, args.limit, args.user) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..69de461 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +pandas +requests