f65b35f881
- Job history tracking with SQLite table - Dry-run mode (--dry-run) to test scrape rules - Plugin system with 3 built-in plugins (sentiment, dedupe, keywords) - REST API server (--api) for Metabase/Grafana integration - Parquet export (--export-parquet) for DuckDB/warehouses - SQLite maintenance (--backup, --vacuum) - Dashboard Integrations tab with external tools guides - Updated Dockerfile and docker-compose.yml for cloud deployment - Comprehensive README documentation
45 lines
1.1 KiB
Python
45 lines
1.1 KiB
Python
"""
|
|
Deduplicator Plugin
|
|
Removes duplicate posts based on permalink.
|
|
"""
|
|
from plugins import Plugin
|
|
|
|
|
|
class Deduplicator(Plugin):
|
|
"""Remove duplicate posts by permalink."""
|
|
|
|
name = "deduplicator"
|
|
description = "Removes duplicate posts by permalink"
|
|
enabled = True
|
|
|
|
def process_posts(self, posts):
|
|
"""Remove duplicate posts."""
|
|
seen = set()
|
|
unique = []
|
|
duplicates = 0
|
|
|
|
for post in posts:
|
|
key = post.get('permalink')
|
|
if key and key not in seen:
|
|
seen.add(key)
|
|
unique.append(post)
|
|
else:
|
|
duplicates += 1
|
|
|
|
if duplicates > 0:
|
|
print(f" 🔄 Removed {duplicates} duplicate posts")
|
|
|
|
return unique
|
|
|
|
def process_comments(self, comments):
|
|
"""Remove duplicate comments."""
|
|
seen = set()
|
|
unique = []
|
|
|
|
for comment in comments:
|
|
key = comment.get('comment_id')
|
|
if key and key not in seen:
|
|
seen.add(key)
|
|
unique.append(comment)
|
|
|
|
return unique
|