f65b35f881
- Job history tracking with SQLite table - Dry-run mode (--dry-run) to test scrape rules - Plugin system with 3 built-in plugins (sentiment, dedupe, keywords) - REST API server (--api) for Metabase/Grafana integration - Parquet export (--export-parquet) for DuckDB/warehouses - SQLite maintenance (--backup, --vacuum) - Dashboard Integrations tab with external tools guides - Updated Dockerfile and docker-compose.yml for cloud deployment - Comprehensive README documentation
35 lines
1.1 KiB
Python
35 lines
1.1 KiB
Python
"""
|
|
Keyword Extractor Plugin
|
|
Extracts and tags posts with top keywords.
|
|
"""
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
|
|
from plugins import Plugin
|
|
from analytics.sentiment import extract_keywords
|
|
|
|
|
|
class KeywordExtractor(Plugin):
|
|
"""Extract and add keywords to posts."""
|
|
|
|
name = "keyword_extractor"
|
|
description = "Adds top keywords to each post"
|
|
enabled = True
|
|
top_n = 5 # Number of keywords per post
|
|
|
|
def process_posts(self, posts):
|
|
"""Add keywords to each post."""
|
|
for post in posts:
|
|
text = f"{post.get('title', '')} {post.get('selftext', '')}"
|
|
keywords = extract_keywords([text], top_n=self.top_n)
|
|
post['keywords'] = ','.join([kw for kw, count in keywords])
|
|
|
|
# Also extract global keywords
|
|
all_texts = [f"{p.get('title', '')} {p.get('selftext', '')}" for p in posts]
|
|
global_keywords = extract_keywords(all_texts, top_n=10)
|
|
|
|
print(f" 🏷️ Top keywords: {', '.join([kw for kw, _ in global_keywords[:5]])}")
|
|
|
|
return posts
|