471 lines
16 KiB
JavaScript
471 lines
16 KiB
JavaScript
import { fetchWithCors } from '../utils/corsProxy';
|
|
import { RSS_TO_JSON_API, debugLog, debugWarn } from '../utils/constants';
|
|
|
|
/**
|
|
* Determine source type from URL
|
|
* @param {string} url - The URL to analyze
|
|
* @returns {string} - 'json' | 'rss' | 'html'
|
|
*/
|
|
export function detectSourceType(url) {
|
|
const urlLower = url.toLowerCase();
|
|
|
|
if (urlLower.includes('.json')) return 'json';
|
|
if (urlLower.includes('.rss') ||
|
|
urlLower.includes('.xml') ||
|
|
urlLower.includes('/feed') ||
|
|
urlLower.includes('rss')) return 'rss';
|
|
|
|
return 'html'; // Default to HTML scraping
|
|
}
|
|
|
|
/**
|
|
* Parse JSON endpoint
|
|
* @param {string} url - The JSON URL
|
|
* @param {string} corsProxy - CORS proxy URL
|
|
* @returns {Promise<any>} - Parsed JSON data
|
|
*/
|
|
export async function parseJSON(url, corsProxy) {
|
|
try {
|
|
// Reddit's JSON API blocks unauthenticated browser CORS requests (returns 403).
|
|
// Route through the local proxy server, which fetches server-side with a proper
|
|
// User-Agent header that Reddit accepts.
|
|
if (url.includes('reddit.com')) {
|
|
const proxyPort = import.meta.env.REACT_APP_PROXY_PORT || '5374';
|
|
const proxyUrl = `http://localhost:${proxyPort}/api/scrape?url=${encodeURIComponent(url)}`;
|
|
const response = await fetch(proxyUrl);
|
|
if (!response.ok) {
|
|
throw new Error(`Local proxy returned ${response.status} for Reddit URL`);
|
|
}
|
|
const text = await response.text();
|
|
return JSON.parse(text);
|
|
}
|
|
|
|
const data = await fetchWithCors(url);
|
|
return data;
|
|
} catch (error) {
|
|
console.error(`Error parsing JSON from ${url}:`, error);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Parse RSS feed
|
|
* @param {string} url - The RSS feed URL
|
|
* @returns {Promise<any>} - Parsed RSS data
|
|
*/
|
|
export async function parseRSS(url) {
|
|
try {
|
|
// Use RSS to JSON service
|
|
const rssUrl = `${RSS_TO_JSON_API}?rss_url=${encodeURIComponent(url)}`;
|
|
const response = await fetch(rssUrl);
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`RSS parse failed: ${response.status}`);
|
|
}
|
|
|
|
const data = await response.json();
|
|
|
|
// Check if RSS2JSON returned an error
|
|
if (data.status !== 'ok') {
|
|
throw new Error(data.message || 'RSS feed parsing failed');
|
|
}
|
|
|
|
return data;
|
|
} catch (error) {
|
|
console.error(`Error parsing RSS from ${url}:`, error);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Strip HTML tags and extract meaningful text content
|
|
* @param {string} html - Raw HTML string
|
|
* @returns {string} - Cleaned text content
|
|
*/
|
|
export function cleanHTML(html) {
|
|
if (typeof html !== 'string') return '';
|
|
|
|
let text = html;
|
|
|
|
// Remove script, style, nav, header, footer, aside blocks entirely
|
|
text = text.replace(/<script[\s\S]*?<\/script>/gi, '');
|
|
text = text.replace(/<style[\s\S]*?<\/style>/gi, '');
|
|
text = text.replace(/<nav[\s\S]*?<\/nav>/gi, '');
|
|
text = text.replace(/<footer[\s\S]*?<\/footer>/gi, '');
|
|
text = text.replace(/<aside[\s\S]*?<\/aside>/gi, '');
|
|
text = text.replace(/<noscript[\s\S]*?<\/noscript>/gi, '');
|
|
text = text.replace(/<svg[\s\S]*?<\/svg>/gi, '');
|
|
|
|
// Replace block-level tags with newlines to preserve structure
|
|
text = text.replace(/<\/?(div|p|br|h[1-6]|li|tr|blockquote|section|article)[^>]*>/gi, '\n');
|
|
|
|
// Strip all remaining HTML tags
|
|
text = text.replace(/<[^>]+>/g, ' ');
|
|
|
|
// Decode common HTML entities
|
|
text = text.replace(/&/g, '&');
|
|
text = text.replace(/</g, '<');
|
|
text = text.replace(/>/g, '>');
|
|
text = text.replace(/"/g, '"');
|
|
text = text.replace(/'/g, "'");
|
|
text = text.replace(/ /g, ' ');
|
|
text = text.replace(/&#\d+;/g, '');
|
|
|
|
// Collapse whitespace: multiple spaces → single, multiple newlines → double
|
|
text = text.replace(/[ \t]+/g, ' ');
|
|
text = text.replace(/\n[ \t]+/g, '\n');
|
|
text = text.replace(/\n{3,}/g, '\n\n');
|
|
|
|
return text.trim();
|
|
}
|
|
|
|
/**
|
|
* Parse HTML content via the local proxy server (avoids CORS issues)
|
|
* Falls back to fetchWithCors if the proxy is unavailable.
|
|
* @param {string} url - The HTML page URL
|
|
* @param {string} corsProxy - CORS proxy URL (unused, kept for API compat)
|
|
* @returns {Promise<Object>} - Object with URL and cleaned text content
|
|
*/
|
|
export async function parseHTML(url, corsProxy) {
|
|
let rawHTML;
|
|
|
|
try {
|
|
// Use local proxy server to avoid CORS issues with arbitrary websites
|
|
const proxyPort = import.meta.env.REACT_APP_PROXY_PORT || '5374';
|
|
const proxyUrl = `http://localhost:${proxyPort}/api/scrape?url=${encodeURIComponent(url)}`;
|
|
const response = await fetch(proxyUrl);
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`Scrape proxy returned ${response.status}`);
|
|
}
|
|
|
|
rawHTML = await response.text();
|
|
} catch (proxyError) {
|
|
debugWarn(`Local scrape proxy failed for ${url}, falling back to CORS proxy:`, proxyError.message);
|
|
try {
|
|
const html = await fetchWithCors(url);
|
|
rawHTML = typeof html === 'string' ? html : JSON.stringify(html);
|
|
} catch (corsError) {
|
|
console.error(`All fetch methods failed for ${url}:`, corsError);
|
|
throw corsError;
|
|
}
|
|
}
|
|
|
|
const cleanedText = cleanHTML(rawHTML);
|
|
|
|
return {
|
|
url: url,
|
|
html: rawHTML,
|
|
cleanedText: cleanedText,
|
|
type: 'html',
|
|
timestamp: Date.now()
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Parse a source based on its detected type
|
|
* @param {string} url - The URL to parse
|
|
* @param {string} type - Optional type override
|
|
* @returns {Promise<any>} - Parsed data
|
|
*/
|
|
export async function parseSource(url, type = null) {
|
|
const sourceType = type || detectSourceType(url);
|
|
|
|
switch(sourceType) {
|
|
case 'json':
|
|
return await parseJSON(url);
|
|
|
|
case 'rss':
|
|
return await parseRSS(url);
|
|
|
|
case 'html':
|
|
return await parseHTML(url);
|
|
|
|
default:
|
|
throw new Error(`Unknown source type: ${sourceType}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Extract meaningful content from HTML using Claude
|
|
* This will be called later when Claude service is integrated
|
|
* @param {string} html - The HTML content
|
|
* @param {string} url - The source URL
|
|
* @returns {Promise<Object>} - Extracted content
|
|
*/
|
|
export async function extractWithClaude(html, url) {
|
|
// This function will be implemented when Claude service is ready
|
|
// For now, return a placeholder structure
|
|
return {
|
|
url: url,
|
|
content: html.substring(0, 5000), // Limit content for now
|
|
extractedAt: Date.now(),
|
|
needsClaudeProcessing: true
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Condense data for Claude to reduce payload size
|
|
* @param {Object} data - Full source data
|
|
* @returns {Object} - Condensed data suitable for Claude
|
|
*/
|
|
export function condenseForClaude(data) {
|
|
if (!data) return null;
|
|
|
|
// Handle weather data - already compact
|
|
if (data.type === 'weather') {
|
|
return {
|
|
type: 'weather',
|
|
location: data.location,
|
|
current: {
|
|
temp: data.current?.temp,
|
|
description: data.current?.description,
|
|
humidity: data.current?.humidity,
|
|
windSpeed: data.current?.windSpeed
|
|
},
|
|
forecast: data.forecast?.slice(0, 3)?.map(day => ({
|
|
date: day.date,
|
|
maxTemp: day.maxTemp,
|
|
minTemp: day.minTemp,
|
|
description: day.description
|
|
}))
|
|
};
|
|
}
|
|
|
|
// Handle Reddit data - needs major condensing
|
|
if (data.type === 'reddit') {
|
|
return {
|
|
type: 'reddit',
|
|
url: data.url,
|
|
posts: data.posts?.slice(0, 10)?.map(post => ({
|
|
title: post.title,
|
|
score: post.score,
|
|
comments: post.comments,
|
|
subreddit: post.subreddit,
|
|
// Truncate selftext to 200 chars
|
|
summary: post.selftext ?
|
|
(post.selftext.length > 200 ?
|
|
post.selftext.substring(0, 197) + '...' :
|
|
post.selftext) : null,
|
|
url: post.url
|
|
}))
|
|
};
|
|
}
|
|
|
|
// Handle RSS feeds
|
|
if (data.type === 'rss') {
|
|
return {
|
|
type: 'rss',
|
|
feedTitle: data.feedTitle,
|
|
items: data.items?.slice(0, 10)?.map(item => ({
|
|
title: item.title,
|
|
// Truncate description to 150 chars
|
|
summary: item.description ?
|
|
(item.description.length > 150 ?
|
|
item.description.substring(0, 147) + '...' :
|
|
item.description) : null,
|
|
pubDate: item.pubDate
|
|
}))
|
|
};
|
|
}
|
|
|
|
// Handle webpage type (Claude-summarized HTML)
|
|
if (data.type === 'webpage') {
|
|
return {
|
|
type: 'webpage',
|
|
url: data.url,
|
|
title: data.title,
|
|
summary: data.summary,
|
|
items: data.items?.slice(0, 5)?.map(item => ({
|
|
title: item.title,
|
|
date: item.date,
|
|
summary: item.summary ?
|
|
(item.summary.length > 200 ?
|
|
item.summary.substring(0, 197) + '...' :
|
|
item.summary) : null
|
|
}))
|
|
};
|
|
}
|
|
|
|
// Default for unknown types - just truncate if it's large
|
|
if (data.type === 'unknown' && data.data) {
|
|
const dataStr = JSON.stringify(data.data);
|
|
if (dataStr.length > 1000) {
|
|
return {
|
|
type: 'unknown',
|
|
url: data.url,
|
|
summary: 'Data truncated for size',
|
|
sample: dataStr.substring(0, 500) + '...'
|
|
};
|
|
}
|
|
}
|
|
|
|
return data;
|
|
}
|
|
|
|
/**
|
|
* Condense multiple sources for Claude
|
|
* @param {Object} sources - Object containing all source data
|
|
* @returns {Object} - Condensed sources object
|
|
*/
|
|
export function condenseSourcesForClaude(sources) {
|
|
const condensed = {};
|
|
|
|
Object.entries(sources).forEach(([key, value]) => {
|
|
if (Array.isArray(value)) {
|
|
// Handle row arrays
|
|
condensed[key] = value.map(item => condenseForClaude(item));
|
|
} else {
|
|
// Handle individual source objects
|
|
condensed[key] = condenseForClaude(value);
|
|
}
|
|
});
|
|
|
|
return condensed;
|
|
}
|
|
|
|
/**
|
|
* Clean and standardize source data
|
|
* @param {any} data - Raw source data
|
|
* @param {string} sourceUrl - The source URL
|
|
* @returns {Object} - Standardized data object
|
|
*/
|
|
export function standardizeSourceData(data, sourceUrl) {
|
|
// Detect the source and format accordingly
|
|
if (sourceUrl.includes('reddit.com') && data.data) {
|
|
// Reddit JSON format
|
|
return {
|
|
type: 'reddit',
|
|
url: sourceUrl,
|
|
posts: data.data.children?.map(child => {
|
|
const post = child.data;
|
|
|
|
// Extract image URLs from different possible locations
|
|
let images = [];
|
|
let mainImage = null;
|
|
|
|
// Check for direct image URL
|
|
if (post.url && post.url.match(/\.(jpg|jpeg|png|gif|webp)$/i)) {
|
|
mainImage = post.url;
|
|
} else if (post.url_overridden_by_dest && post.url_overridden_by_dest.match(/\.(jpg|jpeg|png|gif|webp)$/i)) {
|
|
mainImage = post.url_overridden_by_dest;
|
|
}
|
|
|
|
// Check preview object for images (most reliable source for Reddit images)
|
|
if (!mainImage && post.preview && post.preview.images && post.preview.images.length > 0) {
|
|
const previewImage = post.preview.images[0];
|
|
|
|
// Get the smallest resolution that's still reasonable quality
|
|
// Resolutions array is sorted from smallest to largest
|
|
if (previewImage.resolutions && previewImage.resolutions.length > 0) {
|
|
// Get resolution around 320-640px wide for optimal loading
|
|
const targetRes = previewImage.resolutions.find(r => r.width >= 320 && r.width <= 640)
|
|
|| previewImage.resolutions[previewImage.resolutions.length - 1]
|
|
|| previewImage.resolutions[0];
|
|
mainImage = targetRes.url.replace(/&/g, '&');
|
|
} else if (previewImage.source) {
|
|
// Fallback to source if no resolutions
|
|
mainImage = previewImage.source.url.replace(/&/g, '&');
|
|
}
|
|
}
|
|
|
|
// Check for gallery images
|
|
if (post.is_gallery && post.media_metadata) {
|
|
images = Object.values(post.media_metadata).map(media => {
|
|
// Get smallest reasonable resolution for gallery images
|
|
if (media.p && media.p.length > 0) {
|
|
// Find preview around 320-640px or use middle resolution
|
|
const targetPreview = media.p.find(p => p.x >= 320 && p.x <= 640)
|
|
|| media.p[Math.floor(media.p.length / 2)]
|
|
|| media.p[0];
|
|
return targetPreview.u.replace(/&/g, '&');
|
|
} else if (media.s && media.s.u) {
|
|
// Fallback to source
|
|
return media.s.u.replace(/&/g, '&');
|
|
}
|
|
return null;
|
|
}).filter(url => url !== null);
|
|
|
|
// Use first gallery image as main if no other main image
|
|
if (!mainImage && images.length > 0) {
|
|
mainImage = images[0];
|
|
}
|
|
}
|
|
|
|
// Check for regular thumbnail (not "self", "default", "nsfw", "spoiler", etc.)
|
|
let thumbnail = null;
|
|
if (post.thumbnail && post.thumbnail.startsWith('http')) {
|
|
thumbnail = post.thumbnail;
|
|
}
|
|
|
|
// Use thumbnail as main image if no better option
|
|
if (!mainImage && thumbnail &&
|
|
!thumbnail.includes('external-preview') &&
|
|
thumbnail !== 'self' &&
|
|
thumbnail !== 'default') {
|
|
mainImage = thumbnail;
|
|
}
|
|
|
|
return {
|
|
title: post.title,
|
|
url: post.url,
|
|
permalink: `https://reddit.com${post.permalink}`,
|
|
author: post.author,
|
|
score: post.score,
|
|
comments: post.num_comments,
|
|
created: post.created_utc,
|
|
thumbnail: thumbnail,
|
|
mainImage: mainImage,
|
|
galleryImages: images,
|
|
isGallery: post.is_gallery || false,
|
|
selftext: post.selftext,
|
|
subreddit: post.subreddit
|
|
};
|
|
}) || [],
|
|
after: data.data.after,
|
|
before: data.data.before
|
|
};
|
|
}
|
|
|
|
|
|
if (data.items && data.feed) {
|
|
// RSS feed format from rss2json
|
|
return {
|
|
type: 'rss',
|
|
url: sourceUrl,
|
|
feedTitle: data.feed.title,
|
|
feedUrl: data.feed.url,
|
|
items: data.items.map(item => ({
|
|
title: item.title,
|
|
link: item.link,
|
|
description: item.description,
|
|
pubDate: item.pubDate,
|
|
author: item.author,
|
|
categories: item.categories
|
|
}))
|
|
};
|
|
}
|
|
|
|
// Handle HTML / webpage data (from parseHTML)
|
|
if (data.type === 'html' && data.cleanedText) {
|
|
return {
|
|
type: 'webpage',
|
|
url: sourceUrl,
|
|
cleanedText: data.cleanedText,
|
|
// Will be populated by Claude summarization later
|
|
title: null,
|
|
summary: null,
|
|
items: [],
|
|
needsSummarization: true,
|
|
fetchedAt: Date.now()
|
|
};
|
|
}
|
|
|
|
// Default format for unknown sources
|
|
return {
|
|
type: 'unknown',
|
|
url: sourceUrl,
|
|
data: data,
|
|
fetchedAt: Date.now()
|
|
};
|
|
}
|