added quicklinks, enhanced webscraping w/ proxy
This commit is contained in:
Drew Peifer
parent
a7879d6056
commit
11d510d81d
10 changed files with 526 additions and 78 deletions
|
|
@ -7,7 +7,7 @@ REACT_APP_ANTHROPIC_API_KEY=sk-ant-your-key-here
|
|||
# Example formats:
|
||||
# - Object format: {"row1":["url1","url2"],"row2":["url3","url4"]}
|
||||
# - Legacy format: url1,url2,url3 (comma-separated)
|
||||
REACT_APP_DATA_SOURCES={"row1":["https://www.reddit.com/r/technology.json","https://www.reddit.com/r/worldnews.json"],"row2":["https://www.reddit.com/r/science.json"]}
|
||||
REACT_APP_DATA_SOURCES={"row1":["https://www.reddit.com/r/technology.json","https://www.reddit.com/r/worldnews.json"],"row2":["https://www.reddit.com/r/science.json"],"Games":["https://www.elitedangerous.com/en-US/UpdateNotes"]}
|
||||
|
||||
# Location for weather (supports "city, state, country" or zip code)
|
||||
REACT_APP_LOCATION=harrisburg, pa, usa
|
||||
|
|
@ -39,3 +39,6 @@ REACT_APP_CHART_DATE_FORMAT=MM/DD/YYYY
|
|||
|
||||
# Optional: Default text-to-speech voice (must match exact voice name from browser)
|
||||
REACT_APP_TEXT_VOICE=Google UK English Female
|
||||
|
||||
# Optional: Quick links (comma-separated URLs displayed as buttons in a bar at the top)
|
||||
REACT_APP_QUICK_LINKS=https://github.com/Drewpeifer,https://drewpeifer.com/,https://www.amazon.com/,https://radio.garden/,https://cloudhiker.net/
|
||||
|
|
|
|||
|
|
@ -55,7 +55,42 @@ app.post('/api/claude', async (req, res) => {
|
|||
}
|
||||
});
|
||||
|
||||
// Proxy endpoint for scraping web pages (avoids CORS issues)
|
||||
app.get('/api/scrape', async (req, res) => {
|
||||
try {
|
||||
const { url } = req.query;
|
||||
|
||||
if (!url) {
|
||||
return res.status(400).json({ error: 'Missing url query parameter' });
|
||||
}
|
||||
|
||||
console.log('=== Scrape proxy request ===');
|
||||
console.log('URL:', url);
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.error('Scrape failed:', response.status, response.statusText);
|
||||
return res.status(response.status).json({ error: `Failed to fetch: ${response.status} ${response.statusText}` });
|
||||
}
|
||||
|
||||
const html = await response.text();
|
||||
console.log('Scraped HTML length:', html.length);
|
||||
res.type('text/html').send(html);
|
||||
} catch (error) {
|
||||
console.error('Scrape proxy error:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
app.listen(PORT, () => {
|
||||
console.log(`Proxy server running on http://localhost:${PORT}`);
|
||||
console.log('This proxy allows the frontend to communicate with the Anthropic API');
|
||||
});
|
||||
console.log('Scrape endpoint available at GET /api/scrape?url=...');
|
||||
});
|
||||
|
|
|
|||
|
|
@ -411,3 +411,73 @@ input, textarea {
|
|||
.app-light .debug-warning {
|
||||
color: #e65100;
|
||||
}
|
||||
|
||||
/* Quick Links Bar */
|
||||
.quick-links-bar {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
padding: 6px 12px;
|
||||
background-color: rgba(0, 0, 0, 0.3);
|
||||
border-bottom: 1px solid rgba(255, 255, 255, 0.08);
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.app-light .quick-links-bar {
|
||||
background-color: rgba(0, 0, 0, 0.04);
|
||||
border-bottom: 1px solid rgba(0, 0, 0, 0.08);
|
||||
}
|
||||
|
||||
.quick-link-btn {
|
||||
display: inline-block;
|
||||
padding: 3px 10px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 500;
|
||||
border-radius: 4px;
|
||||
background-color: rgba(255, 255, 255, 0.08);
|
||||
color: rgba(255, 255, 255, 0.7);
|
||||
text-decoration: none;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
max-width: 180px;
|
||||
transition: background-color 0.2s, color 0.2s;
|
||||
}
|
||||
|
||||
.quick-link-btn:hover {
|
||||
background-color: rgba(255, 255, 255, 0.16);
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.app-light .quick-link-btn {
|
||||
background-color: rgba(0, 0, 0, 0.06);
|
||||
color: rgba(0, 0, 0, 0.6);
|
||||
}
|
||||
|
||||
.app-light .quick-link-btn:hover {
|
||||
background-color: rgba(0, 0, 0, 0.12);
|
||||
color: rgba(0, 0, 0, 0.87);
|
||||
}
|
||||
|
||||
/* Webpage Widget Styles */
|
||||
.webpage-summary {
|
||||
font-size: 0.875rem;
|
||||
color: rgba(255, 255, 255, 0.7);
|
||||
line-height: 1.5;
|
||||
margin: 0 0 12px 0;
|
||||
}
|
||||
|
||||
.app-light .webpage-summary {
|
||||
color: rgba(0, 0, 0, 0.6);
|
||||
}
|
||||
|
||||
.webpage-item-summary {
|
||||
font-size: 0.8rem;
|
||||
color: rgba(255, 255, 255, 0.6);
|
||||
line-height: 1.4;
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.app-light .webpage-item-summary {
|
||||
color: rgba(0, 0, 0, 0.54);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import BriefingWidget from './widgets/BriefingWidget';
|
|||
import WeatherWidget from './widgets/WeatherWidget';
|
||||
import NewsWidget from './widgets/NewsWidget';
|
||||
import ChatInterface from './ChatInterface';
|
||||
import QuickLinksBar from './QuickLinksBar';
|
||||
|
||||
function Dashboard({ themeToggle, isDark }) {
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
|
@ -16,7 +17,8 @@ function Dashboard({ themeToggle, isDark }) {
|
|||
const [sourceData, setSourceData] = useState({});
|
||||
const [error, setError] = useState(null);
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
const [tokenUsage, setTokenUsage] = useState(null);
|
||||
const [briefingTokenUsage, setBriefingTokenUsage] = useState(null);
|
||||
const [webpageTokenUsage, setWebpageTokenUsage] = useState(null);
|
||||
const [cacheInfo, setCacheInfo] = useState(null);
|
||||
const isLoadingRef = useRef(false);
|
||||
|
||||
|
|
@ -57,7 +59,8 @@ function Dashboard({ themeToggle, isDark }) {
|
|||
age: cacheAgeMinutes,
|
||||
timestamp: todaysBriefing.timestamp
|
||||
});
|
||||
setTokenUsage(null); // Clear token usage when using cache
|
||||
setBriefingTokenUsage(null); // Clear token usage when using cache
|
||||
setWebpageTokenUsage(null);
|
||||
|
||||
setLoading(false);
|
||||
return;
|
||||
|
|
@ -71,6 +74,11 @@ function Dashboard({ themeToggle, isDark }) {
|
|||
console.log('Fetched sources:', Object.keys(data.sources || {}));
|
||||
console.log('Failed sources:', data.failed);
|
||||
setSourceData(data.sources || {});
|
||||
|
||||
// Track webpage summarization token usage
|
||||
if (data.webpageTokenUsage) {
|
||||
setWebpageTokenUsage(data.webpageTokenUsage);
|
||||
}
|
||||
|
||||
// If all sources failed, show error but continue
|
||||
if (data.failed > 0 && data.successful === 0) {
|
||||
|
|
@ -108,9 +116,9 @@ function Dashboard({ themeToggle, isDark }) {
|
|||
// Handle result object with text, condensedSummary and usage
|
||||
if (result && typeof result === 'object' && result.text) {
|
||||
setBriefing(result.text);
|
||||
setTokenUsage(result.usage);
|
||||
setBriefingTokenUsage(result.usage);
|
||||
setCacheInfo(null); // Clear cache info when generating fresh briefing
|
||||
console.log('Token usage:', result.usage);
|
||||
console.log('Briefing token usage:', result.usage);
|
||||
} else if (typeof result === 'string') {
|
||||
// Fallback for string responses
|
||||
setBriefing(result);
|
||||
|
|
@ -203,16 +211,38 @@ function Dashboard({ themeToggle, isDark }) {
|
|||
</Alert>
|
||||
)}
|
||||
|
||||
{tokenUsage && (
|
||||
{(briefingTokenUsage || webpageTokenUsage) && (
|
||||
<div className="token-usage-box">
|
||||
<p className="token-usage-text">
|
||||
<strong>API Usage:</strong> {tokenUsage.input_tokens?.toLocaleString()} input + {tokenUsage.output_tokens?.toLocaleString()} output = {(tokenUsage.input_tokens + tokenUsage.output_tokens).toLocaleString()} total tokens
|
||||
{' • '}
|
||||
<strong>Cost:</strong> ${((tokenUsage.input_tokens * 0.003 / 1000) + (tokenUsage.output_tokens * 0.015 / 1000)).toFixed(4)}
|
||||
{' '}
|
||||
<span className="token-usage-caption">
|
||||
(Claude Sonnet 4 pricing: $0.003/1K input, $0.015/1K output)
|
||||
</span>
|
||||
<strong>Total API Usage This Refresh:</strong>
|
||||
</p>
|
||||
|
||||
{briefingTokenUsage && (
|
||||
<p className="token-usage-text" style={{ marginTop: '8px', paddingLeft: '12px' }}>
|
||||
• <strong>Briefing Generation:</strong> {briefingTokenUsage.input_tokens?.toLocaleString()} input + {briefingTokenUsage.output_tokens?.toLocaleString()} output = {(briefingTokenUsage.input_tokens + briefingTokenUsage.output_tokens).toLocaleString()} tokens
|
||||
{' • '}
|
||||
<strong>Cost:</strong> ${((briefingTokenUsage.input_tokens * 0.003 / 1000) + (briefingTokenUsage.output_tokens * 0.015 / 1000)).toFixed(4)}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{webpageTokenUsage && (
|
||||
<p className="token-usage-text" style={{ marginTop: '8px', paddingLeft: '12px' }}>
|
||||
• <strong>Webpage Summarization:</strong> {webpageTokenUsage.input_tokens?.toLocaleString()} input + {webpageTokenUsage.output_tokens?.toLocaleString()} output = {(webpageTokenUsage.input_tokens + webpageTokenUsage.output_tokens).toLocaleString()} tokens
|
||||
{' • '}
|
||||
<strong>Cost:</strong> ${((webpageTokenUsage.input_tokens * 0.003 / 1000) + (webpageTokenUsage.output_tokens * 0.015 / 1000)).toFixed(4)}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{briefingTokenUsage && webpageTokenUsage && (
|
||||
<p className="token-usage-text" style={{ marginTop: '12px', paddingLeft: '12px', fontWeight: 600 }}>
|
||||
<strong>Combined Total:</strong> {((briefingTokenUsage.input_tokens + webpageTokenUsage.input_tokens)).toLocaleString()} input + {((briefingTokenUsage.output_tokens + webpageTokenUsage.output_tokens)).toLocaleString()} output = {((briefingTokenUsage.input_tokens + briefingTokenUsage.output_tokens + webpageTokenUsage.input_tokens + webpageTokenUsage.output_tokens)).toLocaleString()} tokens
|
||||
{' • '}
|
||||
<strong>Total Cost:</strong> ${(((briefingTokenUsage.input_tokens + webpageTokenUsage.input_tokens) * 0.003 / 1000) + ((briefingTokenUsage.output_tokens + webpageTokenUsage.output_tokens) * 0.015 / 1000)).toFixed(4)}
|
||||
</p>
|
||||
)}
|
||||
|
||||
<p className="token-usage-caption" style={{ marginTop: '8px' }}>
|
||||
(Claude Sonnet 4 pricing: $0.003/1K input, $0.015/1K output)
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
|
|
@ -230,6 +260,7 @@ function Dashboard({ themeToggle, isDark }) {
|
|||
)}
|
||||
|
||||
<Box className="widget-grid">
|
||||
<QuickLinksBar />
|
||||
<Box className="widget-row">
|
||||
<BriefingWidget briefing={briefing} styles={{maxWidth:'50%'}}/>
|
||||
<WeatherWidget
|
||||
|
|
|
|||
53
claudash/src/components/QuickLinksBar.jsx
Normal file
53
claudash/src/components/QuickLinksBar.jsx
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
/**
|
||||
* QuickLinksBar - A thin horizontal bar at the top of the screen
|
||||
* displaying configurable quick-link buttons that open in new tabs.
|
||||
*/
|
||||
function QuickLinksBar() {
|
||||
const raw = import.meta.env.REACT_APP_QUICK_LINKS || '';
|
||||
const links = raw.split(',').map(s => s.trim()).filter(Boolean);
|
||||
|
||||
if (links.length === 0) return null;
|
||||
|
||||
/**
|
||||
* Parse a URL into a short, readable label.
|
||||
* - For paths like "https://github.com/Drewpeifer" → "github/Drewpeifer"
|
||||
* - For bare domains like "https://radio.garden/" → "radio.garden"
|
||||
* - Strips www. prefix for brevity
|
||||
*/
|
||||
function parseLabel(url) {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
let host = parsed.hostname.replace(/^www\./, '');
|
||||
const path = parsed.pathname.replace(/\/+$/, ''); // trim trailing slashes
|
||||
|
||||
if (path && path !== '') {
|
||||
// Has a meaningful path — include host (without TLD) + path
|
||||
const hostBase = host.replace(/\.[^.]+$/, ''); // strip last TLD segment
|
||||
return `${hostBase}${path}`;
|
||||
}
|
||||
// No path — just return the host
|
||||
return host;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="quick-links-bar">
|
||||
{links.map((url) => (
|
||||
<a
|
||||
key={url}
|
||||
href={url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="quick-link-btn"
|
||||
title={url}
|
||||
>
|
||||
{parseLabel(url)}
|
||||
</a>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default QuickLinksBar;
|
||||
|
|
@ -198,6 +198,67 @@ function NewsWidget({ data, sx }) {
|
|||
}
|
||||
|
||||
|
||||
// Handle webpage / scraped content (Claude-summarized)
|
||||
if (data.type === 'webpage') {
|
||||
const title = data.title || new URL(data.url).hostname;
|
||||
const contentType = data.contentType || 'Webpage';
|
||||
|
||||
return (
|
||||
<GenericWidget
|
||||
className="news-widget"
|
||||
title={title}
|
||||
sx={{ ...sx }}
|
||||
headerAction={
|
||||
<Chip label={contentType} size="small" variant="outlined" color="secondary" />
|
||||
}
|
||||
>
|
||||
{data.summary && (
|
||||
<p className="webpage-summary">{data.summary}</p>
|
||||
)}
|
||||
|
||||
{data.items && data.items.length > 0 && (
|
||||
<List dense>
|
||||
{data.items.map((item, index) => (
|
||||
<ListItem key={index} disablePadding sx={{ mb: 1.5 }}>
|
||||
<ListItemText
|
||||
primary={
|
||||
<Box>
|
||||
<div className="news-title" style={{ fontWeight: 600 }}>
|
||||
{item.title}
|
||||
</div>
|
||||
</Box>
|
||||
}
|
||||
secondaryTypographyProps={{ component: 'div' }}
|
||||
secondary={
|
||||
<Box>
|
||||
{item.date && (
|
||||
<div className="news-date">{item.date}</div>
|
||||
)}
|
||||
{item.summary && (
|
||||
<div className="webpage-item-summary">{item.summary}</div>
|
||||
)}
|
||||
</Box>
|
||||
}
|
||||
/>
|
||||
</ListItem>
|
||||
))}
|
||||
</List>
|
||||
)}
|
||||
|
||||
<Box sx={{ mt: 1 }}>
|
||||
<Link
|
||||
href={data.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
sx={{ fontSize: '0.75rem', display: 'inline-flex', alignItems: 'center', gap: 0.5 }}
|
||||
>
|
||||
Visit source <OpenInNew sx={{ fontSize: 12 }} />
|
||||
</Link>
|
||||
</Box>
|
||||
</GenericWidget>
|
||||
);
|
||||
}
|
||||
|
||||
// Default fallback for unknown data types
|
||||
return (
|
||||
<GenericWidget title="News" sx={{ ...sx }}>
|
||||
|
|
|
|||
|
|
@ -270,51 +270,96 @@ export async function sendChatMessage(message, conversationHistory = []) {
|
|||
}
|
||||
|
||||
/**
|
||||
* Extract meaningful content from HTML using Claude
|
||||
* Summarize scraped webpage content using Claude.
|
||||
* Produces a structured result with a title, overall summary, and
|
||||
* individual items (e.g., blog posts, update notes, products, etc.)
|
||||
* contextualized to whatever the page content actually is.
|
||||
*
|
||||
* @param {string} cleanedText - Pre-cleaned text content (HTML stripped)
|
||||
* @param {string} url - The source URL (used for context)
|
||||
* @returns {Promise<Object>} Structured summary with title, summary, and items[]
|
||||
*/
|
||||
export async function summarizeWebpage(cleanedText, url) {
|
||||
// Limit content sent to Claude to avoid huge payloads
|
||||
const truncated = cleanedText.length > 12000
|
||||
? cleanedText.substring(0, 12000) + '\n\n[...content truncated...]'
|
||||
: cleanedText;
|
||||
|
||||
const prompt = `You are analyzing the text content scraped from a webpage. Your job is to understand what kind of page this is and summarize it appropriately.
|
||||
|
||||
URL: ${url}
|
||||
|
||||
PAGE TEXT:
|
||||
${truncated}
|
||||
|
||||
Analyze the content and determine what type of page this is (e.g., game update notes, blog, product page, news site, documentation, etc.). Then produce a structured JSON summary contextualized for its type:
|
||||
|
||||
- If it's game update/patch notes: summarize recent updates, what changed, key features added or fixed
|
||||
- If it's a blog: summarize recent posts with dates and key points
|
||||
- If it's a news page: summarize recent stories
|
||||
- If it's a product page: summarize features, pricing, what it offers
|
||||
- For anything else: do your best to extract the most useful information
|
||||
|
||||
Respond with ONLY valid JSON in this exact format:
|
||||
{
|
||||
"title": "Short descriptive title for this source",
|
||||
"contentType": "What kind of content this is (e.g., 'Game Update Notes', 'Blog', 'News')",
|
||||
"summary": "2-3 sentence overall summary of the page content",
|
||||
"items": [
|
||||
{
|
||||
"title": "Title of individual item (update name, post title, etc.)",
|
||||
"date": "Date if available, null otherwise",
|
||||
"summary": "1-3 sentence summary of this specific item"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Include up to 5 items. If the page doesn't have distinct items, create 1-2 items summarizing the main content sections.`;
|
||||
|
||||
try {
|
||||
const result = await callClaude(prompt);
|
||||
const responseText = result.text || result;
|
||||
|
||||
// Strip markdown code blocks if present
|
||||
let jsonText = responseText;
|
||||
if (jsonText.includes('```json')) {
|
||||
jsonText = jsonText.replace(/```json\s*/g, '').replace(/```\s*/g, '');
|
||||
} else if (jsonText.includes('```')) {
|
||||
jsonText = jsonText.replace(/```\s*/g, '');
|
||||
}
|
||||
jsonText = jsonText.trim();
|
||||
|
||||
const parsed = JSON.parse(jsonText);
|
||||
|
||||
return {
|
||||
title: parsed.title || 'Web Content',
|
||||
contentType: parsed.contentType || 'Webpage',
|
||||
summary: parsed.summary || '',
|
||||
items: Array.isArray(parsed.items) ? parsed.items : [],
|
||||
usage: result.usage
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Error summarizing webpage:', error);
|
||||
// Return a graceful fallback
|
||||
return {
|
||||
title: new URL(url).hostname,
|
||||
contentType: 'Webpage',
|
||||
summary: `Content from ${url}. Automatic summarization failed: ${error.message}`,
|
||||
items: [],
|
||||
error: true
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract meaningful content from HTML using Claude (legacy wrapper)
|
||||
* @param {string} html - The HTML content
|
||||
* @param {string} url - The source URL
|
||||
* @returns {Promise<Object>} Extracted content
|
||||
*/
|
||||
export async function extractContentFromHTML(html, url) {
|
||||
const prompt = `Extract the main content from this HTML page.
|
||||
Focus on the primary article, news content, or main information.
|
||||
Ignore navigation, ads, and other peripheral content.
|
||||
|
||||
URL: ${url}
|
||||
|
||||
HTML (truncated to first 10000 chars):
|
||||
${html.substring(0, 10000)}
|
||||
|
||||
Please provide a JSON response with:
|
||||
- title: The main title
|
||||
- summary: A brief summary (2-3 sentences)
|
||||
- mainContent: The main text content (up to 500 words)
|
||||
- metadata: Any relevant metadata (author, date, etc.)
|
||||
|
||||
Respond with valid JSON only.`;
|
||||
|
||||
try {
|
||||
const response = await callClaude(prompt);
|
||||
// Try to parse as JSON
|
||||
try {
|
||||
return JSON.parse(response);
|
||||
} catch {
|
||||
// If not valid JSON, return structured response
|
||||
return {
|
||||
title: 'Content from ' + url,
|
||||
summary: response.substring(0, 200),
|
||||
mainContent: response,
|
||||
metadata: { url, extractedAt: Date.now() }
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error extracting HTML content:', error);
|
||||
return {
|
||||
error: true,
|
||||
message: error.message,
|
||||
url: url
|
||||
};
|
||||
}
|
||||
// Delegate to the new summarizeWebpage function
|
||||
return summarizeWebpage(html, url);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { parseSource, standardizeSourceData } from './sourceParser';
|
|||
import { getCachedData, cacheSourceData } from './storageService';
|
||||
import { fetchWithCors } from '../utils/corsProxy';
|
||||
import { fetchOpenMeteoWeather } from './openMeteoService';
|
||||
import { summarizeWebpage } from './claudeService';
|
||||
import {
|
||||
DEFAULT_DATA_SOURCES,
|
||||
WEATHER_API_BASE,
|
||||
|
|
@ -59,13 +60,45 @@ export async function fetchSingleSource(url) {
|
|||
try {
|
||||
console.log(`Fetching fresh data from ${url}`);
|
||||
const data = await parseSource(url);
|
||||
const standardized = standardizeSourceData(data, url);
|
||||
let standardized = standardizeSourceData(data, url);
|
||||
let tokenUsage = null;
|
||||
|
||||
// If this is a webpage that needs Claude summarization, do it now
|
||||
if (standardized.type === 'webpage' && standardized.needsSummarization && standardized.cleanedText) {
|
||||
console.log(`Summarizing webpage content from ${url} with Claude...`);
|
||||
try {
|
||||
const summary = await summarizeWebpage(standardized.cleanedText, url);
|
||||
standardized = {
|
||||
...standardized,
|
||||
title: summary.title,
|
||||
contentType: summary.contentType,
|
||||
summary: summary.summary,
|
||||
items: summary.items || [],
|
||||
needsSummarization: false,
|
||||
// Remove the large cleaned text from the cached result
|
||||
cleanedText: undefined
|
||||
};
|
||||
// Track token usage from webpage summarization
|
||||
tokenUsage = summary.usage;
|
||||
console.log(`Webpage summarized: "${summary.title}" (${summary.items?.length || 0} items)`);
|
||||
if (tokenUsage) {
|
||||
console.log(`Webpage summarization tokens: ${tokenUsage.input_tokens} input + ${tokenUsage.output_tokens} output`);
|
||||
}
|
||||
} catch (summaryError) {
|
||||
console.error(`Failed to summarize webpage ${url}:`, summaryError);
|
||||
// Keep the standardized data but mark summarization as failed
|
||||
standardized.needsSummarization = false;
|
||||
standardized.title = new URL(url).hostname;
|
||||
standardized.summary = `Failed to summarize: ${summaryError.message}`;
|
||||
standardized.cleanedText = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
// Cache the result
|
||||
const ttl = parseInt(import.meta.env.REACT_APP_REFRESH_INTERVAL) || DEFAULT_REFRESH_INTERVAL;
|
||||
cacheSourceData(cacheKey, standardized, ttl);
|
||||
|
||||
return standardized;
|
||||
return { data: standardized, tokenUsage };
|
||||
} catch (error) {
|
||||
console.error(`Error fetching ${url}:`, error);
|
||||
// Try to return cached data even if expired
|
||||
|
|
@ -147,10 +180,10 @@ export async function fetchAllSources() {
|
|||
// Create fetch promises for all URLs
|
||||
const urlPromises = allUrls.map(url =>
|
||||
fetchSingleSource(url)
|
||||
.then(data => ({ url, data, error: false }))
|
||||
.then(result => ({ url, result, error: false }))
|
||||
.catch(error => ({
|
||||
url,
|
||||
data: null,
|
||||
result: null,
|
||||
error: true,
|
||||
message: error.message
|
||||
}))
|
||||
|
|
@ -170,9 +203,10 @@ export async function fetchAllSources() {
|
|||
const urlResults = await Promise.all(urlPromises);
|
||||
const weatherResult = weatherPromise ? await weatherPromise : null;
|
||||
|
||||
// Create a map of URL to fetched data
|
||||
// Create a map of URL to fetched data and accumulate token usage
|
||||
const urlDataMap = {};
|
||||
const errors = [];
|
||||
const allTokenUsage = [];
|
||||
|
||||
urlResults.forEach(result => {
|
||||
if (result.error) {
|
||||
|
|
@ -181,11 +215,31 @@ export async function fetchAllSources() {
|
|||
error: true,
|
||||
message: result.message
|
||||
});
|
||||
} else if (result.data) {
|
||||
urlDataMap[result.url] = result.data;
|
||||
} else if (result.result) {
|
||||
// Handle new return format: { data, tokenUsage }
|
||||
if (result.result.data) {
|
||||
urlDataMap[result.url] = result.result.data;
|
||||
}
|
||||
if (result.result.tokenUsage) {
|
||||
allTokenUsage.push({
|
||||
source: result.url,
|
||||
usage: result.result.tokenUsage
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Calculate total webpage summarization token usage
|
||||
let totalWebpageTokens = { input_tokens: 0, output_tokens: 0 };
|
||||
allTokenUsage.forEach(({ usage }) => {
|
||||
totalWebpageTokens.input_tokens += usage.input_tokens || 0;
|
||||
totalWebpageTokens.output_tokens += usage.output_tokens || 0;
|
||||
});
|
||||
|
||||
if (allTokenUsage.length > 0) {
|
||||
console.log(`Total webpage summarization usage: ${totalWebpageTokens.input_tokens} input + ${totalWebpageTokens.output_tokens} output tokens from ${allTokenUsage.length} source(s)`);
|
||||
}
|
||||
|
||||
// Organize data by rows
|
||||
const sourceData = {};
|
||||
|
||||
|
|
@ -236,7 +290,8 @@ export async function fetchAllSources() {
|
|||
errors: errors,
|
||||
fetchedAt: Date.now(),
|
||||
successful: allUrls.length - errors.length + (weatherResult && !weatherResult.error ? 1 : 0),
|
||||
failed: errors.length
|
||||
failed: errors.length,
|
||||
webpageTokenUsage: allTokenUsage.length > 0 ? totalWebpageTokens : null
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -64,26 +64,87 @@ export async function parseRSS(url) {
|
|||
}
|
||||
|
||||
/**
|
||||
* Parse HTML content (fetch and prepare for Claude extraction)
|
||||
* Strip HTML tags and extract meaningful text content
|
||||
* @param {string} html - Raw HTML string
|
||||
* @returns {string} - Cleaned text content
|
||||
*/
|
||||
export function cleanHTML(html) {
|
||||
if (typeof html !== 'string') return '';
|
||||
|
||||
let text = html;
|
||||
|
||||
// Remove script, style, nav, header, footer, aside blocks entirely
|
||||
text = text.replace(/<script[\s\S]*?<\/script>/gi, '');
|
||||
text = text.replace(/<style[\s\S]*?<\/style>/gi, '');
|
||||
text = text.replace(/<nav[\s\S]*?<\/nav>/gi, '');
|
||||
text = text.replace(/<footer[\s\S]*?<\/footer>/gi, '');
|
||||
text = text.replace(/<aside[\s\S]*?<\/aside>/gi, '');
|
||||
text = text.replace(/<noscript[\s\S]*?<\/noscript>/gi, '');
|
||||
text = text.replace(/<svg[\s\S]*?<\/svg>/gi, '');
|
||||
|
||||
// Replace block-level tags with newlines to preserve structure
|
||||
text = text.replace(/<\/?(div|p|br|h[1-6]|li|tr|blockquote|section|article)[^>]*>/gi, '\n');
|
||||
|
||||
// Strip all remaining HTML tags
|
||||
text = text.replace(/<[^>]+>/g, ' ');
|
||||
|
||||
// Decode common HTML entities
|
||||
text = text.replace(/&/g, '&');
|
||||
text = text.replace(/</g, '<');
|
||||
text = text.replace(/>/g, '>');
|
||||
text = text.replace(/"/g, '"');
|
||||
text = text.replace(/'/g, "'");
|
||||
text = text.replace(/ /g, ' ');
|
||||
text = text.replace(/&#\d+;/g, '');
|
||||
|
||||
// Collapse whitespace: multiple spaces → single, multiple newlines → double
|
||||
text = text.replace(/[ \t]+/g, ' ');
|
||||
text = text.replace(/\n[ \t]+/g, '\n');
|
||||
text = text.replace(/\n{3,}/g, '\n\n');
|
||||
|
||||
return text.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse HTML content via the local proxy server (avoids CORS issues)
|
||||
* Falls back to fetchWithCors if the proxy is unavailable.
|
||||
* @param {string} url - The HTML page URL
|
||||
* @param {string} corsProxy - CORS proxy URL
|
||||
* @returns {Promise<Object>} - Object with URL and HTML content
|
||||
* @param {string} corsProxy - CORS proxy URL (unused, kept for API compat)
|
||||
* @returns {Promise<Object>} - Object with URL and cleaned text content
|
||||
*/
|
||||
export async function parseHTML(url, corsProxy) {
|
||||
try {
|
||||
const html = await fetchWithCors(url);
|
||||
let rawHTML;
|
||||
|
||||
// Return structured data for Claude to process
|
||||
return {
|
||||
url: url,
|
||||
html: typeof html === 'string' ? html : JSON.stringify(html),
|
||||
type: 'html',
|
||||
timestamp: Date.now()
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`Error fetching HTML from ${url}:`, error);
|
||||
throw error;
|
||||
try {
|
||||
// Use local proxy server to avoid CORS issues with arbitrary websites
|
||||
const proxyUrl = `http://localhost:3001/api/scrape?url=${encodeURIComponent(url)}`;
|
||||
const response = await fetch(proxyUrl);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Scrape proxy returned ${response.status}`);
|
||||
}
|
||||
|
||||
rawHTML = await response.text();
|
||||
} catch (proxyError) {
|
||||
console.warn(`Local scrape proxy failed for ${url}, falling back to CORS proxy:`, proxyError.message);
|
||||
try {
|
||||
const html = await fetchWithCors(url);
|
||||
rawHTML = typeof html === 'string' ? html : JSON.stringify(html);
|
||||
} catch (corsError) {
|
||||
console.error(`All fetch methods failed for ${url}:`, corsError);
|
||||
throw corsError;
|
||||
}
|
||||
}
|
||||
|
||||
const cleanedText = cleanHTML(rawHTML);
|
||||
|
||||
return {
|
||||
url: url,
|
||||
html: rawHTML,
|
||||
cleanedText: cleanedText,
|
||||
type: 'html',
|
||||
timestamp: Date.now()
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -193,6 +254,24 @@ export function condenseForClaude(data) {
|
|||
};
|
||||
}
|
||||
|
||||
// Handle webpage type (Claude-summarized HTML)
|
||||
if (data.type === 'webpage') {
|
||||
return {
|
||||
type: 'webpage',
|
||||
url: data.url,
|
||||
title: data.title,
|
||||
summary: data.summary,
|
||||
items: data.items?.slice(0, 5)?.map(item => ({
|
||||
title: item.title,
|
||||
date: item.date,
|
||||
summary: item.summary ?
|
||||
(item.summary.length > 200 ?
|
||||
item.summary.substring(0, 197) + '...' :
|
||||
item.summary) : null
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
// Default for unknown types - just truncate if it's large
|
||||
if (data.type === 'unknown' && data.data) {
|
||||
const dataStr = JSON.stringify(data.data);
|
||||
|
|
@ -352,6 +431,21 @@ export function standardizeSourceData(data, sourceUrl) {
|
|||
};
|
||||
}
|
||||
|
||||
// Handle HTML / webpage data (from parseHTML)
|
||||
if (data.type === 'html' && data.cleanedText) {
|
||||
return {
|
||||
type: 'webpage',
|
||||
url: sourceUrl,
|
||||
cleanedText: data.cleanedText,
|
||||
// Will be populated by Claude summarization later
|
||||
title: null,
|
||||
summary: null,
|
||||
items: [],
|
||||
needsSummarization: true,
|
||||
fetchedAt: Date.now()
|
||||
};
|
||||
}
|
||||
|
||||
// Default format for unknown sources
|
||||
return {
|
||||
type: 'unknown',
|
||||
|
|
@ -359,4 +453,4 @@ export function standardizeSourceData(data, sourceUrl) {
|
|||
data: data,
|
||||
fetchedAt: Date.now()
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ export default defineConfig(({ mode }) => {
|
|||
'import.meta.env.REACT_APP_CLAUDE_MODEL': JSON.stringify(env.REACT_APP_CLAUDE_MODEL),
|
||||
'import.meta.env.REACT_APP_CLAUDE_INSTRUCTIONS': JSON.stringify(env.REACT_APP_CLAUDE_INSTRUCTIONS),
|
||||
'import.meta.env.REACT_APP_DEBUG_MODE': JSON.stringify(env.REACT_APP_DEBUG_MODE),
|
||||
'import.meta.env.REACT_APP_QUICK_LINKS': JSON.stringify(env.REACT_APP_QUICK_LINKS),
|
||||
}
|
||||
}
|
||||
})
|
||||
Loading…
Reference in a new issue