From 11d510d81d03c0fdee7b2a2e0426904fd8676f96 Mon Sep 17 00:00:00 2001 From: Drew Peifer Date: Thu, 19 Feb 2026 02:57:48 -0500 Subject: [PATCH] added quicklinks, enhanced webscraping w/ proxy --- claudash/.env.example | 5 +- claudash/proxy-server.js | 37 ++++- claudash/src/App.css | 70 ++++++++++ claudash/src/components/Dashboard.jsx | 55 ++++++-- claudash/src/components/QuickLinksBar.jsx | 53 ++++++++ .../src/components/widgets/NewsWidget.jsx | 61 +++++++++ claudash/src/services/claudeService.js | 125 +++++++++++------ claudash/src/services/dataFetcher.js | 71 ++++++++-- claudash/src/services/sourceParser.js | 126 +++++++++++++++--- claudash/vite.config.js | 1 + 10 files changed, 526 insertions(+), 78 deletions(-) create mode 100644 claudash/src/components/QuickLinksBar.jsx diff --git a/claudash/.env.example b/claudash/.env.example index 621da7f..8f2be83 100644 --- a/claudash/.env.example +++ b/claudash/.env.example @@ -7,7 +7,7 @@ REACT_APP_ANTHROPIC_API_KEY=sk-ant-your-key-here # Example formats: # - Object format: {"row1":["url1","url2"],"row2":["url3","url4"]} # - Legacy format: url1,url2,url3 (comma-separated) -REACT_APP_DATA_SOURCES={"row1":["https://www.reddit.com/r/technology.json","https://www.reddit.com/r/worldnews.json"],"row2":["https://www.reddit.com/r/science.json"]} +REACT_APP_DATA_SOURCES={"row1":["https://www.reddit.com/r/technology.json","https://www.reddit.com/r/worldnews.json"],"row2":["https://www.reddit.com/r/science.json"],"Games":["https://www.elitedangerous.com/en-US/UpdateNotes"]} # Location for weather (supports "city, state, country" or zip code) REACT_APP_LOCATION=harrisburg, pa, usa @@ -39,3 +39,6 @@ REACT_APP_CHART_DATE_FORMAT=MM/DD/YYYY # Optional: Default text-to-speech voice (must match exact voice name from browser) REACT_APP_TEXT_VOICE=Google UK English Female + +# Optional: Quick links (comma-separated URLs displayed as buttons in a bar at the top) +REACT_APP_QUICK_LINKS=https://github.com/Drewpeifer,https://drewpeifer.com/,https://www.amazon.com/,https://radio.garden/,https://cloudhiker.net/ diff --git a/claudash/proxy-server.js b/claudash/proxy-server.js index 8fac783..2770c61 100644 --- a/claudash/proxy-server.js +++ b/claudash/proxy-server.js @@ -55,7 +55,42 @@ app.post('/api/claude', async (req, res) => { } }); +// Proxy endpoint for scraping web pages (avoids CORS issues) +app.get('/api/scrape', async (req, res) => { + try { + const { url } = req.query; + + if (!url) { + return res.status(400).json({ error: 'Missing url query parameter' }); + } + + console.log('=== Scrape proxy request ==='); + console.log('URL:', url); + + const response = await fetch(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + } + }); + + if (!response.ok) { + console.error('Scrape failed:', response.status, response.statusText); + return res.status(response.status).json({ error: `Failed to fetch: ${response.status} ${response.statusText}` }); + } + + const html = await response.text(); + console.log('Scraped HTML length:', html.length); + res.type('text/html').send(html); + } catch (error) { + console.error('Scrape proxy error:', error); + res.status(500).json({ error: error.message }); + } +}); + app.listen(PORT, () => { console.log(`Proxy server running on http://localhost:${PORT}`); console.log('This proxy allows the frontend to communicate with the Anthropic API'); -}); \ No newline at end of file + console.log('Scrape endpoint available at GET /api/scrape?url=...'); +}); diff --git a/claudash/src/App.css b/claudash/src/App.css index 840beb3..e230d2b 100644 --- a/claudash/src/App.css +++ b/claudash/src/App.css @@ -411,3 +411,73 @@ input, textarea { .app-light .debug-warning { color: #e65100; } + +/* Quick Links Bar */ +.quick-links-bar { + display: flex; + flex-wrap: wrap; + gap: 6px; + padding: 6px 12px; + background-color: rgba(0, 0, 0, 0.3); + border-bottom: 1px solid rgba(255, 255, 255, 0.08); + align-items: center; +} + +.app-light .quick-links-bar { + background-color: rgba(0, 0, 0, 0.04); + border-bottom: 1px solid rgba(0, 0, 0, 0.08); +} + +.quick-link-btn { + display: inline-block; + padding: 3px 10px; + font-size: 0.75rem; + font-weight: 500; + border-radius: 4px; + background-color: rgba(255, 255, 255, 0.08); + color: rgba(255, 255, 255, 0.7); + text-decoration: none; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: 180px; + transition: background-color 0.2s, color 0.2s; +} + +.quick-link-btn:hover { + background-color: rgba(255, 255, 255, 0.16); + color: #fff; +} + +.app-light .quick-link-btn { + background-color: rgba(0, 0, 0, 0.06); + color: rgba(0, 0, 0, 0.6); +} + +.app-light .quick-link-btn:hover { + background-color: rgba(0, 0, 0, 0.12); + color: rgba(0, 0, 0, 0.87); +} + +/* Webpage Widget Styles */ +.webpage-summary { + font-size: 0.875rem; + color: rgba(255, 255, 255, 0.7); + line-height: 1.5; + margin: 0 0 12px 0; +} + +.app-light .webpage-summary { + color: rgba(0, 0, 0, 0.6); +} + +.webpage-item-summary { + font-size: 0.8rem; + color: rgba(255, 255, 255, 0.6); + line-height: 1.4; + margin-top: 4px; +} + +.app-light .webpage-item-summary { + color: rgba(0, 0, 0, 0.54); +} diff --git a/claudash/src/components/Dashboard.jsx b/claudash/src/components/Dashboard.jsx index f7820c2..e84a972 100644 --- a/claudash/src/components/Dashboard.jsx +++ b/claudash/src/components/Dashboard.jsx @@ -8,6 +8,7 @@ import BriefingWidget from './widgets/BriefingWidget'; import WeatherWidget from './widgets/WeatherWidget'; import NewsWidget from './widgets/NewsWidget'; import ChatInterface from './ChatInterface'; +import QuickLinksBar from './QuickLinksBar'; function Dashboard({ themeToggle, isDark }) { const [loading, setLoading] = useState(true); @@ -16,7 +17,8 @@ function Dashboard({ themeToggle, isDark }) { const [sourceData, setSourceData] = useState({}); const [error, setError] = useState(null); const [refreshing, setRefreshing] = useState(false); - const [tokenUsage, setTokenUsage] = useState(null); + const [briefingTokenUsage, setBriefingTokenUsage] = useState(null); + const [webpageTokenUsage, setWebpageTokenUsage] = useState(null); const [cacheInfo, setCacheInfo] = useState(null); const isLoadingRef = useRef(false); @@ -57,7 +59,8 @@ function Dashboard({ themeToggle, isDark }) { age: cacheAgeMinutes, timestamp: todaysBriefing.timestamp }); - setTokenUsage(null); // Clear token usage when using cache + setBriefingTokenUsage(null); // Clear token usage when using cache + setWebpageTokenUsage(null); setLoading(false); return; @@ -71,6 +74,11 @@ function Dashboard({ themeToggle, isDark }) { console.log('Fetched sources:', Object.keys(data.sources || {})); console.log('Failed sources:', data.failed); setSourceData(data.sources || {}); + + // Track webpage summarization token usage + if (data.webpageTokenUsage) { + setWebpageTokenUsage(data.webpageTokenUsage); + } // If all sources failed, show error but continue if (data.failed > 0 && data.successful === 0) { @@ -108,9 +116,9 @@ function Dashboard({ themeToggle, isDark }) { // Handle result object with text, condensedSummary and usage if (result && typeof result === 'object' && result.text) { setBriefing(result.text); - setTokenUsage(result.usage); + setBriefingTokenUsage(result.usage); setCacheInfo(null); // Clear cache info when generating fresh briefing - console.log('Token usage:', result.usage); + console.log('Briefing token usage:', result.usage); } else if (typeof result === 'string') { // Fallback for string responses setBriefing(result); @@ -203,16 +211,38 @@ function Dashboard({ themeToggle, isDark }) { )} - {tokenUsage && ( + {(briefingTokenUsage || webpageTokenUsage) && (

- API Usage: {tokenUsage.input_tokens?.toLocaleString()} input + {tokenUsage.output_tokens?.toLocaleString()} output = {(tokenUsage.input_tokens + tokenUsage.output_tokens).toLocaleString()} total tokens - {' • '} - Cost: ${((tokenUsage.input_tokens * 0.003 / 1000) + (tokenUsage.output_tokens * 0.015 / 1000)).toFixed(4)} - {' '} - - (Claude Sonnet 4 pricing: $0.003/1K input, $0.015/1K output) - + Total API Usage This Refresh: +

+ + {briefingTokenUsage && ( +

+ • Briefing Generation: {briefingTokenUsage.input_tokens?.toLocaleString()} input + {briefingTokenUsage.output_tokens?.toLocaleString()} output = {(briefingTokenUsage.input_tokens + briefingTokenUsage.output_tokens).toLocaleString()} tokens + {' • '} + Cost: ${((briefingTokenUsage.input_tokens * 0.003 / 1000) + (briefingTokenUsage.output_tokens * 0.015 / 1000)).toFixed(4)} +

+ )} + + {webpageTokenUsage && ( +

+ • Webpage Summarization: {webpageTokenUsage.input_tokens?.toLocaleString()} input + {webpageTokenUsage.output_tokens?.toLocaleString()} output = {(webpageTokenUsage.input_tokens + webpageTokenUsage.output_tokens).toLocaleString()} tokens + {' • '} + Cost: ${((webpageTokenUsage.input_tokens * 0.003 / 1000) + (webpageTokenUsage.output_tokens * 0.015 / 1000)).toFixed(4)} +

+ )} + + {briefingTokenUsage && webpageTokenUsage && ( +

+ Combined Total: {((briefingTokenUsage.input_tokens + webpageTokenUsage.input_tokens)).toLocaleString()} input + {((briefingTokenUsage.output_tokens + webpageTokenUsage.output_tokens)).toLocaleString()} output = {((briefingTokenUsage.input_tokens + briefingTokenUsage.output_tokens + webpageTokenUsage.input_tokens + webpageTokenUsage.output_tokens)).toLocaleString()} tokens + {' • '} + Total Cost: ${(((briefingTokenUsage.input_tokens + webpageTokenUsage.input_tokens) * 0.003 / 1000) + ((briefingTokenUsage.output_tokens + webpageTokenUsage.output_tokens) * 0.015 / 1000)).toFixed(4)} +

+ )} + +

+ (Claude Sonnet 4 pricing: $0.003/1K input, $0.015/1K output)

)} @@ -230,6 +260,7 @@ function Dashboard({ themeToggle, isDark }) { )} + s.trim()).filter(Boolean); + + if (links.length === 0) return null; + + /** + * Parse a URL into a short, readable label. + * - For paths like "https://github.com/Drewpeifer" → "github/Drewpeifer" + * - For bare domains like "https://radio.garden/" → "radio.garden" + * - Strips www. prefix for brevity + */ + function parseLabel(url) { + try { + const parsed = new URL(url); + let host = parsed.hostname.replace(/^www\./, ''); + const path = parsed.pathname.replace(/\/+$/, ''); // trim trailing slashes + + if (path && path !== '') { + // Has a meaningful path — include host (without TLD) + path + const hostBase = host.replace(/\.[^.]+$/, ''); // strip last TLD segment + return `${hostBase}${path}`; + } + // No path — just return the host + return host; + } catch { + return url; + } + } + + return ( +
+ {links.map((url) => ( + + {parseLabel(url)} + + ))} +
+ ); +} + +export default QuickLinksBar; diff --git a/claudash/src/components/widgets/NewsWidget.jsx b/claudash/src/components/widgets/NewsWidget.jsx index c4a90d3..3934f77 100644 --- a/claudash/src/components/widgets/NewsWidget.jsx +++ b/claudash/src/components/widgets/NewsWidget.jsx @@ -198,6 +198,67 @@ function NewsWidget({ data, sx }) { } + // Handle webpage / scraped content (Claude-summarized) + if (data.type === 'webpage') { + const title = data.title || new URL(data.url).hostname; + const contentType = data.contentType || 'Webpage'; + + return ( + + } + > + {data.summary && ( +

{data.summary}

+ )} + + {data.items && data.items.length > 0 && ( + + {data.items.map((item, index) => ( + + +
+ {item.title} +
+ + } + secondaryTypographyProps={{ component: 'div' }} + secondary={ + + {item.date && ( +
{item.date}
+ )} + {item.summary && ( +
{item.summary}
+ )} + + } + /> + + ))} + + )} + + + + Visit source + + + + ); + } + // Default fallback for unknown data types return ( diff --git a/claudash/src/services/claudeService.js b/claudash/src/services/claudeService.js index ecab7b8..52aee52 100644 --- a/claudash/src/services/claudeService.js +++ b/claudash/src/services/claudeService.js @@ -270,51 +270,96 @@ export async function sendChatMessage(message, conversationHistory = []) { } /** - * Extract meaningful content from HTML using Claude + * Summarize scraped webpage content using Claude. + * Produces a structured result with a title, overall summary, and + * individual items (e.g., blog posts, update notes, products, etc.) + * contextualized to whatever the page content actually is. + * + * @param {string} cleanedText - Pre-cleaned text content (HTML stripped) + * @param {string} url - The source URL (used for context) + * @returns {Promise} Structured summary with title, summary, and items[] + */ +export async function summarizeWebpage(cleanedText, url) { + // Limit content sent to Claude to avoid huge payloads + const truncated = cleanedText.length > 12000 + ? cleanedText.substring(0, 12000) + '\n\n[...content truncated...]' + : cleanedText; + + const prompt = `You are analyzing the text content scraped from a webpage. Your job is to understand what kind of page this is and summarize it appropriately. + +URL: ${url} + +PAGE TEXT: +${truncated} + +Analyze the content and determine what type of page this is (e.g., game update notes, blog, product page, news site, documentation, etc.). Then produce a structured JSON summary contextualized for its type: + +- If it's game update/patch notes: summarize recent updates, what changed, key features added or fixed +- If it's a blog: summarize recent posts with dates and key points +- If it's a news page: summarize recent stories +- If it's a product page: summarize features, pricing, what it offers +- For anything else: do your best to extract the most useful information + +Respond with ONLY valid JSON in this exact format: +{ + "title": "Short descriptive title for this source", + "contentType": "What kind of content this is (e.g., 'Game Update Notes', 'Blog', 'News')", + "summary": "2-3 sentence overall summary of the page content", + "items": [ + { + "title": "Title of individual item (update name, post title, etc.)", + "date": "Date if available, null otherwise", + "summary": "1-3 sentence summary of this specific item" + } + ] +} + +Include up to 5 items. If the page doesn't have distinct items, create 1-2 items summarizing the main content sections.`; + + try { + const result = await callClaude(prompt); + const responseText = result.text || result; + + // Strip markdown code blocks if present + let jsonText = responseText; + if (jsonText.includes('```json')) { + jsonText = jsonText.replace(/```json\s*/g, '').replace(/```\s*/g, ''); + } else if (jsonText.includes('```')) { + jsonText = jsonText.replace(/```\s*/g, ''); + } + jsonText = jsonText.trim(); + + const parsed = JSON.parse(jsonText); + + return { + title: parsed.title || 'Web Content', + contentType: parsed.contentType || 'Webpage', + summary: parsed.summary || '', + items: Array.isArray(parsed.items) ? parsed.items : [], + usage: result.usage + }; + } catch (error) { + console.error('Error summarizing webpage:', error); + // Return a graceful fallback + return { + title: new URL(url).hostname, + contentType: 'Webpage', + summary: `Content from ${url}. Automatic summarization failed: ${error.message}`, + items: [], + error: true + }; + } +} + +/** + * Extract meaningful content from HTML using Claude (legacy wrapper) * @param {string} html - The HTML content * @param {string} url - The source URL * @returns {Promise} Extracted content */ export async function extractContentFromHTML(html, url) { - const prompt = `Extract the main content from this HTML page. - Focus on the primary article, news content, or main information. - Ignore navigation, ads, and other peripheral content. - - URL: ${url} - - HTML (truncated to first 10000 chars): - ${html.substring(0, 10000)} - - Please provide a JSON response with: - - title: The main title - - summary: A brief summary (2-3 sentences) - - mainContent: The main text content (up to 500 words) - - metadata: Any relevant metadata (author, date, etc.) - - Respond with valid JSON only.`; - - try { - const response = await callClaude(prompt); - // Try to parse as JSON - try { - return JSON.parse(response); - } catch { - // If not valid JSON, return structured response - return { - title: 'Content from ' + url, - summary: response.substring(0, 200), - mainContent: response, - metadata: { url, extractedAt: Date.now() } - }; - } - } catch (error) { - console.error('Error extracting HTML content:', error); - return { - error: true, - message: error.message, - url: url - }; - } + // Delegate to the new summarizeWebpage function + return summarizeWebpage(html, url); } /** diff --git a/claudash/src/services/dataFetcher.js b/claudash/src/services/dataFetcher.js index a12b8f2..30a25ea 100644 --- a/claudash/src/services/dataFetcher.js +++ b/claudash/src/services/dataFetcher.js @@ -2,6 +2,7 @@ import { parseSource, standardizeSourceData } from './sourceParser'; import { getCachedData, cacheSourceData } from './storageService'; import { fetchWithCors } from '../utils/corsProxy'; import { fetchOpenMeteoWeather } from './openMeteoService'; +import { summarizeWebpage } from './claudeService'; import { DEFAULT_DATA_SOURCES, WEATHER_API_BASE, @@ -59,13 +60,45 @@ export async function fetchSingleSource(url) { try { console.log(`Fetching fresh data from ${url}`); const data = await parseSource(url); - const standardized = standardizeSourceData(data, url); + let standardized = standardizeSourceData(data, url); + let tokenUsage = null; + + // If this is a webpage that needs Claude summarization, do it now + if (standardized.type === 'webpage' && standardized.needsSummarization && standardized.cleanedText) { + console.log(`Summarizing webpage content from ${url} with Claude...`); + try { + const summary = await summarizeWebpage(standardized.cleanedText, url); + standardized = { + ...standardized, + title: summary.title, + contentType: summary.contentType, + summary: summary.summary, + items: summary.items || [], + needsSummarization: false, + // Remove the large cleaned text from the cached result + cleanedText: undefined + }; + // Track token usage from webpage summarization + tokenUsage = summary.usage; + console.log(`Webpage summarized: "${summary.title}" (${summary.items?.length || 0} items)`); + if (tokenUsage) { + console.log(`Webpage summarization tokens: ${tokenUsage.input_tokens} input + ${tokenUsage.output_tokens} output`); + } + } catch (summaryError) { + console.error(`Failed to summarize webpage ${url}:`, summaryError); + // Keep the standardized data but mark summarization as failed + standardized.needsSummarization = false; + standardized.title = new URL(url).hostname; + standardized.summary = `Failed to summarize: ${summaryError.message}`; + standardized.cleanedText = undefined; + } + } // Cache the result const ttl = parseInt(import.meta.env.REACT_APP_REFRESH_INTERVAL) || DEFAULT_REFRESH_INTERVAL; cacheSourceData(cacheKey, standardized, ttl); - return standardized; + return { data: standardized, tokenUsage }; } catch (error) { console.error(`Error fetching ${url}:`, error); // Try to return cached data even if expired @@ -147,10 +180,10 @@ export async function fetchAllSources() { // Create fetch promises for all URLs const urlPromises = allUrls.map(url => fetchSingleSource(url) - .then(data => ({ url, data, error: false })) + .then(result => ({ url, result, error: false })) .catch(error => ({ url, - data: null, + result: null, error: true, message: error.message })) @@ -170,9 +203,10 @@ export async function fetchAllSources() { const urlResults = await Promise.all(urlPromises); const weatherResult = weatherPromise ? await weatherPromise : null; - // Create a map of URL to fetched data + // Create a map of URL to fetched data and accumulate token usage const urlDataMap = {}; const errors = []; + const allTokenUsage = []; urlResults.forEach(result => { if (result.error) { @@ -181,11 +215,31 @@ export async function fetchAllSources() { error: true, message: result.message }); - } else if (result.data) { - urlDataMap[result.url] = result.data; + } else if (result.result) { + // Handle new return format: { data, tokenUsage } + if (result.result.data) { + urlDataMap[result.url] = result.result.data; + } + if (result.result.tokenUsage) { + allTokenUsage.push({ + source: result.url, + usage: result.result.tokenUsage + }); + } } }); + // Calculate total webpage summarization token usage + let totalWebpageTokens = { input_tokens: 0, output_tokens: 0 }; + allTokenUsage.forEach(({ usage }) => { + totalWebpageTokens.input_tokens += usage.input_tokens || 0; + totalWebpageTokens.output_tokens += usage.output_tokens || 0; + }); + + if (allTokenUsage.length > 0) { + console.log(`Total webpage summarization usage: ${totalWebpageTokens.input_tokens} input + ${totalWebpageTokens.output_tokens} output tokens from ${allTokenUsage.length} source(s)`); + } + // Organize data by rows const sourceData = {}; @@ -236,7 +290,8 @@ export async function fetchAllSources() { errors: errors, fetchedAt: Date.now(), successful: allUrls.length - errors.length + (weatherResult && !weatherResult.error ? 1 : 0), - failed: errors.length + failed: errors.length, + webpageTokenUsage: allTokenUsage.length > 0 ? totalWebpageTokens : null }; } diff --git a/claudash/src/services/sourceParser.js b/claudash/src/services/sourceParser.js index d5d579f..313507e 100644 --- a/claudash/src/services/sourceParser.js +++ b/claudash/src/services/sourceParser.js @@ -64,26 +64,87 @@ export async function parseRSS(url) { } /** - * Parse HTML content (fetch and prepare for Claude extraction) + * Strip HTML tags and extract meaningful text content + * @param {string} html - Raw HTML string + * @returns {string} - Cleaned text content + */ +export function cleanHTML(html) { + if (typeof html !== 'string') return ''; + + let text = html; + + // Remove script, style, nav, header, footer, aside blocks entirely + text = text.replace(//gi, ''); + text = text.replace(//gi, ''); + text = text.replace(//gi, ''); + text = text.replace(//gi, ''); + text = text.replace(//gi, ''); + text = text.replace(//gi, ''); + text = text.replace(//gi, ''); + + // Replace block-level tags with newlines to preserve structure + text = text.replace(/<\/?(div|p|br|h[1-6]|li|tr|blockquote|section|article)[^>]*>/gi, '\n'); + + // Strip all remaining HTML tags + text = text.replace(/<[^>]+>/g, ' '); + + // Decode common HTML entities + text = text.replace(/&/g, '&'); + text = text.replace(/</g, '<'); + text = text.replace(/>/g, '>'); + text = text.replace(/"/g, '"'); + text = text.replace(/'/g, "'"); + text = text.replace(/ /g, ' '); + text = text.replace(/&#\d+;/g, ''); + + // Collapse whitespace: multiple spaces → single, multiple newlines → double + text = text.replace(/[ \t]+/g, ' '); + text = text.replace(/\n[ \t]+/g, '\n'); + text = text.replace(/\n{3,}/g, '\n\n'); + + return text.trim(); +} + +/** + * Parse HTML content via the local proxy server (avoids CORS issues) + * Falls back to fetchWithCors if the proxy is unavailable. * @param {string} url - The HTML page URL - * @param {string} corsProxy - CORS proxy URL - * @returns {Promise} - Object with URL and HTML content + * @param {string} corsProxy - CORS proxy URL (unused, kept for API compat) + * @returns {Promise} - Object with URL and cleaned text content */ export async function parseHTML(url, corsProxy) { - try { - const html = await fetchWithCors(url); + let rawHTML; - // Return structured data for Claude to process - return { - url: url, - html: typeof html === 'string' ? html : JSON.stringify(html), - type: 'html', - timestamp: Date.now() - }; - } catch (error) { - console.error(`Error fetching HTML from ${url}:`, error); - throw error; + try { + // Use local proxy server to avoid CORS issues with arbitrary websites + const proxyUrl = `http://localhost:3001/api/scrape?url=${encodeURIComponent(url)}`; + const response = await fetch(proxyUrl); + + if (!response.ok) { + throw new Error(`Scrape proxy returned ${response.status}`); + } + + rawHTML = await response.text(); + } catch (proxyError) { + console.warn(`Local scrape proxy failed for ${url}, falling back to CORS proxy:`, proxyError.message); + try { + const html = await fetchWithCors(url); + rawHTML = typeof html === 'string' ? html : JSON.stringify(html); + } catch (corsError) { + console.error(`All fetch methods failed for ${url}:`, corsError); + throw corsError; + } } + + const cleanedText = cleanHTML(rawHTML); + + return { + url: url, + html: rawHTML, + cleanedText: cleanedText, + type: 'html', + timestamp: Date.now() + }; } /** @@ -193,6 +254,24 @@ export function condenseForClaude(data) { }; } + // Handle webpage type (Claude-summarized HTML) + if (data.type === 'webpage') { + return { + type: 'webpage', + url: data.url, + title: data.title, + summary: data.summary, + items: data.items?.slice(0, 5)?.map(item => ({ + title: item.title, + date: item.date, + summary: item.summary ? + (item.summary.length > 200 ? + item.summary.substring(0, 197) + '...' : + item.summary) : null + })) + }; + } + // Default for unknown types - just truncate if it's large if (data.type === 'unknown' && data.data) { const dataStr = JSON.stringify(data.data); @@ -352,6 +431,21 @@ export function standardizeSourceData(data, sourceUrl) { }; } + // Handle HTML / webpage data (from parseHTML) + if (data.type === 'html' && data.cleanedText) { + return { + type: 'webpage', + url: sourceUrl, + cleanedText: data.cleanedText, + // Will be populated by Claude summarization later + title: null, + summary: null, + items: [], + needsSummarization: true, + fetchedAt: Date.now() + }; + } + // Default format for unknown sources return { type: 'unknown', @@ -359,4 +453,4 @@ export function standardizeSourceData(data, sourceUrl) { data: data, fetchedAt: Date.now() }; -} \ No newline at end of file +} diff --git a/claudash/vite.config.js b/claudash/vite.config.js index ca52ecc..e40bcfa 100644 --- a/claudash/vite.config.js +++ b/claudash/vite.config.js @@ -32,6 +32,7 @@ export default defineConfig(({ mode }) => { 'import.meta.env.REACT_APP_CLAUDE_MODEL': JSON.stringify(env.REACT_APP_CLAUDE_MODEL), 'import.meta.env.REACT_APP_CLAUDE_INSTRUCTIONS': JSON.stringify(env.REACT_APP_CLAUDE_INSTRUCTIONS), 'import.meta.env.REACT_APP_DEBUG_MODE': JSON.stringify(env.REACT_APP_DEBUG_MODE), + 'import.meta.env.REACT_APP_QUICK_LINKS': JSON.stringify(env.REACT_APP_QUICK_LINKS), } } }) \ No newline at end of file