better compression and history storage, limiting posts to 10, layout shifts

claudash/.env.example

@@ -1,9 +1,13 @@
 # REQUIRED: Anthropic API Key
 REACT_APP_ANTHROPIC_API_KEY=sk-ant-your-key-here
 
-# Data sources (comma-separated URLs)
+# Data sources (JSON object with rows as keys, arrays of URLs as values)
+# Each key represents a row of widgets in the UI
 # Supports: .json (auto-parsed), .rss/.xml (RSS feeds), plain URLs (scraped)
-REACT_APP_DATA_SOURCES=https://www.reddit.com/r/technology.json,https://hacker-news.firebaseio.com/v0/topstories.json,https://www.reddit.com/r/worldnews.json
+# Example formats:
+# - Object format: {"row1":["url1","url2"],"row2":["url3","url4"]}
+# - Legacy format: url1,url2,url3 (comma-separated)
+REACT_APP_DATA_SOURCES={"row1":["https://www.reddit.com/r/technology.json","https://www.reddit.com/r/worldnews.json"],"row2":["https://www.reddit.com/r/science.json"]}
 
 # Location for weather (supports "city, state, country" or zip code)
 REACT_APP_LOCATION=harrisburg, pa, usa

claudash/proxy-server.js

@@ -5,7 +5,8 @@ const app = express();
 const PORT = 3001;
 
 app.use(cors());
-app.use(express.json());
+// Increase payload limit to 10MB (default is 100kb)
+app.use(express.json({ limit: '10mb' }));
 
 // Proxy endpoint for Claude API
 app.post('/api/claude', async (req, res) => {

claudash/src/App.css

@@ -53,12 +53,18 @@ h1, h2, h3, h4, h5, h6 {
   flex-direction: row;
   gap: 10px;
 }
+.news-widget {
+  min-width: 25%;
+  max-width: 49%;
+  flex-grow: 2;
+}
 .widget-row.condensed {
   flex-wrap: wrap;
-  justify-content: space-between;
+  justify-content: space-evenly;
 }
-.widget-row.condensed .news-widget{
+.news-widget img {
-  flex-grow: 2;
+  width: 70%;
+  margin: 10px auto;
 }
 
 a {

claudash/src/components/Dashboard.jsx

@@ -91,8 +91,9 @@ function Dashboard({ themeToggle, isDark }) {
                     // Generate today's briefing with Claude
                     const result = await generateBriefing(data, previousBriefing);
                     console.log('Briefing generated');
+                    console.log('Condensed summary:', result.condensedSummary?.substring(0, 100) + '...');
 
-                    // Handle result object with text and usage
+                    // Handle result object with text, condensedSummary and usage
                     if (result && typeof result === 'object' && result.text) {
                         setBriefing(result.text);
                         setTokenUsage(result.usage);
@@ -104,11 +105,12 @@ function Dashboard({ themeToggle, isDark }) {
                         setCacheInfo(null);
                     }
 
-                    // Save to localStorage
+                    // Save to localStorage with condensed summary
                     const today = new Date().toISOString().split('T')[0];
                     console.log('Saving briefing for:', today);
                     const briefingText = result?.text || result;
-                    saveBriefing(today, briefingText, data.sources);
+                    const condensedSummary = result?.condensedSummary || null;
+                    saveBriefing(today, briefingText, data.sources, condensedSummary);
                 } catch (briefingError) {
                     console.error('Failed to generate briefing:', briefingError);
                     setBriefing(`Unable to generate AI briefing: ${briefingError.message}\n\nYou can still view the data widgets below.`);
@@ -218,18 +220,45 @@ function Dashboard({ themeToggle, isDark }) {
                             data={sourceData.weather}
                         />
                     </Box>
-                    <Box className="widget-row condensed">
+                    
-                        {/* Render NewsWidget for each news source */}
+                    {/* Render rows based on the new data structure */}
-                        {Object.entries(sourceData).map(([key, data]) => {
+                    {Object.entries(sourceData).map(([key, data]) => {
-                            if (key === 'weather') return null;
+                        // Skip special entries like weather and legacy flat entries
+                        if (key === 'weather' || key.startsWith('reddit_') || key.startsWith('rss_') || key.startsWith('source_')) {
+                            return null;
+                        }
+                        
+                        // Only process row entries (row1, row2, etc.)
+                        if (key.startsWith('row') && Array.isArray(data)) {
                             return (
-                                <NewsWidget
+                                <Box key={key} className="widget-row condensed">
-                                    key={key}
+                                    {data.map((widgetData, index) => (
-                                    data={data}
+                                        <NewsWidget
-                                />
+                                            key={`${key}-${index}`}
+                                            data={widgetData}
+                                        />
+                                    ))}
+                                </Box>
                             );
-                        })}
+                        }
-                    </Box>
+                        
+                        return null;
+                    })}
+                    
+                    {/* Fallback for legacy format - if no rows detected, render flat structure */}
+                    {!Object.keys(sourceData).some(k => k.startsWith('row')) && (
+                        <Box className="widget-row condensed">
+                            {Object.entries(sourceData).map(([key, data]) => {
+                                if (key === 'weather' || !data) return null;
+                                return (
+                                    <NewsWidget
+                                        key={key}
+                                        data={data}
+                                    />
+                                );
+                            })}
+                        </Box>
+                    )}
                 </Box>
             </Container>
 

claudash/src/components/widgets/NewsWidget.jsx

@@ -13,80 +13,121 @@ function NewsWidget({ data, sx }) {
     );
   }
 
-  // Handle Reddit data
+    // Handle Reddit data
-  if (data.type === 'reddit' && data.posts) {
+    if (data.type === 'reddit' && data.posts) {
-    const subreddit = data.url?.match(/\/r\/([^\/\.]+)/)?.[1] || 'Reddit';
+        const subreddit = data.url?.match(/\/r\/([^\/\.]+)/)?.[1] || 'Reddit';
 
-    return (
+        return (
-      <GenericWidget className="news-widget" title={`r/${subreddit}`} sx={{ ...sx }}>
+            <GenericWidget className="news-widget" title={`r/${subreddit}`} sx={{ ...sx }}>
-        <List dense>
+                <List dense>
-          {data.posts.slice(0, 5).map((post, index) => (
+                    {data.posts.slice(0, 5).map((post, index) => {
-            <ListItem key={index} disablePadding sx={{ mb: 1 }}>
+                        // Determine if we should show an image
-              <ListItemText
+                        const hasImage = post.mainImage && 
-                primary={
+                            !post.mainImage.includes('external-preview') && // Skip Reddit's external preview placeholder
-                  <Box sx={{ display: 'flex', alignItems: 'start', gap: 1 }}>
+                            post.mainImage !== 'spoiler' &&
-                    {post.thumbnail && post.thumbnail.startsWith('http') && (
+                            post.mainImage !== 'nsfw' &&
-                      <Link
+                            post.mainImage !== 'default';
-                        href={post.url}
+
-                        target="_blank"
+                        return (
-                        rel="noopener noreferrer"
+                            <ListItem key={index} disablePadding sx={{ mb: 2 }}>
-                      >
+                                <ListItemText
-                        <Box
+                                    primary={
-                          component="img"
+                                        <Box>
-                          src={post.thumbnail}
+                                            {/* Image section - Medium size, above title */}
-                          alt={post.title}
+                                            {hasImage && (
-                          sx={{
+                                                <Link
-                            width: 70,
+                                                    href={post.permalink || post.url}
-                            height: 70,
+                                                    target="_blank"
-                            objectFit: 'cover',
+                                                    rel="noopener noreferrer"
-                            borderRadius: 1,
+                                                    sx={{ display: 'block', mb: 1 }}
-                            flexShrink: 0
+                                                >
-                          }}
+                                                    <Box
-                        />
+                                                        component="img"
-                      </Link>
+                                                        src={post.mainImage}
-                    )}
+                                                        alt={post.title}
-                    <Link
+                                                        onError={(e) => {
-                      href={post.url}
+                                                            e.target.style.display = 'none';
-                      target="_blank"
+                                                        }}
-                      rel="noopener noreferrer"
+                                                        sx={{
-                      sx={{
+                                                            width: '100%',
-                        textDecoration: 'none',
+                                                            maxWidth: 400,
-                        color: 'text.primary',
+                                                            height: 'auto',
-                        '&:hover': { textDecoration: 'underline' }
+                                                            maxHeight: 250,
-                      }}
+                                                            objectFit: 'cover',
-                    >
+                                                            borderRadius: 1,
-                      <div className="news-title">
+                                                            display: 'block',
-                        {post.title}
+                                                            backgroundColor: 'action.hover',
-                        <OpenInNew sx={{ fontSize: 14 }} />
+                                                            cursor: 'pointer',
-                      </div>
+                                                            transition: 'transform 0.2s',
-                    </Link>
+                                                            '&:hover': {
-                  </Box>
+                                                                transform: 'scale(1.02)'
-                }
+                                                            }
-                secondaryTypographyProps={{ component: 'div' }}
+                                                        }}
-                secondary={
+                                                    />
-                  <Box sx={{ display: 'flex', gap: 1, mt: 0.5 }}>
+                                                    {post.isGallery && post.galleryImages.length > 1 && (
-                    <Chip
+                                                        <Chip
-                      icon={<TrendingUp />}
+                                                            label={`Gallery (${post.galleryImages.length} images)`}
-                      label={post.score}
+                                                            size="small"
-                      size="small"
+                                                            sx={{ mt: -3.5, ml: 1, position: 'relative' }}
-                      variant="outlined"
+                                                        />
-                    />
+                                                    )}
-                    <Chip
+                                                </Link>
-                      icon={<Comment />}
+                                            )}
-                      label={post.comments}
+                                            
-                      size="small"
+                                            {/* Title section */}
-                      variant="outlined"
+                                            <Link
-                    />
+                                                href={post.permalink || post.url}
-                    <span className="news-author">by {post.author}</span>
+                                                target="_blank"
-                  </Box>
+                                                rel="noopener noreferrer"
-                }
+                                                sx={{
-              />
+                                                    textDecoration: 'none',
-            </ListItem>
+                                                    color: 'text.primary',
-          ))}
+                                                    '&:hover': { textDecoration: 'underline' }
-        </List>
+                                                }}
-      </GenericWidget>
+                                            >
-    );
+                                                <div className="news-title" style={{ display: 'flex', alignItems: 'center', gap: 4 }}>
-  }
+                                                    {post.title}
+                                                    <OpenInNew sx={{ fontSize: 14, marginLeft: 0.5 }} />
+                                                </div>
+                                            </Link>
+                                        </Box>
+                                    }
+                                    secondaryTypographyProps={{ component: 'div' }}
+                                    secondary={
+                                        <Box sx={{ display: 'flex', gap: 1, mt: 0.5, flexWrap: 'wrap', alignItems: 'center' }}>
+                                            <Chip
+                                                icon={<TrendingUp />}
+                                                label={`${post.score?.toLocaleString() || 0} upvotes`}
+                                                size="small"
+                                                variant="outlined"
+                                                color="primary"
+                                            />
+                                            <Chip
+                                                icon={<Comment />}
+                                                label={`${post.comments?.toLocaleString() || 0} comments`}
+                                                size="small"
+                                                variant="outlined"
+                                            />
+                                            <span className="news-author" style={{ lineHeight: '24px', fontSize: '0.875rem', opacity: 0.7 }}>
+                                                {post.created && (() => {
+                                                    const now = Date.now() / 1000; // Convert to seconds
+                                                    const diff = now - post.created;
+                                                    const hours = Math.floor(diff / 3600);
+                                                    const days = Math.floor(hours / 24);
+                                                    const timeAgo = days > 0 ? `${days}d ago` : `${hours}h ago`;
+                                                    return timeAgo;
+                                                })()} • by {post.author} in r/{post.subreddit}
+                                            </span>
+                                        </Box>
+                                    }
+                                />
+                            </ListItem>
+                        );
+                    })}
+                </List>
+            </GenericWidget>
+        );
+    }
 
   // Handle RSS feed data
   if (data.type === 'rss' && data.items) {
@@ -156,19 +197,6 @@ function NewsWidget({ data, sx }) {
     );
   }
 
-  // Handle Hacker News data
-  if (data.type === 'hackernews' && data.storyIds) {
-    return (
-      <GenericWidget title="Hacker News" sx={{ ...sx }}>
-        <p className="widget-info">
-          {data.storyIds.length} top stories loaded
-        </p>
-        <p className="widget-caption">
-          (Story details will load when API is configured)
-        </p>
-      </GenericWidget>
-    );
-  }
 
   // Default fallback for unknown data types
   return (

claudash/src/services/claudeService.js

@@ -3,6 +3,7 @@ import {
   DEFAULT_CLAUDE_MODEL,
   MAX_TOKENS
 } from '../utils/constants';
+import { condenseSourcesForClaude } from './sourceParser';
 
 /**
  * Make a request to Claude API
@@ -77,7 +78,7 @@ async function callClaude(prompt, messages = []) {
 /**
  * Build briefing prompt with memory
  * @param {Object} currentData - Current source data
- * @param {Object} previousBriefing - Previous briefing object
+ * @param {Object} previousBriefing - Previous briefing object (condensed summary)
  * @returns {string} The prompt
  */
 function buildBriefingPrompt(currentData, previousBriefing) {
@@ -95,16 +96,22 @@ function buildBriefingPrompt(currentData, previousBriefing) {
 CURRENT DATA:
 `;
   
-  // Add all current source data
+  // CONDENSE DATA BEFORE SENDING TO CLAUDE
-  Object.entries(currentData.sources || currentData).forEach(([source, data]) => {
+  const condensedData = condenseSourcesForClaude(currentData.sources || currentData);
+  console.log('Original data size:', JSON.stringify(currentData).length);
+  console.log('Condensed data size:', JSON.stringify(condensedData).length);
+  console.log('Size reduction:', Math.round((1 - JSON.stringify(condensedData).length / JSON.stringify(currentData).length) * 100) + '%');
+  
+  // Add all condensed source data
+  Object.entries(condensedData).forEach(([source, data]) => {
     if (data && !data.error) {
       prompt += `\n${source.toUpperCase()}:\n${JSON.stringify(data, null, 2)}\n`;
     }
   });
 
-  // Add historical context if available
+  // Add historical context if available (now using condensed summary)
-  if (previousBriefing && previousBriefing.briefing) {
+  if (previousBriefing && previousBriefing.condensedSummary) {
-    prompt += `\n\nYESTERDAY'S BRIEFING:\n${previousBriefing.briefing}\n`;
+    prompt += `\n\nPREVIOUS BRIEFING SUMMARY:\n${previousBriefing.condensedSummary}\n`;
     prompt += `\nHighlight what's changed, any follow-up stories, or interesting trends.`;
   }
 
@@ -122,29 +129,69 @@ Focus on the most newsworthy and interesting items. Don't try to mention everyth
 }
 
 /**
- * Generate daily briefing
+ * Generate a condensed summary of a briefing for historical storage
+ * @param {string} fullBriefing - The full briefing text
+ * @returns {Promise<string>} Condensed summary (50-75 words)
+ */
+async function generateCondensedSummary(fullBriefing) {
+  const prompt = `Create an ultra-condensed summary of this briefing in 50-75 words.
+Focus only on the key topics and main stories mentioned. 
+This will be used as historical context for tomorrow's briefing generation.
+
+FULL BRIEFING:
+${fullBriefing}
+
+Provide ONLY the condensed summary, no preamble or explanation.`;
+
+  try {
+    const result = await callClaude(prompt);
+    return result.text;
+  } catch (error) {
+    console.error('Error generating condensed summary:', error);
+    // Fallback to first 300 chars of original
+    return fullBriefing.substring(0, 300) + '...';
+  }
+}
+
+/**
+ * Generate daily briefing with condensed summary
  * @param {Object} currentData - Current data from all sources
  * @param {Object} previousBriefing - Previous briefing for comparison
- * @returns {Promise<string>} Generated briefing text
+ * @returns {Promise<Object>} Object containing full briefing and condensed summary
  */
 export async function generateBriefing(currentData, previousBriefing) {
   if (!currentData || Object.keys(currentData.sources || currentData).length === 0) {
-    return 'No data available to generate briefing. Please check your data sources configuration.';
+    return {
+      text: 'No data available to generate briefing. Please check your data sources configuration.',
+      condensedSummary: 'No data available.'
+    };
   }
 
   try {
     const prompt = buildBriefingPrompt(currentData, previousBriefing);
     const result = await callClaude(prompt);
-    return result; // Returns { text, usage }
+    
+    // Generate condensed summary for historical storage
+    const condensedSummary = await generateCondensedSummary(result.text);
+    
+    return {
+      ...result,
+      condensedSummary: condensedSummary
+    };
   } catch (error) {
     console.error('Error generating briefing:', error);
 
     // Return a fallback briefing
-    return `Good morning! I encountered an issue generating your personalized briefing today.
+    const fallbackText = `Good morning! I encountered an issue generating your personalized briefing today.
 
     Error: ${error.message}
 
     Please check your API key configuration and try again.`;
+    
+    return {
+      text: fallbackText,
+      condensedSummary: `Error: ${error.message}`
+    };
   }
 }
 

claudash/src/services/dataFetcher.js

@@ -10,12 +10,25 @@ import {
 } from '../utils/constants';
 
 /**
- * Parse environment variable array
+ * Parse environment variable - supports both array and object formats
- * @param {string} envValue - Comma-separated values
+ * @param {string} envValue - Comma-separated values or JSON object
- * @returns {Array} Parsed array
+ * @returns {Array|Object} Parsed data
  */
-function parseEnvArray(envValue) {
+function parseEnvSources(envValue) {
   if (!envValue) return [];
+  
+  // Try to parse as JSON object first (new format)
+  try {
+    const parsed = JSON.parse(envValue);
+    if (typeof parsed === 'object' && !Array.isArray(parsed)) {
+      // It's an object with rows
+      return parsed;
+    }
+  } catch (e) {
+    // Not valid JSON, try comma-separated format (legacy)
+  }
+  
+  // Fall back to comma-separated format
   return envValue.split(',').map(s => s.trim()).filter(Boolean);
 }
 
@@ -102,12 +115,12 @@ export async function fetchWeather(location) {
 
 /**
  * Fetch from all configured sources
- * @returns {Promise<Object>} Object with all source data
+ * @returns {Promise<Object>} Object with all source data organized by rows
  */
 export async function fetchAllSources() {
   // Get configured sources from environment or use defaults
   const sourcesEnv = import.meta.env.REACT_APP_DATA_SOURCES;
-  const sources = sourcesEnv ? parseEnvArray(sourcesEnv) : DEFAULT_DATA_SOURCES;
+  const sources = sourcesEnv ? parseEnvSources(sourcesEnv) : DEFAULT_DATA_SOURCES;
 
   // Get location for weather
   const location = import.meta.env.REACT_APP_LOCATION;
@@ -115,69 +128,100 @@ export async function fetchAllSources() {
   console.log('Fetching from sources:', sources);
   console.log('Weather location:', location);
 
-  // Create fetch promises
+  let sourcesByRow = {};
-  const promises = sources.map(url =>
+  let allUrls = [];
-    fetchSingleSource(url).catch(error => ({
+
-      error: true,
+  // Check if sources is an object (new format) or array (legacy format)
-      url: url,
+  if (typeof sources === 'object' && !Array.isArray(sources)) {
-      message: error.message
+    // New format: { "row1": [...urls], "row2": [...urls] }
-    }))
+    sourcesByRow = sources;
+    // Flatten all URLs for fetching
+    allUrls = Object.values(sources).flat();
+  } else {
+    // Legacy format: flat array of URLs
+    // Put all in a single row
+    sourcesByRow = { row1: Array.isArray(sources) ? sources : [] };
+    allUrls = Array.isArray(sources) ? sources : [];
+  }
+
+  // Create fetch promises for all URLs
+  const urlPromises = allUrls.map(url =>
+    fetchSingleSource(url)
+      .then(data => ({ url, data, error: false }))
+      .catch(error => ({
+        url,
+        data: null,
+        error: true,
+        message: error.message
+      }))
   );
 
   // Add weather fetch if location is configured
+  let weatherPromise = null;
   if (location) {
-    promises.push(
+    weatherPromise = fetchWeather(location).catch(error => ({
-      fetchWeather(location).catch(error => ({
+      error: true,
-        error: true,
+      type: 'weather',
-        type: 'weather',
+      message: error.message
-        message: error.message
+    }));
-      }))
-    );
   }
 
   // Fetch all in parallel
-  const results = await Promise.allSettled(promises);
+  const urlResults = await Promise.all(urlPromises);
+  const weatherResult = weatherPromise ? await weatherPromise : null;
 
-  // Process results
+  // Create a map of URL to fetched data
-  const sourceData = {};
+  const urlDataMap = {};
   const errors = [];
 
-  results.forEach((result, index) => {
+  urlResults.forEach(result => {
-    if (result.status === 'fulfilled') {
+    if (result.error) {
-      const data = result.value;
+      errors.push({
-      // Handle null or error responses
+        url: result.url,
-      if (!data || data.error) {
+        error: true,
-        if (data) {
+        message: result.message
-          errors.push(data);
+      });
-        } else {
+    } else if (result.data) {
-          errors.push({
+      urlDataMap[result.url] = result.data;
-            error: true,
+    }
-            index: index,
+  });
-            reason: 'Source returned null'
+
-          });
+  // Organize data by rows
-        }
+  const sourceData = {};
-      } else {
+  
-        // Assign to appropriate key
+  // Add weather as a special entry
-        if (data.type === 'weather') {
+  if (weatherResult && !weatherResult.error) {
-          sourceData.weather = data;
+    sourceData.weather = weatherResult;
-        } else if (data.type === 'reddit') {
+  } else if (weatherResult && weatherResult.error) {
+    errors.push(weatherResult);
+  }
+
+  // Process each row
+  Object.entries(sourcesByRow).forEach(([rowKey, urls]) => {
+    sourceData[rowKey] = [];
+    
+    urls.forEach(url => {
+      if (urlDataMap[url]) {
+        sourceData[rowKey].push(urlDataMap[url]);
+      }
+    });
+  });
+
+  // For backward compatibility, also add flat structure
+  let flatIndex = 0;
+  Object.values(sourceData).forEach(rowData => {
+    if (Array.isArray(rowData)) {
+      rowData.forEach(data => {
+        if (data.type === 'reddit') {
           const key = `reddit_${Object.keys(sourceData).filter(k => k.startsWith('reddit')).length}`;
           sourceData[key] = data;
-        } else if (data.type === 'hackernews') {
-          sourceData.hackernews = data;
         } else if (data.type === 'rss') {
           const key = `rss_${Object.keys(sourceData).filter(k => k.startsWith('rss')).length}`;
           sourceData[key] = data;
         } else {
-          const key = `source_${index}`;
+          const key = `source_${flatIndex++}`;
           sourceData[key] = data;
         }
-      }
-    } else {
-      errors.push({
-        error: true,
-        index: index,
-        reason: result.reason?.message || 'Unknown error'
       });
     }
   });
@@ -191,38 +235,8 @@ export async function fetchAllSources() {
     sources: sourceData,
     errors: errors,
     fetchedAt: Date.now(),
-    successful: Object.keys(sourceData).length,
+    successful: allUrls.length - errors.length + (weatherResult && !weatherResult.error ? 1 : 0),
     failed: errors.length
   };
 }
 
-/**
- * Fetch a specific Hacker News story by ID
- * @param {number} storyId - Story ID
- * @returns {Promise<Object>} Story data
- */
-export async function fetchHackerNewsStory(storyId) {
-  try {
-    const url = `https://hacker-news.firebaseio.com/v0/item/${storyId}.json`;
-    return await fetchWithCors(url);
-  } catch (error) {
-    console.error(`Error fetching HN story ${storyId}:`, error);
-    return null;
-  }
-}
-
-/**
- * Fetch multiple Hacker News stories
- * @param {Array<number>} storyIds - Array of story IDs
- * @param {number} limit - Maximum number of stories to fetch
- * @returns {Promise<Array>} Array of story data
- */
-export async function fetchHackerNewsStories(storyIds, limit = 10) {
-  const idsToFetch = storyIds.slice(0, limit);
-  const promises = idsToFetch.map(id => fetchHackerNewsStory(id));
-  const results = await Promise.allSettled(promises);
-
-  return results
-    .filter(r => r.status === 'fulfilled' && r.value)
-    .map(r => r.value);
-}

claudash/src/services/sourceParser.js

@@ -9,9 +9,9 @@ import { RSS_TO_JSON_API } from '../utils/constants';
 export function detectSourceType(url) {
   const urlLower = url.toLowerCase();
 
-  if (urlLower.endsWith('.json')) return 'json';
+  if (urlLower.includes('.json')) return 'json';
-  if (urlLower.endsWith('.rss') ||
+  if (urlLower.includes('.rss') ||
-      urlLower.endsWith('.xml') ||
+      urlLower.includes('.xml') ||
       urlLower.includes('/feed') ||
       urlLower.includes('rss')) return 'rss';
 
@@ -128,6 +128,108 @@ export async function extractWithClaude(html, url) {
   };
 }
 
+/**
+ * Condense data for Claude to reduce payload size
+ * @param {Object} data - Full source data
+ * @returns {Object} - Condensed data suitable for Claude
+ */
+export function condenseForClaude(data) {
+    if (!data) return null;
+    
+    // Handle weather data - already compact
+    if (data.type === 'weather') {
+        return {
+            type: 'weather',
+            location: data.location,
+            current: {
+                temp: data.current?.temp,
+                description: data.current?.description,
+                humidity: data.current?.humidity,
+                windSpeed: data.current?.windSpeed
+            },
+            forecast: data.forecast?.slice(0, 3)?.map(day => ({
+                date: day.date,
+                maxTemp: day.maxTemp,
+                minTemp: day.minTemp,
+                description: day.description
+            }))
+        };
+    }
+    
+    // Handle Reddit data - needs major condensing
+    if (data.type === 'reddit') {
+        return {
+            type: 'reddit',
+            url: data.url,
+            posts: data.posts?.slice(0, 10)?.map(post => ({
+                title: post.title,
+                score: post.score,
+                comments: post.comments,
+                subreddit: post.subreddit,
+                // Truncate selftext to 200 chars
+                summary: post.selftext ? 
+                    (post.selftext.length > 200 ? 
+                        post.selftext.substring(0, 197) + '...' : 
+                        post.selftext) : null,
+                url: post.url
+            }))
+        };
+    }
+    
+    // Handle RSS feeds
+    if (data.type === 'rss') {
+        return {
+            type: 'rss',
+            feedTitle: data.feedTitle,
+            items: data.items?.slice(0, 10)?.map(item => ({
+                title: item.title,
+                // Truncate description to 150 chars
+                summary: item.description ? 
+                    (item.description.length > 150 ? 
+                        item.description.substring(0, 147) + '...' : 
+                        item.description) : null,
+                pubDate: item.pubDate
+            }))
+        };
+    }
+    
+    // Default for unknown types - just truncate if it's large
+    if (data.type === 'unknown' && data.data) {
+        const dataStr = JSON.stringify(data.data);
+        if (dataStr.length > 1000) {
+            return {
+                type: 'unknown',
+                url: data.url,
+                summary: 'Data truncated for size',
+                sample: dataStr.substring(0, 500) + '...'
+            };
+        }
+    }
+    
+    return data;
+}
+
+/**
+ * Condense multiple sources for Claude
+ * @param {Object} sources - Object containing all source data
+ * @returns {Object} - Condensed sources object
+ */
+export function condenseSourcesForClaude(sources) {
+    const condensed = {};
+    
+    Object.entries(sources).forEach(([key, value]) => {
+        if (Array.isArray(value)) {
+            // Handle row arrays
+            condensed[key] = value.map(item => condenseForClaude(item));
+        } else {
+            // Handle individual source objects
+            condensed[key] = condenseForClaude(value);
+        }
+    });
+    
+    return condensed;
+}
+
 /**
  * Clean and standardize source data
  * @param {any} data - Raw source data
@@ -135,34 +237,102 @@ export async function extractWithClaude(html, url) {
  * @returns {Object} - Standardized data object
  */
 export function standardizeSourceData(data, sourceUrl) {
-  // Detect the source and format accordingly
+    // Detect the source and format accordingly
-  if (sourceUrl.includes('reddit.com') && data.data) {
+    if (sourceUrl.includes('reddit.com') && data.data) {
-    // Reddit JSON format
+        // Reddit JSON format
-    return {
+        return {
-      type: 'reddit',
+            type: 'reddit',
-      url: sourceUrl,
+            url: sourceUrl,
-      posts: data.data.children?.map(child => ({
+            posts: data.data.children?.map(child => {
-        title: child.data.title,
+                const post = child.data;
-        url: child.data.url,
+                
-        author: child.data.author,
+                // Extract image URLs from different possible locations
-        score: child.data.score,
+                let images = [];
-        comments: child.data.num_comments,
+                let mainImage = null;
-        created: child.data.created_utc
+                
-      })) || [],
+                // Check for direct image URL
-      after: data.data.after,
+                if (post.url && post.url.match(/\.(jpg|jpeg|png|gif|webp)$/i)) {
-      before: data.data.before
+                    mainImage = post.url;
-    };
+                } else if (post.url_overridden_by_dest && post.url_overridden_by_dest.match(/\.(jpg|jpeg|png|gif|webp)$/i)) {
-  }
+                    mainImage = post.url_overridden_by_dest;
+                }
+                
+                // Check preview object for images (most reliable source for Reddit images)
+                if (!mainImage && post.preview && post.preview.images && post.preview.images.length > 0) {
+                    const previewImage = post.preview.images[0];
+                    
+                    // Get the smallest resolution that's still reasonable quality
+                    // Resolutions array is sorted from smallest to largest
+                    if (previewImage.resolutions && previewImage.resolutions.length > 0) {
+                        // Get resolution around 320-640px wide for optimal loading
+                        const targetRes = previewImage.resolutions.find(r => r.width >= 320 && r.width <= 640) 
+                                         || previewImage.resolutions[previewImage.resolutions.length - 1]
+                                         || previewImage.resolutions[0];
+                        mainImage = targetRes.url.replace(/&amp;/g, '&');
+                    } else if (previewImage.source) {
+                        // Fallback to source if no resolutions
+                        mainImage = previewImage.source.url.replace(/&amp;/g, '&');
+                    }
+                }
+                
+                // Check for gallery images
+                if (post.is_gallery && post.media_metadata) {
+                    images = Object.values(post.media_metadata).map(media => {
+                        // Get smallest reasonable resolution for gallery images
+                        if (media.p && media.p.length > 0) {
+                            // Find preview around 320-640px or use middle resolution
+                            const targetPreview = media.p.find(p => p.x >= 320 && p.x <= 640)
+                                                || media.p[Math.floor(media.p.length / 2)]
+                                                || media.p[0];
+                            return targetPreview.u.replace(/&amp;/g, '&');
+                        } else if (media.s && media.s.u) {
+                            // Fallback to source
+                            return media.s.u.replace(/&amp;/g, '&');
+                        }
+                        return null;
+                    }).filter(url => url !== null);
+                    
+                    // Use first gallery image as main if no other main image
+                    if (!mainImage && images.length > 0) {
+                        mainImage = images[0];
+                    }
+                }
+                
+                // Check for regular thumbnail (not "self", "default", "nsfw", "spoiler", etc.)
+                let thumbnail = null;
+                if (post.thumbnail && post.thumbnail.startsWith('http')) {
+                    thumbnail = post.thumbnail;
+                }
+                
+                // Use thumbnail as main image if no better option
+                if (!mainImage && thumbnail && 
+                    !thumbnail.includes('external-preview') && 
+                    thumbnail !== 'self' && 
+                    thumbnail !== 'default') {
+                    mainImage = thumbnail;
+                }
+                
+                return {
+                    title: post.title,
+                    url: post.url,
+                    permalink: `https://reddit.com${post.permalink}`,
+                    author: post.author,
+                    score: post.score,
+                    comments: post.num_comments,
+                    created: post.created_utc,
+                    thumbnail: thumbnail,
+                    mainImage: mainImage,
+                    galleryImages: images,
+                    isGallery: post.is_gallery || false,
+                    selftext: post.selftext,
+                    subreddit: post.subreddit
+                };
+            }) || [],
+            after: data.data.after,
+            before: data.data.before
+        };
+    }
 
-  if (sourceUrl.includes('hacker-news') && Array.isArray(data)) {
-    // Hacker News top stories (just IDs)
-    return {
-      type: 'hackernews',
-      url: sourceUrl,
-      storyIds: data.slice(0, 30), // Get top 30 stories
-      fetchedAt: Date.now()
-    };
-  }
 
   if (data.items && data.feed) {
     // RSS feed format from rss2json

claudash/src/services/storageService.js

@@ -27,18 +27,20 @@ function saveStorage(storage) {
 }
 
 /**
- * Save today's briefing
+ * Save today's briefing with condensed summary
  * @param {string} date - Date in YYYY-MM-DD format
  * @param {string} briefingText - The generated briefing text
  * @param {Object} sourceData - Raw data from all sources
+ * @param {string} condensedSummary - Condensed summary for historical context (optional)
  */
-export function saveBriefing(date, briefingText, sourceData) {
+export function saveBriefing(date, briefingText, sourceData, condensedSummary = null) {
   const storage = getStorage();
 
   storage.briefings[date] = {
     timestamp: Date.now(),
     briefing: briefingText,
-    sources: sourceData
+    sources: sourceData,
+    condensedSummary: condensedSummary  // Store condensed version for historical lookups
   };
 
   saveStorage(storage);

claudash/src/utils/constants.js

@@ -20,7 +20,6 @@ export const RSS_TO_JSON_API = 'https://api.rss2json.com/v1/api.json';
 // Default data sources (fallback if none provided)
 export const DEFAULT_DATA_SOURCES = [
   'https://www.reddit.com/r/technology.json',
-  'https://hacker-news.firebaseio.com/v0/topstories.json',
   'https://www.reddit.com/r/worldnews.json'
 ];