better compression and history storage, limiting posts to 10, layout shifts
This commit is contained in:
Drew Peifer
parent
fdc7912a0d
commit
8bbdcf3673
10 changed files with 533 additions and 233 deletions
|
|
@ -1,9 +1,13 @@
|
|||
# REQUIRED: Anthropic API Key
|
||||
REACT_APP_ANTHROPIC_API_KEY=sk-ant-your-key-here
|
||||
|
||||
# Data sources (comma-separated URLs)
|
||||
# Data sources (JSON object with rows as keys, arrays of URLs as values)
|
||||
# Each key represents a row of widgets in the UI
|
||||
# Supports: .json (auto-parsed), .rss/.xml (RSS feeds), plain URLs (scraped)
|
||||
REACT_APP_DATA_SOURCES=https://www.reddit.com/r/technology.json,https://hacker-news.firebaseio.com/v0/topstories.json,https://www.reddit.com/r/worldnews.json
|
||||
# Example formats:
|
||||
# - Object format: {"row1":["url1","url2"],"row2":["url3","url4"]}
|
||||
# - Legacy format: url1,url2,url3 (comma-separated)
|
||||
REACT_APP_DATA_SOURCES={"row1":["https://www.reddit.com/r/technology.json","https://www.reddit.com/r/worldnews.json"],"row2":["https://www.reddit.com/r/science.json"]}
|
||||
|
||||
# Location for weather (supports "city, state, country" or zip code)
|
||||
REACT_APP_LOCATION=harrisburg, pa, usa
|
||||
|
|
|
|||
|
|
@ -5,7 +5,8 @@ const app = express();
|
|||
const PORT = 3001;
|
||||
|
||||
app.use(cors());
|
||||
app.use(express.json());
|
||||
// Increase payload limit to 10MB (default is 100kb)
|
||||
app.use(express.json({ limit: '10mb' }));
|
||||
|
||||
// Proxy endpoint for Claude API
|
||||
app.post('/api/claude', async (req, res) => {
|
||||
|
|
|
|||
|
|
@ -53,12 +53,18 @@ h1, h2, h3, h4, h5, h6 {
|
|||
flex-direction: row;
|
||||
gap: 10px;
|
||||
}
|
||||
.news-widget {
|
||||
min-width: 25%;
|
||||
max-width: 49%;
|
||||
flex-grow: 2;
|
||||
}
|
||||
.widget-row.condensed {
|
||||
flex-wrap: wrap;
|
||||
justify-content: space-between;
|
||||
justify-content: space-evenly;
|
||||
}
|
||||
.widget-row.condensed .news-widget{
|
||||
flex-grow: 2;
|
||||
.news-widget img {
|
||||
width: 70%;
|
||||
margin: 10px auto;
|
||||
}
|
||||
|
||||
a {
|
||||
|
|
|
|||
|
|
@ -91,8 +91,9 @@ function Dashboard({ themeToggle, isDark }) {
|
|||
// Generate today's briefing with Claude
|
||||
const result = await generateBriefing(data, previousBriefing);
|
||||
console.log('Briefing generated');
|
||||
console.log('Condensed summary:', result.condensedSummary?.substring(0, 100) + '...');
|
||||
|
||||
// Handle result object with text and usage
|
||||
// Handle result object with text, condensedSummary and usage
|
||||
if (result && typeof result === 'object' && result.text) {
|
||||
setBriefing(result.text);
|
||||
setTokenUsage(result.usage);
|
||||
|
|
@ -104,11 +105,12 @@ function Dashboard({ themeToggle, isDark }) {
|
|||
setCacheInfo(null);
|
||||
}
|
||||
|
||||
// Save to localStorage
|
||||
// Save to localStorage with condensed summary
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
console.log('Saving briefing for:', today);
|
||||
const briefingText = result?.text || result;
|
||||
saveBriefing(today, briefingText, data.sources);
|
||||
const condensedSummary = result?.condensedSummary || null;
|
||||
saveBriefing(today, briefingText, data.sources, condensedSummary);
|
||||
} catch (briefingError) {
|
||||
console.error('Failed to generate briefing:', briefingError);
|
||||
setBriefing(`Unable to generate AI briefing: ${briefingError.message}\n\nYou can still view the data widgets below.`);
|
||||
|
|
@ -218,18 +220,45 @@ function Dashboard({ themeToggle, isDark }) {
|
|||
data={sourceData.weather}
|
||||
/>
|
||||
</Box>
|
||||
<Box className="widget-row condensed">
|
||||
{/* Render NewsWidget for each news source */}
|
||||
{Object.entries(sourceData).map(([key, data]) => {
|
||||
if (key === 'weather') return null;
|
||||
|
||||
{/* Render rows based on the new data structure */}
|
||||
{Object.entries(sourceData).map(([key, data]) => {
|
||||
// Skip special entries like weather and legacy flat entries
|
||||
if (key === 'weather' || key.startsWith('reddit_') || key.startsWith('rss_') || key.startsWith('source_')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Only process row entries (row1, row2, etc.)
|
||||
if (key.startsWith('row') && Array.isArray(data)) {
|
||||
return (
|
||||
<NewsWidget
|
||||
key={key}
|
||||
data={data}
|
||||
/>
|
||||
<Box key={key} className="widget-row condensed">
|
||||
{data.map((widgetData, index) => (
|
||||
<NewsWidget
|
||||
key={`${key}-${index}`}
|
||||
data={widgetData}
|
||||
/>
|
||||
))}
|
||||
</Box>
|
||||
);
|
||||
})}
|
||||
</Box>
|
||||
}
|
||||
|
||||
return null;
|
||||
})}
|
||||
|
||||
{/* Fallback for legacy format - if no rows detected, render flat structure */}
|
||||
{!Object.keys(sourceData).some(k => k.startsWith('row')) && (
|
||||
<Box className="widget-row condensed">
|
||||
{Object.entries(sourceData).map(([key, data]) => {
|
||||
if (key === 'weather' || !data) return null;
|
||||
return (
|
||||
<NewsWidget
|
||||
key={key}
|
||||
data={data}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</Box>
|
||||
)}
|
||||
</Box>
|
||||
</Container>
|
||||
|
||||
|
|
|
|||
|
|
@ -13,80 +13,121 @@ function NewsWidget({ data, sx }) {
|
|||
);
|
||||
}
|
||||
|
||||
// Handle Reddit data
|
||||
if (data.type === 'reddit' && data.posts) {
|
||||
const subreddit = data.url?.match(/\/r\/([^\/\.]+)/)?.[1] || 'Reddit';
|
||||
// Handle Reddit data
|
||||
if (data.type === 'reddit' && data.posts) {
|
||||
const subreddit = data.url?.match(/\/r\/([^\/\.]+)/)?.[1] || 'Reddit';
|
||||
|
||||
return (
|
||||
<GenericWidget className="news-widget" title={`r/${subreddit}`} sx={{ ...sx }}>
|
||||
<List dense>
|
||||
{data.posts.slice(0, 5).map((post, index) => (
|
||||
<ListItem key={index} disablePadding sx={{ mb: 1 }}>
|
||||
<ListItemText
|
||||
primary={
|
||||
<Box sx={{ display: 'flex', alignItems: 'start', gap: 1 }}>
|
||||
{post.thumbnail && post.thumbnail.startsWith('http') && (
|
||||
<Link
|
||||
href={post.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
<Box
|
||||
component="img"
|
||||
src={post.thumbnail}
|
||||
alt={post.title}
|
||||
sx={{
|
||||
width: 70,
|
||||
height: 70,
|
||||
objectFit: 'cover',
|
||||
borderRadius: 1,
|
||||
flexShrink: 0
|
||||
}}
|
||||
/>
|
||||
</Link>
|
||||
)}
|
||||
<Link
|
||||
href={post.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
sx={{
|
||||
textDecoration: 'none',
|
||||
color: 'text.primary',
|
||||
'&:hover': { textDecoration: 'underline' }
|
||||
}}
|
||||
>
|
||||
<div className="news-title">
|
||||
{post.title}
|
||||
<OpenInNew sx={{ fontSize: 14 }} />
|
||||
</div>
|
||||
</Link>
|
||||
</Box>
|
||||
}
|
||||
secondaryTypographyProps={{ component: 'div' }}
|
||||
secondary={
|
||||
<Box sx={{ display: 'flex', gap: 1, mt: 0.5 }}>
|
||||
<Chip
|
||||
icon={<TrendingUp />}
|
||||
label={post.score}
|
||||
size="small"
|
||||
variant="outlined"
|
||||
/>
|
||||
<Chip
|
||||
icon={<Comment />}
|
||||
label={post.comments}
|
||||
size="small"
|
||||
variant="outlined"
|
||||
/>
|
||||
<span className="news-author">by {post.author}</span>
|
||||
</Box>
|
||||
}
|
||||
/>
|
||||
</ListItem>
|
||||
))}
|
||||
</List>
|
||||
</GenericWidget>
|
||||
);
|
||||
}
|
||||
return (
|
||||
<GenericWidget className="news-widget" title={`r/${subreddit}`} sx={{ ...sx }}>
|
||||
<List dense>
|
||||
{data.posts.slice(0, 5).map((post, index) => {
|
||||
// Determine if we should show an image
|
||||
const hasImage = post.mainImage &&
|
||||
!post.mainImage.includes('external-preview') && // Skip Reddit's external preview placeholder
|
||||
post.mainImage !== 'spoiler' &&
|
||||
post.mainImage !== 'nsfw' &&
|
||||
post.mainImage !== 'default';
|
||||
|
||||
return (
|
||||
<ListItem key={index} disablePadding sx={{ mb: 2 }}>
|
||||
<ListItemText
|
||||
primary={
|
||||
<Box>
|
||||
{/* Image section - Medium size, above title */}
|
||||
{hasImage && (
|
||||
<Link
|
||||
href={post.permalink || post.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
sx={{ display: 'block', mb: 1 }}
|
||||
>
|
||||
<Box
|
||||
component="img"
|
||||
src={post.mainImage}
|
||||
alt={post.title}
|
||||
onError={(e) => {
|
||||
e.target.style.display = 'none';
|
||||
}}
|
||||
sx={{
|
||||
width: '100%',
|
||||
maxWidth: 400,
|
||||
height: 'auto',
|
||||
maxHeight: 250,
|
||||
objectFit: 'cover',
|
||||
borderRadius: 1,
|
||||
display: 'block',
|
||||
backgroundColor: 'action.hover',
|
||||
cursor: 'pointer',
|
||||
transition: 'transform 0.2s',
|
||||
'&:hover': {
|
||||
transform: 'scale(1.02)'
|
||||
}
|
||||
}}
|
||||
/>
|
||||
{post.isGallery && post.galleryImages.length > 1 && (
|
||||
<Chip
|
||||
label={`Gallery (${post.galleryImages.length} images)`}
|
||||
size="small"
|
||||
sx={{ mt: -3.5, ml: 1, position: 'relative' }}
|
||||
/>
|
||||
)}
|
||||
</Link>
|
||||
)}
|
||||
|
||||
{/* Title section */}
|
||||
<Link
|
||||
href={post.permalink || post.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
sx={{
|
||||
textDecoration: 'none',
|
||||
color: 'text.primary',
|
||||
'&:hover': { textDecoration: 'underline' }
|
||||
}}
|
||||
>
|
||||
<div className="news-title" style={{ display: 'flex', alignItems: 'center', gap: 4 }}>
|
||||
{post.title}
|
||||
<OpenInNew sx={{ fontSize: 14, marginLeft: 0.5 }} />
|
||||
</div>
|
||||
</Link>
|
||||
</Box>
|
||||
}
|
||||
secondaryTypographyProps={{ component: 'div' }}
|
||||
secondary={
|
||||
<Box sx={{ display: 'flex', gap: 1, mt: 0.5, flexWrap: 'wrap', alignItems: 'center' }}>
|
||||
<Chip
|
||||
icon={<TrendingUp />}
|
||||
label={`${post.score?.toLocaleString() || 0} upvotes`}
|
||||
size="small"
|
||||
variant="outlined"
|
||||
color="primary"
|
||||
/>
|
||||
<Chip
|
||||
icon={<Comment />}
|
||||
label={`${post.comments?.toLocaleString() || 0} comments`}
|
||||
size="small"
|
||||
variant="outlined"
|
||||
/>
|
||||
<span className="news-author" style={{ lineHeight: '24px', fontSize: '0.875rem', opacity: 0.7 }}>
|
||||
{post.created && (() => {
|
||||
const now = Date.now() / 1000; // Convert to seconds
|
||||
const diff = now - post.created;
|
||||
const hours = Math.floor(diff / 3600);
|
||||
const days = Math.floor(hours / 24);
|
||||
const timeAgo = days > 0 ? `${days}d ago` : `${hours}h ago`;
|
||||
return timeAgo;
|
||||
})()} • by {post.author} in r/{post.subreddit}
|
||||
</span>
|
||||
</Box>
|
||||
}
|
||||
/>
|
||||
</ListItem>
|
||||
);
|
||||
})}
|
||||
</List>
|
||||
</GenericWidget>
|
||||
);
|
||||
}
|
||||
|
||||
// Handle RSS feed data
|
||||
if (data.type === 'rss' && data.items) {
|
||||
|
|
@ -156,19 +197,6 @@ function NewsWidget({ data, sx }) {
|
|||
);
|
||||
}
|
||||
|
||||
// Handle Hacker News data
|
||||
if (data.type === 'hackernews' && data.storyIds) {
|
||||
return (
|
||||
<GenericWidget title="Hacker News" sx={{ ...sx }}>
|
||||
<p className="widget-info">
|
||||
{data.storyIds.length} top stories loaded
|
||||
</p>
|
||||
<p className="widget-caption">
|
||||
(Story details will load when API is configured)
|
||||
</p>
|
||||
</GenericWidget>
|
||||
);
|
||||
}
|
||||
|
||||
// Default fallback for unknown data types
|
||||
return (
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import {
|
|||
DEFAULT_CLAUDE_MODEL,
|
||||
MAX_TOKENS
|
||||
} from '../utils/constants';
|
||||
import { condenseSourcesForClaude } from './sourceParser';
|
||||
|
||||
/**
|
||||
* Make a request to Claude API
|
||||
|
|
@ -77,7 +78,7 @@ async function callClaude(prompt, messages = []) {
|
|||
/**
|
||||
* Build briefing prompt with memory
|
||||
* @param {Object} currentData - Current source data
|
||||
* @param {Object} previousBriefing - Previous briefing object
|
||||
* @param {Object} previousBriefing - Previous briefing object (condensed summary)
|
||||
* @returns {string} The prompt
|
||||
*/
|
||||
function buildBriefingPrompt(currentData, previousBriefing) {
|
||||
|
|
@ -94,17 +95,23 @@ function buildBriefingPrompt(currentData, previousBriefing) {
|
|||
|
||||
CURRENT DATA:
|
||||
`;
|
||||
|
||||
// Add all current source data
|
||||
Object.entries(currentData.sources || currentData).forEach(([source, data]) => {
|
||||
|
||||
// CONDENSE DATA BEFORE SENDING TO CLAUDE
|
||||
const condensedData = condenseSourcesForClaude(currentData.sources || currentData);
|
||||
console.log('Original data size:', JSON.stringify(currentData).length);
|
||||
console.log('Condensed data size:', JSON.stringify(condensedData).length);
|
||||
console.log('Size reduction:', Math.round((1 - JSON.stringify(condensedData).length / JSON.stringify(currentData).length) * 100) + '%');
|
||||
|
||||
// Add all condensed source data
|
||||
Object.entries(condensedData).forEach(([source, data]) => {
|
||||
if (data && !data.error) {
|
||||
prompt += `\n${source.toUpperCase()}:\n${JSON.stringify(data, null, 2)}\n`;
|
||||
}
|
||||
});
|
||||
|
||||
// Add historical context if available
|
||||
if (previousBriefing && previousBriefing.briefing) {
|
||||
prompt += `\n\nYESTERDAY'S BRIEFING:\n${previousBriefing.briefing}\n`;
|
||||
// Add historical context if available (now using condensed summary)
|
||||
if (previousBriefing && previousBriefing.condensedSummary) {
|
||||
prompt += `\n\nPREVIOUS BRIEFING SUMMARY:\n${previousBriefing.condensedSummary}\n`;
|
||||
prompt += `\nHighlight what's changed, any follow-up stories, or interesting trends.`;
|
||||
}
|
||||
|
||||
|
|
@ -122,29 +129,69 @@ Focus on the most newsworthy and interesting items. Don't try to mention everyth
|
|||
}
|
||||
|
||||
/**
|
||||
* Generate daily briefing
|
||||
* Generate a condensed summary of a briefing for historical storage
|
||||
* @param {string} fullBriefing - The full briefing text
|
||||
* @returns {Promise<string>} Condensed summary (50-75 words)
|
||||
*/
|
||||
async function generateCondensedSummary(fullBriefing) {
|
||||
const prompt = `Create an ultra-condensed summary of this briefing in 50-75 words.
|
||||
Focus only on the key topics and main stories mentioned.
|
||||
This will be used as historical context for tomorrow's briefing generation.
|
||||
|
||||
FULL BRIEFING:
|
||||
${fullBriefing}
|
||||
|
||||
Provide ONLY the condensed summary, no preamble or explanation.`;
|
||||
|
||||
try {
|
||||
const result = await callClaude(prompt);
|
||||
return result.text;
|
||||
} catch (error) {
|
||||
console.error('Error generating condensed summary:', error);
|
||||
// Fallback to first 300 chars of original
|
||||
return fullBriefing.substring(0, 300) + '...';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate daily briefing with condensed summary
|
||||
* @param {Object} currentData - Current data from all sources
|
||||
* @param {Object} previousBriefing - Previous briefing for comparison
|
||||
* @returns {Promise<string>} Generated briefing text
|
||||
* @returns {Promise<Object>} Object containing full briefing and condensed summary
|
||||
*/
|
||||
export async function generateBriefing(currentData, previousBriefing) {
|
||||
if (!currentData || Object.keys(currentData.sources || currentData).length === 0) {
|
||||
return 'No data available to generate briefing. Please check your data sources configuration.';
|
||||
return {
|
||||
text: 'No data available to generate briefing. Please check your data sources configuration.',
|
||||
condensedSummary: 'No data available.'
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const prompt = buildBriefingPrompt(currentData, previousBriefing);
|
||||
const result = await callClaude(prompt);
|
||||
return result; // Returns { text, usage }
|
||||
|
||||
// Generate condensed summary for historical storage
|
||||
const condensedSummary = await generateCondensedSummary(result.text);
|
||||
|
||||
return {
|
||||
...result,
|
||||
condensedSummary: condensedSummary
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('Error generating briefing:', error);
|
||||
|
||||
// Return a fallback briefing
|
||||
return `Good morning! I encountered an issue generating your personalized briefing today.
|
||||
const fallbackText = `Good morning! I encountered an issue generating your personalized briefing today.
|
||||
|
||||
Error: ${error.message}
|
||||
|
||||
Please check your API key configuration and try again.`;
|
||||
|
||||
return {
|
||||
text: fallbackText,
|
||||
condensedSummary: `Error: ${error.message}`
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,12 +10,25 @@ import {
|
|||
} from '../utils/constants';
|
||||
|
||||
/**
|
||||
* Parse environment variable array
|
||||
* @param {string} envValue - Comma-separated values
|
||||
* @returns {Array} Parsed array
|
||||
* Parse environment variable - supports both array and object formats
|
||||
* @param {string} envValue - Comma-separated values or JSON object
|
||||
* @returns {Array|Object} Parsed data
|
||||
*/
|
||||
function parseEnvArray(envValue) {
|
||||
function parseEnvSources(envValue) {
|
||||
if (!envValue) return [];
|
||||
|
||||
// Try to parse as JSON object first (new format)
|
||||
try {
|
||||
const parsed = JSON.parse(envValue);
|
||||
if (typeof parsed === 'object' && !Array.isArray(parsed)) {
|
||||
// It's an object with rows
|
||||
return parsed;
|
||||
}
|
||||
} catch (e) {
|
||||
// Not valid JSON, try comma-separated format (legacy)
|
||||
}
|
||||
|
||||
// Fall back to comma-separated format
|
||||
return envValue.split(',').map(s => s.trim()).filter(Boolean);
|
||||
}
|
||||
|
||||
|
|
@ -102,12 +115,12 @@ export async function fetchWeather(location) {
|
|||
|
||||
/**
|
||||
* Fetch from all configured sources
|
||||
* @returns {Promise<Object>} Object with all source data
|
||||
* @returns {Promise<Object>} Object with all source data organized by rows
|
||||
*/
|
||||
export async function fetchAllSources() {
|
||||
// Get configured sources from environment or use defaults
|
||||
const sourcesEnv = import.meta.env.REACT_APP_DATA_SOURCES;
|
||||
const sources = sourcesEnv ? parseEnvArray(sourcesEnv) : DEFAULT_DATA_SOURCES;
|
||||
const sources = sourcesEnv ? parseEnvSources(sourcesEnv) : DEFAULT_DATA_SOURCES;
|
||||
|
||||
// Get location for weather
|
||||
const location = import.meta.env.REACT_APP_LOCATION;
|
||||
|
|
@ -115,69 +128,100 @@ export async function fetchAllSources() {
|
|||
console.log('Fetching from sources:', sources);
|
||||
console.log('Weather location:', location);
|
||||
|
||||
// Create fetch promises
|
||||
const promises = sources.map(url =>
|
||||
fetchSingleSource(url).catch(error => ({
|
||||
error: true,
|
||||
url: url,
|
||||
message: error.message
|
||||
}))
|
||||
let sourcesByRow = {};
|
||||
let allUrls = [];
|
||||
|
||||
// Check if sources is an object (new format) or array (legacy format)
|
||||
if (typeof sources === 'object' && !Array.isArray(sources)) {
|
||||
// New format: { "row1": [...urls], "row2": [...urls] }
|
||||
sourcesByRow = sources;
|
||||
// Flatten all URLs for fetching
|
||||
allUrls = Object.values(sources).flat();
|
||||
} else {
|
||||
// Legacy format: flat array of URLs
|
||||
// Put all in a single row
|
||||
sourcesByRow = { row1: Array.isArray(sources) ? sources : [] };
|
||||
allUrls = Array.isArray(sources) ? sources : [];
|
||||
}
|
||||
|
||||
// Create fetch promises for all URLs
|
||||
const urlPromises = allUrls.map(url =>
|
||||
fetchSingleSource(url)
|
||||
.then(data => ({ url, data, error: false }))
|
||||
.catch(error => ({
|
||||
url,
|
||||
data: null,
|
||||
error: true,
|
||||
message: error.message
|
||||
}))
|
||||
);
|
||||
|
||||
// Add weather fetch if location is configured
|
||||
let weatherPromise = null;
|
||||
if (location) {
|
||||
promises.push(
|
||||
fetchWeather(location).catch(error => ({
|
||||
error: true,
|
||||
type: 'weather',
|
||||
message: error.message
|
||||
}))
|
||||
);
|
||||
weatherPromise = fetchWeather(location).catch(error => ({
|
||||
error: true,
|
||||
type: 'weather',
|
||||
message: error.message
|
||||
}));
|
||||
}
|
||||
|
||||
// Fetch all in parallel
|
||||
const results = await Promise.allSettled(promises);
|
||||
const urlResults = await Promise.all(urlPromises);
|
||||
const weatherResult = weatherPromise ? await weatherPromise : null;
|
||||
|
||||
// Process results
|
||||
const sourceData = {};
|
||||
// Create a map of URL to fetched data
|
||||
const urlDataMap = {};
|
||||
const errors = [];
|
||||
|
||||
results.forEach((result, index) => {
|
||||
if (result.status === 'fulfilled') {
|
||||
const data = result.value;
|
||||
// Handle null or error responses
|
||||
if (!data || data.error) {
|
||||
if (data) {
|
||||
errors.push(data);
|
||||
} else {
|
||||
errors.push({
|
||||
error: true,
|
||||
index: index,
|
||||
reason: 'Source returned null'
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Assign to appropriate key
|
||||
if (data.type === 'weather') {
|
||||
sourceData.weather = data;
|
||||
} else if (data.type === 'reddit') {
|
||||
urlResults.forEach(result => {
|
||||
if (result.error) {
|
||||
errors.push({
|
||||
url: result.url,
|
||||
error: true,
|
||||
message: result.message
|
||||
});
|
||||
} else if (result.data) {
|
||||
urlDataMap[result.url] = result.data;
|
||||
}
|
||||
});
|
||||
|
||||
// Organize data by rows
|
||||
const sourceData = {};
|
||||
|
||||
// Add weather as a special entry
|
||||
if (weatherResult && !weatherResult.error) {
|
||||
sourceData.weather = weatherResult;
|
||||
} else if (weatherResult && weatherResult.error) {
|
||||
errors.push(weatherResult);
|
||||
}
|
||||
|
||||
// Process each row
|
||||
Object.entries(sourcesByRow).forEach(([rowKey, urls]) => {
|
||||
sourceData[rowKey] = [];
|
||||
|
||||
urls.forEach(url => {
|
||||
if (urlDataMap[url]) {
|
||||
sourceData[rowKey].push(urlDataMap[url]);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// For backward compatibility, also add flat structure
|
||||
let flatIndex = 0;
|
||||
Object.values(sourceData).forEach(rowData => {
|
||||
if (Array.isArray(rowData)) {
|
||||
rowData.forEach(data => {
|
||||
if (data.type === 'reddit') {
|
||||
const key = `reddit_${Object.keys(sourceData).filter(k => k.startsWith('reddit')).length}`;
|
||||
sourceData[key] = data;
|
||||
} else if (data.type === 'hackernews') {
|
||||
sourceData.hackernews = data;
|
||||
} else if (data.type === 'rss') {
|
||||
const key = `rss_${Object.keys(sourceData).filter(k => k.startsWith('rss')).length}`;
|
||||
sourceData[key] = data;
|
||||
} else {
|
||||
const key = `source_${index}`;
|
||||
const key = `source_${flatIndex++}`;
|
||||
sourceData[key] = data;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
errors.push({
|
||||
error: true,
|
||||
index: index,
|
||||
reason: result.reason?.message || 'Unknown error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
|
@ -191,38 +235,8 @@ export async function fetchAllSources() {
|
|||
sources: sourceData,
|
||||
errors: errors,
|
||||
fetchedAt: Date.now(),
|
||||
successful: Object.keys(sourceData).length,
|
||||
successful: allUrls.length - errors.length + (weatherResult && !weatherResult.error ? 1 : 0),
|
||||
failed: errors.length
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a specific Hacker News story by ID
|
||||
* @param {number} storyId - Story ID
|
||||
* @returns {Promise<Object>} Story data
|
||||
*/
|
||||
export async function fetchHackerNewsStory(storyId) {
|
||||
try {
|
||||
const url = `https://hacker-news.firebaseio.com/v0/item/${storyId}.json`;
|
||||
return await fetchWithCors(url);
|
||||
} catch (error) {
|
||||
console.error(`Error fetching HN story ${storyId}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch multiple Hacker News stories
|
||||
* @param {Array<number>} storyIds - Array of story IDs
|
||||
* @param {number} limit - Maximum number of stories to fetch
|
||||
* @returns {Promise<Array>} Array of story data
|
||||
*/
|
||||
export async function fetchHackerNewsStories(storyIds, limit = 10) {
|
||||
const idsToFetch = storyIds.slice(0, limit);
|
||||
const promises = idsToFetch.map(id => fetchHackerNewsStory(id));
|
||||
const results = await Promise.allSettled(promises);
|
||||
|
||||
return results
|
||||
.filter(r => r.status === 'fulfilled' && r.value)
|
||||
.map(r => r.value);
|
||||
}
|
||||
|
|
@ -9,9 +9,9 @@ import { RSS_TO_JSON_API } from '../utils/constants';
|
|||
export function detectSourceType(url) {
|
||||
const urlLower = url.toLowerCase();
|
||||
|
||||
if (urlLower.endsWith('.json')) return 'json';
|
||||
if (urlLower.endsWith('.rss') ||
|
||||
urlLower.endsWith('.xml') ||
|
||||
if (urlLower.includes('.json')) return 'json';
|
||||
if (urlLower.includes('.rss') ||
|
||||
urlLower.includes('.xml') ||
|
||||
urlLower.includes('/feed') ||
|
||||
urlLower.includes('rss')) return 'rss';
|
||||
|
||||
|
|
@ -128,6 +128,108 @@ export async function extractWithClaude(html, url) {
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Condense data for Claude to reduce payload size
|
||||
* @param {Object} data - Full source data
|
||||
* @returns {Object} - Condensed data suitable for Claude
|
||||
*/
|
||||
export function condenseForClaude(data) {
|
||||
if (!data) return null;
|
||||
|
||||
// Handle weather data - already compact
|
||||
if (data.type === 'weather') {
|
||||
return {
|
||||
type: 'weather',
|
||||
location: data.location,
|
||||
current: {
|
||||
temp: data.current?.temp,
|
||||
description: data.current?.description,
|
||||
humidity: data.current?.humidity,
|
||||
windSpeed: data.current?.windSpeed
|
||||
},
|
||||
forecast: data.forecast?.slice(0, 3)?.map(day => ({
|
||||
date: day.date,
|
||||
maxTemp: day.maxTemp,
|
||||
minTemp: day.minTemp,
|
||||
description: day.description
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
// Handle Reddit data - needs major condensing
|
||||
if (data.type === 'reddit') {
|
||||
return {
|
||||
type: 'reddit',
|
||||
url: data.url,
|
||||
posts: data.posts?.slice(0, 10)?.map(post => ({
|
||||
title: post.title,
|
||||
score: post.score,
|
||||
comments: post.comments,
|
||||
subreddit: post.subreddit,
|
||||
// Truncate selftext to 200 chars
|
||||
summary: post.selftext ?
|
||||
(post.selftext.length > 200 ?
|
||||
post.selftext.substring(0, 197) + '...' :
|
||||
post.selftext) : null,
|
||||
url: post.url
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
// Handle RSS feeds
|
||||
if (data.type === 'rss') {
|
||||
return {
|
||||
type: 'rss',
|
||||
feedTitle: data.feedTitle,
|
||||
items: data.items?.slice(0, 10)?.map(item => ({
|
||||
title: item.title,
|
||||
// Truncate description to 150 chars
|
||||
summary: item.description ?
|
||||
(item.description.length > 150 ?
|
||||
item.description.substring(0, 147) + '...' :
|
||||
item.description) : null,
|
||||
pubDate: item.pubDate
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
// Default for unknown types - just truncate if it's large
|
||||
if (data.type === 'unknown' && data.data) {
|
||||
const dataStr = JSON.stringify(data.data);
|
||||
if (dataStr.length > 1000) {
|
||||
return {
|
||||
type: 'unknown',
|
||||
url: data.url,
|
||||
summary: 'Data truncated for size',
|
||||
sample: dataStr.substring(0, 500) + '...'
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Condense multiple sources for Claude
|
||||
* @param {Object} sources - Object containing all source data
|
||||
* @returns {Object} - Condensed sources object
|
||||
*/
|
||||
export function condenseSourcesForClaude(sources) {
|
||||
const condensed = {};
|
||||
|
||||
Object.entries(sources).forEach(([key, value]) => {
|
||||
if (Array.isArray(value)) {
|
||||
// Handle row arrays
|
||||
condensed[key] = value.map(item => condenseForClaude(item));
|
||||
} else {
|
||||
// Handle individual source objects
|
||||
condensed[key] = condenseForClaude(value);
|
||||
}
|
||||
});
|
||||
|
||||
return condensed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean and standardize source data
|
||||
* @param {any} data - Raw source data
|
||||
|
|
@ -135,34 +237,102 @@ export async function extractWithClaude(html, url) {
|
|||
* @returns {Object} - Standardized data object
|
||||
*/
|
||||
export function standardizeSourceData(data, sourceUrl) {
|
||||
// Detect the source and format accordingly
|
||||
if (sourceUrl.includes('reddit.com') && data.data) {
|
||||
// Reddit JSON format
|
||||
return {
|
||||
type: 'reddit',
|
||||
url: sourceUrl,
|
||||
posts: data.data.children?.map(child => ({
|
||||
title: child.data.title,
|
||||
url: child.data.url,
|
||||
author: child.data.author,
|
||||
score: child.data.score,
|
||||
comments: child.data.num_comments,
|
||||
created: child.data.created_utc
|
||||
})) || [],
|
||||
after: data.data.after,
|
||||
before: data.data.before
|
||||
};
|
||||
}
|
||||
// Detect the source and format accordingly
|
||||
if (sourceUrl.includes('reddit.com') && data.data) {
|
||||
// Reddit JSON format
|
||||
return {
|
||||
type: 'reddit',
|
||||
url: sourceUrl,
|
||||
posts: data.data.children?.map(child => {
|
||||
const post = child.data;
|
||||
|
||||
// Extract image URLs from different possible locations
|
||||
let images = [];
|
||||
let mainImage = null;
|
||||
|
||||
// Check for direct image URL
|
||||
if (post.url && post.url.match(/\.(jpg|jpeg|png|gif|webp)$/i)) {
|
||||
mainImage = post.url;
|
||||
} else if (post.url_overridden_by_dest && post.url_overridden_by_dest.match(/\.(jpg|jpeg|png|gif|webp)$/i)) {
|
||||
mainImage = post.url_overridden_by_dest;
|
||||
}
|
||||
|
||||
// Check preview object for images (most reliable source for Reddit images)
|
||||
if (!mainImage && post.preview && post.preview.images && post.preview.images.length > 0) {
|
||||
const previewImage = post.preview.images[0];
|
||||
|
||||
// Get the smallest resolution that's still reasonable quality
|
||||
// Resolutions array is sorted from smallest to largest
|
||||
if (previewImage.resolutions && previewImage.resolutions.length > 0) {
|
||||
// Get resolution around 320-640px wide for optimal loading
|
||||
const targetRes = previewImage.resolutions.find(r => r.width >= 320 && r.width <= 640)
|
||||
|| previewImage.resolutions[previewImage.resolutions.length - 1]
|
||||
|| previewImage.resolutions[0];
|
||||
mainImage = targetRes.url.replace(/&/g, '&');
|
||||
} else if (previewImage.source) {
|
||||
// Fallback to source if no resolutions
|
||||
mainImage = previewImage.source.url.replace(/&/g, '&');
|
||||
}
|
||||
}
|
||||
|
||||
// Check for gallery images
|
||||
if (post.is_gallery && post.media_metadata) {
|
||||
images = Object.values(post.media_metadata).map(media => {
|
||||
// Get smallest reasonable resolution for gallery images
|
||||
if (media.p && media.p.length > 0) {
|
||||
// Find preview around 320-640px or use middle resolution
|
||||
const targetPreview = media.p.find(p => p.x >= 320 && p.x <= 640)
|
||||
|| media.p[Math.floor(media.p.length / 2)]
|
||||
|| media.p[0];
|
||||
return targetPreview.u.replace(/&/g, '&');
|
||||
} else if (media.s && media.s.u) {
|
||||
// Fallback to source
|
||||
return media.s.u.replace(/&/g, '&');
|
||||
}
|
||||
return null;
|
||||
}).filter(url => url !== null);
|
||||
|
||||
// Use first gallery image as main if no other main image
|
||||
if (!mainImage && images.length > 0) {
|
||||
mainImage = images[0];
|
||||
}
|
||||
}
|
||||
|
||||
// Check for regular thumbnail (not "self", "default", "nsfw", "spoiler", etc.)
|
||||
let thumbnail = null;
|
||||
if (post.thumbnail && post.thumbnail.startsWith('http')) {
|
||||
thumbnail = post.thumbnail;
|
||||
}
|
||||
|
||||
// Use thumbnail as main image if no better option
|
||||
if (!mainImage && thumbnail &&
|
||||
!thumbnail.includes('external-preview') &&
|
||||
thumbnail !== 'self' &&
|
||||
thumbnail !== 'default') {
|
||||
mainImage = thumbnail;
|
||||
}
|
||||
|
||||
return {
|
||||
title: post.title,
|
||||
url: post.url,
|
||||
permalink: `https://reddit.com${post.permalink}`,
|
||||
author: post.author,
|
||||
score: post.score,
|
||||
comments: post.num_comments,
|
||||
created: post.created_utc,
|
||||
thumbnail: thumbnail,
|
||||
mainImage: mainImage,
|
||||
galleryImages: images,
|
||||
isGallery: post.is_gallery || false,
|
||||
selftext: post.selftext,
|
||||
subreddit: post.subreddit
|
||||
};
|
||||
}) || [],
|
||||
after: data.data.after,
|
||||
before: data.data.before
|
||||
};
|
||||
}
|
||||
|
||||
if (sourceUrl.includes('hacker-news') && Array.isArray(data)) {
|
||||
// Hacker News top stories (just IDs)
|
||||
return {
|
||||
type: 'hackernews',
|
||||
url: sourceUrl,
|
||||
storyIds: data.slice(0, 30), // Get top 30 stories
|
||||
fetchedAt: Date.now()
|
||||
};
|
||||
}
|
||||
|
||||
if (data.items && data.feed) {
|
||||
// RSS feed format from rss2json
|
||||
|
|
|
|||
|
|
@ -27,18 +27,20 @@ function saveStorage(storage) {
|
|||
}
|
||||
|
||||
/**
|
||||
* Save today's briefing
|
||||
* Save today's briefing with condensed summary
|
||||
* @param {string} date - Date in YYYY-MM-DD format
|
||||
* @param {string} briefingText - The generated briefing text
|
||||
* @param {Object} sourceData - Raw data from all sources
|
||||
* @param {string} condensedSummary - Condensed summary for historical context (optional)
|
||||
*/
|
||||
export function saveBriefing(date, briefingText, sourceData) {
|
||||
export function saveBriefing(date, briefingText, sourceData, condensedSummary = null) {
|
||||
const storage = getStorage();
|
||||
|
||||
storage.briefings[date] = {
|
||||
timestamp: Date.now(),
|
||||
briefing: briefingText,
|
||||
sources: sourceData
|
||||
sources: sourceData,
|
||||
condensedSummary: condensedSummary // Store condensed version for historical lookups
|
||||
};
|
||||
|
||||
saveStorage(storage);
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ export const RSS_TO_JSON_API = 'https://api.rss2json.com/v1/api.json';
|
|||
// Default data sources (fallback if none provided)
|
||||
export const DEFAULT_DATA_SOURCES = [
|
||||
'https://www.reddit.com/r/technology.json',
|
||||
'https://hacker-news.firebaseio.com/v0/topstories.json',
|
||||
'https://www.reddit.com/r/worldnews.json'
|
||||
];
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue