Add Vertex AI embeddings + semantic search for Learning Hub
- New: Vector search with pgvector extension (cosine similarity) - Embeddings: Vertex AI text-embedding-005 (768 dims, HIPAA-eligible) - 3 search modes: keyword, semantic, hybrid (best of both) - Auto-generate embeddings on content create/update - Admin endpoints: /api/admin/learning/embeddings/generate (backfill), /status - User endpoints: /api/learning/search/semantic, /search/hybrid - Falls back to OpenAI embeddings if Vertex not configured - Supports LiteLLM proxy routing Models tested: - vertex_ai/text-embedding-005 (768 dims, English+code) ✅ - vertex_ai/gemini-embedding-001 (3072 dims, multilingual) ✅ - vertex_ai/text-multilingual-embedding-002 (768 dims) ✅
This commit is contained in:
ifedan-ed
parent
0658b31df3
commit
106e4baf17
5 changed files with 575 additions and 7 deletions
21
.env.example
21
.env.example
|
|
@ -125,6 +125,27 @@ NEXTCLOUD_URL=https://cloud.yourdomain.com
|
|||
# S3_SECRET_ACCESS_KEY=minio-secret-key
|
||||
# S3_FORCE_PATH_STYLE=true
|
||||
|
||||
# ============================================================
|
||||
# EMBEDDINGS (for Learning Hub semantic search)
|
||||
# ============================================================
|
||||
# Enables vector-based semantic search in Learning Hub
|
||||
# Requires pgvector extension: apt-get install postgresql-16-pgvector
|
||||
|
||||
# Default model (Vertex AI text-embedding-005, 768 dims, English + code optimized)
|
||||
EMBEDDING_MODEL=vertex_ai/text-embedding-005
|
||||
EMBEDDING_DIMENSIONS=768
|
||||
|
||||
# Other Vertex AI embedding models:
|
||||
# - vertex_ai/text-embedding-005 → 768 dims, English + code (recommended)
|
||||
# - vertex_ai/gemini-embedding-001 → up to 3072 dims, multilingual + code
|
||||
# - vertex_ai/text-multilingual-embedding-002 → 768 dims, multilingual focus
|
||||
#
|
||||
# LiteLLM usage (if using LiteLLM proxy):
|
||||
# EMBEDDING_MODEL=text-embedding-005 # LiteLLM will route to configured provider
|
||||
#
|
||||
# OpenAI fallback (NOT HIPAA-eligible):
|
||||
# Uses text-embedding-3-small if OPENAI_API_KEY is set and no Vertex/LiteLLM configured
|
||||
|
||||
# ============================================================
|
||||
# DATABASE
|
||||
# ============================================================
|
||||
|
|
|
|||
|
|
@ -18,6 +18,14 @@ pool.query('SELECT NOW()')
|
|||
async function initDatabase() {
|
||||
var client = await pool.connect();
|
||||
try {
|
||||
// Enable pgvector extension for embeddings
|
||||
try {
|
||||
await client.query('CREATE EXTENSION IF NOT EXISTS vector');
|
||||
console.log('✅ pgvector extension: enabled');
|
||||
} catch (err) {
|
||||
console.warn('⚠️ pgvector extension not available. Vector search disabled. Install: apt-get install postgresql-16-pgvector');
|
||||
}
|
||||
|
||||
await client.query(`
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id SERIAL PRIMARY KEY,
|
||||
|
|
@ -249,6 +257,36 @@ async function initDatabase() {
|
|||
CREATE INDEX IF NOT EXISTS idx_user_docs_user ON user_documents(user_id);
|
||||
`); } catch(e) {}
|
||||
|
||||
// Add embedding column to learning_content for vector search (768 dims = Vertex AI text-embedding-005)
|
||||
try {
|
||||
await client.query('ALTER TABLE learning_content ADD COLUMN IF NOT EXISTS embedding vector(768)');
|
||||
console.log('✅ learning_content.embedding: added');
|
||||
} catch(e) {
|
||||
console.warn('⚠️ Could not add embedding column:', e.message);
|
||||
}
|
||||
|
||||
// Create IVFFLAT index for fast similarity search (after data is populated)
|
||||
try {
|
||||
var indexExists = await client.query(
|
||||
"SELECT 1 FROM pg_indexes WHERE indexname = 'idx_learning_content_embedding'"
|
||||
);
|
||||
if (indexExists.rows.length === 0) {
|
||||
// Check if we have at least some embeddings before creating index
|
||||
var embCount = await client.query('SELECT COUNT(*) as count FROM learning_content WHERE embedding IS NOT NULL');
|
||||
if (parseInt(embCount.rows[0]?.count || 0) >= 10) {
|
||||
// IVFFLAT requires lists parameter — use sqrt(rows) as heuristic
|
||||
var lists = Math.max(10, Math.floor(Math.sqrt(embCount.rows[0].count)));
|
||||
await client.query(
|
||||
`CREATE INDEX idx_learning_content_embedding ON learning_content
|
||||
USING ivfflat (embedding vector_cosine_ops) WITH (lists = ${lists})`
|
||||
);
|
||||
console.log('✅ Vector search index: created');
|
||||
}
|
||||
}
|
||||
} catch(e) {
|
||||
// Index creation can fail if not enough data or pgvector not installed — safe to ignore
|
||||
}
|
||||
|
||||
// Seed all default config values (ON CONFLICT DO NOTHING — never overwrites admin changes)
|
||||
var defaults = [
|
||||
// Core
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ var express = require('express');
|
|||
var router = express.Router();
|
||||
var db = require('../db/database');
|
||||
var { authMiddleware, moderatorMiddleware } = require('../middleware/auth');
|
||||
var { generateContentEmbedding, isEmbeddingsAvailable } = require('../utils/embeddings');
|
||||
|
||||
router.use(authMiddleware);
|
||||
router.use(moderatorMiddleware);
|
||||
|
|
@ -149,7 +150,22 @@ router.post('/content', async function(req, res) {
|
|||
[title.trim(), slug, body || '', category_id || null, subject || '', content_type || 'article', published ? true : false, req.user.id]
|
||||
);
|
||||
|
||||
res.json({ success: true, id: result.lastInsertRowid, slug: slug });
|
||||
var contentId = result.lastInsertRowid;
|
||||
|
||||
// Generate embedding asynchronously (don't block response)
|
||||
if (isEmbeddingsAvailable() && body && body.trim()) {
|
||||
generateContentEmbedding({ title: title.trim(), subject: subject || '', body: body })
|
||||
.then(function(embedding) {
|
||||
return db.query(
|
||||
'UPDATE learning_content SET embedding = $1 WHERE id = $2',
|
||||
[JSON.stringify(embedding), contentId]
|
||||
);
|
||||
})
|
||||
.then(function() { console.log('[Embeddings] Generated for content ID:', contentId); })
|
||||
.catch(function(err) { console.error('[Embeddings] Failed for content ID ' + contentId + ':', err.message); });
|
||||
}
|
||||
|
||||
res.json({ success: true, id: contentId, slug: slug });
|
||||
} catch (err) { console.error('[LearningAdmin]', err.message); res.status(500).json({ error: 'Internal server error' }); }
|
||||
});
|
||||
|
||||
|
|
@ -160,19 +176,38 @@ router.put('/content/:id', async function(req, res) {
|
|||
|
||||
var { title, body, category_id, subject, content_type, published } = req.body;
|
||||
|
||||
var newTitle = title !== undefined ? title : item.title;
|
||||
var newBody = body !== undefined ? body : item.body;
|
||||
var newSubject = subject !== undefined ? subject : item.subject;
|
||||
|
||||
await db.run(
|
||||
'UPDATE learning_content SET title = ?, body = ?, category_id = ?, subject = ?, content_type = ?, published = ?, updated_at = NOW() WHERE id = ?',
|
||||
[
|
||||
title !== undefined ? title : item.title,
|
||||
body !== undefined ? body : item.body,
|
||||
newTitle,
|
||||
newBody,
|
||||
category_id !== undefined ? (category_id || null) : item.category_id,
|
||||
subject !== undefined ? subject : item.subject,
|
||||
newSubject,
|
||||
content_type !== undefined ? content_type : item.content_type,
|
||||
published !== undefined ? published : item.published,
|
||||
item.id
|
||||
]
|
||||
);
|
||||
|
||||
// Regenerate embedding if title/body/subject changed (async, don't block)
|
||||
if (isEmbeddingsAvailable() && (title !== undefined || body !== undefined || subject !== undefined)) {
|
||||
if (newBody && newBody.trim()) {
|
||||
generateContentEmbedding({ title: newTitle, subject: newSubject, body: newBody })
|
||||
.then(function(embedding) {
|
||||
return db.query(
|
||||
'UPDATE learning_content SET embedding = $1 WHERE id = $2',
|
||||
[JSON.stringify(embedding), item.id]
|
||||
);
|
||||
})
|
||||
.then(function() { console.log('[Embeddings] Regenerated for content ID:', item.id); })
|
||||
.catch(function(err) { console.error('[Embeddings] Failed to regenerate for content ID ' + item.id + ':', err.message); });
|
||||
}
|
||||
}
|
||||
|
||||
res.json({ success: true });
|
||||
} catch (err) { console.error('[LearningAdmin]', err.message); res.status(500).json({ error: 'Internal server error' }); }
|
||||
});
|
||||
|
|
@ -264,6 +299,7 @@ router.get('/stats', async function(req, res) {
|
|||
var totalCategories = await db.get('SELECT COUNT(*) as count FROM learning_categories', []);
|
||||
var totalQuizzes = await db.get("SELECT COUNT(DISTINCT content_id) as count FROM learning_questions", []);
|
||||
var totalAttempts = await db.get('SELECT COUNT(*) as count FROM learning_progress', []);
|
||||
var withEmbeddings = await db.get('SELECT COUNT(*) as count FROM learning_content WHERE embedding IS NOT NULL', []);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
|
|
@ -272,10 +308,112 @@ router.get('/stats', async function(req, res) {
|
|||
publishedContent: parseInt(published.count),
|
||||
totalCategories: parseInt(totalCategories.count),
|
||||
totalQuizzes: parseInt(totalQuizzes.count),
|
||||
totalAttempts: parseInt(totalAttempts.count)
|
||||
totalAttempts: parseInt(totalAttempts.count),
|
||||
withEmbeddings: parseInt(withEmbeddings.count),
|
||||
embeddingsEnabled: isEmbeddingsAvailable()
|
||||
}
|
||||
});
|
||||
} catch (err) { console.error('[LearningAdmin]', err.message); res.status(500).json({ error: 'Internal server error' }); }
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// EMBEDDINGS — Backfill & Management
|
||||
// ============================================================
|
||||
|
||||
// Generate embeddings for all content (or just missing ones)
|
||||
router.post('/embeddings/generate', async function(req, res) {
|
||||
try {
|
||||
if (!isEmbeddingsAvailable()) {
|
||||
return res.status(400).json({ error: 'Embeddings not configured. Set LITELLM_API_BASE, VERTEX_PROJECT, or OPENAI_API_KEY' });
|
||||
}
|
||||
|
||||
var { regenerateAll } = req.body;
|
||||
|
||||
// Get content without embeddings (or all if regenerateAll=true)
|
||||
var whereClause = regenerateAll ? '' : 'WHERE embedding IS NULL';
|
||||
var content = await db.all(
|
||||
'SELECT id, title, subject, body FROM learning_content ' + whereClause + ' ORDER BY id ASC',
|
||||
[]
|
||||
);
|
||||
|
||||
if (content.length === 0) {
|
||||
return res.json({ success: true, message: 'All content already has embeddings', processed: 0 });
|
||||
}
|
||||
|
||||
// Process in background
|
||||
var processed = 0;
|
||||
var failed = 0;
|
||||
|
||||
console.log('[Embeddings] Starting batch generation for ' + content.length + ' items...');
|
||||
|
||||
// Don't await — run in background
|
||||
(async function() {
|
||||
for (var i = 0; i < content.length; i++) {
|
||||
var item = content[i];
|
||||
try {
|
||||
if (!item.body || !item.body.trim()) {
|
||||
console.log('[Embeddings] Skipping empty content ID:', item.id);
|
||||
continue;
|
||||
}
|
||||
var embedding = await generateContentEmbedding(item);
|
||||
await db.query(
|
||||
'UPDATE learning_content SET embedding = $1 WHERE id = $2',
|
||||
[JSON.stringify(embedding), item.id]
|
||||
);
|
||||
processed++;
|
||||
console.log('[Embeddings] Generated ' + processed + '/' + content.length + ' (ID: ' + item.id + ')');
|
||||
} catch (err) {
|
||||
failed++;
|
||||
console.error('[Embeddings] Failed for content ID ' + item.id + ':', err.message);
|
||||
}
|
||||
}
|
||||
console.log('[Embeddings] Batch complete: ' + processed + ' succeeded, ' + failed + ' failed');
|
||||
|
||||
// Create index if we have enough embeddings now
|
||||
if (processed >= 10) {
|
||||
try {
|
||||
var lists = Math.max(10, Math.floor(Math.sqrt(processed)));
|
||||
await db.query(
|
||||
'CREATE INDEX IF NOT EXISTS idx_learning_content_embedding ON learning_content USING ivfflat (embedding vector_cosine_ops) WITH (lists = ' + lists + ')'
|
||||
);
|
||||
console.log('[Embeddings] Vector index created/updated');
|
||||
} catch (e) {
|
||||
console.error('[Embeddings] Index creation failed:', e.message);
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
message: 'Embedding generation started in background',
|
||||
total: content.length
|
||||
});
|
||||
|
||||
} catch (err) {
|
||||
console.error('[LearningAdmin]', err.message);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Check embedding status
|
||||
router.get('/embeddings/status', async function(req, res) {
|
||||
try {
|
||||
var total = await db.get('SELECT COUNT(*) as count FROM learning_content', []);
|
||||
var withEmbeddings = await db.get('SELECT COUNT(*) as count FROM learning_content WHERE embedding IS NOT NULL', []);
|
||||
var missing = parseInt(total.count) - parseInt(withEmbeddings.count);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
enabled: isEmbeddingsAvailable(),
|
||||
total: parseInt(total.count),
|
||||
withEmbeddings: parseInt(withEmbeddings.count),
|
||||
missing: missing,
|
||||
model: process.env.EMBEDDING_MODEL || 'vertex_ai/text-embedding-005',
|
||||
dimensions: parseInt(process.env.EMBEDDING_DIMENSIONS) || 768
|
||||
});
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ var express = require('express');
|
|||
var router = express.Router();
|
||||
var db = require('../db/database');
|
||||
var { authMiddleware } = require('../middleware/auth');
|
||||
var { searchSimilar, isEmbeddingsAvailable } = require('../utils/embeddings');
|
||||
|
||||
router.use(authMiddleware);
|
||||
|
||||
|
|
@ -228,7 +229,7 @@ router.post('/submit-quiz', async function(req, res) {
|
|||
});
|
||||
|
||||
// ============================================================
|
||||
// SEARCH CONTENT
|
||||
// SEARCH CONTENT (keyword-based)
|
||||
// ============================================================
|
||||
router.get('/search', async function(req, res) {
|
||||
try {
|
||||
|
|
@ -247,8 +248,122 @@ router.get('/search', async function(req, res) {
|
|||
[pattern, pattern, pattern]
|
||||
);
|
||||
|
||||
res.json({ success: true, content: content });
|
||||
res.json({ success: true, content: content, method: 'keyword' });
|
||||
} catch (err) { console.error('[LearningHub]', err.message); res.status(500).json({ error: 'Internal server error' }); }
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// SEMANTIC SEARCH (vector-based)
|
||||
// ============================================================
|
||||
router.get('/search/semantic', async function(req, res) {
|
||||
try {
|
||||
if (!isEmbeddingsAvailable()) {
|
||||
return res.status(400).json({ error: 'Semantic search not available. Embeddings not configured.' });
|
||||
}
|
||||
|
||||
var q = (req.query.q || '').trim();
|
||||
if (!q) return res.json({ success: true, content: [], method: 'semantic' });
|
||||
|
||||
var limit = Math.min(parseInt(req.query.limit) || 10, 50);
|
||||
var threshold = parseFloat(req.query.threshold) || 0.5;
|
||||
|
||||
var results = await searchSimilar(q, {
|
||||
limit: limit,
|
||||
threshold: threshold,
|
||||
contentType: req.query.contentType || null
|
||||
});
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
content: results,
|
||||
method: 'semantic',
|
||||
query: q
|
||||
});
|
||||
|
||||
} catch (err) {
|
||||
console.error('[LearningHub] Semantic search error:', err.message);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ============================================================
|
||||
// HYBRID SEARCH (keyword + semantic combined)
|
||||
// ============================================================
|
||||
router.get('/search/hybrid', async function(req, res) {
|
||||
try {
|
||||
var q = (req.query.q || '').trim();
|
||||
if (!q) return res.json({ success: true, content: [], method: 'hybrid' });
|
||||
|
||||
var limit = Math.min(parseInt(req.query.limit) || 20, 50);
|
||||
|
||||
// 1. Get keyword matches
|
||||
var pattern = '%' + q + '%';
|
||||
var keywordResults = await db.all(
|
||||
`SELECT c.id, c.title, c.slug, c.subject, c.content_type, c.created_at,
|
||||
cat.name as category_name, cat.slug as category_slug,
|
||||
(SELECT COUNT(*) FROM learning_questions lq WHERE lq.content_id = c.id) as question_count,
|
||||
1.0 as score, 'keyword' as match_type
|
||||
FROM learning_content c
|
||||
LEFT JOIN learning_categories cat ON c.category_id = cat.id
|
||||
WHERE c.published = true AND (c.title ILIKE ? OR c.subject ILIKE ? OR c.body ILIKE ?)
|
||||
LIMIT 15`,
|
||||
[pattern, pattern, pattern]
|
||||
);
|
||||
|
||||
// 2. Get semantic matches (if available)
|
||||
var semanticResults = [];
|
||||
if (isEmbeddingsAvailable()) {
|
||||
try {
|
||||
semanticResults = await searchSimilar(q, {
|
||||
limit: 15,
|
||||
threshold: 0.4
|
||||
});
|
||||
// Add match_type
|
||||
semanticResults = semanticResults.map(function(r) {
|
||||
return Object.assign(r, { score: r.similarity, match_type: 'semantic' });
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[LearningHub] Semantic search in hybrid failed:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Merge and deduplicate (favor semantic if both match)
|
||||
var seen = {};
|
||||
var combined = [];
|
||||
|
||||
// Add semantic results first (higher quality)
|
||||
semanticResults.forEach(function(r) {
|
||||
if (!seen[r.id]) {
|
||||
seen[r.id] = true;
|
||||
combined.push(r);
|
||||
}
|
||||
});
|
||||
|
||||
// Add keyword results if not already included
|
||||
keywordResults.forEach(function(r) {
|
||||
if (!seen[r.id]) {
|
||||
seen[r.id] = true;
|
||||
combined.push(r);
|
||||
}
|
||||
});
|
||||
|
||||
// Sort by score descending, limit results
|
||||
combined.sort(function(a, b) { return (b.score || 0) - (a.score || 0); });
|
||||
combined = combined.slice(0, limit);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
content: combined,
|
||||
method: 'hybrid',
|
||||
query: q,
|
||||
keywordCount: keywordResults.length,
|
||||
semanticCount: semanticResults.length
|
||||
});
|
||||
|
||||
} catch (err) {
|
||||
console.error('[LearningHub] Hybrid search error:', err.message);
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
|
|
|
|||
256
src/utils/embeddings.js
Normal file
256
src/utils/embeddings.js
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
// ============================================================
|
||||
// EMBEDDINGS UTILITY — Generate & search with Vertex AI embeddings
|
||||
// Supports: Vertex AI (direct), LiteLLM proxy, OpenAI fallback
|
||||
// ============================================================
|
||||
|
||||
var axios = require('axios');
|
||||
|
||||
// Vertex AI embedding models (via LiteLLM or direct)
|
||||
// gemini-embedding-001: 768 dims, multilingual + code, best quality
|
||||
// text-embedding-005: 768 dims, English + code optimized
|
||||
// text-multilingual-embedding-002: 768 dims, multilingual focus
|
||||
var DEFAULT_MODEL = 'vertex_ai/text-embedding-005';
|
||||
var DEFAULT_DIMS = 768;
|
||||
|
||||
/**
|
||||
* Generate embedding for text using configured provider
|
||||
* @param {string} text - Text to embed (max ~2000 tokens)
|
||||
* @param {object} opts - Options: { model, dimensions }
|
||||
* @returns {Promise<number[]>} - Embedding vector
|
||||
*/
|
||||
async function generateEmbedding(text, opts) {
|
||||
opts = opts || {};
|
||||
var model = opts.model || process.env.EMBEDDING_MODEL || DEFAULT_MODEL;
|
||||
var dimensions = opts.dimensions || parseInt(process.env.EMBEDDING_DIMENSIONS) || DEFAULT_DIMS;
|
||||
|
||||
// Truncate text to ~2000 tokens (~8000 chars) to avoid API errors
|
||||
var truncated = text.substring(0, 8000);
|
||||
if (!truncated.trim()) {
|
||||
throw new Error('Empty text provided for embedding');
|
||||
}
|
||||
|
||||
// Try LiteLLM first if configured
|
||||
if (process.env.LITELLM_API_BASE) {
|
||||
return await generateEmbeddingLiteLLM(truncated, model, dimensions);
|
||||
}
|
||||
|
||||
// Try Vertex AI direct if configured
|
||||
if (process.env.GOOGLE_APPLICATION_CREDENTIALS || process.env.VERTEX_PROJECT) {
|
||||
return await generateEmbeddingVertexDirect(truncated, model, dimensions);
|
||||
}
|
||||
|
||||
// Fallback to OpenAI if configured
|
||||
if (process.env.OPENAI_API_KEY) {
|
||||
return await generateEmbeddingOpenAI(truncated, model, dimensions);
|
||||
}
|
||||
|
||||
throw new Error('No embedding provider configured. Set LITELLM_API_BASE, VERTEX_PROJECT, or OPENAI_API_KEY');
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embedding via LiteLLM proxy
|
||||
*/
|
||||
async function generateEmbeddingLiteLLM(text, model, dimensions) {
|
||||
try {
|
||||
var base = process.env.LITELLM_API_BASE.replace(/\/+$/, '');
|
||||
var headers = { 'Content-Type': 'application/json' };
|
||||
if (process.env.LITELLM_API_KEY) {
|
||||
headers['Authorization'] = 'Bearer ' + process.env.LITELLM_API_KEY;
|
||||
}
|
||||
|
||||
var payload = {
|
||||
model: model,
|
||||
input: text
|
||||
};
|
||||
|
||||
// Only include dimensions if model supports it (some models have fixed dims)
|
||||
if (dimensions && model.includes('text-embedding-005')) {
|
||||
payload.dimensions = dimensions;
|
||||
}
|
||||
|
||||
var response = await axios.post(base + '/embeddings', payload, {
|
||||
headers: headers,
|
||||
timeout: 30000
|
||||
});
|
||||
|
||||
if (!response.data || !response.data.data || !response.data.data[0]) {
|
||||
throw new Error('Invalid response from LiteLLM embeddings API');
|
||||
}
|
||||
|
||||
return response.data.data[0].embedding;
|
||||
} catch (err) {
|
||||
console.error('[Embeddings] LiteLLM error:', err.response?.data || err.message);
|
||||
throw new Error('LiteLLM embedding failed: ' + (err.response?.data?.error || err.message));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embedding via Vertex AI direct (using @google-cloud/vertexai)
|
||||
*/
|
||||
async function generateEmbeddingVertexDirect(text, model, dimensions) {
|
||||
try {
|
||||
var { VertexAI } = require('@google-cloud/vertexai');
|
||||
|
||||
var project = process.env.VERTEX_PROJECT || process.env.GOOGLE_CLOUD_PROJECT;
|
||||
var location = process.env.VERTEX_LOCATION || 'us-central1';
|
||||
|
||||
if (!project) {
|
||||
throw new Error('VERTEX_PROJECT or GOOGLE_CLOUD_PROJECT not set');
|
||||
}
|
||||
|
||||
var vertexAI = new VertexAI({ project: project, location: location });
|
||||
|
||||
// Extract model name (strip vertex_ai/ prefix if present)
|
||||
var modelName = model.replace(/^vertex_ai\//, '');
|
||||
|
||||
// For text-embedding-005, we can specify output dimensions
|
||||
var request = {
|
||||
instances: [{ content: text }]
|
||||
};
|
||||
|
||||
if (dimensions && modelName.includes('text-embedding-005')) {
|
||||
request.parameters = { outputDimensionality: dimensions };
|
||||
}
|
||||
|
||||
// Use predictText API for embeddings
|
||||
var predictionClient = vertexAI.preview.getPredictionServiceClient();
|
||||
var endpoint = `projects/${project}/locations/${location}/publishers/google/models/${modelName}`;
|
||||
|
||||
var [response] = await predictionClient.predict({
|
||||
endpoint: endpoint,
|
||||
instances: [{ content: text }],
|
||||
parameters: request.parameters || {}
|
||||
});
|
||||
|
||||
if (!response || !response.predictions || !response.predictions[0]) {
|
||||
throw new Error('Invalid response from Vertex AI');
|
||||
}
|
||||
|
||||
var prediction = response.predictions[0];
|
||||
return prediction.embeddings?.values || prediction.values || prediction;
|
||||
|
||||
} catch (err) {
|
||||
console.error('[Embeddings] Vertex AI direct error:', err.message);
|
||||
throw new Error('Vertex AI embedding failed: ' + err.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embedding via OpenAI (fallback)
|
||||
*/
|
||||
async function generateEmbeddingOpenAI(text, model, dimensions) {
|
||||
try {
|
||||
var openai = require('openai');
|
||||
var client = new openai.OpenAI({ apiKey: process.env.OPENAI_API_KEY });
|
||||
|
||||
// Use OpenAI's text-embedding-3-small model (1536 dims by default)
|
||||
var embModel = 'text-embedding-3-small';
|
||||
var response = await client.embeddings.create({
|
||||
model: embModel,
|
||||
input: text,
|
||||
dimensions: dimensions || 768 // OpenAI supports custom dimensions
|
||||
});
|
||||
|
||||
return response.data[0].embedding;
|
||||
} catch (err) {
|
||||
console.error('[Embeddings] OpenAI error:', err.message);
|
||||
throw new Error('OpenAI embedding failed: ' + err.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search for similar content using cosine similarity
|
||||
* @param {string} queryText - Search query
|
||||
* @param {object} opts - Options: { limit, threshold, contentType }
|
||||
* @returns {Promise<Array>} - Matching content with similarity scores
|
||||
*/
|
||||
async function searchSimilar(queryText, opts) {
|
||||
opts = opts || {};
|
||||
var limit = opts.limit || 10;
|
||||
var threshold = opts.threshold || 0.5; // Cosine similarity threshold (0-1)
|
||||
|
||||
var db = require('../db/database');
|
||||
|
||||
// Generate embedding for query
|
||||
var queryEmbedding = await generateEmbedding(queryText);
|
||||
|
||||
// Build WHERE clause for filtering
|
||||
var whereClause = 'WHERE c.published = true AND c.embedding IS NOT NULL';
|
||||
var params = [JSON.stringify(queryEmbedding), threshold, limit];
|
||||
var paramIdx = 4;
|
||||
|
||||
if (opts.contentType) {
|
||||
whereClause += ' AND c.content_type = $' + paramIdx;
|
||||
params.push(opts.contentType);
|
||||
paramIdx++;
|
||||
}
|
||||
|
||||
if (opts.categoryId) {
|
||||
whereClause += ' AND c.category_id = $' + paramIdx;
|
||||
params.push(opts.categoryId);
|
||||
}
|
||||
|
||||
// Query with cosine similarity using pgvector
|
||||
// 1 - (a <=> b) converts distance to similarity (higher = more similar)
|
||||
var sql = `
|
||||
SELECT
|
||||
c.id, c.title, c.slug, c.subject, c.content_type, c.created_at,
|
||||
cat.name as category_name, cat.slug as category_slug,
|
||||
1 - (c.embedding <=> $1::vector) as similarity,
|
||||
(SELECT COUNT(*) FROM learning_questions q WHERE q.content_id = c.id) as question_count
|
||||
FROM learning_content c
|
||||
LEFT JOIN learning_categories cat ON c.category_id = cat.id
|
||||
${whereClause}
|
||||
AND 1 - (c.embedding <=> $1::vector) >= $2
|
||||
ORDER BY c.embedding <=> $1::vector
|
||||
LIMIT $3
|
||||
`;
|
||||
|
||||
var results = await db.all(sql, params);
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate embedding for learning content (combines title + subject + body)
|
||||
* @param {object} content - { title, subject, body }
|
||||
* @returns {Promise<number[]>} - Embedding vector
|
||||
*/
|
||||
async function generateContentEmbedding(content) {
|
||||
// Combine title, subject, and body (weighted toward title)
|
||||
var text = [
|
||||
content.title || '',
|
||||
content.title || '', // Title twice for emphasis
|
||||
content.subject || '',
|
||||
stripHtml(content.body || '').substring(0, 6000)
|
||||
].filter(Boolean).join('\n\n');
|
||||
|
||||
return await generateEmbedding(text);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip HTML tags from string
|
||||
*/
|
||||
function stripHtml(html) {
|
||||
return html.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if embeddings are available (provider configured)
|
||||
*/
|
||||
function isEmbeddingsAvailable() {
|
||||
return !!(
|
||||
process.env.LITELLM_API_BASE ||
|
||||
process.env.VERTEX_PROJECT ||
|
||||
process.env.GOOGLE_CLOUD_PROJECT ||
|
||||
process.env.OPENAI_API_KEY
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
generateEmbedding,
|
||||
generateContentEmbedding,
|
||||
searchSimilar,
|
||||
isEmbeddingsAvailable,
|
||||
DEFAULT_MODEL,
|
||||
DEFAULT_DIMS
|
||||
};
|
||||
Loading…
Reference in a new issue