v10: Local Whisper transcription, bigger text areas, flexible AI memory

- Add local Whisper (whisper.cpp / faster-whisper) as transcription provider
  Set TRANSCRIBE_PROVIDER=local with configurable model size and binary path
- Upgrade all refine/instruction inputs to resizable textareas across
  encounter, dictation, hospital course, chart review, well visit, sick visit
- Make AI memory injection flexible: physician preferences and corrections
  are now actively applied (not just "formatting reference"), while still
  overridable by current prompt instructions
This commit is contained in:
Daniel Onyejesi 2026-03-28 22:00:30 +00:00
parent 1191ba0d2d
commit 044c809ff3
15 changed files with 211 additions and 16 deletions

View file

@ -33,6 +33,15 @@ OPENAI_API_KEY=sk-your-openai-key
# Leave unset to auto-detect (uses AWS when AWS_BEDROCK_REGION is configured).
# TRANSCRIBE_PROVIDER=aws
# Option C: Local Whisper (privacy-first, no cloud API needed)
# Requires whisper.cpp or faster-whisper installed on the server.
# TRANSCRIBE_PROVIDER=local
# WHISPER_MODEL_SIZE=small # tiny, base, small, medium, large
# WHISPER_BINARY=whisper-cpp # or: whisper, faster-whisper
# WHISPER_MODEL_PATH= # custom path to .bin model file
# WHISPER_LANGUAGE=en
# WHISPER_THREADS=4 # defaults to CPU count - 1
# Amazon Transcribe Medical — better accuracy for clinical dictation
# Knows drug names, diagnoses, procedures, SOAP terminology
# HIPAA eligible (ensure your AWS account has a BAA)

View file

@ -80,7 +80,7 @@
<div class="card">
<div class="card-header"><h3><i class="fas fa-comment-dots"></i> Instructions (optional)</h3></div>
<input type="text" id="cr-instructions" class="full-input" placeholder="e.g., 'Focus on thyroid management', 'Include medication changes'">
<textarea id="cr-instructions" class="full-input" rows="3" placeholder="e.g., 'Focus on thyroid management', 'Include medication changes'"></textarea>
</div>
<button id="cr-generate-btn" class="btn-generate btn-generate-green"><i class="fas fa-wand-magic-sparkles"></i> Generate Chart Review</button>
@ -97,7 +97,7 @@
</div>
<div id="cr-review-text" class="output-text" contenteditable="true"></div>
<div class="refine-bar">
<input type="text" id="cr-refine-input" class="refine-input" placeholder="Tell AI to modify...">
<textarea id="cr-refine-input" class="refine-input" rows="2" placeholder="Tell AI to modify..."></textarea>
<button class="btn-sm btn-primary" id="cr-refine-btn"><i class="fas fa-edit"></i> Refine</button>
<button class="btn-sm btn-ghost" id="cr-shorten-btn"><i class="fas fa-compress"></i> Shorter</button>
</div>

View file

@ -76,7 +76,7 @@
</div>
<div id="dict-hpi-text" class="output-text" contenteditable="true"></div>
<div class="refine-bar">
<input type="text" id="dict-refine-input" class="refine-input" placeholder="Tell AI to modify...">
<textarea id="dict-refine-input" class="refine-input" rows="2" placeholder="Tell AI to modify..."></textarea>
<button class="btn-sm btn-primary" id="dict-refine-btn"><i class="fas fa-edit"></i> Refine</button>
<button class="btn-sm btn-ghost" id="dict-shorten-btn"><i class="fas fa-compress"></i> Shorter</button>
</div>

View file

@ -73,7 +73,7 @@
</div>
<div id="enc-hpi-text" class="output-text" contenteditable="true"></div>
<div class="refine-bar">
<input type="text" id="enc-refine-input" class="refine-input" placeholder="Tell AI to modify (e.g., 'make it shorter', 'add that patient has asthma history')">
<textarea id="enc-refine-input" class="refine-input" rows="2" placeholder="Tell AI to modify (e.g., 'make it shorter', 'add that patient has asthma history')"></textarea>
<button class="btn-sm btn-primary" id="enc-refine-btn"><i class="fas fa-edit"></i> Refine</button>
<button class="btn-sm btn-ghost" id="enc-shorten-btn"><i class="fas fa-compress"></i> Shorter</button>
</div>

View file

@ -124,7 +124,7 @@
<!-- AI Instructions -->
<div class="card">
<div class="card-header"><h3><i class="fas fa-comment-dots"></i> Additional Instructions (optional)</h3></div>
<input type="text" id="hc-instructions" class="full-input" placeholder="e.g., 'Focus on respiratory course', 'Patient was transferred from outside hospital'">
<textarea id="hc-instructions" class="full-input" rows="3" placeholder="e.g., 'Focus on respiratory course', 'Patient was transferred from outside hospital'"></textarea>
</div>
<div class="action-row">
@ -145,7 +145,7 @@
</div>
<div id="hc-course-text" class="output-text" contenteditable="true"></div>
<div class="refine-bar">
<input type="text" id="hc-refine-input" class="refine-input" placeholder="Tell AI to modify or add discharge day info...">
<textarea id="hc-refine-input" class="refine-input" rows="2" placeholder="Tell AI to modify or add discharge day info..."></textarea>
<button class="btn-sm btn-primary" id="hc-refine-btn"><i class="fas fa-edit"></i> Refine</button>
<button class="btn-sm btn-ghost" id="hc-shorten-btn"><i class="fas fa-compress"></i> Shorter</button>
<button class="btn-sm btn-warning" id="hc-clarify-btn"><i class="fas fa-circle-question"></i> What's Missing?</button>

View file

@ -99,7 +99,7 @@
</div>
<div id="sick-note-text" class="output-text" contenteditable="true"></div>
<div class="refine-bar">
<input type="text" id="sick-refine-input" class="refine-input" placeholder="Tell AI to modify (e.g., 'add that patient received amoxicillin')">
<textarea id="sick-refine-input" class="refine-input" rows="2" placeholder="Tell AI to modify (e.g., 'add that patient received amoxicillin')"></textarea>
<button class="btn-sm btn-primary" id="sick-refine-btn"><i class="fas fa-edit"></i> Refine</button>
<button class="btn-sm btn-ghost" id="sick-shorten-btn"><i class="fas fa-compress"></i> Shorter</button>
</div>

View file

@ -297,7 +297,7 @@
</div>
<div id="wv-note-text" class="output-text" contenteditable="true"></div>
<div class="refine-bar">
<input type="text" id="wv-refine-input" class="refine-input" placeholder="Tell AI to modify (e.g., 'add that patient has asthma', 'expand the plan section')">
<textarea id="wv-refine-input" class="refine-input" rows="2" placeholder="Tell AI to modify (e.g., 'add that patient has asthma', 'expand the plan section')"></textarea>
<button class="btn-sm btn-primary" id="wv-refine-btn"><i class="fas fa-edit"></i> Refine</button>
<button class="btn-sm btn-ghost" id="wv-shorten-btn"><i class="fas fa-compress"></i> Shorter</button>
</div>

View file

@ -156,8 +156,9 @@ body{font-family:'Inter',system-ui,sans-serif;background:var(--g50);color:var(--
.model-tag{font-size:10px;padding:2px 7px;background:var(--g200);border-radius:4px;color:var(--g600);}
/* REFINE BAR */
.refine-bar{display:flex;gap:6px;padding:10px 16px;border-top:1px solid var(--g200);background:var(--g50);align-items:center;flex-wrap:wrap;}
.refine-input{flex:1;min-width:200px;padding:7px 12px;border:1.5px solid var(--g300);border-radius:6px;font-size:13px;font-family:inherit;}
.refine-bar{display:flex;gap:6px;padding:10px 16px;border-top:1px solid var(--g200);background:var(--g50);align-items:flex-start;flex-wrap:wrap;}
.refine-input{flex:1;min-width:200px;padding:7px 12px;border:1.5px solid var(--g300);border-radius:6px;font-size:13px;font-family:inherit;resize:vertical;}
textarea.full-input{resize:vertical;}
.refine-input:focus{outline:none;border-color:var(--blue);box-shadow:0 0 0 3px var(--blue-light);}
/* CLARIFY BOX */

View file

@ -81,7 +81,7 @@ router.post('/generate-hospital-course', authMiddleware, async (req, res) => {
prompt += `\n\nADDITIONAL INSTRUCTIONS FROM PHYSICIAN:\n${additionalInstructions}`;
}
if (physicianMemories) clinicalData += '\n\n[PHYSICIAN TEMPLATES — use as formatting reference only]\n' + physicianMemories + '\n[END TEMPLATES]';
if (physicianMemories) clinicalData += '\n\n[PHYSICIAN PREFERENCES & LEARNED PATTERNS — Apply these preferences to your output. These reflect how this physician writes notes, their preferred style, terminology, and corrections from past outputs. Adapt your response accordingly, but user instructions in the current prompt take priority if they conflict.]\n' + physicianMemories + '\n[END PREFERENCES]';
const result = await callAI([
{ role: 'system', content: prompt },

View file

@ -13,7 +13,7 @@ router.post('/generate-hpi-encounter', authMiddleware, async (req, res) => {
const prompt = setting === 'inpatient' ? PROMPTS.hpiInpatient : PROMPTS.hpiEncounter;
var context = `Patient: ${patientAge || 'Unknown'}, ${patientGender || 'Unknown'}\nSetting: ${setting || 'outpatient'}\n\nTRANSCRIPT:\n${transcript}`;
if (physicianMemories) context += '\n\n[PHYSICIAN TEMPLATES — use as formatting reference only]\n' + physicianMemories + '\n[END TEMPLATES]';
if (physicianMemories) context += '\n\n[PHYSICIAN PREFERENCES & LEARNED PATTERNS — Apply these preferences to your output. These reflect how this physician writes notes, their preferred style, terminology, and corrections from past outputs. Adapt your response accordingly, but user instructions in the current prompt take priority if they conflict.]\n' + physicianMemories + '\n[END PREFERENCES]';
const result = await callAI([
{ role: 'system', content: prompt },
@ -35,7 +35,7 @@ router.post('/generate-hpi-dictation', authMiddleware, async (req, res) => {
const prompt = setting === 'inpatient' ? PROMPTS.hpiInpatient : PROMPTS.hpiDictation;
var context = `Patient: ${patientAge || 'Unknown'}, ${patientGender || 'Unknown'}\nSetting: ${setting || 'outpatient'}\n\nDICTATION:\n${transcript}`;
if (physicianMemories) context += '\n\n[PHYSICIAN TEMPLATES — use as formatting reference only]\n' + physicianMemories + '\n[END TEMPLATES]';
if (physicianMemories) context += '\n\n[PHYSICIAN PREFERENCES & LEARNED PATTERNS — Apply these preferences to your output. These reflect how this physician writes notes, their preferred style, terminology, and corrections from past outputs. Adapt your response accordingly, but user instructions in the current prompt take priority if they conflict.]\n' + physicianMemories + '\n[END PREFERENCES]';
const result = await callAI([
{ role: 'system', content: prompt },

View file

@ -49,7 +49,7 @@ router.post('/sick-visit/note', authMiddleware, async function(req, res) {
}
if (physicianMemories) {
context += '[PHYSICIAN TEMPLATES — use as formatting reference only]\n' + physicianMemories + '\n[END TEMPLATES]\n\n';
context += '[PHYSICIAN PREFERENCES & LEARNED PATTERNS — Apply these preferences to your output. These reflect how this physician writes notes, their preferred style, terminology, and corrections from past outputs. Adapt your response accordingly, but user instructions in the current prompt take priority if they conflict.]\n' + physicianMemories + '\n[END PREFERENCES]\n\n';
}
var result = await callAI([

View file

@ -21,7 +21,7 @@ router.post('/generate-soap', authMiddleware, async (req, res) => {
}
var context = `Patient: ${patientAge || 'Unknown'}, ${patientGender || 'Unknown'}\n\nINPUT:\n${transcript}`;
if (physicianMemories) context += '\n\n[PHYSICIAN TEMPLATES — use as formatting reference only]\n' + physicianMemories + '\n[END TEMPLATES]';
if (physicianMemories) context += '\n\n[PHYSICIAN PREFERENCES & LEARNED PATTERNS — Apply these preferences to your output. These reflect how this physician writes notes, their preferred style, terminology, and corrections from past outputs. Adapt your response accordingly, but user instructions in the current prompt take priority if they conflict.]\n' + physicianMemories + '\n[END PREFERENCES]';
const result = await callAI([
{ role: 'system', content: prompt },

View file

@ -3,6 +3,7 @@ const router = express.Router();
const multer = require('multer');
const { whisperClient } = require('../utils/ai');
const { transcribeWithAWS, isAWSTranscribeConfigured } = require('../utils/transcribeAWS');
const { transcribeWithLocal, isLocalWhisperConfigured } = require('../utils/transcribeLocal');
const { authMiddleware } = require('../middleware/auth');
const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 25 * 1024 * 1024 } });
@ -14,6 +15,7 @@ const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 25
// otherwise → use OpenAI Whisper
function getTranscribeProvider() {
var env = process.env.TRANSCRIBE_PROVIDER;
if (env === 'local') return 'local';
if (env === 'aws') return 'aws';
if (env === 'openai') return 'openai';
// Auto-detect: prefer AWS when Bedrock is already configured
@ -29,6 +31,11 @@ router.post('/transcribe', authMiddleware, upload.single('audio'), async (req, r
try {
if (!req.file) return res.status(400).json({ error: 'No audio' });
if (provider === 'local') {
var text = await transcribeWithLocal(req.file.buffer, req.file.mimetype || 'audio/webm');
return res.json({ success: true, text: text, provider: 'local-whisper' });
}
if (provider === 'aws') {
if (!isAWSTranscribeConfigured()) {
return res.status(400).json({ error: 'AWS Transcribe not configured. Set AWS_BEDROCK_REGION.' });

View file

@ -187,7 +187,7 @@ router.post('/well-visit/note', authMiddleware, async function(req, res) {
context += diagnoses + '\n\n';
}
if (physicianMemories) {
context += '[PHYSICIAN TEMPLATES — use as formatting reference only]\n' + physicianMemories + '\n[END TEMPLATES]\n\n';
context += '[PHYSICIAN PREFERENCES & LEARNED PATTERNS — Apply these preferences to your output. These reflect how this physician writes notes, their preferred style, terminology, and corrections from past outputs. Adapt your response accordingly, but user instructions in the current prompt take priority if they conflict.]\n' + physicianMemories + '\n[END PREFERENCES]\n\n';
}
// Add growth reference and feeding guidance for this age

View file

@ -0,0 +1,178 @@
// ============================================================
// LOCAL WHISPER TRANSCRIPTION — uses whisper.cpp or faster-whisper
// ============================================================
// Requires: whisper.cpp binary or faster-whisper-cli installed on the host.
// Set TRANSCRIBE_PROVIDER=local and optionally WHISPER_MODEL_SIZE, WHISPER_BINARY.
var { execFile } = require('child_process');
var fs = require('fs');
var path = require('path');
var os = require('os');
var crypto = require('crypto');
var logger = require('./logger');
var WHISPER_BINARY = process.env.WHISPER_BINARY || 'whisper-cpp';
var WHISPER_MODEL_SIZE = process.env.WHISPER_MODEL_SIZE || 'small';
var WHISPER_MODEL_PATH = process.env.WHISPER_MODEL_PATH || '';
var WHISPER_LANGUAGE = process.env.WHISPER_LANGUAGE || 'en';
var WHISPER_THREADS = process.env.WHISPER_THREADS || String(Math.max(2, os.cpus().length - 1));
/**
* Check if local Whisper is available
*/
function isLocalWhisperConfigured() {
return process.env.TRANSCRIBE_PROVIDER === 'local';
}
/**
* Detect which whisper binary is available
* Supports: whisper-cpp, whisper, faster-whisper
*/
function detectBinary(callback) {
var candidates = [WHISPER_BINARY, 'whisper-cpp', 'whisper', 'faster-whisper'];
var tried = 0;
function tryNext() {
if (tried >= candidates.length) return callback(null);
var bin = candidates[tried++];
execFile(bin, ['--help'], { timeout: 5000 }, function(err) {
if (!err || (err.code !== 'ENOENT' && err.code !== 127)) return callback(bin);
tryNext();
});
}
tryNext();
}
/**
* Convert audio buffer to WAV using ffmpeg (whisper.cpp needs 16kHz mono WAV)
*/
function convertToWav(inputPath, callback) {
var wavPath = inputPath + '.wav';
execFile('ffmpeg', [
'-i', inputPath,
'-ar', '16000',
'-ac', '1',
'-f', 'wav',
'-y', wavPath
], { timeout: 30000 }, function(err) {
if (err) return callback(err, null);
callback(null, wavPath);
});
}
/**
* Transcribe audio buffer using local Whisper
* @param {Buffer} audioBuffer - Audio data
* @param {string} mimeType - MIME type of the audio
* @returns {Promise<string>} Transcribed text
*/
function transcribeWithLocal(audioBuffer, mimeType) {
return new Promise(function(resolve, reject) {
// Write buffer to temp file
var ext = '.webm';
if (mimeType && mimeType.includes('wav')) ext = '.wav';
else if (mimeType && mimeType.includes('mp3')) ext = '.mp3';
else if (mimeType && mimeType.includes('ogg')) ext = '.ogg';
else if (mimeType && mimeType.includes('mp4')) ext = '.mp4';
var tmpDir = os.tmpdir();
var tmpFile = path.join(tmpDir, 'whisper_' + crypto.randomBytes(8).toString('hex') + ext);
fs.writeFile(tmpFile, audioBuffer, function(err) {
if (err) return reject(new Error('Failed to write temp audio: ' + err.message));
// Convert to WAV for whisper.cpp compatibility
convertToWav(tmpFile, function(convErr, wavPath) {
var audioPath = convErr ? tmpFile : wavPath;
detectBinary(function(binary) {
if (!binary) {
cleanup(tmpFile, wavPath);
return reject(new Error(
'Local Whisper not found. Install whisper.cpp (https://github.com/ggerganov/whisper.cpp) ' +
'or faster-whisper (pip install faster-whisper). Set WHISPER_BINARY env var if using a custom path.'
));
}
var args = buildArgs(binary, audioPath);
logger.info('[LocalWhisper] Running: ' + binary + ' ' + args.join(' '));
execFile(binary, args, { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, function(execErr, stdout, stderr) {
cleanup(tmpFile, wavPath);
if (execErr) {
logger.error('[LocalWhisper] Error:', execErr.message);
if (stderr) logger.error('[LocalWhisper] stderr:', stderr.substring(0, 500));
return reject(new Error('Local Whisper transcription failed: ' + execErr.message));
}
// Parse output — whisper.cpp outputs timestamped lines like [00:00:00.000 --> 00:00:05.000] text
var text = parseOutput(stdout);
if (!text || !text.trim()) {
return reject(new Error('Local Whisper returned empty transcription'));
}
resolve(text.trim());
});
});
});
});
});
}
/**
* Build command-line args based on the detected binary
*/
function buildArgs(binary, audioPath) {
if (binary.includes('faster-whisper')) {
// faster-whisper CLI
var args = ['--model', WHISPER_MODEL_SIZE, '--language', WHISPER_LANGUAGE];
if (WHISPER_THREADS) args.push('--threads', WHISPER_THREADS);
args.push(audioPath);
return args;
}
// whisper.cpp style
var args = ['-f', audioPath, '-l', WHISPER_LANGUAGE, '-t', WHISPER_THREADS, '--no-timestamps'];
if (WHISPER_MODEL_PATH) {
args.push('-m', WHISPER_MODEL_PATH);
} else {
// whisper.cpp uses model names like ggml-small.bin
args.push('-m', 'models/ggml-' + WHISPER_MODEL_SIZE + '.bin');
}
// Medical vocabulary hint via initial prompt
args.push('--prompt', 'Medical patient encounter. Pediatric. Clinical terms, diagnoses, medications.');
return args;
}
/**
* Parse whisper output into clean text
*/
function parseOutput(stdout) {
if (!stdout) return '';
// whisper.cpp with --no-timestamps outputs plain text
// whisper.cpp with timestamps: [00:00:00.000 --> 00:00:05.000] Hello world
var lines = stdout.split('\n');
var text = [];
for (var i = 0; i < lines.length; i++) {
var line = lines[i];
// Strip timestamp prefix if present
var match = line.match(/^\[[\d:.]+\s*-->\s*[\d:.]+\]\s*(.*)$/);
if (match) {
text.push(match[1].trim());
} else if (line.trim() && !line.startsWith('whisper_') && !line.startsWith('main:')) {
// Plain text line (skip whisper.cpp log lines)
text.push(line.trim());
}
}
return text.join(' ');
}
/**
* Clean up temp files
*/
function cleanup(tmpFile, wavPath) {
try { fs.unlinkSync(tmpFile); } catch(e) {}
if (wavPath) { try { fs.unlinkSync(wavPath); } catch(e) {} }
}
module.exports = { transcribeWithLocal, isLocalWhisperConfigured };