danvics/pediatric-ai-scribe-v3
1
0
Fork 0
Code Issues Pull requests Projects Releases 14 Packages Wiki Activity Actions

remove browser whisper transcription

Browse source
This commit is contained in:
Daniel 2026-05-08 07:33:12 +02:00
parent 59fa229b59
commit 20fc6798a9
9 changed files with 50 additions and 341 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
Dockerfile
View file

@ -8,7 +8,7 @@ FROM node:20-alpine
WORKDIR /app
# ffmpeg: audio conversion for AWS Transcribe (WebM → PCM)
# curl: download Whisper models for browser-based transcription
# curl: HTTP helper used by the OpenBao entrypoint and health/debug tooling
# jq: JSON parsing for the entrypoint's OpenBao secret-fetch step
RUN apk add --no-cache ffmpeg curl jq
@ -30,22 +30,6 @@ RUN chmod +x /app/docker-entrypoint.sh
RUN mkdir -p /app/data/logs
# Download Browser Whisper (COMPLETE self-hosting - zero CDN dependencies)
# Library + Models all bundled and served from our server
RUN mkdir -p /app/public/models/Xenova/whisper-tiny.en/onnx && \
cd /app/public/models && \
echo "Downloading transformers.js library (worker-compatible build)..." && \
curl -sL -o transformers.min.js https://cdn.jsdelivr.net/npm/@xenova/transformers@2.0.0/dist/transformers.min.js && \
cd Xenova/whisper-tiny.en && \
echo "Downloading Whisper model files..." && \
curl -sL -o config.json https://huggingface.co/Xenova/whisper-tiny.en/resolve/main/config.json && \
curl -sL -o tokenizer.json https://huggingface.co/Xenova/whisper-tiny.en/resolve/main/tokenizer.json && \
curl -sL -o preprocessor_config.json https://huggingface.co/Xenova/whisper-tiny.en/resolve/main/preprocessor_config.json && \
curl -sL -o generation_config.json https://huggingface.co/Xenova/whisper-tiny.en/resolve/main/generation_config.json && \
curl -sL -o onnx/encoder_model_quantized.onnx https://huggingface.co/Xenova/whisper-tiny.en/resolve/main/onnx/encoder_model_quantized.onnx && \
curl -sL -o onnx/decoder_model_merged_quantized.onnx https://huggingface.co/Xenova/whisper-tiny.en/resolve/main/onnx/decoder_model_merged_quantized.onnx && \
echo "✅ Browser Whisper: 100% self-hosted (library: 760KB, models: 42MB)"
EXPOSE 3000
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s \
@ -56,4 +40,3 @@ HEALTHCHECK --interval=30s --timeout=5s --start-period=20s \
# unset, so legacy .env-only deployments continue to work unchanged.
ENTRYPOINT ["/app/docker-entrypoint.sh"]
CMD ["node", "server.js"]

11
public/components/faq.html
View file

@ -101,14 +101,6 @@
</div>
</div>
<div class="faq-item">
<button class="faq-question">What is Browser Whisper?</button>
<div class="faq-answer">
<p>Browser Whisper runs the Whisper AI model entirely in your browser using WebAssembly. Your audio never leaves your device, making it the most private transcription option. You can enable it in <strong>Settings &gt; Browser Whisper</strong>.</p>
<p>It works offline and is HIPAA-safe since no data is transmitted. The tradeoff is that it is slower than cloud-based transcription and requires downloading the model (~40&ndash;240 MB) on first use.</p>
</div>
</div>
<div class="faq-item">
<button class="faq-question">Can I use the app on my phone?</button>
<div class="faq-answer">
@ -176,7 +168,7 @@
<li>No patient data is stored long-term on the server</li>
<li>Every action is audit-logged (who accessed what, when)</li>
<li>Two-factor authentication (2FA) and session management are available</li>
<li>Browser Whisper keeps audio entirely on your device</li>
<li>Server transcription can be configured with HIPAA-eligible providers</li>
</ul>
<p>For HIPAA compliance, ensure your administrator has configured a BAA-covered AI provider (such as AWS Bedrock, Google Vertex AI, or Azure OpenAI).</p>
</div>
@ -343,4 +335,3 @@
</div>
</div>

29
public/components/settings.html
View file

@ -34,33 +34,6 @@
</div>
</div>
<!-- Browser Whisper -->
<div class="settings-section card" id="browser-whisper-section">
<h3><i class="fas fa-microchip"></i> Browser Transcription (Local Whisper)</h3>
<p style="font-size:13px;color:var(--g600);">Transcribes audio entirely in your browser — no audio sent to any server. Powered by OpenAI Whisper running in WebAssembly. Model is downloaded once and cached locally.</p>
<div style="display:flex;align-items:center;gap:12px;flex-wrap:wrap;margin-bottom:12px;">
<label style="font-size:13px;font-weight:600;">Enable browser transcription:</label>
<label style="display:flex;align-items:center;gap:6px;cursor:pointer;">
<input type="checkbox" id="browser-whisper-enabled" style="accent-color:var(--blue);width:16px;height:16px;">
<span style="font-size:13px;" id="browser-whisper-status">Off</span>
</label>
</div>
<div id="browser-whisper-model-row" style="display:flex;align-items:center;gap:12px;flex-wrap:wrap;margin-bottom:12px;">
<label style="font-size:13px;font-weight:600;">Model:</label>
<select id="browser-whisper-model" style="font-size:13px;padding:5px 8px;border:1px solid var(--g300);border-radius:6px;">
<option value="Xenova/whisper-tiny.en">Tiny (~39MB) — fastest, ~2-3s</option>
<option value="Xenova/whisper-base.en">Base (~74MB) — balanced, ~3-5s</option>
<option value="Xenova/whisper-small.en">Small (~244MB) — best quality, ~6-10s</option>
</select>
<button id="btn-whisper-preload" class="btn-sm btn-ghost"><i class="fas fa-download"></i> Pre-download model</button>
</div>
<div id="browser-whisper-progress" style="display:none;font-size:12px;color:var(--g500);margin-top:4px;">
<i class="fas fa-spinner fa-spin"></i> <span id="browser-whisper-progress-text">Loading...</span>
</div>
<p style="font-size:12px;color:var(--g400);margin:8px 0 0;"><i class="fas fa-info-circle"></i> When enabled, overrides server transcription. Falls back to server if browser transcription fails.</p>
<p style="font-size:11px;color:var(--orange);margin:4px 0 0;display:none;" id="browser-whisper-csp-warning"><i class="fas fa-exclamation-triangle"></i> <strong>Network/Firewall Issue:</strong> If model download fails, check that <code>cdn.jsdelivr.net</code> and <code>huggingface.co</code> are not blocked. Server transcription will be used as fallback.</p>
</div>
<!-- Web Speech Recognition (Real-time Streaming) -->
<div class="settings-section card" id="web-speech-section" style="border-left:3px solid var(--orange);">
<h3><i class="fas fa-wave-square"></i> Real-Time Streaming Transcription</h3>
@ -82,7 +55,7 @@
<p style="margin:0;" id="web-speech-browser-info">Detecting...</p>
</div>
<p style="font-size:11px;color:var(--g400);margin:8px 0 0;"><i class="fas fa-info-circle"></i> <strong>Trade-off:</strong> Immediate transcription vs. privacy. For maximum privacy, use Browser Whisper (offline batch mode) or Server transcription with HIPAA-eligible provider.</p>
<p style="font-size:11px;color:var(--g400);margin:8px 0 0;"><i class="fas fa-info-circle"></i> <strong>Trade-off:</strong> Immediate transcription vs. privacy. For maximum privacy, use server transcription with a HIPAA-eligible provider.</p>
</div>
<!-- Change Password — hidden by default; unhidden only for users with a real password -->

5
public/index.html
View file

@ -393,9 +393,9 @@
<ul style="font-size:12.5px;color:var(--g700);line-height:1.8;padding-left:18px;margin:0 0 12px;">
<li>Multiple AI providers with HIPAA-compliant options (BAA-covered)</li>
<li>Multiple transcription engines including medical-specific models</li>
<li>Browser Whisper for fully offline, private transcription</li>
<li>Configurable speech-to-text with server-side providers</li>
<li>Text-to-speech for reading notes aloud</li>
<li>AI learns from your edits over time (correction tracking)</li>
<li>AI can follow your saved note templates and formatting preferences</li>
<li>Customizable AI prompts for each note type</li>
</ul>
@ -476,7 +476,6 @@
crossorigin="anonymous" referrerpolicy="no-referrer" defer></script>
<script defer src="/js/milestonesData.js"></script>
<script defer src="/js/audioBackup.js"></script>
<script defer src="/js/browserWhisper.js"></script>
<script defer src="/js/speechRecognition.js"></script>
<script defer src="/js/transcriptionSettings.js"></script>
<script defer src="/js/voicePreferences.js"></script>

116
public/js/app.js
View file

@ -163,7 +163,6 @@ document.addEventListener('DOMContentLoaded', function() {
if (typeof loadSavedEncountersList === 'function') loadSavedEncountersList();
if (typeof renderAudioBackups === 'function') renderAudioBackups();
if (typeof loadDocuments === 'function') loadDocuments();
initBrowserWhisperSettings();
}
if (e.detail && e.detail.tab === 'faq') {
// Wire FAQ accordion after component loads
@ -183,105 +182,6 @@ document.addEventListener('DOMContentLoaded', function() {
}
});
// ── Browser Whisper settings UI ───────────────────────────
function initBrowserWhisperSettings() {
var chk = document.getElementById('browser-whisper-enabled');
var sel = document.getElementById('browser-whisper-model');
var pre = document.getElementById('btn-whisper-preload');
var stat = document.getElementById('browser-whisper-status');
var prog = document.getElementById('browser-whisper-progress');
var pt = document.getElementById('browser-whisper-progress-text');
var sec = document.getElementById('browser-whisper-section');
if (!chk) return;
var supported = typeof BrowserWhisper !== 'undefined' && BrowserWhisper.isSupported();
if (!supported) {
if (sec) sec.innerHTML += '<p style="color:var(--red);font-size:12px;margin:8px 0 0;">Not supported in this browser. Use Chrome or Edge.</p>';
if (chk) chk.disabled = true;
return;
}
// Restore saved state
chk.checked = BrowserWhisper.isEnabled();
sel.value = BrowserWhisper.getModel();
stat.textContent = chk.checked ? 'On — audio stays on device' : 'Off';
chk.addEventListener('change', function() {
BrowserWhisper.setEnabled(chk.checked);
stat.textContent = chk.checked ? 'On — audio stays on device' : 'Off';
if (chk.checked) {
BrowserWhisper.preload(function(file, pct) {
if (!prog || !pt) return;
if (pct >= 100) { prog.style.display = 'none'; return; }
prog.style.display = 'block';
pt.textContent = file + (pct > 0 ? ' ' + pct + '%' : '');
});
}
});
sel.addEventListener('change', function() {
BrowserWhisper.setModel(sel.value);
});
if (pre) {
pre.addEventListener('click', function(e) {
console.log('[BrowserWhisper] Pre-download button clicked!');
e.preventDefault();
if (!prog || !pt) {
console.error('[BrowserWhisper] Progress elements not found');
showToast('UI elements missing - check page load', 'error');
return;
}
if (!BrowserWhisper || !BrowserWhisper.isSupported()) {
console.error('[BrowserWhisper] Not supported');
showToast('Browser Whisper not supported in this browser', 'error');
return;
}
console.log('[BrowserWhisper] Starting preload...');
prog.style.display = 'block';
pt.textContent = 'Initializing...';
BrowserWhisper.setEnabled(true);
chk.checked = true;
stat.textContent = 'On — audio stays on device';
// Set timeout in case it gets stuck
var timeout = setTimeout(function() {
console.warn('[BrowserWhisper] 30s elapsed - still downloading, check Network tab');
showToast('Download in progress - check browser console', 'info');
}, 30000);
try {
BrowserWhisper.preload(function(file, pct) {
console.log('[BrowserWhisper] Progress:', file, pct + '%');
if (pct >= 100) {
clearTimeout(timeout);
prog.style.display = 'none';
showToast('Whisper model ready!', 'success');
return;
}
prog.style.display = 'block';
pt.textContent = file + (pct > 0 ? ' ' + pct + '%' : '');
});
} catch (err) {
clearTimeout(timeout);
console.error('[BrowserWhisper] Preload error:', err);
prog.style.display = 'none';
pt.textContent = '';
// Show CSP/network warning
var cspWarning = document.getElementById('browser-whisper-csp-warning');
if (cspWarning) cspWarning.style.display = 'block';
showToast('Download blocked by network/firewall. Server transcription will be used.', 'warning');
}
});
}
}
// --- MODEL SELECTOR ---
var modelSelect = document.getElementById('global-model-select');
var costBadge = document.getElementById('model-cost-badge');
@ -742,22 +642,6 @@ function checkTranscribeStatus() {
}
function transcribeAudio(blob) {
// Browser Whisper — local, zero network, HIPAA-safe
if (typeof BrowserWhisper !== 'undefined' && BrowserWhisper.isEnabled()) {
var startTime = Date.now();
return BrowserWhisper.transcribe(blob)
.then(function(text) {
var elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
showToast('Transcribed locally (' + elapsed + 's)', 'success');
window._lastAudioBackupId = null;
return { success: true, text: text, provider: 'browser-whisper' };
})
.catch(function(err) {
console.warn('[BrowserWhisper] Failed:', err.message, '— falling back to server');
if (typeof saveAudioBackup === 'function') saveAudioBackup(blob, 'browser-whisper-failed').catch(function() {});
return _serverTranscribe(blob);
});
}
return _serverTranscribe(blob);
}

155
public/js/browserWhisper.js
View file

@ -1,155 +0,0 @@
// ============================================================
// BROWSER WHISPER — client-side transcription, zero server calls
// Uses @xenova/transformers running Whisper in WebAssembly.
// Audio is processed entirely in-browser, never transmitted.
//
// Models (cached in IndexedDB after first download):
// Xenova/whisper-tiny.en — ~39MB — fastest, ~2-3s/clip
// Xenova/whisper-base.en — ~74MB — balanced, ~3-5s/clip
// Xenova/whisper-small.en — ~244MB — best quality, ~6-10s/clip
// ============================================================
(function() {
var STORAGE_ENABLED = 'ped_browser_whisper';
var STORAGE_MODEL = 'ped_whisper_model';
var DEFAULT_MODEL = 'Xenova/whisper-tiny.en';
var _worker = null;
var _ready = false;
var _loading = false;
var _modelLoaded = null;
var _pending = null; // { resolve, reject }
window.BrowserWhisper = {
isSupported: function() {
return typeof Worker !== 'undefined' &&
typeof AudioContext !== 'undefined' &&
typeof WebAssembly !== 'undefined';
},
isEnabled: function() {
try { return localStorage.getItem(STORAGE_ENABLED) === '1'; } catch(e) { return false; }
},
setEnabled: function(val) {
try { localStorage.setItem(STORAGE_ENABLED, val ? '1' : '0'); } catch(e) {}
},
getModel: function() {
try { return localStorage.getItem(STORAGE_MODEL) || DEFAULT_MODEL; } catch(e) { return DEFAULT_MODEL; }
},
setModel: function(m) {
try { localStorage.setItem(STORAGE_MODEL, m); } catch(e) {}
// Force reload next time
_ready = false;
_modelLoaded = null;
},
// Pre-warm: load model in background before first recording
preload: function(onProgress) {
if (!this.isSupported() || !this.isEnabled()) return;
_initWorker(this.getModel(), onProgress || function() {});
},
// Transcribe a Blob (WebM, WAV, etc.)
transcribe: function(blob) {
var self = this;
if (!this.isSupported()) return Promise.reject(new Error('WebAssembly not supported'));
if (!this.isEnabled()) return Promise.reject(new Error('Browser Whisper not enabled'));
return _blobToFloat32(blob).then(function(float32) {
return new Promise(function(resolve, reject) {
var model = self.getModel();
// If worker is ready with same model, send immediately
if (_ready && _modelLoaded === model) {
_pending = { resolve: resolve, reject: reject };
_worker.postMessage({ type: 'transcribe', audio: float32, model: model }, [float32.buffer]);
return;
}
// Need to (re)load model first
_pending = { resolve: resolve, reject: reject };
_initWorker(model, function() {}, function() {
_worker.postMessage({ type: 'transcribe', audio: float32, model: model }, [float32.buffer]);
});
});
});
}
};
// ── Internal ──────────────────────────────────────────────
function _initWorker(model, onProgress, onReady) {
if (_loading && _modelLoaded === model) return; // already loading same model
if (_worker) { _worker.terminate(); _worker = null; }
_ready = false;
_loading = true;
_modelLoaded = model;
_worker = new Worker('/js/whisperWorker.js');
_worker.addEventListener('message', function(e) {
var d = e.data;
if (d.type === 'loading') {
if (onProgress) onProgress('Downloading Whisper model (' + d.model.split('/').pop() + ')...', 0);
}
if (d.type === 'progress') {
if (onProgress) onProgress(d.file.split('/').pop() || 'model', d.progress);
}
if (d.type === 'ready') {
_ready = true;
_loading = false;
if (onProgress) onProgress('', 100);
if (onReady) onReady();
}
if (d.type === 'result') {
if (_pending) { _pending.resolve(d.text); _pending = null; }
}
if (d.type === 'error') {
_loading = false;
_ready = false;
console.error('[BrowserWhisper] Worker error:', d.message);
// Show user-friendly message
if (typeof showToast === 'function') {
showToast('Browser Whisper blocked by network/firewall. Using server transcription.', 'warning');
}
if (_pending) { _pending.reject(new Error(d.message)); _pending = null; }
if (onProgress) onProgress('', 0);
}
});
_worker.addEventListener('error', function(err) {
_loading = false;
_ready = false;
console.error('[BrowserWhisper] Worker crashed:', err);
if (typeof showToast === 'function') {
showToast('Browser Whisper unavailable. Using server transcription.', 'warning');
}
if (_pending) { _pending.reject(err); _pending = null; }
if (onProgress) onProgress('', 0);
});
_worker.postMessage({ type: 'load', model: model });
}
// Convert audio Blob → Float32Array at 16kHz mono (what Whisper expects)
function _blobToFloat32(blob) {
return blob.arrayBuffer().then(function(buf) {
var ctx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
return ctx.decodeAudioData(buf).then(function(audioBuffer) {
ctx.close();
return audioBuffer.getChannelData(0); // mono
}).catch(function(err) {
ctx.close();
throw new Error('Audio decode failed: ' + err.message);
});
});
}
})();

12
server.js
View file

@ -24,25 +24,17 @@ app.use(helmet({
contentSecurityPolicy: {
directives: {
defaultSrc: ["'self'"],
// 'wasm-unsafe-eval' required for WebAssembly (Whisper in-browser transcription)
// cdn.jsdelivr.net required for @xenova/transformers worker script
// 'unsafe-eval' needed for transformers.js dynamic imports in worker
scriptSrc: ["'self'", "'wasm-unsafe-eval'", "'unsafe-eval'", 'https://cdn.jsdelivr.net', 'https://cdnjs.cloudflare.com', 'https://challenges.cloudflare.com'],
scriptSrc: ["'self'", 'https://cdn.jsdelivr.net', 'https://cdnjs.cloudflare.com', 'https://challenges.cloudflare.com'],
scriptSrcAttr: ["'none'"],
styleSrc: ["'self'", "'unsafe-inline'", 'https://fonts.googleapis.com', 'https://cdnjs.cloudflare.com'],
fontSrc: ["'self'", 'https://fonts.gstatic.com', 'https://cdnjs.cloudflare.com'],
imgSrc: ["'self'", 'data:', 'blob:'],
mediaSrc: ["'self'", 'blob:'],
connectSrc: ["'self'", 'https://cdnjs.cloudflare.com', 'https://fonts.googleapis.com', 'https://fonts.gstatic.com', 'https://www.google.com', 'wss://www.google.com', 'https://clinicaltables.nlm.nih.gov',
// HuggingFace CDN for Whisper model downloads
'https://huggingface.co', 'https://cdn-lfs.huggingface.co', 'https://cdn-lfs-us-1.huggingface.co', 'https://cdn-lfs-us-2.huggingface.co',
// jsdelivr for worker importScripts
'https://cdn.jsdelivr.net',
// Cloudflare Turnstile
'https://challenges.cloudflare.com'],
workerSrc: ["'self'", 'blob:'],
// Allow workers to load scripts from jsdelivr
childSrc: ["'self'", 'blob:', 'https://cdn.jsdelivr.net'],
childSrc: ["'self'", 'blob:'],
frameSrc: ["'self'", 'https://challenges.cloudflare.com'],
objectSrc: ["'none'"],
// Disable helmet's default upgrade-insecure-requests so tests can hit

8
src/routes/memories.js
View file

@ -25,6 +25,11 @@ var VALID_CATEGORIES = [
'template_soap', 'template_hpi', 'template_wellvisit', 'template_sickvisit', 'template_ed'
];
var AI_CONTEXT_CATEGORIES = [
'physical_exam', 'ros', 'encounter_format', 'family_history', 'assessment_plan',
'template_soap', 'template_hpi', 'template_wellvisit', 'template_sickvisit', 'template_ed'
];
// ── GET all memories for current user ───────────────────────────────────
router.get('/memories', async function(req, res) {
try {
@ -103,8 +108,9 @@ router.get('/memories/context', async function(req, res) {
"SELECT category, name, content FROM user_memories WHERE user_id = $1 AND category NOT LIKE 'correction_%' ORDER BY category, id",
[req.user.id]
);
if (rows.length === 0) return res.json({ success: true, context: '' });
rows.forEach(decryptMemory);
rows = rows.filter(function(r) { return AI_CONTEXT_CATEGORIES.indexOf(r.category) !== -1; });
if (rows.length === 0) return res.json({ success: true, context: '' });
var context = '\n\nPHYSICIAN TEMPLATES AND PREFERENCES:\n';
rows.forEach(function(r) {

36
test/transcription-memory-policy.test.js Normal file
View file

@ -0,0 +1,36 @@
const { test } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const root = path.join(__dirname, '..');
function read(relativePath) {
return fs.readFileSync(path.join(root, relativePath), 'utf8');
}
test('AI memory context is limited to saved template categories', () => {
const route = read('src/routes/memories.js');
assert.match(route, /var AI_CONTEXT_CATEGORIES = \[/);
assert.match(route, /'physical_exam'/);
assert.match(route, /'template_ed'/);
assert.match(route, /rows = rows\.filter\(function\(r\) \{ return AI_CONTEXT_CATEGORIES\.indexOf\(r\.category\) !== -1; \}\)/);
assert.doesNotMatch(route.match(/var AI_CONTEXT_CATEGORIES = \[[\s\S]*?\];/)[0], /'custom'/);
});
test('browser Whisper is removed from public runtime and user settings', () => {
assert.equal(fs.existsSync(path.join(root, 'public/js/browserWhisper.js')), false);
const publicRuntimeFiles = [
'public/index.html',
'public/js/app.js',
'public/components/settings.html',
'public/components/faq.html'
];
publicRuntimeFiles.forEach((file) => {
assert.doesNotMatch(read(file), /BrowserWhisper|browser-whisper|Browser Whisper|Xenova\/whisper|transformers\.min\.js/, file);
});
const dockerfile = read('Dockerfile');
assert.doesNotMatch(dockerfile, /Xenova\/whisper|transformers\.min\.js|Browser Whisper/);
const server = read('server.js');
assert.doesNotMatch(server, /wasm-unsafe-eval|unsafe-eval|huggingface\.co|cdn-lfs|transformers/);
});