From 59bf0a5cd8b2f4d484cc02ba8c762a53a54a8767 Mon Sep 17 00:00:00 2001 From: Daniel Onyejesi Date: Tue, 31 Mar 2026 07:30:17 -0400 Subject: [PATCH] Add browser-side Whisper transcription (local, zero network, HIPAA-safe) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - whisperWorker.js: Web Worker running @xenova/transformers Whisper in WASM - browserWhisper.js: main-thread manager — audio→Float32 conversion, worker lifecycle - transcribeAudio() checks BrowserWhisper.isEnabled() first, falls back to server - Settings UI: enable/disable, model picker (tiny/base/small), pre-download button - CSP: add wasm-unsafe-eval, cdn.jsdelivr.net, HuggingFace CDN domains - Default: whisper-tiny.en (~39MB, ~2-3s per clip) --- public/components/settings.html | 26 ++++++ public/index.html | 1 + public/js/app.js | 81 +++++++++++++++++- public/js/browserWhisper.js | 143 ++++++++++++++++++++++++++++++++ public/js/whisperWorker.js | 53 ++++++++++++ server.js | 10 ++- 6 files changed, 310 insertions(+), 4 deletions(-) create mode 100644 public/js/browserWhisper.js create mode 100644 public/js/whisperWorker.js diff --git a/public/components/settings.html b/public/components/settings.html index ac4ad91..9dcf50e 100644 --- a/public/components/settings.html +++ b/public/components/settings.html @@ -5,6 +5,32 @@
+ +
+

Browser Transcription (Local Whisper)

+

Transcribes audio entirely in your browser — no audio sent to any server. Powered by OpenAI Whisper running in WebAssembly. Model is downloaded once and cached locally.

+
+ + +
+
+ + + +
+ +

When enabled, overrides server transcription. Falls back to server if browser transcription fails.

+
+

Two-Factor Authentication

diff --git a/public/index.html b/public/index.html index 15ffaae..17918e7 100644 --- a/public/index.html +++ b/public/index.html @@ -302,6 +302,7 @@ + diff --git a/public/js/app.js b/public/js/app.js index 66a5055..a12dc9e 100644 --- a/public/js/app.js +++ b/public/js/app.js @@ -157,9 +157,67 @@ document.addEventListener('DOMContentLoaded', function() { if (typeof loadSavedEncountersList === 'function') loadSavedEncountersList(); if (typeof renderAudioBackups === 'function') renderAudioBackups(); if (typeof loadDocuments === 'function') loadDocuments(); + initBrowserWhisperSettings(); } }); + // ── Browser Whisper settings UI ─────────────────────────── + function initBrowserWhisperSettings() { + var chk = document.getElementById('browser-whisper-enabled'); + var sel = document.getElementById('browser-whisper-model'); + var pre = document.getElementById('btn-whisper-preload'); + var stat = document.getElementById('browser-whisper-status'); + var prog = document.getElementById('browser-whisper-progress'); + var pt = document.getElementById('browser-whisper-progress-text'); + var sec = document.getElementById('browser-whisper-section'); + if (!chk) return; + + var supported = typeof BrowserWhisper !== 'undefined' && BrowserWhisper.isSupported(); + if (!supported) { + if (sec) sec.innerHTML += '

Not supported in this browser. Use Chrome or Edge.

'; + if (chk) chk.disabled = true; + return; + } + + // Restore saved state + chk.checked = BrowserWhisper.isEnabled(); + sel.value = BrowserWhisper.getModel(); + stat.textContent = chk.checked ? 'On — audio stays on device' : 'Off'; + + chk.addEventListener('change', function() { + BrowserWhisper.setEnabled(chk.checked); + stat.textContent = chk.checked ? 'On — audio stays on device' : 'Off'; + if (chk.checked) { + BrowserWhisper.preload(function(file, pct) { + if (!prog || !pt) return; + if (pct >= 100) { prog.style.display = 'none'; return; } + prog.style.display = 'block'; + pt.textContent = file + (pct > 0 ? ' ' + pct + '%' : ''); + }); + } + }); + + sel.addEventListener('change', function() { + BrowserWhisper.setModel(sel.value); + }); + + if (pre) { + pre.addEventListener('click', function() { + if (!prog || !pt) return; + prog.style.display = 'block'; + pt.textContent = 'Starting download...'; + BrowserWhisper.setEnabled(true); + chk.checked = true; + stat.textContent = 'On — audio stays on device'; + BrowserWhisper.preload(function(file, pct) { + if (pct >= 100) { prog.style.display = 'none'; showToast('Whisper model ready', 'success'); return; } + prog.style.display = 'block'; + pt.textContent = file + (pct > 0 ? ' ' + pct + '%' : ''); + }); + }); + } + } + // --- MODEL SELECTOR --- var modelSelect = document.getElementById('global-model-select'); var costBadge = document.getElementById('model-cost-badge'); @@ -507,6 +565,28 @@ function checkTranscribeStatus() { } function transcribeAudio(blob) { + // Browser Whisper — local, zero network, HIPAA-safe + if (typeof BrowserWhisper !== 'undefined' && BrowserWhisper.isEnabled()) { + var startTime = Date.now(); + return BrowserWhisper.transcribe(blob) + .then(function(text) { + var elapsed = ((Date.now() - startTime) / 1000).toFixed(1); + showToast('Transcribed locally (' + elapsed + 's)', 'success'); + if (window._lastAudioBackupId && typeof deleteAudioBackup === 'function') { + deleteAudioBackup(window._lastAudioBackupId); + window._lastAudioBackupId = null; + } + return { success: true, text: text, provider: 'browser-whisper' }; + }) + .catch(function(err) { + console.warn('[BrowserWhisper] Failed:', err.message, '— falling back to server'); + return _serverTranscribe(blob); + }); + } + return _serverTranscribe(blob); +} + +function _serverTranscribe(blob) { // If no server transcription is configured, skip upload entirely if (window._transcribeAvailable === false) { return Promise.resolve({ success: false, noProvider: true, error: 'No transcription API configured — using live transcript' }); @@ -524,7 +604,6 @@ function transcribeAudio(blob) { if (data.success && data.provider) { showToast('Transcribed via ' + data.provider + ' (' + elapsed + 's)', 'info'); } - // Delete audio backup on successful transcription if (data.success && window._lastAudioBackupId) { if (typeof deleteAudioBackup === 'function') deleteAudioBackup(window._lastAudioBackupId); window._lastAudioBackupId = null; diff --git a/public/js/browserWhisper.js b/public/js/browserWhisper.js new file mode 100644 index 0000000..0d771c4 --- /dev/null +++ b/public/js/browserWhisper.js @@ -0,0 +1,143 @@ +// ============================================================ +// BROWSER WHISPER — client-side transcription, zero server calls +// Uses @xenova/transformers running Whisper in WebAssembly. +// Audio is processed entirely in-browser, never transmitted. +// +// Models (cached in IndexedDB after first download): +// Xenova/whisper-tiny.en — ~39MB — fastest, ~2-3s/clip +// Xenova/whisper-base.en — ~74MB — balanced, ~3-5s/clip +// Xenova/whisper-small.en — ~244MB — best quality, ~6-10s/clip +// ============================================================ + +(function() { + + var STORAGE_ENABLED = 'ped_browser_whisper'; + var STORAGE_MODEL = 'ped_whisper_model'; + var DEFAULT_MODEL = 'Xenova/whisper-tiny.en'; + + var _worker = null; + var _ready = false; + var _loading = false; + var _modelLoaded = null; + var _pending = null; // { resolve, reject } + + window.BrowserWhisper = { + + isSupported: function() { + return typeof Worker !== 'undefined' && + typeof AudioContext !== 'undefined' && + typeof WebAssembly !== 'undefined'; + }, + + isEnabled: function() { + try { return localStorage.getItem(STORAGE_ENABLED) === '1'; } catch(e) { return false; } + }, + + setEnabled: function(val) { + try { localStorage.setItem(STORAGE_ENABLED, val ? '1' : '0'); } catch(e) {} + }, + + getModel: function() { + try { return localStorage.getItem(STORAGE_MODEL) || DEFAULT_MODEL; } catch(e) { return DEFAULT_MODEL; } + }, + + setModel: function(m) { + try { localStorage.setItem(STORAGE_MODEL, m); } catch(e) {} + // Force reload next time + _ready = false; + _modelLoaded = null; + }, + + // Pre-warm: load model in background before first recording + preload: function(onProgress) { + if (!this.isSupported() || !this.isEnabled()) return; + _initWorker(this.getModel(), onProgress || function() {}); + }, + + // Transcribe a Blob (WebM, WAV, etc.) + transcribe: function(blob) { + var self = this; + if (!this.isSupported()) return Promise.reject(new Error('WebAssembly not supported')); + if (!this.isEnabled()) return Promise.reject(new Error('Browser Whisper not enabled')); + + return _blobToFloat32(blob).then(function(float32) { + return new Promise(function(resolve, reject) { + var model = self.getModel(); + + // If worker is ready with same model, send immediately + if (_ready && _modelLoaded === model) { + _pending = { resolve: resolve, reject: reject }; + _worker.postMessage({ type: 'transcribe', audio: float32, model: model }, [float32.buffer]); + return; + } + + // Need to (re)load model first + _pending = { resolve: resolve, reject: reject }; + _initWorker(model, function() {}, function() { + _worker.postMessage({ type: 'transcribe', audio: float32, model: model }, [float32.buffer]); + }); + }); + }); + } + }; + + // ── Internal ────────────────────────────────────────────── + + function _initWorker(model, onProgress, onReady) { + if (_loading && _modelLoaded === model) return; // already loading same model + + if (_worker) { _worker.terminate(); _worker = null; } + _ready = false; + _loading = true; + _modelLoaded = model; + + _worker = new Worker('/js/whisperWorker.js'); + + _worker.addEventListener('message', function(e) { + var d = e.data; + + if (d.type === 'loading') { + if (onProgress) onProgress('Downloading Whisper model (' + d.model.split('/').pop() + ')...', 0); + } + if (d.type === 'progress') { + if (onProgress) onProgress(d.file.split('/').pop() || 'model', d.progress); + } + if (d.type === 'ready') { + _ready = true; + _loading = false; + if (onProgress) onProgress('', 100); + if (onReady) onReady(); + } + if (d.type === 'result') { + if (_pending) { _pending.resolve(d.text); _pending = null; } + } + if (d.type === 'error') { + _loading = false; + if (_pending) { _pending.reject(new Error(d.message)); _pending = null; } + } + }); + + _worker.addEventListener('error', function(err) { + _loading = false; + _ready = false; + if (_pending) { _pending.reject(err); _pending = null; } + }); + + _worker.postMessage({ type: 'load', model: model }); + } + + // Convert audio Blob → Float32Array at 16kHz mono (what Whisper expects) + function _blobToFloat32(blob) { + return blob.arrayBuffer().then(function(buf) { + var ctx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 }); + return ctx.decodeAudioData(buf).then(function(audioBuffer) { + ctx.close(); + return audioBuffer.getChannelData(0); // mono + }).catch(function(err) { + ctx.close(); + throw new Error('Audio decode failed: ' + err.message); + }); + }); + } + +})(); diff --git a/public/js/whisperWorker.js b/public/js/whisperWorker.js new file mode 100644 index 0000000..6a39ab4 --- /dev/null +++ b/public/js/whisperWorker.js @@ -0,0 +1,53 @@ +// ============================================================ +// WHISPER WORKER — runs @xenova/transformers in a Web Worker +// Audio never leaves the device. Model cached in IndexedDB. +// ============================================================ + +importScripts('https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2'); + +var T = self.transformers || transformers; +T.env.allowLocalModels = false; +T.env.useBrowserCache = true; + +var _pipe = null; +var _loadedModel = null; + +async function load(modelName) { + if (_pipe && _loadedModel === modelName) return; + self.postMessage({ type: 'loading', model: modelName }); + _pipe = await T.pipeline('automatic-speech-recognition', modelName, { + progress_callback: function(p) { + if (p.status === 'downloading' || p.status === 'progress') { + self.postMessage({ type: 'progress', file: p.file || '', progress: Math.round(p.progress || 0) }); + } + } + }); + _loadedModel = modelName; + self.postMessage({ type: 'ready', model: modelName }); +} + +self.addEventListener('message', async function(e) { + var d = e.data; + + if (d.type === 'load') { + try { await load(d.model || 'Xenova/whisper-tiny.en'); } + catch(err) { self.postMessage({ type: 'error', message: err.message }); } + return; + } + + if (d.type === 'transcribe') { + try { + await load(d.model || 'Xenova/whisper-tiny.en'); + var result = await _pipe(d.audio, { + language: 'english', + task: 'transcribe', + chunk_length_s: 30, + stride_length_s: 5, + return_timestamps: false + }); + self.postMessage({ type: 'result', text: result.text.trim() }); + } catch(err) { + self.postMessage({ type: 'error', message: err.message }); + } + } +}); diff --git a/server.js b/server.js index d61b75d..59ca4af 100644 --- a/server.js +++ b/server.js @@ -19,14 +19,18 @@ app.use(helmet({ contentSecurityPolicy: { directives: { defaultSrc: ["'self'"], - scriptSrc: ["'self'"], + // 'wasm-unsafe-eval' required for WebAssembly (Whisper in-browser transcription) + // cdn.jsdelivr.net required for @xenova/transformers worker script + scriptSrc: ["'self'", "'wasm-unsafe-eval'", 'https://cdn.jsdelivr.net'], scriptSrcAttr: ["'none'"], styleSrc: ["'self'", "'unsafe-inline'", 'https://fonts.googleapis.com', 'https://cdnjs.cloudflare.com'], fontSrc: ["'self'", 'https://fonts.gstatic.com', 'https://cdnjs.cloudflare.com'], imgSrc: ["'self'", 'data:', 'blob:'], mediaSrc: ["'self'", 'blob:'], - connectSrc: ["'self'", 'https://cdnjs.cloudflare.com', 'https://fonts.googleapis.com', 'https://fonts.gstatic.com', 'https://www.google.com', 'wss://www.google.com', 'https://clinicaltables.nlm.nih.gov'], - workerSrc: ["'self'"], + connectSrc: ["'self'", 'https://cdnjs.cloudflare.com', 'https://fonts.googleapis.com', 'https://fonts.gstatic.com', 'https://www.google.com', 'wss://www.google.com', 'https://clinicaltables.nlm.nih.gov', + // HuggingFace CDN for Whisper model downloads + 'https://huggingface.co', 'https://cdn-lfs.huggingface.co', 'https://cdn-lfs-us-1.huggingface.co', 'https://cdn-lfs-us-2.huggingface.co'], + workerSrc: ["'self'", 'blob:'], frameSrc: ["'none'"], objectSrc: ["'none'"], }