Add browser-side Whisper transcription (local, zero network, HIPAA-safe)

- whisperWorker.js: Web Worker running @xenova/transformers Whisper in WASM - browserWhisper.js: main-thread manager — audio→Float32 conversion, worker lifecycle - transcribeAudio() checks BrowserWhisper.isEnabled() first, falls back to server - Settings UI: enable/disable, model picker (tiny/base/small), pre-download button - CSP: add wasm-unsafe-eval, cdn.jsdelivr.net, HuggingFace CDN domains - Default: whisper-tiny.en (~39MB, ~2-3s per clip)
2026-03-31 07:30:17 -04:00 · 2026-03-31 07:30:17 -04:00 · 59bf0a5cd8
commit 59bf0a5cd8
parent 1e8aaf75f8
6 changed files with 310 additions and 4 deletions
--- a/public/components/settings.html
+++ b/public/components/settings.html
@ -5,6 +5,32 @@

        <div class="settings-page">

+          <!-- Browser Whisper -->
+          <div class="settings-section card" id="browser-whisper-section">
+            <h3><i class="fas fa-microchip"></i> Browser Transcription (Local Whisper)</h3>
+            <p style="font-size:13px;color:var(--g600);">Transcribes audio entirely in your browser — no audio sent to any server. Powered by OpenAI Whisper running in WebAssembly. Model is downloaded once and cached locally.</p>
+            <div style="display:flex;align-items:center;gap:12px;flex-wrap:wrap;margin-bottom:12px;">
+              <label style="font-size:13px;font-weight:600;">Enable browser transcription:</label>
+              <label style="display:flex;align-items:center;gap:6px;cursor:pointer;">
+                <input type="checkbox" id="browser-whisper-enabled" style="accent-color:var(--blue);width:16px;height:16px;">
+                <span style="font-size:13px;" id="browser-whisper-status">Off</span>
+              </label>
+            </div>
+            <div id="browser-whisper-model-row" style="display:flex;align-items:center;gap:12px;flex-wrap:wrap;margin-bottom:12px;">
+              <label style="font-size:13px;font-weight:600;">Model:</label>
+              <select id="browser-whisper-model" style="font-size:13px;padding:5px 8px;border:1px solid var(--g300);border-radius:6px;">
+                <option value="Xenova/whisper-tiny.en">Tiny (~39MB) — fastest, ~2-3s</option>
+                <option value="Xenova/whisper-base.en">Base (~74MB) — balanced, ~3-5s</option>
+                <option value="Xenova/whisper-small.en">Small (~244MB) — best quality, ~6-10s</option>
+              </select>
+              <button id="btn-whisper-preload" class="btn-sm btn-ghost"><i class="fas fa-download"></i> Pre-download model</button>
+            </div>
+            <div id="browser-whisper-progress" style="display:none;font-size:12px;color:var(--g500);margin-top:4px;">
+              <i class="fas fa-spinner fa-spin"></i> <span id="browser-whisper-progress-text">Loading...</span>
+            </div>
+            <p style="font-size:12px;color:var(--g400);margin:8px 0 0;"><i class="fas fa-info-circle"></i> When enabled, overrides server transcription. Falls back to server if browser transcription fails.</p>
+          </div>
+
          <!-- 2FA -->
          <div class="settings-section card">
            <h3><i class="fas fa-shield-halved"></i> Two-Factor Authentication</h3>
--- a/public/index.html
+++ b/public/index.html
@ -302,6 +302,7 @@
  <script defer src="/js/pediatricScheduleData.js"></script>
  <script defer src="/js/audioBackup.js"></script>
  <script defer src="/js/correctionTracker.js"></script>
+  <script defer src="/js/browserWhisper.js"></script>
  <script defer src="/js/app.js"></script>
  <script defer src="/js/auth.js"></script>
  <script defer src="/js/liveEncounter.js"></script>
--- a/public/js/app.js
+++ b/public/js/app.js
@ -157,9 +157,67 @@ document.addEventListener('DOMContentLoaded', function() {
      if (typeof loadSavedEncountersList === 'function') loadSavedEncountersList();
      if (typeof renderAudioBackups === 'function') renderAudioBackups();
      if (typeof loadDocuments === 'function') loadDocuments();
+      initBrowserWhisperSettings();
    }
  });

+  // ── Browser Whisper settings UI ───────────────────────────
+  function initBrowserWhisperSettings() {
+    var chk  = document.getElementById('browser-whisper-enabled');
+    var sel  = document.getElementById('browser-whisper-model');
+    var pre  = document.getElementById('btn-whisper-preload');
+    var stat = document.getElementById('browser-whisper-status');
+    var prog = document.getElementById('browser-whisper-progress');
+    var pt   = document.getElementById('browser-whisper-progress-text');
+    var sec  = document.getElementById('browser-whisper-section');
+    if (!chk) return;
+
+    var supported = typeof BrowserWhisper !== 'undefined' && BrowserWhisper.isSupported();
+    if (!supported) {
+      if (sec) sec.innerHTML += '<p style="color:var(--red);font-size:12px;margin:8px 0 0;">Not supported in this browser. Use Chrome or Edge.</p>';
+      if (chk) chk.disabled = true;
+      return;
+    }
+
+    // Restore saved state
+    chk.checked = BrowserWhisper.isEnabled();
+    sel.value   = BrowserWhisper.getModel();
+    stat.textContent = chk.checked ? 'On — audio stays on device' : 'Off';
+
+    chk.addEventListener('change', function() {
+      BrowserWhisper.setEnabled(chk.checked);
+      stat.textContent = chk.checked ? 'On — audio stays on device' : 'Off';
+      if (chk.checked) {
+        BrowserWhisper.preload(function(file, pct) {
+          if (!prog || !pt) return;
+          if (pct >= 100) { prog.style.display = 'none'; return; }
+          prog.style.display = 'block';
+          pt.textContent = file + (pct > 0 ? ' ' + pct + '%' : '');
+        });
+      }
+    });
+
+    sel.addEventListener('change', function() {
+      BrowserWhisper.setModel(sel.value);
+    });
+
+    if (pre) {
+      pre.addEventListener('click', function() {
+        if (!prog || !pt) return;
+        prog.style.display = 'block';
+        pt.textContent = 'Starting download...';
+        BrowserWhisper.setEnabled(true);
+        chk.checked = true;
+        stat.textContent = 'On — audio stays on device';
+        BrowserWhisper.preload(function(file, pct) {
+          if (pct >= 100) { prog.style.display = 'none'; showToast('Whisper model ready', 'success'); return; }
+          prog.style.display = 'block';
+          pt.textContent = file + (pct > 0 ? ' ' + pct + '%' : '');
+        });
+      });
+    }
+  }
+
  // --- MODEL SELECTOR ---
  var modelSelect = document.getElementById('global-model-select');
  var costBadge = document.getElementById('model-cost-badge');
@ -507,6 +565,28 @@ function checkTranscribeStatus() {
 }

 function transcribeAudio(blob) {
+  // Browser Whisper — local, zero network, HIPAA-safe
+  if (typeof BrowserWhisper !== 'undefined' && BrowserWhisper.isEnabled()) {
+    var startTime = Date.now();
+    return BrowserWhisper.transcribe(blob)
+      .then(function(text) {
+        var elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+        showToast('Transcribed locally (' + elapsed + 's)', 'success');
+        if (window._lastAudioBackupId && typeof deleteAudioBackup === 'function') {
+          deleteAudioBackup(window._lastAudioBackupId);
+          window._lastAudioBackupId = null;
+        }
+        return { success: true, text: text, provider: 'browser-whisper' };
+      })
+      .catch(function(err) {
+        console.warn('[BrowserWhisper] Failed:', err.message, '— falling back to server');
+        return _serverTranscribe(blob);
+      });
+  }
+  return _serverTranscribe(blob);
+}
+
+function _serverTranscribe(blob) {
  // If no server transcription is configured, skip upload entirely
  if (window._transcribeAvailable === false) {
    return Promise.resolve({ success: false, noProvider: true, error: 'No transcription API configured — using live transcript' });
@ -524,7 +604,6 @@ function transcribeAudio(blob) {
    if (data.success && data.provider) {
      showToast('Transcribed via ' + data.provider + ' (' + elapsed + 's)', 'info');
    }
-    // Delete audio backup on successful transcription
    if (data.success && window._lastAudioBackupId) {
      if (typeof deleteAudioBackup === 'function') deleteAudioBackup(window._lastAudioBackupId);
      window._lastAudioBackupId = null;
--- a/public/js/browserWhisper.js
+++ b/public/js/browserWhisper.js
@ -0,0 +1,143 @@
+// ============================================================
+// BROWSER WHISPER — client-side transcription, zero server calls
+// Uses @xenova/transformers running Whisper in WebAssembly.
+// Audio is processed entirely in-browser, never transmitted.
+//
+// Models (cached in IndexedDB after first download):
+//   Xenova/whisper-tiny.en  — ~39MB  — fastest, ~2-3s/clip
+//   Xenova/whisper-base.en  — ~74MB  — balanced, ~3-5s/clip
+//   Xenova/whisper-small.en — ~244MB — best quality, ~6-10s/clip
+// ============================================================
+
+(function() {
+
+  var STORAGE_ENABLED = 'ped_browser_whisper';
+  var STORAGE_MODEL   = 'ped_whisper_model';
+  var DEFAULT_MODEL   = 'Xenova/whisper-tiny.en';
+
+  var _worker   = null;
+  var _ready    = false;
+  var _loading  = false;
+  var _modelLoaded = null;
+  var _pending  = null; // { resolve, reject }
+
+  window.BrowserWhisper = {
+
+    isSupported: function() {
+      return typeof Worker !== 'undefined' &&
+             typeof AudioContext !== 'undefined' &&
+             typeof WebAssembly !== 'undefined';
+    },
+
+    isEnabled: function() {
+      try { return localStorage.getItem(STORAGE_ENABLED) === '1'; } catch(e) { return false; }
+    },
+
+    setEnabled: function(val) {
+      try { localStorage.setItem(STORAGE_ENABLED, val ? '1' : '0'); } catch(e) {}
+    },
+
+    getModel: function() {
+      try { return localStorage.getItem(STORAGE_MODEL) || DEFAULT_MODEL; } catch(e) { return DEFAULT_MODEL; }
+    },
+
+    setModel: function(m) {
+      try { localStorage.setItem(STORAGE_MODEL, m); } catch(e) {}
+      // Force reload next time
+      _ready = false;
+      _modelLoaded = null;
+    },
+
+    // Pre-warm: load model in background before first recording
+    preload: function(onProgress) {
+      if (!this.isSupported() || !this.isEnabled()) return;
+      _initWorker(this.getModel(), onProgress || function() {});
+    },
+
+    // Transcribe a Blob (WebM, WAV, etc.)
+    transcribe: function(blob) {
+      var self = this;
+      if (!this.isSupported()) return Promise.reject(new Error('WebAssembly not supported'));
+      if (!this.isEnabled()) return Promise.reject(new Error('Browser Whisper not enabled'));
+
+      return _blobToFloat32(blob).then(function(float32) {
+        return new Promise(function(resolve, reject) {
+          var model = self.getModel();
+
+          // If worker is ready with same model, send immediately
+          if (_ready && _modelLoaded === model) {
+            _pending = { resolve: resolve, reject: reject };
+            _worker.postMessage({ type: 'transcribe', audio: float32, model: model }, [float32.buffer]);
+            return;
+          }
+
+          // Need to (re)load model first
+          _pending = { resolve: resolve, reject: reject };
+          _initWorker(model, function() {}, function() {
+            _worker.postMessage({ type: 'transcribe', audio: float32, model: model }, [float32.buffer]);
+          });
+        });
+      });
+    }
+  };
+
+  // ── Internal ──────────────────────────────────────────────
+
+  function _initWorker(model, onProgress, onReady) {
+    if (_loading && _modelLoaded === model) return; // already loading same model
+
+    if (_worker) { _worker.terminate(); _worker = null; }
+    _ready = false;
+    _loading = true;
+    _modelLoaded = model;
+
+    _worker = new Worker('/js/whisperWorker.js');
+
+    _worker.addEventListener('message', function(e) {
+      var d = e.data;
+
+      if (d.type === 'loading') {
+        if (onProgress) onProgress('Downloading Whisper model (' + d.model.split('/').pop() + ')...', 0);
+      }
+      if (d.type === 'progress') {
+        if (onProgress) onProgress(d.file.split('/').pop() || 'model', d.progress);
+      }
+      if (d.type === 'ready') {
+        _ready = true;
+        _loading = false;
+        if (onProgress) onProgress('', 100);
+        if (onReady) onReady();
+      }
+      if (d.type === 'result') {
+        if (_pending) { _pending.resolve(d.text); _pending = null; }
+      }
+      if (d.type === 'error') {
+        _loading = false;
+        if (_pending) { _pending.reject(new Error(d.message)); _pending = null; }
+      }
+    });
+
+    _worker.addEventListener('error', function(err) {
+      _loading = false;
+      _ready = false;
+      if (_pending) { _pending.reject(err); _pending = null; }
+    });
+
+    _worker.postMessage({ type: 'load', model: model });
+  }
+
+  // Convert audio Blob → Float32Array at 16kHz mono (what Whisper expects)
+  function _blobToFloat32(blob) {
+    return blob.arrayBuffer().then(function(buf) {
+      var ctx = new (window.AudioContext || window.webkitAudioContext)({ sampleRate: 16000 });
+      return ctx.decodeAudioData(buf).then(function(audioBuffer) {
+        ctx.close();
+        return audioBuffer.getChannelData(0); // mono
+      }).catch(function(err) {
+        ctx.close();
+        throw new Error('Audio decode failed: ' + err.message);
+      });
+    });
+  }
+
+})();
--- a/public/js/whisperWorker.js
+++ b/public/js/whisperWorker.js
@ -0,0 +1,53 @@
+// ============================================================
+// WHISPER WORKER — runs @xenova/transformers in a Web Worker
+// Audio never leaves the device. Model cached in IndexedDB.
+// ============================================================
+
+importScripts('https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2');
+
+var T = self.transformers || transformers;
+T.env.allowLocalModels = false;
+T.env.useBrowserCache = true;
+
+var _pipe = null;
+var _loadedModel = null;
+
+async function load(modelName) {
+  if (_pipe && _loadedModel === modelName) return;
+  self.postMessage({ type: 'loading', model: modelName });
+  _pipe = await T.pipeline('automatic-speech-recognition', modelName, {
+    progress_callback: function(p) {
+      if (p.status === 'downloading' || p.status === 'progress') {
+        self.postMessage({ type: 'progress', file: p.file || '', progress: Math.round(p.progress || 0) });
+      }
+    }
+  });
+  _loadedModel = modelName;
+  self.postMessage({ type: 'ready', model: modelName });
+}
+
+self.addEventListener('message', async function(e) {
+  var d = e.data;
+
+  if (d.type === 'load') {
+    try { await load(d.model || 'Xenova/whisper-tiny.en'); }
+    catch(err) { self.postMessage({ type: 'error', message: err.message }); }
+    return;
+  }
+
+  if (d.type === 'transcribe') {
+    try {
+      await load(d.model || 'Xenova/whisper-tiny.en');
+      var result = await _pipe(d.audio, {
+        language: 'english',
+        task: 'transcribe',
+        chunk_length_s: 30,
+        stride_length_s: 5,
+        return_timestamps: false
+      });
+      self.postMessage({ type: 'result', text: result.text.trim() });
+    } catch(err) {
+      self.postMessage({ type: 'error', message: err.message });
+    }
+  }
+});
--- a/server.js
+++ b/server.js
@ -19,14 +19,18 @@ app.use(helmet({
  contentSecurityPolicy: {
    directives: {
      defaultSrc: ["'self'"],
-      scriptSrc: ["'self'"],
+      // 'wasm-unsafe-eval' required for WebAssembly (Whisper in-browser transcription)
+      // cdn.jsdelivr.net required for @xenova/transformers worker script
+      scriptSrc: ["'self'", "'wasm-unsafe-eval'", 'https://cdn.jsdelivr.net'],
      scriptSrcAttr: ["'none'"],
      styleSrc: ["'self'", "'unsafe-inline'", 'https://fonts.googleapis.com', 'https://cdnjs.cloudflare.com'],
      fontSrc: ["'self'", 'https://fonts.gstatic.com', 'https://cdnjs.cloudflare.com'],
      imgSrc: ["'self'", 'data:', 'blob:'],
      mediaSrc: ["'self'", 'blob:'],
-      connectSrc: ["'self'", 'https://cdnjs.cloudflare.com', 'https://fonts.googleapis.com', 'https://fonts.gstatic.com', 'https://www.google.com', 'wss://www.google.com', 'https://clinicaltables.nlm.nih.gov'],
-      workerSrc: ["'self'"],
+      connectSrc: ["'self'", 'https://cdnjs.cloudflare.com', 'https://fonts.googleapis.com', 'https://fonts.gstatic.com', 'https://www.google.com', 'wss://www.google.com', 'https://clinicaltables.nlm.nih.gov',
+        // HuggingFace CDN for Whisper model downloads
+        'https://huggingface.co', 'https://cdn-lfs.huggingface.co', 'https://cdn-lfs-us-1.huggingface.co', 'https://cdn-lfs-us-2.huggingface.co'],
+      workerSrc: ["'self'", 'blob:'],
      frameSrc: ["'none'"],
      objectSrc: ["'none'"],
    }