fix: hanging up the voice call stops the voice, and the answer is read not its markup
Some checks failed
Forgejo Android APK / Root app tests (push) Successful in 1m0s
Forgejo Docker Build / Root app tests (push) Successful in 47s
Forgejo Android APK / Build signed APK (push) Successful in 2m3s
Forgejo Docker Build / Build Docker image (push) Successful in 20s
Forgejo Docker Build / Deploy to the host (push) Failing after 0s
Some checks failed
Forgejo Android APK / Root app tests (push) Successful in 1m0s
Forgejo Docker Build / Root app tests (push) Successful in 47s
Forgejo Android APK / Build signed APK (push) Successful in 2m3s
Forgejo Docker Build / Build Docker image (push) Successful in 20s
Forgejo Docker Build / Deploy to the host (push) Failing after 0s
Ending the call stopped listening and aborted the request but never stopped playback, and the <audio> element was a local variable with no handle kept — so nothing could stop it and the answer talked on until the page was reloaded. conversationMode now holds whatever is speaking, whichever of the two players it is, and hanging up silences it before anything else. Starting a new turn does the same, because speaking over the previous answer is how an assistant talks past you, and a reply that arrives after the overlay has closed no longer starts talking into a closed call. The other half: the raw answer went straight to the speaker. It is markdown, so a browser voice reads "#" and "**" aloud or stumbles over them. speakableText() now reduces it to what a person would read out — headings and bullets become sentences, links keep their words, emphasis and code fences are dropped, tables are dropped entirely because a table read aloud is noise, and a removed citation marker leaves no gap before the punctuation it preceded. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU
This commit is contained in:
parent
fa2e7523d6
commit
0e17f553fc
2 changed files with 120 additions and 7 deletions
|
|
@ -1138,7 +1138,22 @@ import {
|
|||
}).catch(function() {});
|
||||
}
|
||||
|
||||
var conversationMode = { active: false, listening: false, recorder: null, recognition: null, sessionFinals: '' };
|
||||
var conversationMode = { active: false, listening: false, recorder: null, recognition: null,
|
||||
sessionFinals: '', speaking: null };
|
||||
|
||||
// Whatever is currently reading an answer aloud, stopped. Without a handle on
|
||||
// the <audio> element there was nothing to stop: ending the call left the
|
||||
// answer talking until the page was reloaded.
|
||||
function conversationStopSpeaking() {
|
||||
try { if (window.speechSynthesis) window.speechSynthesis.cancel(); } catch (e) {}
|
||||
var playing = conversationMode.speaking;
|
||||
conversationMode.speaking = null;
|
||||
if (!playing) return;
|
||||
try { playing.pause(); playing.src = ''; } catch (e) {}
|
||||
if (playing.dataset && playing.dataset.objectUrl) {
|
||||
try { URL.revokeObjectURL(playing.dataset.objectUrl); } catch (e) {}
|
||||
}
|
||||
}
|
||||
|
||||
function openConversationMode() {
|
||||
if (conversationMode.active) return;
|
||||
|
|
@ -1179,6 +1194,8 @@ import {
|
|||
|
||||
function conversationStartListening() {
|
||||
unlockAudioPlayback();
|
||||
// Speaking over the last answer is how a voice assistant talks past you.
|
||||
conversationStopSpeaking();
|
||||
var status = document.getElementById('assistant-voice-status');
|
||||
if (status) status.textContent = 'Listening…';
|
||||
conversationMode.listening = true;
|
||||
|
|
@ -1246,7 +1263,35 @@ import {
|
|||
speakAnswerVoice(String(detail.answer || ''));
|
||||
}
|
||||
|
||||
function speakAnswerVoice(text) {
|
||||
// What a person would read out, not what the model wrote. Passed raw, a
|
||||
// browser voice reads "#" and "**" aloud or stumbles over them — the answer
|
||||
// is markdown and the speaker is not a renderer.
|
||||
function speakableText(markdown) {
|
||||
return String(markdown || '')
|
||||
.replace(/```[\s\S]*?```/g, ' ') // code blocks say nothing useful
|
||||
.replace(/`([^`]+)`/g, '$1')
|
||||
.replace(/!\[[^\]]*\]\([^)]*\)/g, ' ') // images
|
||||
.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1') // links keep their words
|
||||
.replace(/^\s{0,3}#{1,6}\s+/gm, '') // headings are just sentences
|
||||
.replace(/^\s{0,3}>\s?/gm, '')
|
||||
.replace(/^\s*[-*+]\s+/gm, '') // bullets
|
||||
.replace(/^\s*\d+\.\s+/gm, '')
|
||||
.replace(/^\s*\|.*\|\s*$/gm, ' ') // a table read aloud is noise
|
||||
.replace(/^\s*[-:|\s]{3,}\s*$/gm, ' ')
|
||||
.replace(/(\*\*|__)(.*?)\1/g, '$2')
|
||||
.replace(/(\*|_)(.*?)\1/g, '$2')
|
||||
.replace(/~~(.*?)~~/g, '$1')
|
||||
.replace(/^\s*[-*_]{3,}\s*$/gm, ' ') // horizontal rules
|
||||
.replace(/\[\^?\d+\]/g, ' ') // citation markers
|
||||
.replace(/[ \t]+/g, ' ')
|
||||
.replace(/ ([.,;:!?])/g, '$1') // the gap a removed marker leaves
|
||||
.replace(/\n[ \t]*\n+/g, '\n')
|
||||
.replace(/\n{2,}/g, '\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function speakAnswerVoice(markdown) {
|
||||
var text = speakableText(markdown);
|
||||
var done = function(msg) {
|
||||
var st = document.getElementById('assistant-voice-status');
|
||||
if (st) st.textContent = msg || 'Tap to speak';
|
||||
|
|
@ -1256,9 +1301,10 @@ import {
|
|||
if (typeof window !== 'undefined' && window.speechSynthesis && typeof SpeechSynthesisUtterance === 'function') {
|
||||
try {
|
||||
var utter = new SpeechSynthesisUtterance(text);
|
||||
utter.onend = function() { done('Tap to speak'); };
|
||||
utter.onerror = function() { done('Tap to speak'); };
|
||||
utter.onend = function() { conversationMode.speaking = null; done('Tap to speak'); };
|
||||
utter.onerror = function() { conversationMode.speaking = null; done('Tap to speak'); };
|
||||
window.speechSynthesis.cancel();
|
||||
conversationMode.speaking = utter;
|
||||
window.speechSynthesis.speak(utter);
|
||||
return;
|
||||
} catch (e) { /* fall through to the model voice */ }
|
||||
|
|
@ -1271,14 +1317,20 @@ import {
|
|||
.then(function(blob) {
|
||||
var url = URL.createObjectURL(blob);
|
||||
var audio = new Audio(url);
|
||||
audio.onended = function() { URL.revokeObjectURL(url); done(); };
|
||||
audio.onerror = function() { URL.revokeObjectURL(url); done('Voice reply unavailable — tap to speak'); };
|
||||
audio.play().catch(function() { URL.revokeObjectURL(url); done('Voice reply unavailable — tap to speak'); });
|
||||
audio.dataset.objectUrl = url;
|
||||
// The reply arrives after the call may already have ended. Nothing
|
||||
// should start talking into a closed overlay.
|
||||
if (!conversationMode.active) { URL.revokeObjectURL(url); return; }
|
||||
conversationMode.speaking = audio;
|
||||
audio.onended = function() { conversationMode.speaking = null; URL.revokeObjectURL(url); done(); };
|
||||
audio.onerror = function() { conversationMode.speaking = null; URL.revokeObjectURL(url); done('Voice reply unavailable — tap to speak'); };
|
||||
audio.play().catch(function() { conversationMode.speaking = null; URL.revokeObjectURL(url); done('Voice reply unavailable — tap to speak'); });
|
||||
}).catch(function() { done('Voice reply unavailable — tap to speak'); });
|
||||
}
|
||||
|
||||
function endConversationMode() {
|
||||
conversationMode.active = false;
|
||||
conversationStopSpeaking();
|
||||
conversationStopListening();
|
||||
if (activeAssistantRequest) activeAssistantRequest.abort();
|
||||
document.removeEventListener('assistant-answer-done', conversationAnswerDone);
|
||||
|
|
|
|||
61
test/assistant-voice-mode.test.js
Normal file
61
test/assistant-voice-mode.test.js
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
// ============================================================
|
||||
// ASSISTANT VOICE MODE
|
||||
// ============================================================
|
||||
// The hands-free overlay: tap, speak, hear the answer. Two things about it were
|
||||
// wrong in ways a person notices immediately.
|
||||
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const read = p => fs.readFileSync(path.join(__dirname, '..', p), 'utf8');
|
||||
const src = read('public/js/clinicalAssistant.js');
|
||||
|
||||
// The function under test is a closure inside a browser IIFE, so it is pulled
|
||||
// out by source rather than imported.
|
||||
const speakableText = (() => {
|
||||
const from = src.indexOf('function speakableText');
|
||||
const to = src.indexOf('function speakAnswerVoice');
|
||||
// eslint-disable-next-line no-eval
|
||||
return eval(src.slice(from, to) + '; speakableText');
|
||||
})();
|
||||
|
||||
test('ending the call stops the voice', () => {
|
||||
// It stopped listening and aborted the request but never stopped playback,
|
||||
// and the <audio> element was a local with no handle kept — so the answer
|
||||
// talked on until the page was reloaded.
|
||||
assert.match(src, /function conversationStopSpeaking\(\)/);
|
||||
assert.match(src, /if \(window\.speechSynthesis\) window\.speechSynthesis\.cancel\(\)/);
|
||||
assert.match(src, /playing\.pause\(\); playing\.src = '';/);
|
||||
|
||||
const end = src.slice(src.indexOf('function endConversationMode'));
|
||||
assert.ok(end.indexOf('conversationStopSpeaking()') < end.indexOf('conversationStopListening()'),
|
||||
'hanging up silences the answer');
|
||||
// Speaking over the previous answer is how an assistant talks past you.
|
||||
assert.match(src, /unlockAudioPlayback\(\);\s*\n\s*\/\/[^\n]*\n\s*conversationStopSpeaking\(\);/);
|
||||
// A reply that arrives after the overlay closed must not start talking.
|
||||
assert.match(src, /if \(!conversationMode\.active\) \{ URL\.revokeObjectURL\(url\); return; \}/);
|
||||
// Both players are tracked, or only one of them can be stopped.
|
||||
assert.match(src, /conversationMode\.speaking = utter;/);
|
||||
assert.match(src, /conversationMode\.speaking = audio;/);
|
||||
});
|
||||
|
||||
test('the answer is read, not its markup', () => {
|
||||
// The raw answer went straight to the speaker. It is markdown, and a browser
|
||||
// voice reads "#" and "**" aloud or stumbles over them.
|
||||
assert.match(src, /var text = speakableText\(markdown\);/);
|
||||
|
||||
assert.equal(speakableText('## Croup\n\nA **viral** illness.'), 'Croup\nA viral illness.');
|
||||
assert.equal(speakableText('- One\n- Two'), 'One\nTwo');
|
||||
assert.equal(speakableText('1. First\n2. Second'), 'First\nSecond');
|
||||
assert.equal(speakableText('See [the guideline](https://x.y).'), 'See the guideline.');
|
||||
assert.equal(speakableText('Use `epinephrine` now.'), 'Use epinephrine now.');
|
||||
// A table read aloud is noise; the prose around it survives.
|
||||
assert.equal(speakableText('| A | B |\n|---|---|\n| 1 | 2 |\n\nAfter.'), 'After.');
|
||||
// Citation markers leave no gap before the punctuation they preceded.
|
||||
assert.equal(speakableText('Clear [1] and also [^2].'), 'Clear and also.');
|
||||
assert.equal(speakableText('---\n\n> Quoted.'), 'Quoted.');
|
||||
assert.equal(speakableText('```\ncode\n```\n\nThen.'), 'Then.');
|
||||
assert.equal(speakableText(''), '');
|
||||
assert.equal(speakableText(null), '');
|
||||
});
|
||||
Loading…
Reference in a new issue