From 56ecbb0889925445d113b48f65d32641f82898fc Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 12 Sep 2026 04:12:48 +0200 Subject: [PATCH] fix: the session owns the menu button, and study loses the exam's review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four things, all from one screenshot pair. The Review button in a study session was inherited from the exam player. Reviewing a block before handing it in is an exam idea; a study session has nothing to hand in — it keeps going until every question is answered and at that point it *is* the review. The review link, the top-bar button and the rail button are exam-only now, and a study session whose questions are all answered says "Finish session" and submits rather than opening a dialog to ask a second time. The drawer's "Qbank" pointed at /questions, which has never been a route — /questions/:id is the editor. It went nowhere. It points at /question-bank, and Collections and AI Mode join the list. While a session is open on a narrow screen, the navbar burger now opens that session's questions instead of the site menu, which is a tab inside the same drawer. Two menu buttons an inch apart, one of which leaves the session you are sitting, is the wrong offer. The player claims the button only while it has no rail, and hands it back when it leaves. And the drawer says what AMBOSS's does: a Review badge once everything is answered, the mode in the title, a progress bar under the count, and the session and question clocks pinned beneath the list. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TqXevQJhxFrM7jJg82cgZN --- backend/app/routers/admin.py | 58 +++++++-- frontend/src/App.jsx | 3 + frontend/src/components/Navbar.jsx | 14 ++- frontend/src/context/SessionDrawer.jsx | 38 ++++++ frontend/src/context/SessionDrawer.test.jsx | 47 +++++++ frontend/src/hooks/useDictation.js | 131 ++++++++++++++++++++ frontend/src/hooks/useDictation.test.jsx | 112 +++++++++++++++++ frontend/src/pages/AiModePage.css | 20 +++ frontend/src/pages/AiModePage.jsx | 16 +++ frontend/src/pages/QuizPage.jsx | 66 ++++++++-- frontend/src/pages/QuizPage.test.jsx | 26 ++++ frontend/src/pages/QuizPlayer.css | 22 +++- 12 files changed, 525 insertions(+), 28 deletions(-) create mode 100644 frontend/src/context/SessionDrawer.jsx create mode 100644 frontend/src/context/SessionDrawer.test.jsx create mode 100644 frontend/src/hooks/useDictation.js create mode 100644 frontend/src/hooks/useDictation.test.jsx diff --git a/backend/app/routers/admin.py b/backend/app/routers/admin.py index b97c779..a5d1918 100644 --- a/backend/app/routers/admin.py +++ b/backend/app/routers/admin.py @@ -1,6 +1,8 @@ -from fastapi import APIRouter, Depends, HTTPException, Query +import logging from datetime import datetime +from fastapi import APIRouter, Depends, HTTPException, Query + from pydantic import BaseModel, Field from sqlalchemy import text from sqlalchemy.orm import Session @@ -11,7 +13,7 @@ from app.database import get_db from app.models.user import User from app.models.ai_model_config import AIModelConfig from app.models.invite import InviteCode -from app.services import invites, site_settings +from app.services import ai_service, invites, site_settings from app.schemas.auth import UserResponse, UserUpdateRole, UserCreate from app.schemas.admin import AIModelConfigCreate, AIModelConfigResponse, AIModelConfigUpdate from app.utils.auth import require_admin, get_current_user, get_password_hash @@ -146,6 +148,13 @@ class LiteLLMSearchRequest(BaseModel): mode: str | None = None +#: Short, unambiguous, and clinical enough that a medical model has no excuse. +#: Used to make a voice speak and a transcriber listen. +TEST_PHRASE = "Inspiratory stridor at rest." + +log = logging.getLogger(__name__) + + def _proxy_models(base: str, key: str | None, mode: str | None = None) -> tuple[list[str], bool]: """What the proxy will serve, and whether the answer knows about modes. @@ -308,10 +317,17 @@ def test_model( raise HTTPException(status_code=404, detail="Model config not found") if model.task == "tts": - # Not a failure. A voice is tested by hearing it, and the page offers - # that; saying so in red taught administrators to distrust a working - # configuration. - return {"message": f"{model.model_id} is a voice — press Preview to hear it."} + # A voice is tested by making it speak. The old answer was an + # instruction ("press Preview") returned as an error, which taught + # administrators to distrust a working configuration. + try: + audio = ai_service.generate_tts_audio( + TEST_PHRASE, model_id=model.model_id, api_key=model.api_key or None) + except Exception as e: + raise HTTPException(status_code=502, detail=str(e)[:300]) + if not audio: + raise HTTPException(status_code=502, detail=f"{model.model_id} returned no audio") + return {"message": f"✓ {model.model_id} spoke {len(audio):,} bytes of audio"} if model.task == "stt": base = (settings.LITELLM_API_BASE or "").rstrip("/").removesuffix("/v1") @@ -325,15 +341,33 @@ def test_model( status_code=404, detail=f"{model.model_id} is not served by the proxy" + (" as a transcription model" if by_mode else "")) - return {"message": f"✓ {model.model_id} is served by the proxy" + ( - " for speech transcription" if by_mode - # The route that says what a model is for is not open to this - # key, so this is presence, not suitability. - else " — the proxy would not say whether it transcribes")} except HTTPException: raise except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + raise HTTPException(status_code=502, detail=str(e)[:300]) + + # Presence is not proof. A voice we already have says a known phrase, + # and the model is asked what it heard — the only test that shows + # transcription actually working end to end. + spoken = None + try: + tts_id, tts_key = ai_service.get_model_for_task(db, "tts") + if tts_id: + spoken = ai_service.generate_tts_audio(TEST_PHRASE, model_id=tts_id, api_key=tts_key) + except Exception: + log.warning("Could not synthesise audio to test %s", model.model_id, exc_info=True) + if not spoken: + return {"message": f"✓ {model.model_id} is served by the proxy. No voice is " + "configured, so it could not be given anything to hear."} + try: + heard = ai_service.transcribe_audio( + spoken, filename="test.mp3", content_type="audio/mpeg", + model_id=model.model_id, api_key=model.api_key or None) + except Exception as e: + raise HTTPException(status_code=502, detail=str(e)[:300]) + if not heard: + raise HTTPException(status_code=502, detail=f"{model.model_id} heard nothing") + return {"message": f"✓ {model.model_id} heard “{heard.strip()}”"} try: import litellm diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 7bbdeba..20e776a 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -1,6 +1,7 @@ import { Suspense } from 'react' import { BrowserRouter, Routes, Route, Navigate, Outlet, Link, useLocation, useParams } from 'react-router-dom' import { AuthProvider, useAuth } from './context/AuthContext' +import { SessionDrawerProvider } from './context/SessionDrawer' import { ThemeProvider } from './context/ThemeContext' import Navbar from './components/Navbar' import SiteFooter from './components/SiteFooter' @@ -238,7 +239,9 @@ export default function App() { + + diff --git a/frontend/src/components/Navbar.jsx b/frontend/src/components/Navbar.jsx index 9f61b8a..4bf5b47 100644 --- a/frontend/src/components/Navbar.jsx +++ b/frontend/src/components/Navbar.jsx @@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from 'react' import { Link, useLocation } from 'react-router-dom' import ScrollStrip from './ScrollStrip' import { useAuth } from '../context/AuthContext' +import { useSessionDrawer } from '../context/SessionDrawer' import api from '../api/client' import ExamSwitcher from './ExamSwitcher' import GlobalSearch from './GlobalSearch' @@ -185,6 +186,8 @@ export default function Navbar({ onSignIn, onRegister }) { // The section bar is wanted when you decide to go elsewhere and in the way // every other moment, so it leaves on the way down and returns on the way up. const sectionBarHidden = useHidingBar() + // Registered by the quiz player while it is on screen without a rail. + const sessionDrawer = useSessionDrawer() const [jobs, setJobs] = useState([]) const location = useLocation() const isModerator = user?.role === 'admin' || user?.role === 'moderator' @@ -258,11 +261,16 @@ export default function Navbar({ onSignIn, onRegister }) { + {/* While a session is open on a narrow screen, this button + belongs to the session: it opens the list of questions, and + the site menu is a tab inside that drawer. Two menu buttons + an inch apart, one of which leaves the session you are + sitting, is the wrong offer. */} + ) +} + +/** Stands in for the quiz player. */ +function Player({ claim, onOpen }) { + const { register } = useSessionDrawer() + useEffect(() => (claim ? register(onOpen) : undefined), [claim, register, onOpen]) + return null +} + +it('hands the button to a session that claims it, and back when it leaves', async () => { + const onOpen = vi.fn() + const { rerender } = render( + ) + + await userEvent.click(screen.getByRole('button', { name: 'Session menu' })) + expect(onOpen).toHaveBeenCalled() + expect(document.body.dataset.siteMenu).toBeUndefined() + + // The player leaves; the site menu is the button's job again. + rerender() + await userEvent.click(screen.getByRole('button', { name: 'Menu' })) + expect(document.body.dataset.siteMenu).toBe('open') + document.body.removeAttribute('data-site-menu') +}) + +it('a player that never claims it leaves the site menu alone', async () => { + const onOpen = vi.fn() + render() + await userEvent.click(screen.getByRole('button', { name: 'Menu' })) + expect(onOpen).not.toHaveBeenCalled() + expect(document.body.dataset.siteMenu).toBe('open') + document.body.removeAttribute('data-site-menu') +}) diff --git a/frontend/src/hooks/useDictation.js b/frontend/src/hooks/useDictation.js new file mode 100644 index 0000000..32ed506 --- /dev/null +++ b/frontend/src/hooks/useDictation.js @@ -0,0 +1,131 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import api from '../api/client' + +/** + * Speaking instead of typing. + * + * Two ways to do it, and the browser's own is tried first: `SpeechRecognition` + * transcribes as you talk, costs nothing, and needs no round trip. Where it is + * missing — Firefox, and every browser on iOS that is not Safari — the audio + * is recorded and sent to our own transcriber instead, which is slower but + * works everywhere a microphone does. + * + * The fallback is also taken when the browser's recogniser fails mid-sentence, + * because a half-finished dictation is worse than a slow one. + */ + +/** Read when asked, not when this module loads: a constant captured at import + time cannot be seen to be right or wrong. */ +const recognition = () => (typeof window === 'undefined' + ? null + : window.SpeechRecognition || window.webkitSpeechRecognition || null) + +/** Recording needs both halves; iOS Safari shipped one without the other for + years, and a button that throws is worse than a button that is not there. */ +const canRecord = () => typeof window !== 'undefined' + && typeof window.MediaRecorder !== 'undefined' + && Boolean(navigator?.mediaDevices?.getUserMedia) + +/** Whether anything here can work at all. */ +export const canDictate = () => Boolean(recognition()) || canRecord() + +export default function useDictation({ onText, onError } = {}) { + const [listening, setListening] = useState(false) + const [transcribing, setTranscribing] = useState(false) + const recogniser = useRef(null) + const recorder = useRef(null) + const chunks = useRef([]) + const stream = useRef(null) + const heard = useRef('') + + const cleanup = useCallback(() => { + stream.current?.getTracks().forEach(track => track.stop()) + stream.current = null + recorder.current = null + recogniser.current = null + setListening(false) + }, []) + + useEffect(() => cleanup, [cleanup]) + + /** Record, then send it to our own transcriber. */ + const record = useCallback(async () => { + if (!canRecord()) { + onError?.('This browser cannot record audio. Try typing instead.') + cleanup() + return + } + try { + stream.current = await navigator.mediaDevices.getUserMedia({ audio: true }) + } catch { + onError?.('No microphone. Check the permission for this site.') + return + } + chunks.current = [] + const media = new MediaRecorder(stream.current) + recorder.current = media + media.ondataavailable = event => { if (event.data.size) chunks.current.push(event.data) } + media.onstop = async () => { + const blob = new Blob(chunks.current, { type: media.mimeType || 'audio/webm' }) + cleanup() + if (!blob.size) return + setTranscribing(true) + try { + const form = new FormData() + form.append('file', blob, 'dictation.webm') + const res = await api.post('/tts/transcribe', form) + if (res.data?.text) onText?.(res.data.text) + } catch (err) { + const detail = err?.response?.data?.detail + onError?.(typeof detail === 'string' ? detail : 'Could not transcribe that') + } finally { + setTranscribing(false) + } + } + media.start() + setListening(true) + }, [cleanup, onError, onText]) + + const start = useCallback(async () => { + if (listening || transcribing) return + const Engine = recognition() + if (!Engine) { await record(); return } + + heard.current = '' + const engine = new Engine() + engine.lang = navigator.language || 'en-US' + engine.interimResults = false + engine.continuous = true + engine.onresult = (event) => { + for (let i = event.resultIndex; i < event.results.length; i += 1) { + if (event.results[i].isFinal) heard.current += event.results[i][0].transcript + } + } + engine.onerror = (event) => { + cleanup() + // Refused permission is the learner's answer, not a fault to route around. + if (event.error === 'not-allowed' || event.error === 'service-not-allowed') { + onError?.('No microphone. Check the permission for this site.') + return + } + if (canRecord()) record() + else onError?.('Could not hear anything. Try typing instead.') + } + engine.onend = () => { + setListening(false) + const text = heard.current.trim() + if (text) onText?.(text) + } + recogniser.current = engine + engine.start() + setListening(true) + }, [cleanup, listening, onError, onText, record, transcribing]) + + const stop = useCallback(() => { + if (recogniser.current) { recogniser.current.stop(); return } + if (recorder.current?.state === 'recording') recorder.current.stop() + else cleanup() + }, [cleanup]) + + return { listening, transcribing, start, stop, toggle: () => (listening ? stop() : start()) } +} diff --git a/frontend/src/hooks/useDictation.test.jsx b/frontend/src/hooks/useDictation.test.jsx new file mode 100644 index 0000000..e29c987 --- /dev/null +++ b/frontend/src/hooks/useDictation.test.jsx @@ -0,0 +1,112 @@ +import { render, screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import useDictation, { canDictate } from './useDictation' +import api from '../api/client' + +vi.mock('../api/client', () => ({ default: { post: vi.fn() } })) + +function Harness({ onError }) { + const { listening, transcribing, toggle } = useDictation({ + onText: text => { document.title = text }, onError, + }) + return ( + + ) +} + +let engines = [] + +const withBrowserRecogniser = () => { + engines = [] + window.SpeechRecognition = function Recogniser() { + const self = this + self.start = vi.fn(() => { engines.push(self) }) + self.stop = vi.fn(() => self.onend?.()) + engines.push(self) + return self + } +} + +const withMediaRecorderOnly = () => { + delete window.SpeechRecognition + delete window.webkitSpeechRecognition + const track = { stop: vi.fn() } + navigator.mediaDevices = { getUserMedia: vi.fn().mockResolvedValue({ getTracks: () => [track] }) } + window.MediaRecorder = function Recorder() { + const self = this + self.state = 'recording' + self.mimeType = 'audio/webm' + self.start = vi.fn() + self.stop = vi.fn(() => { + self.ondataavailable?.({ data: new Blob(['x'], { type: 'audio/webm' }) }) + self.onstop?.() + }) + return self + } +} + +beforeEach(() => { vi.clearAllMocks(); document.title = '' }) +afterEach(() => { + delete window.SpeechRecognition + delete window.webkitSpeechRecognition + delete window.MediaRecorder + delete navigator.mediaDevices +}) + +it('uses the browser recogniser when there is one, and never calls the server', async () => { + withBrowserRecogniser() + render() + await userEvent.click(screen.getByRole('button')) + expect(screen.getByRole('button')).toHaveTextContent('listening') + + const engine = engines[engines.length - 1] + engine.onresult({ resultIndex: 0, results: [{ isFinal: true, 0: { transcript: 'What causes stridor' } }] }) + await userEvent.click(screen.getByRole('button')) + await waitFor(() => expect(document.title).toBe('What causes stridor')) + // Free and local: nothing was uploaded. + expect(api.post).not.toHaveBeenCalled() +}) + +it('records and sends to our own transcriber where the browser has none', async () => { + withMediaRecorderOnly() + api.post.mockResolvedValue({ data: { text: 'A neonate with bilious vomiting' } }) + render() + await userEvent.click(screen.getByRole('button')) + await waitFor(() => expect(screen.getByRole('button')).toHaveTextContent('listening')) + await userEvent.click(screen.getByRole('button')) + await waitFor(() => expect(api.post).toHaveBeenCalledWith('/tts/transcribe', expect.any(FormData))) + await waitFor(() => expect(document.title).toBe('A neonate with bilious vomiting')) +}) + +it('falls back to recording when the browser recogniser fails mid-sentence', async () => { + withBrowserRecogniser() + const track = { stop: vi.fn() } + navigator.mediaDevices = { getUserMedia: vi.fn().mockResolvedValue({ getTracks: () => [track] }) } + window.MediaRecorder = function Recorder() { this.start = vi.fn(); this.state = 'recording'; return this } + render() + await userEvent.click(screen.getByRole('button')) + engines[engines.length - 1].onerror({ error: 'network' }) + // A half-finished dictation is worse than a slow one. + await waitFor(() => expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalled()) +}) + +it('a refused microphone is an answer, not a fault to route around', async () => { + withBrowserRecogniser() + navigator.mediaDevices = { getUserMedia: vi.fn() } + const onError = vi.fn() + render() + await userEvent.click(screen.getByRole('button')) + engines[engines.length - 1].onerror({ error: 'not-allowed' }) + expect(onError).toHaveBeenCalledWith('No microphone. Check the permission for this site.') + expect(navigator.mediaDevices.getUserMedia).not.toHaveBeenCalled() +}) + +it('offers nothing at all where neither exists', () => { + delete window.SpeechRecognition + delete window.webkitSpeechRecognition + delete navigator.mediaDevices + expect(canDictate()).toBe(false) +}) diff --git a/frontend/src/pages/AiModePage.css b/frontend/src/pages/AiModePage.css index 9e4ac7d..19990a5 100644 --- a/frontend/src/pages/AiModePage.css +++ b/frontend/src/pages/AiModePage.css @@ -104,3 +104,23 @@ } .ai-practise:hover:not(:disabled) { background: var(--option-sel-bg); } .ai-practise:disabled { opacity: 0.6; cursor: default; } + +/* Speaking instead of typing. Red while it is listening, because a microphone + you have forgotten is on is the one thing this must never be. */ +.ai-mic { + flex-shrink: 0; width: 42px; height: 42px; font-size: 1rem; cursor: pointer; + border: 1px solid var(--border); border-radius: 10px; + background: var(--card-bg); color: var(--text-muted); +} +.ai-mic:hover:not(:disabled) { border-color: var(--primary); color: var(--primary); } +.ai-mic.is-live { + background: var(--wrong-bg); border-color: var(--wrong-fg); color: var(--wrong-fg); + animation: ai-mic-pulse 1.4s ease-in-out infinite; +} +.ai-mic:disabled { opacity: 0.6; cursor: default; } + +@keyframes ai-mic-pulse { + 0%, 100% { box-shadow: 0 0 0 0 rgba(220, 38, 38, 0.35); } + 50% { box-shadow: 0 0 0 6px rgba(220, 38, 38, 0); } +} +@media (prefers-reduced-motion: reduce) { .ai-mic.is-live { animation: none; } } diff --git a/frontend/src/pages/AiModePage.jsx b/frontend/src/pages/AiModePage.jsx index 77ea14c..93dfc14 100644 --- a/frontend/src/pages/AiModePage.jsx +++ b/frontend/src/pages/AiModePage.jsx @@ -3,6 +3,7 @@ import { Link, useNavigate } from 'react-router-dom' import ReactMarkdown from 'react-markdown' import remarkGfm from 'remark-gfm' import api from '../api/client' +import useDictation, { canDictate } from '../hooks/useDictation' import './AiModePage.css' const apiError = (err, fallback) => { @@ -86,6 +87,12 @@ export default function AiModePage() { const [practising, setPractising] = useState(null) const endRef = useRef(null) const navigate = useNavigate() + // Speaking instead of typing. The browser's own recogniser where there is + // one, our transcriber where there is not. + const dictation = useDictation({ + onText: text => setDraft(d => (d ? `${d} ${text}` : text)), + onError: setError, + }) const loadThreads = useCallback(() => api.get('/ai/conversations') .then(res => { setThreads(res.data || []); return res.data || [] }) @@ -236,6 +243,15 @@ export default function AiModePage() { placeholder="Ask about anything in your library…" onChange={e => setDraft(e.target.value)} onKeyDown={e => { if (e.key === 'Enter' && !e.shiftKey) send(e) }} /> + {canDictate() && ( + + )} diff --git a/frontend/src/pages/QuizPage.jsx b/frontend/src/pages/QuizPage.jsx index c59cc4f..b4cefc8 100644 --- a/frontend/src/pages/QuizPage.jsx +++ b/frontend/src/pages/QuizPage.jsx @@ -9,6 +9,7 @@ import { useAuth } from '../context/AuthContext' import api from '../api/client' import useMediaQuery from '../hooks/useMediaQuery' import useAwayDetector from '../hooks/useAwayDetector' +import { useSessionDrawer } from '../context/SessionDrawer' import FigureStrip from '../components/FigureStrip' import FeedbackForm from '../components/FeedbackForm' import ShareSession from '../components/ShareSession' @@ -412,7 +413,18 @@ export default function QuizPage() { // The session rail is the navigator whenever there is room for it; the // dropdown only exists for screens too narrow to show it. Matches the // 1150px breakpoint in QuizPlayer.css that hides the rail. + const sessionDrawer = useSessionDrawer() const hasRail = useMediaQuery('(min-width: 1151px)') + + // The site's burger opens this session's questions while there is no rail. + // Only while there is no rail: on a desktop the list is already beside the + // question, and taking the button over there would hide the site menu for + // no reason. + const { register: registerDrawer } = sessionDrawer + useEffect(() => { + if (hasRail) return undefined + return registerDrawer(() => { setDrawerTab('questions'); setNavOpen(true) }) + }, [hasRail, registerDrawer]) const [expandedImagePath, setExpandedImagePath] = useState('') const [imageZoom, setImageZoom] = useState(1) const [startedAt, setStartedAt] = useState(null) @@ -1227,11 +1239,15 @@ const timerStarted = timeLeft !== null )} {isLast ? ( - // The end of the block. The dialog it opens names how many are still - // unanswered and that they count against you, which is the warning - // worth giving before anything is handed in. - ) : ( /* Skip, when nothing has been chosen. Moving on from a question you @@ -1417,8 +1433,10 @@ const timerStarted = timeLeft !== null
e.target === e.currentTarget && setNavOpen(false)}>
+ {/* The same burger that opened it closes it: on a phone this + drawer is what that button does while a session is open. */} + onClick={() => setNavOpen(false)}>☰
@@ -1430,17 +1448,35 @@ const timerStarted = timeLeft !== null {drawerTab === 'questions' ? ( <>
- {quiz.title} + {/* Everything answered means there is nothing left to sit; + what you are doing now is reading it back. */} + {answeredCount >= totalCount && totalCount > 0 && ( + Review + )} + {isStudy ? 'Study mode' : 'Exam mode'}: {quiz.title} {answeredCount}/{totalCount} +
{questions.map((q, i) => )}
+ {/* How long this is taking, where it is being read — the same + two figures the desktop shows beside the explanation. */} +
+ setClockPaused(v => !v)} /> +
) : (
@@ -1821,7 +1860,7 @@ const timerStarted = timeLeft !== null {answeredCount} answered {totalCount - answeredCount} remaining
- {answeredCount > 0 && !isLast && ( + {!isStudy && answeredCount > 0 && !isLast && ( {quizNavigation('bottom')} - {answeredCount > 0 && ( + {/* Reviewing a block before handing it in is an exam idea. A study + session has nothing to hand in: it keeps going until every + question is answered, and at that point it is the review. This + button sat in both because the study player inherited the exam + player's bar. */} + {!isStudy && answeredCount > 0 && (