diff --git a/backend/app/routers/admin.py b/backend/app/routers/admin.py index b97c779..a5d1918 100644 --- a/backend/app/routers/admin.py +++ b/backend/app/routers/admin.py @@ -1,6 +1,8 @@ -from fastapi import APIRouter, Depends, HTTPException, Query +import logging from datetime import datetime +from fastapi import APIRouter, Depends, HTTPException, Query + from pydantic import BaseModel, Field from sqlalchemy import text from sqlalchemy.orm import Session @@ -11,7 +13,7 @@ from app.database import get_db from app.models.user import User from app.models.ai_model_config import AIModelConfig from app.models.invite import InviteCode -from app.services import invites, site_settings +from app.services import ai_service, invites, site_settings from app.schemas.auth import UserResponse, UserUpdateRole, UserCreate from app.schemas.admin import AIModelConfigCreate, AIModelConfigResponse, AIModelConfigUpdate from app.utils.auth import require_admin, get_current_user, get_password_hash @@ -146,6 +148,13 @@ class LiteLLMSearchRequest(BaseModel): mode: str | None = None +#: Short, unambiguous, and clinical enough that a medical model has no excuse. +#: Used to make a voice speak and a transcriber listen. +TEST_PHRASE = "Inspiratory stridor at rest." + +log = logging.getLogger(__name__) + + def _proxy_models(base: str, key: str | None, mode: str | None = None) -> tuple[list[str], bool]: """What the proxy will serve, and whether the answer knows about modes. @@ -308,10 +317,17 @@ def test_model( raise HTTPException(status_code=404, detail="Model config not found") if model.task == "tts": - # Not a failure. A voice is tested by hearing it, and the page offers - # that; saying so in red taught administrators to distrust a working - # configuration. - return {"message": f"{model.model_id} is a voice — press Preview to hear it."} + # A voice is tested by making it speak. The old answer was an + # instruction ("press Preview") returned as an error, which taught + # administrators to distrust a working configuration. + try: + audio = ai_service.generate_tts_audio( + TEST_PHRASE, model_id=model.model_id, api_key=model.api_key or None) + except Exception as e: + raise HTTPException(status_code=502, detail=str(e)[:300]) + if not audio: + raise HTTPException(status_code=502, detail=f"{model.model_id} returned no audio") + return {"message": f"✓ {model.model_id} spoke {len(audio):,} bytes of audio"} if model.task == "stt": base = (settings.LITELLM_API_BASE or "").rstrip("/").removesuffix("/v1") @@ -325,15 +341,33 @@ def test_model( status_code=404, detail=f"{model.model_id} is not served by the proxy" + (" as a transcription model" if by_mode else "")) - return {"message": f"✓ {model.model_id} is served by the proxy" + ( - " for speech transcription" if by_mode - # The route that says what a model is for is not open to this - # key, so this is presence, not suitability. - else " — the proxy would not say whether it transcribes")} except HTTPException: raise except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + raise HTTPException(status_code=502, detail=str(e)[:300]) + + # Presence is not proof. A voice we already have says a known phrase, + # and the model is asked what it heard — the only test that shows + # transcription actually working end to end. + spoken = None + try: + tts_id, tts_key = ai_service.get_model_for_task(db, "tts") + if tts_id: + spoken = ai_service.generate_tts_audio(TEST_PHRASE, model_id=tts_id, api_key=tts_key) + except Exception: + log.warning("Could not synthesise audio to test %s", model.model_id, exc_info=True) + if not spoken: + return {"message": f"✓ {model.model_id} is served by the proxy. No voice is " + "configured, so it could not be given anything to hear."} + try: + heard = ai_service.transcribe_audio( + spoken, filename="test.mp3", content_type="audio/mpeg", + model_id=model.model_id, api_key=model.api_key or None) + except Exception as e: + raise HTTPException(status_code=502, detail=str(e)[:300]) + if not heard: + raise HTTPException(status_code=502, detail=f"{model.model_id} heard nothing") + return {"message": f"✓ {model.model_id} heard “{heard.strip()}”"} try: import litellm diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 7bbdeba..20e776a 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -1,6 +1,7 @@ import { Suspense } from 'react' import { BrowserRouter, Routes, Route, Navigate, Outlet, Link, useLocation, useParams } from 'react-router-dom' import { AuthProvider, useAuth } from './context/AuthContext' +import { SessionDrawerProvider } from './context/SessionDrawer' import { ThemeProvider } from './context/ThemeContext' import Navbar from './components/Navbar' import SiteFooter from './components/SiteFooter' @@ -238,7 +239,9 @@ export default function App() { + + diff --git a/frontend/src/components/Navbar.jsx b/frontend/src/components/Navbar.jsx index 9f61b8a..4bf5b47 100644 --- a/frontend/src/components/Navbar.jsx +++ b/frontend/src/components/Navbar.jsx @@ -2,6 +2,7 @@ import { useEffect, useRef, useState } from 'react' import { Link, useLocation } from 'react-router-dom' import ScrollStrip from './ScrollStrip' import { useAuth } from '../context/AuthContext' +import { useSessionDrawer } from '../context/SessionDrawer' import api from '../api/client' import ExamSwitcher from './ExamSwitcher' import GlobalSearch from './GlobalSearch' @@ -185,6 +186,8 @@ export default function Navbar({ onSignIn, onRegister }) { // The section bar is wanted when you decide to go elsewhere and in the way // every other moment, so it leaves on the way down and returns on the way up. const sectionBarHidden = useHidingBar() + // Registered by the quiz player while it is on screen without a rail. + const sessionDrawer = useSessionDrawer() const [jobs, setJobs] = useState([]) const location = useLocation() const isModerator = user?.role === 'admin' || user?.role === 'moderator' @@ -258,11 +261,16 @@ export default function Navbar({ onSignIn, onRegister }) { + {/* While a session is open on a narrow screen, this button + belongs to the session: it opens the list of questions, and + the site menu is a tab inside that drawer. Two menu buttons + an inch apart, one of which leaves the session you are + sitting, is the wrong offer. */} + ) +} + +/** Stands in for the quiz player. */ +function Player({ claim, onOpen }) { + const { register } = useSessionDrawer() + useEffect(() => (claim ? register(onOpen) : undefined), [claim, register, onOpen]) + return null +} + +it('hands the button to a session that claims it, and back when it leaves', async () => { + const onOpen = vi.fn() + const { rerender } = render( + ) + + await userEvent.click(screen.getByRole('button', { name: 'Session menu' })) + expect(onOpen).toHaveBeenCalled() + expect(document.body.dataset.siteMenu).toBeUndefined() + + // The player leaves; the site menu is the button's job again. + rerender() + await userEvent.click(screen.getByRole('button', { name: 'Menu' })) + expect(document.body.dataset.siteMenu).toBe('open') + document.body.removeAttribute('data-site-menu') +}) + +it('a player that never claims it leaves the site menu alone', async () => { + const onOpen = vi.fn() + render() + await userEvent.click(screen.getByRole('button', { name: 'Menu' })) + expect(onOpen).not.toHaveBeenCalled() + expect(document.body.dataset.siteMenu).toBe('open') + document.body.removeAttribute('data-site-menu') +}) diff --git a/frontend/src/hooks/useDictation.js b/frontend/src/hooks/useDictation.js new file mode 100644 index 0000000..32ed506 --- /dev/null +++ b/frontend/src/hooks/useDictation.js @@ -0,0 +1,131 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import api from '../api/client' + +/** + * Speaking instead of typing. + * + * Two ways to do it, and the browser's own is tried first: `SpeechRecognition` + * transcribes as you talk, costs nothing, and needs no round trip. Where it is + * missing — Firefox, and every browser on iOS that is not Safari — the audio + * is recorded and sent to our own transcriber instead, which is slower but + * works everywhere a microphone does. + * + * The fallback is also taken when the browser's recogniser fails mid-sentence, + * because a half-finished dictation is worse than a slow one. + */ + +/** Read when asked, not when this module loads: a constant captured at import + time cannot be seen to be right or wrong. */ +const recognition = () => (typeof window === 'undefined' + ? null + : window.SpeechRecognition || window.webkitSpeechRecognition || null) + +/** Recording needs both halves; iOS Safari shipped one without the other for + years, and a button that throws is worse than a button that is not there. */ +const canRecord = () => typeof window !== 'undefined' + && typeof window.MediaRecorder !== 'undefined' + && Boolean(navigator?.mediaDevices?.getUserMedia) + +/** Whether anything here can work at all. */ +export const canDictate = () => Boolean(recognition()) || canRecord() + +export default function useDictation({ onText, onError } = {}) { + const [listening, setListening] = useState(false) + const [transcribing, setTranscribing] = useState(false) + const recogniser = useRef(null) + const recorder = useRef(null) + const chunks = useRef([]) + const stream = useRef(null) + const heard = useRef('') + + const cleanup = useCallback(() => { + stream.current?.getTracks().forEach(track => track.stop()) + stream.current = null + recorder.current = null + recogniser.current = null + setListening(false) + }, []) + + useEffect(() => cleanup, [cleanup]) + + /** Record, then send it to our own transcriber. */ + const record = useCallback(async () => { + if (!canRecord()) { + onError?.('This browser cannot record audio. Try typing instead.') + cleanup() + return + } + try { + stream.current = await navigator.mediaDevices.getUserMedia({ audio: true }) + } catch { + onError?.('No microphone. Check the permission for this site.') + return + } + chunks.current = [] + const media = new MediaRecorder(stream.current) + recorder.current = media + media.ondataavailable = event => { if (event.data.size) chunks.current.push(event.data) } + media.onstop = async () => { + const blob = new Blob(chunks.current, { type: media.mimeType || 'audio/webm' }) + cleanup() + if (!blob.size) return + setTranscribing(true) + try { + const form = new FormData() + form.append('file', blob, 'dictation.webm') + const res = await api.post('/tts/transcribe', form) + if (res.data?.text) onText?.(res.data.text) + } catch (err) { + const detail = err?.response?.data?.detail + onError?.(typeof detail === 'string' ? detail : 'Could not transcribe that') + } finally { + setTranscribing(false) + } + } + media.start() + setListening(true) + }, [cleanup, onError, onText]) + + const start = useCallback(async () => { + if (listening || transcribing) return + const Engine = recognition() + if (!Engine) { await record(); return } + + heard.current = '' + const engine = new Engine() + engine.lang = navigator.language || 'en-US' + engine.interimResults = false + engine.continuous = true + engine.onresult = (event) => { + for (let i = event.resultIndex; i < event.results.length; i += 1) { + if (event.results[i].isFinal) heard.current += event.results[i][0].transcript + } + } + engine.onerror = (event) => { + cleanup() + // Refused permission is the learner's answer, not a fault to route around. + if (event.error === 'not-allowed' || event.error === 'service-not-allowed') { + onError?.('No microphone. Check the permission for this site.') + return + } + if (canRecord()) record() + else onError?.('Could not hear anything. Try typing instead.') + } + engine.onend = () => { + setListening(false) + const text = heard.current.trim() + if (text) onText?.(text) + } + recogniser.current = engine + engine.start() + setListening(true) + }, [cleanup, listening, onError, onText, record, transcribing]) + + const stop = useCallback(() => { + if (recogniser.current) { recogniser.current.stop(); return } + if (recorder.current?.state === 'recording') recorder.current.stop() + else cleanup() + }, [cleanup]) + + return { listening, transcribing, start, stop, toggle: () => (listening ? stop() : start()) } +} diff --git a/frontend/src/hooks/useDictation.test.jsx b/frontend/src/hooks/useDictation.test.jsx new file mode 100644 index 0000000..e29c987 --- /dev/null +++ b/frontend/src/hooks/useDictation.test.jsx @@ -0,0 +1,112 @@ +import { render, screen, waitFor } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import useDictation, { canDictate } from './useDictation' +import api from '../api/client' + +vi.mock('../api/client', () => ({ default: { post: vi.fn() } })) + +function Harness({ onError }) { + const { listening, transcribing, toggle } = useDictation({ + onText: text => { document.title = text }, onError, + }) + return ( + + ) +} + +let engines = [] + +const withBrowserRecogniser = () => { + engines = [] + window.SpeechRecognition = function Recogniser() { + const self = this + self.start = vi.fn(() => { engines.push(self) }) + self.stop = vi.fn(() => self.onend?.()) + engines.push(self) + return self + } +} + +const withMediaRecorderOnly = () => { + delete window.SpeechRecognition + delete window.webkitSpeechRecognition + const track = { stop: vi.fn() } + navigator.mediaDevices = { getUserMedia: vi.fn().mockResolvedValue({ getTracks: () => [track] }) } + window.MediaRecorder = function Recorder() { + const self = this + self.state = 'recording' + self.mimeType = 'audio/webm' + self.start = vi.fn() + self.stop = vi.fn(() => { + self.ondataavailable?.({ data: new Blob(['x'], { type: 'audio/webm' }) }) + self.onstop?.() + }) + return self + } +} + +beforeEach(() => { vi.clearAllMocks(); document.title = '' }) +afterEach(() => { + delete window.SpeechRecognition + delete window.webkitSpeechRecognition + delete window.MediaRecorder + delete navigator.mediaDevices +}) + +it('uses the browser recogniser when there is one, and never calls the server', async () => { + withBrowserRecogniser() + render() + await userEvent.click(screen.getByRole('button')) + expect(screen.getByRole('button')).toHaveTextContent('listening') + + const engine = engines[engines.length - 1] + engine.onresult({ resultIndex: 0, results: [{ isFinal: true, 0: { transcript: 'What causes stridor' } }] }) + await userEvent.click(screen.getByRole('button')) + await waitFor(() => expect(document.title).toBe('What causes stridor')) + // Free and local: nothing was uploaded. + expect(api.post).not.toHaveBeenCalled() +}) + +it('records and sends to our own transcriber where the browser has none', async () => { + withMediaRecorderOnly() + api.post.mockResolvedValue({ data: { text: 'A neonate with bilious vomiting' } }) + render() + await userEvent.click(screen.getByRole('button')) + await waitFor(() => expect(screen.getByRole('button')).toHaveTextContent('listening')) + await userEvent.click(screen.getByRole('button')) + await waitFor(() => expect(api.post).toHaveBeenCalledWith('/tts/transcribe', expect.any(FormData))) + await waitFor(() => expect(document.title).toBe('A neonate with bilious vomiting')) +}) + +it('falls back to recording when the browser recogniser fails mid-sentence', async () => { + withBrowserRecogniser() + const track = { stop: vi.fn() } + navigator.mediaDevices = { getUserMedia: vi.fn().mockResolvedValue({ getTracks: () => [track] }) } + window.MediaRecorder = function Recorder() { this.start = vi.fn(); this.state = 'recording'; return this } + render() + await userEvent.click(screen.getByRole('button')) + engines[engines.length - 1].onerror({ error: 'network' }) + // A half-finished dictation is worse than a slow one. + await waitFor(() => expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalled()) +}) + +it('a refused microphone is an answer, not a fault to route around', async () => { + withBrowserRecogniser() + navigator.mediaDevices = { getUserMedia: vi.fn() } + const onError = vi.fn() + render() + await userEvent.click(screen.getByRole('button')) + engines[engines.length - 1].onerror({ error: 'not-allowed' }) + expect(onError).toHaveBeenCalledWith('No microphone. Check the permission for this site.') + expect(navigator.mediaDevices.getUserMedia).not.toHaveBeenCalled() +}) + +it('offers nothing at all where neither exists', () => { + delete window.SpeechRecognition + delete window.webkitSpeechRecognition + delete navigator.mediaDevices + expect(canDictate()).toBe(false) +}) diff --git a/frontend/src/pages/AiModePage.css b/frontend/src/pages/AiModePage.css index 9e4ac7d..19990a5 100644 --- a/frontend/src/pages/AiModePage.css +++ b/frontend/src/pages/AiModePage.css @@ -104,3 +104,23 @@ } .ai-practise:hover:not(:disabled) { background: var(--option-sel-bg); } .ai-practise:disabled { opacity: 0.6; cursor: default; } + +/* Speaking instead of typing. Red while it is listening, because a microphone + you have forgotten is on is the one thing this must never be. */ +.ai-mic { + flex-shrink: 0; width: 42px; height: 42px; font-size: 1rem; cursor: pointer; + border: 1px solid var(--border); border-radius: 10px; + background: var(--card-bg); color: var(--text-muted); +} +.ai-mic:hover:not(:disabled) { border-color: var(--primary); color: var(--primary); } +.ai-mic.is-live { + background: var(--wrong-bg); border-color: var(--wrong-fg); color: var(--wrong-fg); + animation: ai-mic-pulse 1.4s ease-in-out infinite; +} +.ai-mic:disabled { opacity: 0.6; cursor: default; } + +@keyframes ai-mic-pulse { + 0%, 100% { box-shadow: 0 0 0 0 rgba(220, 38, 38, 0.35); } + 50% { box-shadow: 0 0 0 6px rgba(220, 38, 38, 0); } +} +@media (prefers-reduced-motion: reduce) { .ai-mic.is-live { animation: none; } } diff --git a/frontend/src/pages/AiModePage.jsx b/frontend/src/pages/AiModePage.jsx index 77ea14c..93dfc14 100644 --- a/frontend/src/pages/AiModePage.jsx +++ b/frontend/src/pages/AiModePage.jsx @@ -3,6 +3,7 @@ import { Link, useNavigate } from 'react-router-dom' import ReactMarkdown from 'react-markdown' import remarkGfm from 'remark-gfm' import api from '../api/client' +import useDictation, { canDictate } from '../hooks/useDictation' import './AiModePage.css' const apiError = (err, fallback) => { @@ -86,6 +87,12 @@ export default function AiModePage() { const [practising, setPractising] = useState(null) const endRef = useRef(null) const navigate = useNavigate() + // Speaking instead of typing. The browser's own recogniser where there is + // one, our transcriber where there is not. + const dictation = useDictation({ + onText: text => setDraft(d => (d ? `${d} ${text}` : text)), + onError: setError, + }) const loadThreads = useCallback(() => api.get('/ai/conversations') .then(res => { setThreads(res.data || []); return res.data || [] }) @@ -236,6 +243,15 @@ export default function AiModePage() { placeholder="Ask about anything in your library…" onChange={e => setDraft(e.target.value)} onKeyDown={e => { if (e.key === 'Enter' && !e.shiftKey) send(e) }} /> + {canDictate() && ( + + )} diff --git a/frontend/src/pages/QuizPage.jsx b/frontend/src/pages/QuizPage.jsx index c59cc4f..b4cefc8 100644 --- a/frontend/src/pages/QuizPage.jsx +++ b/frontend/src/pages/QuizPage.jsx @@ -9,6 +9,7 @@ import { useAuth } from '../context/AuthContext' import api from '../api/client' import useMediaQuery from '../hooks/useMediaQuery' import useAwayDetector from '../hooks/useAwayDetector' +import { useSessionDrawer } from '../context/SessionDrawer' import FigureStrip from '../components/FigureStrip' import FeedbackForm from '../components/FeedbackForm' import ShareSession from '../components/ShareSession' @@ -412,7 +413,18 @@ export default function QuizPage() { // The session rail is the navigator whenever there is room for it; the // dropdown only exists for screens too narrow to show it. Matches the // 1150px breakpoint in QuizPlayer.css that hides the rail. + const sessionDrawer = useSessionDrawer() const hasRail = useMediaQuery('(min-width: 1151px)') + + // The site's burger opens this session's questions while there is no rail. + // Only while there is no rail: on a desktop the list is already beside the + // question, and taking the button over there would hide the site menu for + // no reason. + const { register: registerDrawer } = sessionDrawer + useEffect(() => { + if (hasRail) return undefined + return registerDrawer(() => { setDrawerTab('questions'); setNavOpen(true) }) + }, [hasRail, registerDrawer]) const [expandedImagePath, setExpandedImagePath] = useState('') const [imageZoom, setImageZoom] = useState(1) const [startedAt, setStartedAt] = useState(null) @@ -1227,11 +1239,15 @@ const timerStarted = timeLeft !== null )} {isLast ? ( - // The end of the block. The dialog it opens names how many are still - // unanswered and that they count against you, which is the warning - // worth giving before anything is handed in. - ) : ( /* Skip, when nothing has been chosen. Moving on from a question you @@ -1417,8 +1433,10 @@ const timerStarted = timeLeft !== null
e.target === e.currentTarget && setNavOpen(false)}>
+ {/* The same burger that opened it closes it: on a phone this + drawer is what that button does while a session is open. */} + onClick={() => setNavOpen(false)}>☰
@@ -1430,17 +1448,35 @@ const timerStarted = timeLeft !== null {drawerTab === 'questions' ? ( <>
- {quiz.title} + {/* Everything answered means there is nothing left to sit; + what you are doing now is reading it back. */} + {answeredCount >= totalCount && totalCount > 0 && ( + Review + )} + {isStudy ? 'Study mode' : 'Exam mode'}: {quiz.title} {answeredCount}/{totalCount} +
{questions.map((q, i) => )}
+ {/* How long this is taking, where it is being read — the same + two figures the desktop shows beside the explanation. */} +
+ setClockPaused(v => !v)} /> +
) : (
@@ -1821,7 +1860,7 @@ const timerStarted = timeLeft !== null {answeredCount} answered {totalCount - answeredCount} remaining
- {answeredCount > 0 && !isLast && ( + {!isStudy && answeredCount > 0 && !isLast && ( {quizNavigation('bottom')} - {answeredCount > 0 && ( + {/* Reviewing a block before handing it in is an exam idea. A study + session has nothing to hand in: it keeps going until every + question is answered, and at that point it is the review. This + button sat in both because the study player inherited the exam + player's bar. */} + {!isStudy && answeredCount > 0 && (