3418ed023b
Three things landed together; the message names all of them, because a commit that mentions one is a commit nobody finds the other two in. **Figures.** Thirty-four JPEG 2000 files — 21 on questions, the rest unattached in the media library — are WebP now, with `questions.image_path`, `questions.explanation_image_path` and `media_assets.path` repointed together. Serving already converted them on the way out, so nothing was broken; this removes the step and makes what is stored the same thing that is served. The originals stay: they are the only copy of what came out of the PDF, they cost a few megabytes between them, and a conversion nobody can undo is not one to run against a live bank. Paths are found by what the columns say rather than by listing a bucket, because three tables record them and updating two would be worse than none. **The openai SDK is gone.** Ten call sites — one more than the map said, the Celery article drafter — every one of them a POST with a JSON body, and not one reading usage, cost, tool calls or logprobs. Every other call to the same proxy was already plain httpx: embeddings, the ChromaDB embedding function, speech both ways, model discovery, the vision probe. So this deletes an abstraction rather than swapping one for another, and leaves one HTTP client instead of two. `chat()` and `achat()` return the message content; a `ProxyError` carries the status and the first 500 characters of the body, which is where the proxy explains itself. Behaviour is preserved deliberately, including a 600-second fallback timeout for the four call sites that were running on the SDK's ten-minute default. Lowering that is a real change and belongs in its own commit. Proved against the live proxy on both services rather than only against mocks: a completion, an async completion, a real 400 the vision probe still classifies as a refusal, 407 models read from the catalogue, and a word read off an image. **Voice.** A chosen voice is honoured whatever serves it. The prefix check only accepted a locally served one, so a site adding a hosted voice would offer it in Settings, save the learner's choice, and then quietly read every question in the default voice. The list has always come from the database — adding a voice is a row in Settings → AI models, never a code change. And the sign-in page stops offering a locked door: `signup-policy` reports whether registration is open at all, and the Sign up link goes when it is not. The switch existed and the only way to discover it was to fill the form in. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TqXevQJhxFrM7jJg82cgZN
268 lines
12 KiB
Python
268 lines
12 KiB
Python
"""What happens to an image when the model doing the job cannot see one.
|
|
|
|
No network and no models: the proxy's catalogue and the completions client are
|
|
both stubbed, because what is worth testing is not what a model says about a
|
|
picture but which model is asked, how often, and what the caller is told when
|
|
nobody can be.
|
|
"""
|
|
import os
|
|
import unittest
|
|
from unittest.mock import Mock, patch
|
|
|
|
os.environ.setdefault("DATABASE_URL", "sqlite://")
|
|
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from app.config import settings
|
|
from app.models.ai_model_config import AIModelConfig
|
|
from app.services import vision_service
|
|
from app.services.ai_service import ProxyError
|
|
from app.services.vision_service import Image, VisionUnavailable
|
|
|
|
|
|
def catalogue_rows(*pairs):
|
|
"""A /model/info body, in the shape the live proxy answers with."""
|
|
return {"data": [{"model_name": name, "model_info": {"supports_vision": says}}
|
|
for name, says in pairs]}
|
|
|
|
|
|
def stub_chat(reply="A chest radiograph with a right lower lobe opacity."):
|
|
"""`ai_service.chat` hands back the message text itself, so the stub is the
|
|
text — there is no envelope left to imitate."""
|
|
return Mock(return_value=reply)
|
|
|
|
|
|
class VisionFallbackTests(unittest.TestCase):
|
|
def setUp(self):
|
|
engine = create_engine("sqlite://")
|
|
AIModelConfig.__table__.create(engine)
|
|
self.db = sessionmaker(bind=engine)()
|
|
self.image = Image(data=b"\x89PNG-not-really", media_type="image/png",
|
|
caption="the figure printed with the question stem")
|
|
|
|
# Redis is not part of what is being tested, and a cache that answers
|
|
# would hide the calls these tests are counting.
|
|
patch.object(vision_service, "_redis", return_value=None).start()
|
|
self.reset_caches()
|
|
|
|
def tearDown(self):
|
|
patch.stopall()
|
|
self.db.close()
|
|
self.reset_caches()
|
|
|
|
def reset_caches(self):
|
|
vision_service._catalogue = {}
|
|
vision_service._catalogue_at = 0.0
|
|
vision_service._probes.clear()
|
|
|
|
def catalogue(self, *pairs):
|
|
vision_service._catalogue = dict(pairs)
|
|
vision_service._catalogue_at = vision_service.time.monotonic()
|
|
|
|
def tool_model(self, model_id="tool-vision"):
|
|
self.db.add(AIModelConfig(name=model_id, model_id=model_id, task="tool",
|
|
is_active=True, is_default=True))
|
|
self.db.commit()
|
|
|
|
def test_a_model_that_can_see_is_handed_the_image_itself(self):
|
|
self.catalogue(("seeing-model", True))
|
|
create = stub_chat()
|
|
with patch.object(vision_service, "chat", create):
|
|
parts, handoff = vision_service.image_context(
|
|
self.db, [self.image], model_id="seeing-model")
|
|
|
|
self.assertEqual([part["type"] for part in parts], ["image_url"])
|
|
self.assertTrue(parts[0]["image_url"]["url"].startswith("data:image/png;base64,"))
|
|
self.assertFalse(handoff.delegated)
|
|
self.assertIsNone(handoff.tool_model)
|
|
create.assert_not_called()
|
|
|
|
def test_a_model_that_cannot_see_gets_the_tool_model_s_description(self):
|
|
self.catalogue(("blind-model", False), ("tool-vision", True))
|
|
self.tool_model()
|
|
create = stub_chat()
|
|
with patch.object(vision_service, "chat", create):
|
|
parts, handoff = vision_service.image_context(
|
|
self.db, [self.image], model_id="blind-model", context="A 4-year-old…")
|
|
|
|
self.assertTrue(handoff.delegated)
|
|
self.assertEqual(handoff.tool_model, "tool-vision")
|
|
self.assertEqual(handoff.as_dict()["primary_model"], "blind-model")
|
|
|
|
# The tool model saw the picture; the primary is given words.
|
|
create.assert_called_once()
|
|
sent = create.call_args.kwargs
|
|
self.assertEqual(sent["model"], "tool-vision")
|
|
self.assertEqual([part["type"] for part in sent["messages"][0]["content"]],
|
|
["image_url", "text"])
|
|
self.assertEqual([part["type"] for part in parts], ["text"])
|
|
self.assertIn("right lower lobe opacity", parts[0]["text"])
|
|
self.assertIn("tool-vision", parts[0]["text"])
|
|
|
|
def test_no_tool_model_is_an_error_naming_the_fix(self):
|
|
self.catalogue(("blind-model", False))
|
|
with self.assertRaises(VisionUnavailable) as raised:
|
|
vision_service.image_context(self.db, [self.image], model_id="blind-model")
|
|
message = str(raised.exception)
|
|
self.assertIn("cannot read images", message)
|
|
self.assertIn("Settings → AI models", message)
|
|
|
|
def test_a_tool_model_that_cannot_see_either_is_refused_before_it_is_called(self):
|
|
self.catalogue(("blind-model", False), ("also-blind", False))
|
|
self.tool_model("also-blind")
|
|
create = stub_chat()
|
|
with patch.object(vision_service, "chat", create):
|
|
with self.assertRaises(VisionUnavailable) as raised:
|
|
vision_service.image_context(self.db, [self.image], model_id="blind-model")
|
|
self.assertIn("also-blind", str(raised.exception))
|
|
create.assert_not_called()
|
|
|
|
def test_a_failing_tool_model_is_an_error_rather_than_a_text_only_answer(self):
|
|
self.catalogue(("blind-model", False), ("tool-vision", True))
|
|
self.tool_model()
|
|
create = Mock(side_effect=RuntimeError("upstream 500"))
|
|
with patch.object(vision_service, "chat", create):
|
|
with self.assertRaises(VisionUnavailable) as raised:
|
|
vision_service.image_context(self.db, [self.image], model_id="blind-model")
|
|
self.assertIn("upstream 500", str(raised.exception))
|
|
|
|
|
|
class CapabilityLookupTests(unittest.TestCase):
|
|
def setUp(self):
|
|
patch.object(vision_service, "_redis", return_value=None).start()
|
|
self.base = settings.LITELLM_API_BASE
|
|
settings.LITELLM_API_BASE = "http://proxy.invalid"
|
|
vision_service._catalogue = {}
|
|
vision_service._catalogue_at = 0.0
|
|
vision_service._probes.clear()
|
|
|
|
def tearDown(self):
|
|
patch.stopall()
|
|
settings.LITELLM_API_BASE = self.base
|
|
vision_service._catalogue = {}
|
|
vision_service._catalogue_at = 0.0
|
|
vision_service._probes.clear()
|
|
|
|
def test_the_catalogue_is_read_once_for_many_questions(self):
|
|
response = Mock(json=Mock(return_value=catalogue_rows(
|
|
("sees", True), ("blind", False), ("unlisted-in-the-cost-map", None))))
|
|
with patch("httpx.get", return_value=response) as fetch:
|
|
for _ in range(5):
|
|
self.assertTrue(vision_service.can_see("sees"))
|
|
self.assertFalse(vision_service.can_see("blind"))
|
|
fetch.assert_called_once()
|
|
|
|
def test_one_deployment_that_cannot_see_decides_the_alias(self):
|
|
# `best-chat` fans out to six deployments and a request lands on any of
|
|
# them, so a single false is the answer for the name.
|
|
response = Mock(json=Mock(return_value=catalogue_rows(
|
|
("best-chat", True), ("best-chat", None), ("best-chat", False))))
|
|
with patch("httpx.get", return_value=response):
|
|
self.assertFalse(vision_service.can_see("best-chat"))
|
|
|
|
def test_a_model_the_catalogue_says_nothing_about_is_probed_once(self):
|
|
response = Mock(json=Mock(return_value=catalogue_rows(("quiet", None))))
|
|
create = stub_chat("ok")
|
|
with patch("httpx.get", return_value=response), \
|
|
patch.object(vision_service, "chat", create):
|
|
self.assertTrue(vision_service.can_see("quiet"))
|
|
self.assertTrue(vision_service.can_see("quiet"))
|
|
create.assert_called_once()
|
|
self.assertEqual(create.call_args.kwargs["max_tokens"], 1)
|
|
|
|
def test_a_refused_probe_is_remembered_and_a_broken_one_is_not(self):
|
|
# The real error the proxy layer raises, not a stand-in: the probe sorts
|
|
# refusals by `status_code`, and that ProxyError carries one is the
|
|
# whole reason it exists.
|
|
response = Mock(json=Mock(return_value=catalogue_rows(("quiet", None))))
|
|
create = Mock(side_effect=ProxyError(400, "image input not supported"))
|
|
with patch("httpx.get", return_value=response), \
|
|
patch.object(vision_service, "chat", create):
|
|
self.assertFalse(vision_service.can_see("quiet"))
|
|
self.assertFalse(vision_service.can_see("quiet"))
|
|
create.assert_called_once()
|
|
|
|
# A timeout says nothing about the model, so nothing is kept: asking
|
|
# again asks the model again.
|
|
vision_service._probes.clear()
|
|
create.reset_mock()
|
|
create.side_effect = TimeoutError("proxy is down")
|
|
with patch("httpx.get", return_value=response), \
|
|
patch.object(vision_service, "chat", create):
|
|
self.assertFalse(vision_service.can_see("quiet"))
|
|
self.assertFalse(vision_service.can_see("quiet"))
|
|
self.assertEqual(create.call_count, 2)
|
|
|
|
def test_a_5xx_probe_is_not_remembered_either(self):
|
|
# Same shape as a refusal, opposite meaning: the proxy broke, the model
|
|
# said nothing, so the verdict must not be cached.
|
|
response = Mock(json=Mock(return_value=catalogue_rows(("quiet", None))))
|
|
create = Mock(side_effect=ProxyError(503, "upstream unavailable"))
|
|
with patch("httpx.get", return_value=response), \
|
|
patch.object(vision_service, "chat", create):
|
|
self.assertFalse(vision_service.can_see("quiet"))
|
|
self.assertFalse(vision_service.can_see("quiet"))
|
|
self.assertEqual(create.call_count, 2)
|
|
|
|
|
|
class TutorFigureTests(unittest.TestCase):
|
|
"""The tutor is the first job in the app where a picture reaches a model.
|
|
|
|
Borrowed fixtures: every question in them is printed with a stem figure and
|
|
an explanation figure, which is exactly the case being tested.
|
|
"""
|
|
|
|
def setUp(self):
|
|
import test_related_privacy as fixtures
|
|
from app.routers import teach
|
|
|
|
self.teach = teach
|
|
self.privacy = fixtures.PrivacyTests()
|
|
self.privacy.setUp()
|
|
self.privacy.can_see.return_value = False
|
|
patch.object(vision_service, "_redis", return_value=None).start()
|
|
# The tool model is checked against the catalogue before it is trusted,
|
|
# and this suite reaches no proxy: an empty catalogue is "nothing known
|
|
# against it", which is the case being tested.
|
|
patch.object(vision_service, "catalogue", return_value={}).start()
|
|
vision_service._catalogue, vision_service._catalogue_at = {}, 0.0
|
|
|
|
def tearDown(self):
|
|
self.privacy.tearDown()
|
|
vision_service._catalogue, vision_service._catalogue_at = {}, 0.0
|
|
|
|
def chat(self):
|
|
return self.privacy.client.post("/teach/chat", json={
|
|
"question_id": 1, "messages": [{"role": "user", "content": "Explain"}]})
|
|
|
|
def sent_to_the_tutor(self):
|
|
return self.privacy.ai.call_args.kwargs["messages"]
|
|
|
|
def test_a_blind_tutor_is_given_the_tool_model_s_description(self):
|
|
self.privacy.db.add(AIModelConfig(name="tool-vision", model_id="tool-vision",
|
|
task="tool", is_active=True, is_default=True))
|
|
self.privacy.db.commit()
|
|
create = stub_chat("A frontal chest radiograph.")
|
|
with patch.object(vision_service, "chat", create):
|
|
response = self.chat()
|
|
|
|
self.assertEqual(response.status_code, 200, response.text)
|
|
self.assertTrue(response.json()["vision"]["delegated"])
|
|
self.assertEqual(response.json()["vision"]["tool_model"], "tool-vision")
|
|
# Both figures described, and the descriptions are what the tutor read.
|
|
self.assertEqual(create.call_count, 2)
|
|
figures = self.sent_to_the_tutor()[1]["content"]
|
|
self.assertEqual([part["type"] for part in figures], ["text", "text"])
|
|
self.assertIn("A frontal chest radiograph.", figures[0]["text"])
|
|
|
|
def test_with_nothing_able_to_read_it_the_tutor_is_told_so(self):
|
|
response = self.chat()
|
|
self.assertEqual(response.status_code, 200, response.text)
|
|
self.assertNotIn("vision", response.json())
|
|
told = self.sent_to_the_tutor()[1]["content"][0]["text"]
|
|
self.assertIn("neither describe nor infer", told)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|