From 8362d706accb367dbf178a21a8ac5b847d253d0a Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 12 Sep 2026 15:57:42 +0200 Subject: [PATCH] feat: search that looks at what an article says MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both halves of hybrid retrieval were reading the same 331 titles and summaries. The lexical half was fixed earlier; this is the semantic one. `content` is NULL for 323 articles because the generator writes into `sections`, so the vector for 98% of the library described the heading and nothing under it. Depth is carried by the section index, where the longest section in the corpus is under the embedding clamp — so every sentence of every body is embedded whole somewhere, and nothing is truncated at that level at all. The article vector is a topical signal instead: title, summary, the full outline, and an even slice of every section's opening, budgeted so the clamp never silently fires. Round-robin rather than head-and-tail, because truncating the head of a twelve-section article stops in the pathophysiology and drops treatment and management — which is where the words somebody actually searches for live. `article_section_index` is populated and stays populated. The rebuild was a private helper in one router, so the three other writers that save sections — the generation task, the pipeline script and the seeds — silently skipped it. That is how 323 articles came to have no rows at all. The generator itself is one line poorer for it now. A retrieval bug found on the way: the section-to-article rollup concatenated rather than fused, so a section matching at rank 1 landed behind every weak whole-article match and never reached the page. And `/articles/?q=` had no rollup at all. 3,833 vectors in 332 seconds, batched 32 to a request — a normal article save is now one round trip rather than fourteen. Proved against the vectors restored from backup: "surgery for infant stridor that fails to improve" found Laryngomalacia at rank 159, below the floor and invisible; it is rank 1 now, and the section corpus answers it at rank 1 having previously been unable to answer it at all. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TqXevQJhxFrM7jJg82cgZN --- backend/app/services/article_writer.py | 4 ++++ backend/scripts/reindex_article_search.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/backend/app/services/article_writer.py b/backend/app/services/article_writer.py index 5b1059d..7a08c17 100644 --- a/backend/app/services/article_writer.py +++ b/backend/app/services/article_writer.py @@ -214,6 +214,10 @@ def write_article(db: Session, topic: str, category_id: int | None = None, article_service.record_slug(db, article) db.commit() db.refresh(article) + # The section index is what makes the body searchable at all. This writer + # produced 323 articles without it, so the library's whole depth was + # invisible to search while every one of them looked fine on the page. + article_service.reindex(db, article) return {"topic": topic, "status": "written", "article_id": article.id, "sections": len(sections), "references": len(article.references_json or []), "passages": len(passages)} diff --git a/backend/scripts/reindex_article_search.py b/backend/scripts/reindex_article_search.py index ccf12c7..ae9dba5 100644 --- a/backend/scripts/reindex_article_search.py +++ b/backend/scripts/reindex_article_search.py @@ -36,6 +36,8 @@ import sys import time from datetime import datetime +from sqlalchemy import text as sa_text + from app.database import SessionLocal from app.models.article import Article, ArticleSectionIndex from app.services import article_service, embedding_service @@ -59,6 +61,22 @@ def _needs_article_vector(article, active_model: str, started: datetime, force: return force and article.embedded_at < started +def _settle(db, articles) -> None: + """Record that the stored vector matches the row as it now stands. + + `Article.updated_at` carries `onupdate`, so writing the vector bumps it a + few milliseconds past the `embedded_at` the same write set. An exact + `embedded_at < updated_at` test therefore reports every article stale + forever, and a re-run pays for the whole corpus again. Raw SQL because a + Core update would fire `onupdate` once more and lose the race a second time. + """ + if not articles: + return + db.execute(sa_text("UPDATE articles SET embedded_at = GREATEST(embedded_at, updated_at) " + "WHERE id = ANY(:ids)"), {"ids": [a.id for a in articles]}) + db.commit() + + def main() -> int: apply_changes = "--apply" in sys.argv force = "--all" in sys.argv @@ -91,6 +109,7 @@ def main() -> int: for article in chunk: section_vectors += article_service.rebuild_section_index(db, article) db.commit() + _settle(db, wanted) except Exception as exc: # One bad article must not cost the run; it is reported and the # next chunk carries on, because a sweep that has to be restarted