diff --git a/backend/alembic/versions/d6e7f8091a2b_search_the_article_body.py b/backend/alembic/versions/d6e7f8091a2b_search_the_article_body.py new file mode 100644 index 0000000..527c079 --- /dev/null +++ b/backend/alembic/versions/d6e7f8091a2b_search_the_article_body.py @@ -0,0 +1,63 @@ +"""Full-text search over what an article actually says. + +`search_vector` weighted title, summary and `content` — but 323 of the 331 +articles have `content` NULL, because everything the generator writes goes into +the `sections` JSON and only the eight hand-seeded samples ever used the column. +So for 98% of the library the body contributed nothing, and a learner searching +for a drug name, a diagnostic criterion or an eponym that appears only in a +section got no result and no indication that the search had not looked there. + +A generated column cannot contain a subquery, so the extraction lives in an +IMMUTABLE function it can call. `content` stays in the expression for the eight +articles that use it. + +Revision ID: d6e7f8091a2b +Revises: c5d6e7f8091a +""" +from alembic import op + +revision = "d6e7f8091a2b" +down_revision = "c5d6e7f8091a" +branch_labels = None +depends_on = None + +EXTRACT = """ +CREATE OR REPLACE FUNCTION article_sections_text(sections json) +RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ + SELECT COALESCE(string_agg(CONCAT_WS(' ', s->>'title', s->>'content'), ' '), '') + FROM json_array_elements(COALESCE(sections, '[]'::json)) AS s +$$; +""" + +OLD = """ + setweight(to_tsvector('english', COALESCE(title, '')), 'A') || + setweight(to_tsvector('english', COALESCE(summary, '')), 'B') || + setweight(to_tsvector('english', COALESCE(content, '')), 'C') +""" + +NEW = """ + setweight(to_tsvector('english', COALESCE(title, '')), 'A') || + setweight(to_tsvector('english', COALESCE(summary, '')), 'B') || + setweight(to_tsvector('english', + COALESCE(content, '') || ' ' || article_sections_text(sections)), 'C') +""" + + +def _rebuild(expression: str) -> None: + # The expression of a generated column cannot be altered in place, and + # dropping the column takes its index with it. + op.execute("DROP INDEX IF EXISTS ix_articles_search_vector") + op.execute("ALTER TABLE articles DROP COLUMN IF EXISTS search_vector") + op.execute(f"ALTER TABLE articles ADD COLUMN search_vector tsvector " + f"GENERATED ALWAYS AS ({expression}) STORED") + op.execute("CREATE INDEX ix_articles_search_vector ON articles USING gin (search_vector)") + + +def upgrade(): + op.execute(EXTRACT) + _rebuild(NEW) + + +def downgrade(): + _rebuild(OLD) + op.execute("DROP FUNCTION IF EXISTS article_sections_text(json)") diff --git a/backend/app/routers/questions.py b/backend/app/routers/questions.py index 48f3d4c..ff624f6 100644 --- a/backend/app/routers/questions.py +++ b/backend/app/routers/questions.py @@ -41,7 +41,11 @@ router = APIRouter() MAX_SEARCH_RESULTS = 500 # A matched test stays a study aid, not a dump of the bank. MAX_MATCHED_QUESTIONS = 30 -MAX_UPLOAD_BYTES = 10 * 1024 * 1024 +# A document here is a query, never content: it is read once to find matching +# questions in the bank and then discarded, so the cap is about how much text +# is worth reading rather than about storage. Two megabytes is a long syllabus +# or a score report; past that somebody is uploading a textbook. +MAX_UPLOAD_BYTES = 2 * 1024 * 1024 MAX_QUERY_CHARS = 6000