Merge pull request #373 from ggozad/feat/rerank-cross-encoder

Add cross-encoder reranking provider support
2026-05-14 16:02:07 +03:00 · 2026-05-14 16:02:07 +03:00 · 9c19de93c0
commit 9c19de93c0
parent 0e4c462885 5de4d50622
12 changed files with 247 additions and 7 deletions
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@ -66,6 +66,8 @@ jobs:
          key: huggingface-${{ runner.os }}-qwen-tokenizer-v1
      - name: Pre-download tokenizer
        run: uv run python -c "from transformers import AutoTokenizer; AutoTokenizer.from_pretrained('Qwen/Qwen3-Embedding-0.6B')"
+      - name: Pre-download cross-encoder test model
+        run: uv run python -c "from sentence_transformers import CrossEncoder; CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')"
      - name: Run tests with coverage
        run: uv run pytest -m "not integration" --cov=haiku --cov-report=xml
      - name: Upload coverage to Codecov
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -1,6 +1,10 @@
 # Changelog
 ## [Unreleased]

+### Added
+
+- **`cross-encoder` reranking provider.** Runs any HuggingFace cross-encoder reranker in-process via `sentence_transformers.CrossEncoder` — no separate server. Useful for BGE (`BAAI/bge-reranker-v2-m3`), Qwen3-Reranker, MS-MARCO MiniLM, and other CrossEncoder-compatible models when vLLM is not an option. New `[cross-encoder]` extra pulls `sentence-transformers`.
+
 ### Fixed

 - **`rebuild --embed-only` no longer buffers the entire corpus in memory.** The previous implementation accumulated every chunk's id, content, content_fts, metadata, and new embedding vector in a single Python list before flushing. The rebuild now stream-copies non-vector columns into a `chunks_rebuild_staging` table (1000 rows / page), recreates the chunks table fresh to honour vector-dim changes, then streams from staging one document at a time, embedding in batches of `embeddings.batch_size` and flushing to the new chunks table every 50 documents.
--- a/docs/configuration/providers.md
+++ b/docs/configuration/providers.md
@ -465,3 +465,24 @@ reranking:
 ```

 **Note:** The Jina Reranker v3 local model is licensed under CC BY-NC 4.0, which restricts commercial use. For commercial applications, use the API mode instead.
+
+### Cross-Encoder (sentence-transformers)
+
+Run any HuggingFace cross-encoder reranker in-process via `sentence-transformers` — no separate server required. Useful when you want a specific model (BGE, Qwen3-Reranker, MS-MARCO MiniLM, etc.) without running vLLM.
+
+Install the extra:
+
+```bash
+uv pip install haiku.rag-slim[cross-encoder]
+```
+
+Then configure with any HuggingFace model id:
+
+```yaml
+reranking:
+  model:
+    provider: cross-encoder
+    name: BAAI/bge-reranker-v2-m3
+```
+
+Other tested models: `Qwen/Qwen3-Reranker-0.6B`, `cross-encoder/ms-marco-MiniLM-L-6-v2`. Any model exposed as a `sentence_transformers.CrossEncoder` works.
--- a/haiku_rag_slim/haiku/rag/client/downloads.py
+++ b/haiku_rag_slim/haiku/rag/client/downloads.py
@ -52,13 +52,14 @@ async def download_models(
    # Sentence-transformers embedder
    if config.embeddings.model.provider == "sentence-transformers":  # pragma: no cover
        try:
-            from sentence_transformers import (  # type: ignore[import-not-found]  # ty: ignore[unresolved-import]
+            from sentence_transformers import (  # type: ignore[import-not-found]
                SentenceTransformer,
            )

            model_name = config.embeddings.model.name
            yield DownloadProgress(model=model_name, status="start")
-            await asyncio.to_thread(SentenceTransformer, model_name)
+            # Wrap in lambda: ty loses ParamSpec inference on third-party __init__.
+            await asyncio.to_thread(lambda: SentenceTransformer(model_name))
            yield DownloadProgress(model=model_name, status="done")
        except ImportError:
            pass
--- a/haiku_rag_slim/haiku/rag/reranking/init.py
+++ b/haiku_rag_slim/haiku/rag/reranking/init.py
@ -67,4 +67,17 @@ def get_reranker(config: AppConfig = Config) -> RerankerBase | None:
        except ImportError:  # pragma: no cover
            return None

+    if config.reranking.model and config.reranking.model.provider == "cross-encoder":
+        try:
+            from haiku.rag.reranking.cross_encoder import CrossEncoderReranker
+
+            name = config.reranking.model.name
+            if not name:
+                raise ValueError(
+                    "cross-encoder reranker requires name in reranking.model"
+                )
+            return CrossEncoderReranker(name)
+        except ImportError:  # pragma: no cover
+            return None
+
    return None
--- a/haiku_rag_slim/haiku/rag/reranking/cross_encoder.py
+++ b/haiku_rag_slim/haiku/rag/reranking/cross_encoder.py
@ -0,0 +1,39 @@
+import asyncio
+
+try:
+    from sentence_transformers import (
+        CrossEncoder,  # pyright: ignore[reportMissingImports]
+    )
+except ImportError as e:  # pragma: no cover
+    raise ImportError(
+        "sentence-transformers is not installed. Install it with "
+        "`pip install sentence-transformers` or use the cross-encoder optional dependency."
+    ) from e
+
+from haiku.rag.reranking.base import RerankerBase
+from haiku.rag.store.models.chunk import Chunk
+
+
+class CrossEncoderReranker(RerankerBase):
+    """Reranker for any sentence-transformers CrossEncoder model.
+
+    Loads the model in-process. Pass any HuggingFace cross-encoder reranker
+    as ``model`` (e.g. ``BAAI/bge-reranker-v2-m3``, ``Qwen/Qwen3-Reranker-0.6B``,
+    ``cross-encoder/ms-marco-MiniLM-L-6-v2``).
+    """
+
+    def __init__(self, model: str):
+        self._model = model
+        self._reranker = CrossEncoder(model)
+
+    async def rerank(
+        self, query: str, chunks: list[Chunk], top_n: int = 10
+    ) -> list[tuple[Chunk, float]]:
+        if not chunks:
+            return []
+
+        documents = [chunk.content for chunk in chunks]
+        rankings = await asyncio.to_thread(
+            lambda: self._reranker.rank(query, documents, top_k=top_n)
+        )
+        return [(chunks[r["corpus_id"]], float(r["score"])) for r in rankings]
--- a/haiku_rag_slim/haiku/rag/reranking/jina_local.py
+++ b/haiku_rag_slim/haiku/rag/reranking/jina_local.py
@ -1,8 +1,10 @@
+import asyncio
+
 try:
    from transformers import (
        AutoModel,  # pyright: ignore[reportMissingImports]
    )
-except ImportError as e:
+except ImportError as e:  # pragma: no cover
    raise ImportError(
        "transformers is not installed. Please install it with `pip install transformers torch` "
        "or use the jina optional dependency."
@ -32,6 +34,8 @@ class JinaLocalReranker(RerankerBase):  # pragma: no cover

        documents = [chunk.content for chunk in chunks]

-        results = self._reranker.rerank(query, documents, top_n=top_n)
+        results = await asyncio.to_thread(
+            lambda: self._reranker.rerank(query, documents, top_n=top_n)
+        )

        return [(chunks[r["index"]], float(r["relevance_score"])) for r in results]
--- a/haiku_rag_slim/haiku/rag/reranking/mxbai.py
+++ b/haiku_rag_slim/haiku/rag/reranking/mxbai.py
@ -1,3 +1,5 @@
+import asyncio
+
 from mxbai_rerank import MxbaiRerankV2  # pyright: ignore[reportMissingImports]

 from haiku.rag.config import Config
@ -22,7 +24,9 @@ class MxBAIReranker(RerankerBase):

        documents = [chunk.content for chunk in chunks]

-        results = self._client.rank(query=query, documents=documents, top_k=top_n)
+        results = await asyncio.to_thread(
+            lambda: self._client.rank(query=query, documents=documents, top_k=top_n)
+        )
        reranked_chunks = []
        for result in results:
            original_chunk = chunks[result.index]
--- a/haiku_rag_slim/pyproject.toml
+++ b/haiku_rag_slim/pyproject.toml
@ -52,6 +52,7 @@ mxbai = ["mxbai-rerank>=0.1.6"]
 cohere = ["cohere>=5.21.1"]
 zeroentropy = ["zeroentropy>=0.1.0a11"]
 jina = ["transformers>=4.40.0", "torch>=2.0.0"]
+cross-encoder = ["sentence-transformers>=3.0.0"]
 # TUI (chat and inspect commands)
 tui = ["textual>=8.2.4", "textual-image>=0.8.5"]
 # Model providers (delegated to pydantic-ai-slim)
--- a/pyproject.toml
+++ b/pyproject.toml
@ -39,6 +39,7 @@ haiku-rag = "haiku.rag.cli:cli"
 [project.optional-dependencies]
 tui = ["textual>=8.2.4"]
 s3 = ["haiku.rag-slim[s3]==0.46.0"]
+cross-encoder = ["haiku.rag-slim[cross-encoder]==0.46.0"]

 [build-system]
 requires = ["hatchling"]
--- a/tests/test_reranker.py
+++ b/tests/test_reranker.py
@ -62,6 +62,18 @@ async def test_mxbai_reranker():
        pytest.skip("MxBAI package not installed")


+@pytest.mark.asyncio
+async def test_mxbai_reranker_empty_chunks():
+    try:
+        from haiku.rag.reranking.mxbai import MxBAIReranker
+
+        reranker = MxBAIReranker()
+        result = await reranker.rerank("query", [], top_n=2)
+        assert result == []
+    except ImportError:
+        pytest.skip("MxBAI package not installed")
+
+
@pytest.mark.asyncio
@pytest.mark.vcr()
 async def test_cohere_reranker():
@ -126,6 +138,15 @@ class TestGetReranker:
        with pytest.raises(ValueError, match="vLLM reranker requires base_url"):
            get_reranker(config)

+    def test_cross_encoder_provider_without_name_raises_error(self):
+        config = AppConfig(
+            reranking=RerankingConfig(
+                model=ModelConfig(provider="cross-encoder", name="")
+            )
+        )
+        with pytest.raises(ValueError, match="cross-encoder reranker requires name"):
+            get_reranker(config)
+
    @pytest.mark.parametrize(
        "provider, model_name, class_module, class_name, extra_model_kwargs, expected_attrs, env_vars",
        [
@ -195,6 +216,15 @@ class TestGetReranker:
                {"_model": "jinaai/jina-reranker-v3"},
                {},
            ),
+            (
+                "cross-encoder",
+                "cross-encoder/ms-marco-MiniLM-L-6-v2",
+                "haiku.rag.reranking.cross_encoder",
+                "CrossEncoderReranker",
+                {},
+                {"_model": "cross-encoder/ms-marco-MiniLM-L-6-v2"},
+                {},
+            ),
        ],
        ids=[
            "mxbai",
@ -204,6 +234,7 @@ class TestGetReranker:
            "zeroentropy-default",
            "jina",
            "jina-local",
+            "cross-encoder",
        ],
    )
    def test_provider(
@ -298,3 +329,33 @@ async def test_jina_local_reranker():
        assert "0" in top_ids or "2" in top_ids  # These chunks mention the book/author
    except ImportError:
        pytest.skip("Jina local dependencies not installed")
+
+
+@pytest.mark.asyncio
+async def test_cross_encoder_reranker():
+    try:
+        from haiku.rag.reranking.cross_encoder import CrossEncoderReranker
+
+        reranker = CrossEncoderReranker("cross-encoder/ms-marco-MiniLM-L-6-v2")
+
+        reranked = await reranker.rerank(
+            "Who wrote 'To Kill a Mockingbird'?", chunks, top_n=2
+        )
+        assert len(reranked) == 2
+        assert all(isinstance(score, float) for chunk, score in reranked)
+        top_ids = [chunk.document_id for chunk, score in reranked]
+        assert "0" in top_ids or "2" in top_ids
+    except ImportError:
+        pytest.skip("sentence-transformers not installed")
+
+
+@pytest.mark.asyncio
+async def test_cross_encoder_reranker_empty_chunks():
+    try:
+        from haiku.rag.reranking.cross_encoder import CrossEncoderReranker
+
+        reranker = CrossEncoderReranker("cross-encoder/ms-marco-MiniLM-L-6-v2")
+        result = await reranker.rerank("query", [], top_n=2)
+        assert result == []
+    except ImportError:
+        pytest.skip("sentence-transformers not installed")
--- a/uv.lock
+++ b/uv.lock
@ -1445,6 +1445,9 @@ dependencies = [
 ]

 [package.optional-dependencies]
+cross-encoder = [
+    { name = "haiku-rag-slim", extra = ["cross-encoder"] },
+]
 s3 = [
    { name = "haiku-rag-slim", extra = ["s3"] },
 ]
@ -1472,11 +1475,12 @@ dev = [

 [package.metadata]
 requires-dist = [
+    { name = "haiku-rag-slim", extras = ["cross-encoder"], marker = "extra == 'cross-encoder'", editable = "haiku_rag_slim" },
    { name = "haiku-rag-slim", extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy", "tui"], editable = "haiku_rag_slim" },
    { name = "haiku-rag-slim", extras = ["s3"], marker = "extra == 's3'", editable = "haiku_rag_slim" },
    { name = "textual", marker = "extra == 'tui'", specifier = ">=8.2.4" },
 ]
-provides-extras = ["tui", "s3"]
+provides-extras = ["tui", "s3", "cross-encoder"]

 [package.metadata.requires-dev]
 dev = [
@ -1557,6 +1561,9 @@ bedrock = [
 cohere = [
    { name = "cohere" },
 ]
+cross-encoder = [
+    { name = "sentence-transformers" },
+]
 docling = [
    { name = "docling" },
    { name = "opencv-python-headless" },
@ -1621,6 +1628,7 @@ requires-dist = [
    { name = "python-dotenv", specifier = ">=1.2.2" },
    { name = "pyyaml", specifier = ">=6.0.3" },
    { name = "rich", specifier = ">=14.3.3" },
+    { name = "sentence-transformers", marker = "extra == 'cross-encoder'", specifier = ">=3.0.0" },
    { name = "textual", marker = "extra == 'tui'", specifier = ">=8.2.4" },
    { name = "textual-image", marker = "extra == 'tui'", specifier = ">=0.8.5" },
    { name = "torch", marker = "extra == 'jina'", specifier = ">=2.0.0" },
@ -1630,7 +1638,7 @@ requires-dist = [
    { name = "zeroentropy", marker = "extra == 'zeroentropy'", specifier = ">=0.1.0a11" },
    { name = "zstandard", marker = "python_full_version < '3.14'", specifier = ">=0.23.0" },
 ]
-provides-extras = ["docling", "s3", "voyageai", "mxbai", "cohere", "zeroentropy", "jina", "tui", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"]
+provides-extras = ["docling", "s3", "voyageai", "mxbai", "cohere", "zeroentropy", "jina", "cross-encoder", "tui", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"]

 [[package]]
 name = "haiku-skills"
@ -1907,6 +1915,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" },
 ]

+[[package]]
+name = "joblib"
+version = "1.5.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" },
+]
+
 [[package]]
 name = "jsonlines"
 version = "4.0.0"
@ -4549,6 +4566,50 @@ torch = [
    { name = "torch" },
 ]

+[[package]]
+name = "scikit-learn"
+version = "1.8.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "joblib" },
+    { name = "numpy" },
+    { name = "scipy" },
+    { name = "threadpoolctl" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/90/74/e6a7cc4b820e95cc38cf36cd74d5aa2b42e8ffc2d21fe5a9a9c45c1c7630/scikit_learn-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5fb63362b5a7ddab88e52b6dbb47dac3fd7dafeee740dc6c8d8a446ddedade8e", size = 8548242, upload-time = "2025-12-10T07:07:51.568Z" },
+    { url = "https://files.pythonhosted.org/packages/49/d8/9be608c6024d021041c7f0b3928d4749a706f4e2c3832bbede4fb4f58c95/scikit_learn-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5025ce924beccb28298246e589c691fe1b8c1c96507e6d27d12c5fadd85bfd76", size = 8079075, upload-time = "2025-12-10T07:07:53.697Z" },
+    { url = "https://files.pythonhosted.org/packages/dd/47/f187b4636ff80cc63f21cd40b7b2d177134acaa10f6bb73746130ee8c2e5/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4496bb2cf7a43ce1a2d7524a79e40bc5da45cf598dbf9545b7e8316ccba47bb4", size = 8660492, upload-time = "2025-12-10T07:07:55.574Z" },
+    { url = "https://files.pythonhosted.org/packages/97/74/b7a304feb2b49df9fafa9382d4d09061a96ee9a9449a7cbea7988dda0828/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0bcfe4d0d14aec44921545fd2af2338c7471de9cb701f1da4c9d85906ab847a", size = 8931904, upload-time = "2025-12-10T07:07:57.666Z" },
+    { url = "https://files.pythonhosted.org/packages/9f/c4/0ab22726a04ede56f689476b760f98f8f46607caecff993017ac1b64aa5d/scikit_learn-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:35c007dedb2ffe38fe3ee7d201ebac4a2deccd2408e8621d53067733e3c74809", size = 8019359, upload-time = "2025-12-10T07:07:59.838Z" },
+    { url = "https://files.pythonhosted.org/packages/24/90/344a67811cfd561d7335c1b96ca21455e7e472d281c3c279c4d3f2300236/scikit_learn-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:8c497fff237d7b4e07e9ef1a640887fa4fb765647f86fbe00f969ff6280ce2bb", size = 7641898, upload-time = "2025-12-10T07:08:01.36Z" },
+    { url = "https://files.pythonhosted.org/packages/03/aa/e22e0768512ce9255eba34775be2e85c2048da73da1193e841707f8f039c/scikit_learn-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d6ae97234d5d7079dc0040990a6f7aeb97cb7fa7e8945f1999a429b23569e0a", size = 8513770, upload-time = "2025-12-10T07:08:03.251Z" },
+    { url = "https://files.pythonhosted.org/packages/58/37/31b83b2594105f61a381fc74ca19e8780ee923be2d496fcd8d2e1147bd99/scikit_learn-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:edec98c5e7c128328124a029bceb09eda2d526997780fef8d65e9a69eead963e", size = 8044458, upload-time = "2025-12-10T07:08:05.336Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/5a/3f1caed8765f33eabb723596666da4ebbf43d11e96550fb18bdec42b467b/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74b66d8689d52ed04c271e1329f0c61635bcaf5b926db9b12d58914cdc01fe57", size = 8610341, upload-time = "2025-12-10T07:08:07.732Z" },
+    { url = "https://files.pythonhosted.org/packages/38/cf/06896db3f71c75902a8e9943b444a56e727418f6b4b4a90c98c934f51ed4/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8fdf95767f989b0cfedb85f7ed8ca215d4be728031f56ff5a519ee1e3276dc2e", size = 8900022, upload-time = "2025-12-10T07:08:09.862Z" },
+    { url = "https://files.pythonhosted.org/packages/1c/f9/9b7563caf3ec8873e17a31401858efab6b39a882daf6c1bfa88879c0aa11/scikit_learn-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:2de443b9373b3b615aec1bb57f9baa6bb3a9bd093f1269ba95c17d870422b271", size = 7989409, upload-time = "2025-12-10T07:08:12.028Z" },
+    { url = "https://files.pythonhosted.org/packages/49/bd/1f4001503650e72c4f6009ac0c4413cb17d2d601cef6f71c0453da2732fc/scikit_learn-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:eddde82a035681427cbedded4e6eff5e57fa59216c2e3e90b10b19ab1d0a65c3", size = 7619760, upload-time = "2025-12-10T07:08:13.688Z" },
+    { url = "https://files.pythonhosted.org/packages/d2/7d/a630359fc9dcc95496588c8d8e3245cc8fd81980251079bc09c70d41d951/scikit_learn-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7cc267b6108f0a1499a734167282c00c4ebf61328566b55ef262d48e9849c735", size = 8826045, upload-time = "2025-12-10T07:08:15.215Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/56/a0c86f6930cfcd1c7054a2bc417e26960bb88d32444fe7f71d5c2cfae891/scikit_learn-1.8.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:fe1c011a640a9f0791146011dfd3c7d9669785f9fed2b2a5f9e207536cf5c2fd", size = 8420324, upload-time = "2025-12-10T07:08:17.561Z" },
+    { url = "https://files.pythonhosted.org/packages/46/1e/05962ea1cebc1cf3876667ecb14c283ef755bf409993c5946ade3b77e303/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72358cce49465d140cc4e7792015bb1f0296a9742d5622c67e31399b75468b9e", size = 8680651, upload-time = "2025-12-10T07:08:19.952Z" },
+    { url = "https://files.pythonhosted.org/packages/fe/56/a85473cd75f200c9759e3a5f0bcab2d116c92a8a02ee08ccd73b870f8bb4/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:80832434a6cc114f5219211eec13dcbc16c2bac0e31ef64c6d346cde3cf054cb", size = 8925045, upload-time = "2025-12-10T07:08:22.11Z" },
+    { url = "https://files.pythonhosted.org/packages/cc/b7/64d8cfa896c64435ae57f4917a548d7ac7a44762ff9802f75a79b77cb633/scikit_learn-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ee787491dbfe082d9c3013f01f5991658b0f38aa8177e4cd4bf434c58f551702", size = 8507994, upload-time = "2025-12-10T07:08:23.943Z" },
+    { url = "https://files.pythonhosted.org/packages/5e/37/e192ea709551799379958b4c4771ec507347027bb7c942662c7fbeba31cb/scikit_learn-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf97c10a3f5a7543f9b88cbf488d33d175e9146115a451ae34568597ba33dcde", size = 7869518, upload-time = "2025-12-10T07:08:25.71Z" },
+    { url = "https://files.pythonhosted.org/packages/24/05/1af2c186174cc92dcab2233f327336058c077d38f6fe2aceb08e6ab4d509/scikit_learn-1.8.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c22a2da7a198c28dd1a6e1136f19c830beab7fdca5b3e5c8bba8394f8a5c45b3", size = 8528667, upload-time = "2025-12-10T07:08:27.541Z" },
+    { url = "https://files.pythonhosted.org/packages/a8/25/01c0af38fe969473fb292bba9dc2b8f9b451f3112ff242c647fee3d0dfe7/scikit_learn-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:6b595b07a03069a2b1740dc08c2299993850ea81cce4fe19b2421e0c970de6b7", size = 8066524, upload-time = "2025-12-10T07:08:29.822Z" },
+    { url = "https://files.pythonhosted.org/packages/be/ce/a0623350aa0b68647333940ee46fe45086c6060ec604874e38e9ab7d8e6c/scikit_learn-1.8.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:29ffc74089f3d5e87dfca4c2c8450f88bdc61b0fc6ed5d267f3988f19a1309f6", size = 8657133, upload-time = "2025-12-10T07:08:31.865Z" },
+    { url = "https://files.pythonhosted.org/packages/b8/cb/861b41341d6f1245e6ca80b1c1a8c4dfce43255b03df034429089ca2a2c5/scikit_learn-1.8.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fb65db5d7531bccf3a4f6bec3462223bea71384e2cda41da0f10b7c292b9e7c4", size = 8923223, upload-time = "2025-12-10T07:08:34.166Z" },
+    { url = "https://files.pythonhosted.org/packages/76/18/a8def8f91b18cd1ba6e05dbe02540168cb24d47e8dcf69e8d00b7da42a08/scikit_learn-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:56079a99c20d230e873ea40753102102734c5953366972a71d5cb39a32bc40c6", size = 8096518, upload-time = "2025-12-10T07:08:36.339Z" },
+    { url = "https://files.pythonhosted.org/packages/d1/77/482076a678458307f0deb44e29891d6022617b2a64c840c725495bee343f/scikit_learn-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:3bad7565bc9cf37ce19a7c0d107742b320c1285df7aab1a6e2d28780df167242", size = 7754546, upload-time = "2025-12-10T07:08:38.128Z" },
+    { url = "https://files.pythonhosted.org/packages/2d/d1/ef294ca754826daa043b2a104e59960abfab4cf653891037d19dd5b6f3cf/scikit_learn-1.8.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:4511be56637e46c25721e83d1a9cea9614e7badc7040c4d573d75fbe257d6fd7", size = 8848305, upload-time = "2025-12-10T07:08:41.013Z" },
+    { url = "https://files.pythonhosted.org/packages/5b/e2/b1f8b05138ee813b8e1a4149f2f0d289547e60851fd1bb268886915adbda/scikit_learn-1.8.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:a69525355a641bf8ef136a7fa447672fb54fe8d60cab5538d9eb7c6438543fb9", size = 8432257, upload-time = "2025-12-10T07:08:42.873Z" },
+    { url = "https://files.pythonhosted.org/packages/26/11/c32b2138a85dcb0c99f6afd13a70a951bfdff8a6ab42d8160522542fb647/scikit_learn-1.8.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c2656924ec73e5939c76ac4c8b026fc203b83d8900362eb2599d8aee80e4880f", size = 8678673, upload-time = "2025-12-10T07:08:45.362Z" },
+    { url = "https://files.pythonhosted.org/packages/c7/57/51f2384575bdec454f4fe4e7a919d696c9ebce914590abf3e52d47607ab8/scikit_learn-1.8.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15fc3b5d19cc2be65404786857f2e13c70c83dd4782676dd6814e3b89dc8f5b9", size = 8922467, upload-time = "2025-12-10T07:08:47.408Z" },
+    { url = "https://files.pythonhosted.org/packages/35/4d/748c9e2872637a57981a04adc038dacaa16ba8ca887b23e34953f0b3f742/scikit_learn-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:00d6f1d66fbcf4eba6e356e1420d33cc06c70a45bb1363cd6f6a8e4ebbbdece2", size = 8774395, upload-time = "2025-12-10T07:08:49.337Z" },
+    { url = "https://files.pythonhosted.org/packages/60/22/d7b2ebe4704a5e50790ba089d5c2ae308ab6bb852719e6c3bd4f04c3a363/scikit_learn-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f28dd15c6bb0b66ba09728cf09fd8736c304be29409bd8445a080c1280619e8c", size = 8002647, upload-time = "2025-12-10T07:08:51.601Z" },
+]
+
 [[package]]
 name = "scipy"
 version = "1.17.1"
@ -4636,6 +4697,25 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/f8/95/12d226ee4d207cb1f77a216baa7e1a8bae2639733c140abe8d0316d23a18/semchunk-3.2.5-py3-none-any.whl", hash = "sha256:fd09cc5f380bd010b8ca773bd81893f7eaf11d37dd8362a83d46cedaf5dae076", size = 13048, upload-time = "2025-10-28T02:12:36.724Z" },
 ]

+[[package]]
+name = "sentence-transformers"
+version = "5.5.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "huggingface-hub" },
+    { name = "numpy" },
+    { name = "scikit-learn" },
+    { name = "scipy" },
+    { name = "torch" },
+    { name = "tqdm" },
+    { name = "transformers" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/2c/27/16d127a61303e05847d878b23687f3371868c76e738557fa80b4373a8c2b/sentence_transformers-5.5.0.tar.gz", hash = "sha256:9cec675e68bfe09d07466d1f13ab06d1d79d60a0f45b154baf433bde6ae159cb", size = 444908, upload-time = "2026-05-12T14:05:42.383Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/55/20/18416624bcbae866ec0b111979766cebabe8e5ff7563ab953ecbaf3ff9e7/sentence_transformers-5.5.0-py3-none-any.whl", hash = "sha256:75313fdcc2397ec4b58297c25d6187fcca5a6b2aeb09570a72eff5a3223d8d58", size = 588665, upload-time = "2026-05-12T14:05:40.899Z" },
+]
+
 [[package]]
 name = "setuptools"
 version = "81.0.0"
@ -4843,6 +4923,15 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/b0/0d/ca8367c100c09850379f83645abd60f47c051e6b1e7b64adb953bce96be9/textual_image-0.8.5-py3-none-any.whl", hash = "sha256:dffc85458ca8744bce3f17bddbf582b1e086eb52d111f1063753e8dd316fa45d", size = 109618, upload-time = "2025-12-26T18:38:07.11Z" },
 ]

+[[package]]
+name = "threadpoolctl"
+version = "3.6.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" },
+]
+
 [[package]]
 name = "tiktoken"
 version = "0.12.0"