diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aa71c1bf..14facbd5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,6 +66,8 @@ jobs: key: huggingface-${{ runner.os }}-qwen-tokenizer-v1 - name: Pre-download tokenizer run: uv run python -c "from transformers import AutoTokenizer; AutoTokenizer.from_pretrained('Qwen/Qwen3-Embedding-0.6B')" + - name: Pre-download cross-encoder test model + run: uv run python -c "from sentence_transformers import CrossEncoder; CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')" - name: Run tests with coverage run: uv run pytest -m "not integration" --cov=haiku --cov-report=xml - name: Upload coverage to Codecov diff --git a/CHANGELOG.md b/CHANGELOG.md index 393b9be7..fecd4781 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog ## [Unreleased] +### Added + +- **`cross-encoder` reranking provider.** Runs any HuggingFace cross-encoder reranker in-process via `sentence_transformers.CrossEncoder` — no separate server. Useful for BGE (`BAAI/bge-reranker-v2-m3`), Qwen3-Reranker, MS-MARCO MiniLM, and other CrossEncoder-compatible models when vLLM is not an option. New `[cross-encoder]` extra pulls `sentence-transformers`. + ### Fixed - **`rebuild --embed-only` no longer buffers the entire corpus in memory.** The previous implementation accumulated every chunk's id, content, content_fts, metadata, and new embedding vector in a single Python list before flushing. The rebuild now stream-copies non-vector columns into a `chunks_rebuild_staging` table (1000 rows / page), recreates the chunks table fresh to honour vector-dim changes, then streams from staging one document at a time, embedding in batches of `embeddings.batch_size` and flushing to the new chunks table every 50 documents. diff --git a/docs/configuration/providers.md b/docs/configuration/providers.md index 176f0dda..19ca08ef 100644 --- a/docs/configuration/providers.md +++ b/docs/configuration/providers.md @@ -465,3 +465,24 @@ reranking: ``` **Note:** The Jina Reranker v3 local model is licensed under CC BY-NC 4.0, which restricts commercial use. For commercial applications, use the API mode instead. + +### Cross-Encoder (sentence-transformers) + +Run any HuggingFace cross-encoder reranker in-process via `sentence-transformers` — no separate server required. Useful when you want a specific model (BGE, Qwen3-Reranker, MS-MARCO MiniLM, etc.) without running vLLM. + +Install the extra: + +```bash +uv pip install haiku.rag-slim[cross-encoder] +``` + +Then configure with any HuggingFace model id: + +```yaml +reranking: + model: + provider: cross-encoder + name: BAAI/bge-reranker-v2-m3 +``` + +Other tested models: `Qwen/Qwen3-Reranker-0.6B`, `cross-encoder/ms-marco-MiniLM-L-6-v2`. Any model exposed as a `sentence_transformers.CrossEncoder` works. diff --git a/haiku_rag_slim/haiku/rag/client/downloads.py b/haiku_rag_slim/haiku/rag/client/downloads.py index 62bd7611..019abc52 100644 --- a/haiku_rag_slim/haiku/rag/client/downloads.py +++ b/haiku_rag_slim/haiku/rag/client/downloads.py @@ -52,13 +52,14 @@ async def download_models( # Sentence-transformers embedder if config.embeddings.model.provider == "sentence-transformers": # pragma: no cover try: - from sentence_transformers import ( # type: ignore[import-not-found] # ty: ignore[unresolved-import] + from sentence_transformers import ( # type: ignore[import-not-found] SentenceTransformer, ) model_name = config.embeddings.model.name yield DownloadProgress(model=model_name, status="start") - await asyncio.to_thread(SentenceTransformer, model_name) + # Wrap in lambda: ty loses ParamSpec inference on third-party __init__. + await asyncio.to_thread(lambda: SentenceTransformer(model_name)) yield DownloadProgress(model=model_name, status="done") except ImportError: pass diff --git a/haiku_rag_slim/haiku/rag/reranking/__init__.py b/haiku_rag_slim/haiku/rag/reranking/__init__.py index f3fbb2c3..85b7b803 100644 --- a/haiku_rag_slim/haiku/rag/reranking/__init__.py +++ b/haiku_rag_slim/haiku/rag/reranking/__init__.py @@ -67,4 +67,17 @@ def get_reranker(config: AppConfig = Config) -> RerankerBase | None: except ImportError: # pragma: no cover return None + if config.reranking.model and config.reranking.model.provider == "cross-encoder": + try: + from haiku.rag.reranking.cross_encoder import CrossEncoderReranker + + name = config.reranking.model.name + if not name: + raise ValueError( + "cross-encoder reranker requires name in reranking.model" + ) + return CrossEncoderReranker(name) + except ImportError: # pragma: no cover + return None + return None diff --git a/haiku_rag_slim/haiku/rag/reranking/cross_encoder.py b/haiku_rag_slim/haiku/rag/reranking/cross_encoder.py new file mode 100644 index 00000000..bbeff5b6 --- /dev/null +++ b/haiku_rag_slim/haiku/rag/reranking/cross_encoder.py @@ -0,0 +1,39 @@ +import asyncio + +try: + from sentence_transformers import ( + CrossEncoder, # pyright: ignore[reportMissingImports] + ) +except ImportError as e: # pragma: no cover + raise ImportError( + "sentence-transformers is not installed. Install it with " + "`pip install sentence-transformers` or use the cross-encoder optional dependency." + ) from e + +from haiku.rag.reranking.base import RerankerBase +from haiku.rag.store.models.chunk import Chunk + + +class CrossEncoderReranker(RerankerBase): + """Reranker for any sentence-transformers CrossEncoder model. + + Loads the model in-process. Pass any HuggingFace cross-encoder reranker + as ``model`` (e.g. ``BAAI/bge-reranker-v2-m3``, ``Qwen/Qwen3-Reranker-0.6B``, + ``cross-encoder/ms-marco-MiniLM-L-6-v2``). + """ + + def __init__(self, model: str): + self._model = model + self._reranker = CrossEncoder(model) + + async def rerank( + self, query: str, chunks: list[Chunk], top_n: int = 10 + ) -> list[tuple[Chunk, float]]: + if not chunks: + return [] + + documents = [chunk.content for chunk in chunks] + rankings = await asyncio.to_thread( + lambda: self._reranker.rank(query, documents, top_k=top_n) + ) + return [(chunks[r["corpus_id"]], float(r["score"])) for r in rankings] diff --git a/haiku_rag_slim/haiku/rag/reranking/jina_local.py b/haiku_rag_slim/haiku/rag/reranking/jina_local.py index b76e6e64..9b8949f7 100644 --- a/haiku_rag_slim/haiku/rag/reranking/jina_local.py +++ b/haiku_rag_slim/haiku/rag/reranking/jina_local.py @@ -1,8 +1,10 @@ +import asyncio + try: from transformers import ( AutoModel, # pyright: ignore[reportMissingImports] ) -except ImportError as e: +except ImportError as e: # pragma: no cover raise ImportError( "transformers is not installed. Please install it with `pip install transformers torch` " "or use the jina optional dependency." @@ -32,6 +34,8 @@ class JinaLocalReranker(RerankerBase): # pragma: no cover documents = [chunk.content for chunk in chunks] - results = self._reranker.rerank(query, documents, top_n=top_n) + results = await asyncio.to_thread( + lambda: self._reranker.rerank(query, documents, top_n=top_n) + ) return [(chunks[r["index"]], float(r["relevance_score"])) for r in results] diff --git a/haiku_rag_slim/haiku/rag/reranking/mxbai.py b/haiku_rag_slim/haiku/rag/reranking/mxbai.py index 6e42f1a4..d73ed751 100644 --- a/haiku_rag_slim/haiku/rag/reranking/mxbai.py +++ b/haiku_rag_slim/haiku/rag/reranking/mxbai.py @@ -1,3 +1,5 @@ +import asyncio + from mxbai_rerank import MxbaiRerankV2 # pyright: ignore[reportMissingImports] from haiku.rag.config import Config @@ -22,7 +24,9 @@ class MxBAIReranker(RerankerBase): documents = [chunk.content for chunk in chunks] - results = self._client.rank(query=query, documents=documents, top_k=top_n) + results = await asyncio.to_thread( + lambda: self._client.rank(query=query, documents=documents, top_k=top_n) + ) reranked_chunks = [] for result in results: original_chunk = chunks[result.index] diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index d7fa0049..002df5b7 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -52,6 +52,7 @@ mxbai = ["mxbai-rerank>=0.1.6"] cohere = ["cohere>=5.21.1"] zeroentropy = ["zeroentropy>=0.1.0a11"] jina = ["transformers>=4.40.0", "torch>=2.0.0"] +cross-encoder = ["sentence-transformers>=3.0.0"] # TUI (chat and inspect commands) tui = ["textual>=8.2.4", "textual-image>=0.8.5"] # Model providers (delegated to pydantic-ai-slim) diff --git a/pyproject.toml b/pyproject.toml index 945e4d83..cf34ca39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ haiku-rag = "haiku.rag.cli:cli" [project.optional-dependencies] tui = ["textual>=8.2.4"] s3 = ["haiku.rag-slim[s3]==0.46.0"] +cross-encoder = ["haiku.rag-slim[cross-encoder]==0.46.0"] [build-system] requires = ["hatchling"] diff --git a/tests/test_reranker.py b/tests/test_reranker.py index 6cd8a607..0fc3c8f0 100644 --- a/tests/test_reranker.py +++ b/tests/test_reranker.py @@ -62,6 +62,18 @@ async def test_mxbai_reranker(): pytest.skip("MxBAI package not installed") +@pytest.mark.asyncio +async def test_mxbai_reranker_empty_chunks(): + try: + from haiku.rag.reranking.mxbai import MxBAIReranker + + reranker = MxBAIReranker() + result = await reranker.rerank("query", [], top_n=2) + assert result == [] + except ImportError: + pytest.skip("MxBAI package not installed") + + @pytest.mark.asyncio @pytest.mark.vcr() async def test_cohere_reranker(): @@ -126,6 +138,15 @@ class TestGetReranker: with pytest.raises(ValueError, match="vLLM reranker requires base_url"): get_reranker(config) + def test_cross_encoder_provider_without_name_raises_error(self): + config = AppConfig( + reranking=RerankingConfig( + model=ModelConfig(provider="cross-encoder", name="") + ) + ) + with pytest.raises(ValueError, match="cross-encoder reranker requires name"): + get_reranker(config) + @pytest.mark.parametrize( "provider, model_name, class_module, class_name, extra_model_kwargs, expected_attrs, env_vars", [ @@ -195,6 +216,15 @@ class TestGetReranker: {"_model": "jinaai/jina-reranker-v3"}, {}, ), + ( + "cross-encoder", + "cross-encoder/ms-marco-MiniLM-L-6-v2", + "haiku.rag.reranking.cross_encoder", + "CrossEncoderReranker", + {}, + {"_model": "cross-encoder/ms-marco-MiniLM-L-6-v2"}, + {}, + ), ], ids=[ "mxbai", @@ -204,6 +234,7 @@ class TestGetReranker: "zeroentropy-default", "jina", "jina-local", + "cross-encoder", ], ) def test_provider( @@ -298,3 +329,33 @@ async def test_jina_local_reranker(): assert "0" in top_ids or "2" in top_ids # These chunks mention the book/author except ImportError: pytest.skip("Jina local dependencies not installed") + + +@pytest.mark.asyncio +async def test_cross_encoder_reranker(): + try: + from haiku.rag.reranking.cross_encoder import CrossEncoderReranker + + reranker = CrossEncoderReranker("cross-encoder/ms-marco-MiniLM-L-6-v2") + + reranked = await reranker.rerank( + "Who wrote 'To Kill a Mockingbird'?", chunks, top_n=2 + ) + assert len(reranked) == 2 + assert all(isinstance(score, float) for chunk, score in reranked) + top_ids = [chunk.document_id for chunk, score in reranked] + assert "0" in top_ids or "2" in top_ids + except ImportError: + pytest.skip("sentence-transformers not installed") + + +@pytest.mark.asyncio +async def test_cross_encoder_reranker_empty_chunks(): + try: + from haiku.rag.reranking.cross_encoder import CrossEncoderReranker + + reranker = CrossEncoderReranker("cross-encoder/ms-marco-MiniLM-L-6-v2") + result = await reranker.rerank("query", [], top_n=2) + assert result == [] + except ImportError: + pytest.skip("sentence-transformers not installed") diff --git a/uv.lock b/uv.lock index 9a18e32e..e7fac1a8 100644 --- a/uv.lock +++ b/uv.lock @@ -1445,6 +1445,9 @@ dependencies = [ ] [package.optional-dependencies] +cross-encoder = [ + { name = "haiku-rag-slim", extra = ["cross-encoder"] }, +] s3 = [ { name = "haiku-rag-slim", extra = ["s3"] }, ] @@ -1472,11 +1475,12 @@ dev = [ [package.metadata] requires-dist = [ + { name = "haiku-rag-slim", extras = ["cross-encoder"], marker = "extra == 'cross-encoder'", editable = "haiku_rag_slim" }, { name = "haiku-rag-slim", extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy", "tui"], editable = "haiku_rag_slim" }, { name = "haiku-rag-slim", extras = ["s3"], marker = "extra == 's3'", editable = "haiku_rag_slim" }, { name = "textual", marker = "extra == 'tui'", specifier = ">=8.2.4" }, ] -provides-extras = ["tui", "s3"] +provides-extras = ["tui", "s3", "cross-encoder"] [package.metadata.requires-dev] dev = [ @@ -1557,6 +1561,9 @@ bedrock = [ cohere = [ { name = "cohere" }, ] +cross-encoder = [ + { name = "sentence-transformers" }, +] docling = [ { name = "docling" }, { name = "opencv-python-headless" }, @@ -1621,6 +1628,7 @@ requires-dist = [ { name = "python-dotenv", specifier = ">=1.2.2" }, { name = "pyyaml", specifier = ">=6.0.3" }, { name = "rich", specifier = ">=14.3.3" }, + { name = "sentence-transformers", marker = "extra == 'cross-encoder'", specifier = ">=3.0.0" }, { name = "textual", marker = "extra == 'tui'", specifier = ">=8.2.4" }, { name = "textual-image", marker = "extra == 'tui'", specifier = ">=0.8.5" }, { name = "torch", marker = "extra == 'jina'", specifier = ">=2.0.0" }, @@ -1630,7 +1638,7 @@ requires-dist = [ { name = "zeroentropy", marker = "extra == 'zeroentropy'", specifier = ">=0.1.0a11" }, { name = "zstandard", marker = "python_full_version < '3.14'", specifier = ">=0.23.0" }, ] -provides-extras = ["docling", "s3", "voyageai", "mxbai", "cohere", "zeroentropy", "jina", "tui", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"] +provides-extras = ["docling", "s3", "voyageai", "mxbai", "cohere", "zeroentropy", "jina", "cross-encoder", "tui", "anthropic", "groq", "google", "mistral", "bedrock", "vertexai"] [[package]] name = "haiku-skills" @@ -1907,6 +1915,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, ] +[[package]] +name = "joblib" +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/41/f2/d34e8b3a08a9cc79a50b2208a93dce981fe615b64d5a4d4abee421d898df/joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3", size = 331603, upload-time = "2025-12-15T08:41:46.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/91/984aca2ec129e2757d1e4e3c81c3fcda9d0f85b74670a094cc443d9ee949/joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713", size = 309071, upload-time = "2025-12-15T08:41:44.973Z" }, +] + [[package]] name = "jsonlines" version = "4.0.0" @@ -4549,6 +4566,50 @@ torch = [ { name = "torch" }, ] +[[package]] +name = "scikit-learn" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "joblib" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "threadpoolctl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/74/e6a7cc4b820e95cc38cf36cd74d5aa2b42e8ffc2d21fe5a9a9c45c1c7630/scikit_learn-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5fb63362b5a7ddab88e52b6dbb47dac3fd7dafeee740dc6c8d8a446ddedade8e", size = 8548242, upload-time = "2025-12-10T07:07:51.568Z" }, + { url = "https://files.pythonhosted.org/packages/49/d8/9be608c6024d021041c7f0b3928d4749a706f4e2c3832bbede4fb4f58c95/scikit_learn-1.8.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5025ce924beccb28298246e589c691fe1b8c1c96507e6d27d12c5fadd85bfd76", size = 8079075, upload-time = "2025-12-10T07:07:53.697Z" }, + { url = "https://files.pythonhosted.org/packages/dd/47/f187b4636ff80cc63f21cd40b7b2d177134acaa10f6bb73746130ee8c2e5/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4496bb2cf7a43ce1a2d7524a79e40bc5da45cf598dbf9545b7e8316ccba47bb4", size = 8660492, upload-time = "2025-12-10T07:07:55.574Z" }, + { url = "https://files.pythonhosted.org/packages/97/74/b7a304feb2b49df9fafa9382d4d09061a96ee9a9449a7cbea7988dda0828/scikit_learn-1.8.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0bcfe4d0d14aec44921545fd2af2338c7471de9cb701f1da4c9d85906ab847a", size = 8931904, upload-time = "2025-12-10T07:07:57.666Z" }, + { url = "https://files.pythonhosted.org/packages/9f/c4/0ab22726a04ede56f689476b760f98f8f46607caecff993017ac1b64aa5d/scikit_learn-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:35c007dedb2ffe38fe3ee7d201ebac4a2deccd2408e8621d53067733e3c74809", size = 8019359, upload-time = "2025-12-10T07:07:59.838Z" }, + { url = "https://files.pythonhosted.org/packages/24/90/344a67811cfd561d7335c1b96ca21455e7e472d281c3c279c4d3f2300236/scikit_learn-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:8c497fff237d7b4e07e9ef1a640887fa4fb765647f86fbe00f969ff6280ce2bb", size = 7641898, upload-time = "2025-12-10T07:08:01.36Z" }, + { url = "https://files.pythonhosted.org/packages/03/aa/e22e0768512ce9255eba34775be2e85c2048da73da1193e841707f8f039c/scikit_learn-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d6ae97234d5d7079dc0040990a6f7aeb97cb7fa7e8945f1999a429b23569e0a", size = 8513770, upload-time = "2025-12-10T07:08:03.251Z" }, + { url = "https://files.pythonhosted.org/packages/58/37/31b83b2594105f61a381fc74ca19e8780ee923be2d496fcd8d2e1147bd99/scikit_learn-1.8.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:edec98c5e7c128328124a029bceb09eda2d526997780fef8d65e9a69eead963e", size = 8044458, upload-time = "2025-12-10T07:08:05.336Z" }, + { url = "https://files.pythonhosted.org/packages/2d/5a/3f1caed8765f33eabb723596666da4ebbf43d11e96550fb18bdec42b467b/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74b66d8689d52ed04c271e1329f0c61635bcaf5b926db9b12d58914cdc01fe57", size = 8610341, upload-time = "2025-12-10T07:08:07.732Z" }, + { url = "https://files.pythonhosted.org/packages/38/cf/06896db3f71c75902a8e9943b444a56e727418f6b4b4a90c98c934f51ed4/scikit_learn-1.8.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8fdf95767f989b0cfedb85f7ed8ca215d4be728031f56ff5a519ee1e3276dc2e", size = 8900022, upload-time = "2025-12-10T07:08:09.862Z" }, + { url = "https://files.pythonhosted.org/packages/1c/f9/9b7563caf3ec8873e17a31401858efab6b39a882daf6c1bfa88879c0aa11/scikit_learn-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:2de443b9373b3b615aec1bb57f9baa6bb3a9bd093f1269ba95c17d870422b271", size = 7989409, upload-time = "2025-12-10T07:08:12.028Z" }, + { url = "https://files.pythonhosted.org/packages/49/bd/1f4001503650e72c4f6009ac0c4413cb17d2d601cef6f71c0453da2732fc/scikit_learn-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:eddde82a035681427cbedded4e6eff5e57fa59216c2e3e90b10b19ab1d0a65c3", size = 7619760, upload-time = "2025-12-10T07:08:13.688Z" }, + { url = "https://files.pythonhosted.org/packages/d2/7d/a630359fc9dcc95496588c8d8e3245cc8fd81980251079bc09c70d41d951/scikit_learn-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7cc267b6108f0a1499a734167282c00c4ebf61328566b55ef262d48e9849c735", size = 8826045, upload-time = "2025-12-10T07:08:15.215Z" }, + { url = "https://files.pythonhosted.org/packages/cc/56/a0c86f6930cfcd1c7054a2bc417e26960bb88d32444fe7f71d5c2cfae891/scikit_learn-1.8.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:fe1c011a640a9f0791146011dfd3c7d9669785f9fed2b2a5f9e207536cf5c2fd", size = 8420324, upload-time = "2025-12-10T07:08:17.561Z" }, + { url = "https://files.pythonhosted.org/packages/46/1e/05962ea1cebc1cf3876667ecb14c283ef755bf409993c5946ade3b77e303/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72358cce49465d140cc4e7792015bb1f0296a9742d5622c67e31399b75468b9e", size = 8680651, upload-time = "2025-12-10T07:08:19.952Z" }, + { url = "https://files.pythonhosted.org/packages/fe/56/a85473cd75f200c9759e3a5f0bcab2d116c92a8a02ee08ccd73b870f8bb4/scikit_learn-1.8.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:80832434a6cc114f5219211eec13dcbc16c2bac0e31ef64c6d346cde3cf054cb", size = 8925045, upload-time = "2025-12-10T07:08:22.11Z" }, + { url = "https://files.pythonhosted.org/packages/cc/b7/64d8cfa896c64435ae57f4917a548d7ac7a44762ff9802f75a79b77cb633/scikit_learn-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:ee787491dbfe082d9c3013f01f5991658b0f38aa8177e4cd4bf434c58f551702", size = 8507994, upload-time = "2025-12-10T07:08:23.943Z" }, + { url = "https://files.pythonhosted.org/packages/5e/37/e192ea709551799379958b4c4771ec507347027bb7c942662c7fbeba31cb/scikit_learn-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf97c10a3f5a7543f9b88cbf488d33d175e9146115a451ae34568597ba33dcde", size = 7869518, upload-time = "2025-12-10T07:08:25.71Z" }, + { url = "https://files.pythonhosted.org/packages/24/05/1af2c186174cc92dcab2233f327336058c077d38f6fe2aceb08e6ab4d509/scikit_learn-1.8.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c22a2da7a198c28dd1a6e1136f19c830beab7fdca5b3e5c8bba8394f8a5c45b3", size = 8528667, upload-time = "2025-12-10T07:08:27.541Z" }, + { url = "https://files.pythonhosted.org/packages/a8/25/01c0af38fe969473fb292bba9dc2b8f9b451f3112ff242c647fee3d0dfe7/scikit_learn-1.8.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:6b595b07a03069a2b1740dc08c2299993850ea81cce4fe19b2421e0c970de6b7", size = 8066524, upload-time = "2025-12-10T07:08:29.822Z" }, + { url = "https://files.pythonhosted.org/packages/be/ce/a0623350aa0b68647333940ee46fe45086c6060ec604874e38e9ab7d8e6c/scikit_learn-1.8.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:29ffc74089f3d5e87dfca4c2c8450f88bdc61b0fc6ed5d267f3988f19a1309f6", size = 8657133, upload-time = "2025-12-10T07:08:31.865Z" }, + { url = "https://files.pythonhosted.org/packages/b8/cb/861b41341d6f1245e6ca80b1c1a8c4dfce43255b03df034429089ca2a2c5/scikit_learn-1.8.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fb65db5d7531bccf3a4f6bec3462223bea71384e2cda41da0f10b7c292b9e7c4", size = 8923223, upload-time = "2025-12-10T07:08:34.166Z" }, + { url = "https://files.pythonhosted.org/packages/76/18/a8def8f91b18cd1ba6e05dbe02540168cb24d47e8dcf69e8d00b7da42a08/scikit_learn-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:56079a99c20d230e873ea40753102102734c5953366972a71d5cb39a32bc40c6", size = 8096518, upload-time = "2025-12-10T07:08:36.339Z" }, + { url = "https://files.pythonhosted.org/packages/d1/77/482076a678458307f0deb44e29891d6022617b2a64c840c725495bee343f/scikit_learn-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:3bad7565bc9cf37ce19a7c0d107742b320c1285df7aab1a6e2d28780df167242", size = 7754546, upload-time = "2025-12-10T07:08:38.128Z" }, + { url = "https://files.pythonhosted.org/packages/2d/d1/ef294ca754826daa043b2a104e59960abfab4cf653891037d19dd5b6f3cf/scikit_learn-1.8.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:4511be56637e46c25721e83d1a9cea9614e7badc7040c4d573d75fbe257d6fd7", size = 8848305, upload-time = "2025-12-10T07:08:41.013Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e2/b1f8b05138ee813b8e1a4149f2f0d289547e60851fd1bb268886915adbda/scikit_learn-1.8.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:a69525355a641bf8ef136a7fa447672fb54fe8d60cab5538d9eb7c6438543fb9", size = 8432257, upload-time = "2025-12-10T07:08:42.873Z" }, + { url = "https://files.pythonhosted.org/packages/26/11/c32b2138a85dcb0c99f6afd13a70a951bfdff8a6ab42d8160522542fb647/scikit_learn-1.8.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c2656924ec73e5939c76ac4c8b026fc203b83d8900362eb2599d8aee80e4880f", size = 8678673, upload-time = "2025-12-10T07:08:45.362Z" }, + { url = "https://files.pythonhosted.org/packages/c7/57/51f2384575bdec454f4fe4e7a919d696c9ebce914590abf3e52d47607ab8/scikit_learn-1.8.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15fc3b5d19cc2be65404786857f2e13c70c83dd4782676dd6814e3b89dc8f5b9", size = 8922467, upload-time = "2025-12-10T07:08:47.408Z" }, + { url = "https://files.pythonhosted.org/packages/35/4d/748c9e2872637a57981a04adc038dacaa16ba8ca887b23e34953f0b3f742/scikit_learn-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:00d6f1d66fbcf4eba6e356e1420d33cc06c70a45bb1363cd6f6a8e4ebbbdece2", size = 8774395, upload-time = "2025-12-10T07:08:49.337Z" }, + { url = "https://files.pythonhosted.org/packages/60/22/d7b2ebe4704a5e50790ba089d5c2ae308ab6bb852719e6c3bd4f04c3a363/scikit_learn-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f28dd15c6bb0b66ba09728cf09fd8736c304be29409bd8445a080c1280619e8c", size = 8002647, upload-time = "2025-12-10T07:08:51.601Z" }, +] + [[package]] name = "scipy" version = "1.17.1" @@ -4636,6 +4697,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f8/95/12d226ee4d207cb1f77a216baa7e1a8bae2639733c140abe8d0316d23a18/semchunk-3.2.5-py3-none-any.whl", hash = "sha256:fd09cc5f380bd010b8ca773bd81893f7eaf11d37dd8362a83d46cedaf5dae076", size = 13048, upload-time = "2025-10-28T02:12:36.724Z" }, ] +[[package]] +name = "sentence-transformers" +version = "5.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "scikit-learn" }, + { name = "scipy" }, + { name = "torch" }, + { name = "tqdm" }, + { name = "transformers" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2c/27/16d127a61303e05847d878b23687f3371868c76e738557fa80b4373a8c2b/sentence_transformers-5.5.0.tar.gz", hash = "sha256:9cec675e68bfe09d07466d1f13ab06d1d79d60a0f45b154baf433bde6ae159cb", size = 444908, upload-time = "2026-05-12T14:05:42.383Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/20/18416624bcbae866ec0b111979766cebabe8e5ff7563ab953ecbaf3ff9e7/sentence_transformers-5.5.0-py3-none-any.whl", hash = "sha256:75313fdcc2397ec4b58297c25d6187fcca5a6b2aeb09570a72eff5a3223d8d58", size = 588665, upload-time = "2026-05-12T14:05:40.899Z" }, +] + [[package]] name = "setuptools" version = "81.0.0" @@ -4843,6 +4923,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b0/0d/ca8367c100c09850379f83645abd60f47c051e6b1e7b64adb953bce96be9/textual_image-0.8.5-py3-none-any.whl", hash = "sha256:dffc85458ca8744bce3f17bddbf582b1e086eb52d111f1063753e8dd316fa45d", size = 109618, upload-time = "2025-12-26T18:38:07.11Z" }, ] +[[package]] +name = "threadpoolctl" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/4d/08c89e34946fce2aec4fbb45c9016efd5f4d7f24af8e5d93296e935631d8/threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e", size = 21274, upload-time = "2025-03-13T13:49:23.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d5/f9a850d79b0851d1d4ef6456097579a9005b31fea68726a4ae5f2d82ddd9/threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb", size = 18638, upload-time = "2025-03-13T13:49:21.846Z" }, +] + [[package]] name = "tiktoken" version = "0.12.0"