Switch to using HuggingFace tokenizers
This commit is contained in:
Yiorgis Gozadinos
parent
fe7d0c5c24
commit
541552215e
7 changed files with 31 additions and 103 deletions
|
|
@ -5,6 +5,11 @@
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
|
- **BREAKING: Chunking Tokenizer**: Switched from tiktoken to HuggingFace tokenizers for consistency with docling-serve
|
||||||
|
- Default tokenizer changed from tiktoken "gpt-4o" to "Qwen/Qwen3-Embedding-0.6B"
|
||||||
|
- New `chunking_tokenizer` config option in `ProcessingConfig` for customization
|
||||||
|
- Removed `tiktoken` dependency
|
||||||
|
- `download-models` CLI command now also downloads the configured HuggingFace tokenizer
|
||||||
- **Evaluations**: Refactored QA benchmark to run entire dataset as single evaluation for better Logfire experiment tracking
|
- **Evaluations**: Refactored QA benchmark to run entire dataset as single evaluation for better Logfire experiment tracking
|
||||||
- **Evaluations**: Added `.env` file loading support via `python-dotenv` dependency
|
- **Evaluations**: Added `.env` file loading support via `python-dotenv` dependency
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,4 @@
|
||||||
from typing import TYPE_CHECKING, ClassVar
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
import tiktoken
|
|
||||||
|
|
||||||
from haiku.rag.config import Config
|
from haiku.rag.config import Config
|
||||||
|
|
||||||
|
|
@ -16,21 +14,25 @@ class Chunker:
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
chunk_size: The maximum size of a chunk in tokens.
|
chunk_size: The maximum size of a chunk in tokens.
|
||||||
|
tokenizer_name: HuggingFace model name for tokenization.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
encoder: ClassVar[tiktoken.Encoding] = tiktoken.encoding_for_model("gpt-4o")
|
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
chunk_size: int = Config.processing.chunk_size,
|
chunk_size: int = Config.processing.chunk_size,
|
||||||
|
tokenizer_name: str = Config.processing.chunking_tokenizer,
|
||||||
):
|
):
|
||||||
from docling_core.transforms.chunker.hybrid_chunker import HybridChunker
|
from docling_core.transforms.chunker.hybrid_chunker import HybridChunker
|
||||||
from docling_core.transforms.chunker.tokenizer.openai import OpenAITokenizer
|
from docling_core.transforms.chunker.tokenizer.huggingface import (
|
||||||
|
HuggingFaceTokenizer,
|
||||||
|
)
|
||||||
|
from transformers import AutoTokenizer
|
||||||
|
|
||||||
self.chunk_size = chunk_size
|
self.chunk_size = chunk_size
|
||||||
tokenizer = OpenAITokenizer(
|
self.tokenizer_name = tokenizer_name
|
||||||
tokenizer=tiktoken.encoding_for_model("gpt-4o"), max_tokens=chunk_size
|
|
||||||
)
|
hf_tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
|
||||||
|
tokenizer = HuggingFaceTokenizer(tokenizer=hf_tokenizer, max_tokens=chunk_size)
|
||||||
|
|
||||||
self.chunker = HybridChunker(tokenizer=tokenizer)
|
self.chunker = HybridChunker(tokenizer=tokenizer)
|
||||||
|
|
||||||
|
|
@ -51,4 +53,4 @@ class Chunker:
|
||||||
return [self.chunker.contextualize(chunk) for chunk in chunks]
|
return [self.chunker.contextualize(chunk) for chunk in chunks]
|
||||||
|
|
||||||
|
|
||||||
chunker = Chunker()
|
chunker = Chunker()
|
||||||
|
|
|
||||||
|
|
@ -55,6 +55,7 @@ class ProcessingConfig(BaseModel):
|
||||||
context_chunk_radius: int = 0
|
context_chunk_radius: int = 0
|
||||||
markdown_preprocessor: str = ""
|
markdown_preprocessor: str = ""
|
||||||
converter: str = "docling-local"
|
converter: str = "docling-local"
|
||||||
|
chunking_tokenizer: str = "Qwen/Qwen3-Embedding-0.6B"
|
||||||
|
|
||||||
|
|
||||||
class OllamaConfig(BaseModel):
|
class OllamaConfig(BaseModel):
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,6 @@
|
||||||
import asyncio
|
|
||||||
import importlib
|
import importlib
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import sys
|
import sys
|
||||||
from collections.abc import Callable
|
|
||||||
from functools import wraps
|
|
||||||
from importlib import metadata
|
from importlib import metadata
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import ModuleType
|
from types import ModuleType
|
||||||
|
|
@ -11,42 +8,6 @@ from types import ModuleType
|
||||||
from packaging.version import Version, parse
|
from packaging.version import Version, parse
|
||||||
|
|
||||||
|
|
||||||
def debounce(wait: float) -> Callable:
|
|
||||||
"""
|
|
||||||
A decorator to debounce a function, ensuring it is called only after a specified delay
|
|
||||||
and always executes after the last call.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
wait (float): The debounce delay in seconds.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Callable: The decorated function.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def decorator(func: Callable) -> Callable:
|
|
||||||
last_call = None
|
|
||||||
task = None
|
|
||||||
|
|
||||||
@wraps(func)
|
|
||||||
async def debounced(*args, **kwargs):
|
|
||||||
nonlocal last_call, task
|
|
||||||
last_call = asyncio.get_event_loop().time()
|
|
||||||
|
|
||||||
if task:
|
|
||||||
task.cancel()
|
|
||||||
|
|
||||||
async def call_func():
|
|
||||||
await asyncio.sleep(wait)
|
|
||||||
if asyncio.get_event_loop().time() - last_call >= wait: # type: ignore
|
|
||||||
await func(*args, **kwargs)
|
|
||||||
|
|
||||||
task = asyncio.create_task(call_func())
|
|
||||||
|
|
||||||
return debounced
|
|
||||||
|
|
||||||
return decorator
|
|
||||||
|
|
||||||
|
|
||||||
def get_default_data_dir() -> Path:
|
def get_default_data_dir() -> Path:
|
||||||
"""Get the user data directory for the current system platform.
|
"""Get the user data directory for the current system platform.
|
||||||
|
|
||||||
|
|
@ -144,7 +105,7 @@ def load_callable(path: str):
|
||||||
|
|
||||||
|
|
||||||
def prefetch_models():
|
def prefetch_models():
|
||||||
"""Prefetch runtime models (Docling + Ollama as configured)."""
|
"""Prefetch runtime models (Docling + Ollama + HuggingFace tokenizer as configured)."""
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from haiku.rag.config import Config
|
from haiku.rag.config import Config
|
||||||
|
|
@ -157,6 +118,11 @@ def prefetch_models():
|
||||||
# Docling not installed, skip downloading docling models
|
# Docling not installed, skip downloading docling models
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
# Download HuggingFace tokenizer
|
||||||
|
from transformers import AutoTokenizer
|
||||||
|
|
||||||
|
AutoTokenizer.from_pretrained(Config.processing.chunking_tokenizer)
|
||||||
|
|
||||||
# Collect Ollama models from config
|
# Collect Ollama models from config
|
||||||
required_models: set[str] = set()
|
required_models: set[str] = set()
|
||||||
if Config.embeddings.provider == "ollama":
|
if Config.embeddings.provider == "ollama":
|
||||||
|
|
@ -179,4 +145,4 @@ def prefetch_models():
|
||||||
"POST", f"{base_url}/api/pull", json={"model": model}
|
"POST", f"{base_url}/api/pull", json={"model": model}
|
||||||
) as r:
|
) as r:
|
||||||
for _ in r.iter_lines():
|
for _ in r.iter_lines():
|
||||||
pass
|
pass
|
||||||
|
|
|
||||||
|
|
@ -31,7 +31,6 @@ dependencies = [
|
||||||
"python-dotenv>=1.2.1",
|
"python-dotenv>=1.2.1",
|
||||||
"pyyaml>=6.0.3",
|
"pyyaml>=6.0.3",
|
||||||
"rich>=14.2.0",
|
"rich>=14.2.0",
|
||||||
"tiktoken>=0.12.0",
|
|
||||||
"typer>=0.19.2,<0.20.0",
|
"typer>=0.19.2,<0.20.0",
|
||||||
"watchfiles>=1.1.1",
|
"watchfiles>=1.1.1",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
import pytest
|
import pytest
|
||||||
from datasets import Dataset
|
from datasets import Dataset
|
||||||
|
from transformers import AutoTokenizer
|
||||||
|
|
||||||
from haiku.rag.chunker import Chunker
|
from haiku.rag.chunker import Chunker
|
||||||
from haiku.rag.config import Config
|
from haiku.rag.config import Config
|
||||||
|
|
@ -20,10 +21,13 @@ async def test_chunker(qa_corpus: Dataset):
|
||||||
# Ensure that the text is split into multiple chunks
|
# Ensure that the text is split into multiple chunks
|
||||||
assert len(chunks) > 1
|
assert len(chunks) > 1
|
||||||
|
|
||||||
|
# Load tokenizer for verification
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(chunker.tokenizer_name)
|
||||||
|
|
||||||
# Ensure that chunks are reasonably sized (allowing more flexibility for structure-aware chunking)
|
# Ensure that chunks are reasonably sized (allowing more flexibility for structure-aware chunking)
|
||||||
total_tokens = 0
|
total_tokens = 0
|
||||||
for chunk in chunks:
|
for chunk in chunks:
|
||||||
encoded_tokens = Chunker.encoder.encode(chunk, disallowed_special=())
|
encoded_tokens = tokenizer.encode(chunk, add_special_tokens=False)
|
||||||
token_count = len(encoded_tokens)
|
token_count = len(encoded_tokens)
|
||||||
total_tokens += token_count
|
total_tokens += token_count
|
||||||
|
|
||||||
|
|
@ -34,7 +38,7 @@ async def test_chunker(qa_corpus: Dataset):
|
||||||
assert token_count > 5 # Ensure chunks aren't too small
|
assert token_count > 5 # Ensure chunks aren't too small
|
||||||
|
|
||||||
# Ensure that all chunks together contain roughly the same content as original
|
# Ensure that all chunks together contain roughly the same content as original
|
||||||
original_tokens = len(Chunker.encoder.encode(doc_text, disallowed_special=()))
|
original_tokens = len(tokenizer.encode(doc_text, add_special_tokens=False))
|
||||||
|
|
||||||
# Due to structure-aware chunking, we might have some variation in token count
|
# Due to structure-aware chunking, we might have some variation in token count
|
||||||
# but it should be reasonable
|
# but it should be reasonable
|
||||||
|
|
|
||||||
49
uv.lock
49
uv.lock
|
|
@ -1206,7 +1206,6 @@ dependencies = [
|
||||||
{ name = "python-dotenv" },
|
{ name = "python-dotenv" },
|
||||||
{ name = "pyyaml" },
|
{ name = "pyyaml" },
|
||||||
{ name = "rich" },
|
{ name = "rich" },
|
||||||
{ name = "tiktoken" },
|
|
||||||
{ name = "typer" },
|
{ name = "typer" },
|
||||||
{ name = "watchfiles" },
|
{ name = "watchfiles" },
|
||||||
]
|
]
|
||||||
|
|
@ -1266,7 +1265,6 @@ requires-dist = [
|
||||||
{ name = "python-dotenv", specifier = ">=1.2.1" },
|
{ name = "python-dotenv", specifier = ">=1.2.1" },
|
||||||
{ name = "pyyaml", specifier = ">=6.0.3" },
|
{ name = "pyyaml", specifier = ">=6.0.3" },
|
||||||
{ name = "rich", specifier = ">=14.2.0" },
|
{ name = "rich", specifier = ">=14.2.0" },
|
||||||
{ name = "tiktoken", specifier = ">=0.12.0" },
|
|
||||||
{ name = "typer", specifier = ">=0.19.2,<0.20.0" },
|
{ name = "typer", specifier = ">=0.19.2,<0.20.0" },
|
||||||
{ name = "voyageai", marker = "extra == 'voyageai'", specifier = ">=0.3.5" },
|
{ name = "voyageai", marker = "extra == 'voyageai'", specifier = ">=0.3.5" },
|
||||||
{ name = "watchfiles", specifier = ">=1.1.1" },
|
{ name = "watchfiles", specifier = ">=1.1.1" },
|
||||||
|
|
@ -4077,53 +4075,6 @@ wheels = [
|
||||||
{ url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" },
|
{ url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tiktoken"
|
|
||||||
version = "0.12.0"
|
|
||||||
source = { registry = "https://pypi.org/simple" }
|
|
||||||
dependencies = [
|
|
||||||
{ name = "regex" },
|
|
||||||
{ name = "requests" },
|
|
||||||
]
|
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/7d/ab/4d017d0f76ec3171d469d80fc03dfbb4e48a4bcaddaa831b31d526f05edc/tiktoken-0.12.0.tar.gz", hash = "sha256:b18ba7ee2b093863978fcb14f74b3707cdc8d4d4d3836853ce7ec60772139931", size = 37806, upload-time = "2025-10-06T20:22:45.419Z" }
|
|
||||||
wheels = [
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/a4/85/be65d39d6b647c79800fd9d29241d081d4eeb06271f383bb87200d74cf76/tiktoken-0.12.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b97f74aca0d78a1ff21b8cd9e9925714c15a9236d6ceacf5c7327c117e6e21e8", size = 1050728, upload-time = "2025-10-06T20:21:52.756Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/4a/42/6573e9129bc55c9bf7300b3a35bef2c6b9117018acca0dc760ac2d93dffe/tiktoken-0.12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2b90f5ad190a4bb7c3eb30c5fa32e1e182ca1ca79f05e49b448438c3e225a49b", size = 994049, upload-time = "2025-10-06T20:21:53.782Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/66/c5/ed88504d2f4a5fd6856990b230b56d85a777feab84e6129af0822f5d0f70/tiktoken-0.12.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:65b26c7a780e2139e73acc193e5c63ac754021f160df919add909c1492c0fb37", size = 1129008, upload-time = "2025-10-06T20:21:54.832Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f4/90/3dae6cc5436137ebd38944d396b5849e167896fc2073da643a49f372dc4f/tiktoken-0.12.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:edde1ec917dfd21c1f2f8046b86348b0f54a2c0547f68149d8600859598769ad", size = 1152665, upload-time = "2025-10-06T20:21:56.129Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/a3/fe/26df24ce53ffde419a42f5f53d755b995c9318908288c17ec3f3448313a3/tiktoken-0.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:35a2f8ddd3824608b3d650a000c1ef71f730d0c56486845705a8248da00f9fe5", size = 1194230, upload-time = "2025-10-06T20:21:57.546Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/20/cc/b064cae1a0e9fac84b0d2c46b89f4e57051a5f41324e385d10225a984c24/tiktoken-0.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:83d16643edb7fa2c99eff2ab7733508aae1eebb03d5dfc46f5565862810f24e3", size = 1254688, upload-time = "2025-10-06T20:21:58.619Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/81/10/b8523105c590c5b8349f2587e2fdfe51a69544bd5a76295fc20f2374f470/tiktoken-0.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffc5288f34a8bc02e1ea7047b8d041104791d2ddbf42d1e5fa07822cbffe16bd", size = 878694, upload-time = "2025-10-06T20:21:59.876Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/00/61/441588ee21e6b5cdf59d6870f86beb9789e532ee9718c251b391b70c68d6/tiktoken-0.12.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:775c2c55de2310cc1bc9a3ad8826761cbdc87770e586fd7b6da7d4589e13dab3", size = 1050802, upload-time = "2025-10-06T20:22:00.96Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/1f/05/dcf94486d5c5c8d34496abe271ac76c5b785507c8eae71b3708f1ad9b45a/tiktoken-0.12.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a01b12f69052fbe4b080a2cfb867c4de12c704b56178edf1d1d7b273561db160", size = 993995, upload-time = "2025-10-06T20:22:02.788Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/a0/70/5163fe5359b943f8db9946b62f19be2305de8c3d78a16f629d4165e2f40e/tiktoken-0.12.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:01d99484dc93b129cd0964f9d34eee953f2737301f18b3c7257bf368d7615baa", size = 1128948, upload-time = "2025-10-06T20:22:03.814Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/0c/da/c028aa0babf77315e1cef357d4d768800c5f8a6de04d0eac0f377cb619fa/tiktoken-0.12.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:4a1a4fcd021f022bfc81904a911d3df0f6543b9e7627b51411da75ff2fe7a1be", size = 1151986, upload-time = "2025-10-06T20:22:05.173Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/a0/5a/886b108b766aa53e295f7216b509be95eb7d60b166049ce2c58416b25f2a/tiktoken-0.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:981a81e39812d57031efdc9ec59fa32b2a5a5524d20d4776574c4b4bd2e9014a", size = 1194222, upload-time = "2025-10-06T20:22:06.265Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f4/f8/4db272048397636ac7a078d22773dd2795b1becee7bc4922fe6207288d57/tiktoken-0.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9baf52f84a3f42eef3ff4e754a0db79a13a27921b457ca9832cf944c6be4f8f3", size = 1255097, upload-time = "2025-10-06T20:22:07.403Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/8e/32/45d02e2e0ea2be3a9ed22afc47d93741247e75018aac967b713b2941f8ea/tiktoken-0.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b8a0cd0c789a61f31bf44851defbd609e8dd1e2c8589c614cc1060940ef1f697", size = 879117, upload-time = "2025-10-06T20:22:08.418Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ce/76/994fc868f88e016e6d05b0da5ac24582a14c47893f4474c3e9744283f1d5/tiktoken-0.12.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:d5f89ea5680066b68bcb797ae85219c72916c922ef0fcdd3480c7d2315ffff16", size = 1050309, upload-time = "2025-10-06T20:22:10.939Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f6/b8/57ef1456504c43a849821920d582a738a461b76a047f352f18c0b26c6516/tiktoken-0.12.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b4e7ed1c6a7a8a60a3230965bdedba8cc58f68926b835e519341413370e0399a", size = 993712, upload-time = "2025-10-06T20:22:12.115Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/72/90/13da56f664286ffbae9dbcfadcc625439142675845baa62715e49b87b68b/tiktoken-0.12.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:fc530a28591a2d74bce821d10b418b26a094bf33839e69042a6e86ddb7a7fb27", size = 1128725, upload-time = "2025-10-06T20:22:13.541Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/05/df/4f80030d44682235bdaecd7346c90f67ae87ec8f3df4a3442cb53834f7e4/tiktoken-0.12.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:06a9f4f49884139013b138920a4c393aa6556b2f8f536345f11819389c703ebb", size = 1151875, upload-time = "2025-10-06T20:22:14.559Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/22/1f/ae535223a8c4ef4c0c1192e3f9b82da660be9eb66b9279e95c99288e9dab/tiktoken-0.12.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:04f0e6a985d95913cabc96a741c5ffec525a2c72e9df086ff17ebe35985c800e", size = 1194451, upload-time = "2025-10-06T20:22:15.545Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/78/a7/f8ead382fce0243cb625c4f266e66c27f65ae65ee9e77f59ea1653b6d730/tiktoken-0.12.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:0ee8f9ae00c41770b5f9b0bb1235474768884ae157de3beb5439ca0fd70f3e25", size = 1253794, upload-time = "2025-10-06T20:22:16.624Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/93/e0/6cc82a562bc6365785a3ff0af27a2a092d57c47d7a81d9e2295d8c36f011/tiktoken-0.12.0-cp313-cp313t-win_amd64.whl", hash = "sha256:dc2dd125a62cb2b3d858484d6c614d136b5b848976794edfb63688d539b8b93f", size = 878777, upload-time = "2025-10-06T20:22:18.036Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/72/05/3abc1db5d2c9aadc4d2c76fa5640134e475e58d9fbb82b5c535dc0de9b01/tiktoken-0.12.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:a90388128df3b3abeb2bfd1895b0681412a8d7dc644142519e6f0a97c2111646", size = 1050188, upload-time = "2025-10-06T20:22:19.563Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/e3/7b/50c2f060412202d6c95f32b20755c7a6273543b125c0985d6fa9465105af/tiktoken-0.12.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:da900aa0ad52247d8794e307d6446bd3cdea8e192769b56276695d34d2c9aa88", size = 993978, upload-time = "2025-10-06T20:22:20.702Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/14/27/bf795595a2b897e271771cd31cb847d479073497344c637966bdf2853da1/tiktoken-0.12.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:285ba9d73ea0d6171e7f9407039a290ca77efcdb026be7769dccc01d2c8d7fff", size = 1129271, upload-time = "2025-10-06T20:22:22.06Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/f5/de/9341a6d7a8f1b448573bbf3425fa57669ac58258a667eb48a25dfe916d70/tiktoken-0.12.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:d186a5c60c6a0213f04a7a802264083dea1bbde92a2d4c7069e1a56630aef830", size = 1151216, upload-time = "2025-10-06T20:22:23.085Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/75/0d/881866647b8d1be4d67cb24e50d0c26f9f807f994aa1510cb9ba2fe5f612/tiktoken-0.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:604831189bd05480f2b885ecd2d1986dc7686f609de48208ebbbddeea071fc0b", size = 1194860, upload-time = "2025-10-06T20:22:24.602Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/b3/1e/b651ec3059474dab649b8d5b69f5c65cd8fcd8918568c1935bd4136c9392/tiktoken-0.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8f317e8530bb3a222547b85a58583238c8f74fd7a7408305f9f63246d1a0958b", size = 1254567, upload-time = "2025-10-06T20:22:25.671Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/80/57/ce64fd16ac390fafde001268c364d559447ba09b509181b2808622420eec/tiktoken-0.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:399c3dd672a6406719d84442299a490420b458c44d3ae65516302a99675888f3", size = 921067, upload-time = "2025-10-06T20:22:26.753Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ac/a4/72eed53e8976a099539cdd5eb36f241987212c29629d0a52c305173e0a68/tiktoken-0.12.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:c2c714c72bc00a38ca969dae79e8266ddec999c7ceccd603cc4f0d04ccd76365", size = 1050473, upload-time = "2025-10-06T20:22:27.775Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/e6/d7/0110b8f54c008466b19672c615f2168896b83706a6611ba6e47313dbc6e9/tiktoken-0.12.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cbb9a3ba275165a2cb0f9a83f5d7025afe6b9d0ab01a22b50f0e74fee2ad253e", size = 993855, upload-time = "2025-10-06T20:22:28.799Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/5f/77/4f268c41a3957c418b084dd576ea2fad2e95da0d8e1ab705372892c2ca22/tiktoken-0.12.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:dfdfaa5ffff8993a3af94d1125870b1d27aed7cb97aa7eb8c1cefdbc87dbee63", size = 1129022, upload-time = "2025-10-06T20:22:29.981Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/4e/2b/fc46c90fe5028bd094cd6ee25a7db321cb91d45dc87531e2bdbb26b4867a/tiktoken-0.12.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:584c3ad3d0c74f5269906eb8a659c8bfc6144a52895d9261cdaf90a0ae5f4de0", size = 1150736, upload-time = "2025-10-06T20:22:30.996Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/28/c0/3c7a39ff68022ddfd7d93f3337ad90389a342f761c4d71de99a3ccc57857/tiktoken-0.12.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:54c891b416a0e36b8e2045b12b33dd66fb34a4fe7965565f1b482da50da3e86a", size = 1194908, upload-time = "2025-10-06T20:22:32.073Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/ab/0d/c1ad6f4016a3968c048545f5d9b8ffebf577774b2ede3e2e352553b685fe/tiktoken-0.12.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5edb8743b88d5be814b1a8a8854494719080c28faaa1ccbef02e87354fe71ef0", size = 1253706, upload-time = "2025-10-06T20:22:33.385Z" },
|
|
||||||
{ url = "https://files.pythonhosted.org/packages/af/df/c7891ef9d2712ad774777271d39fdef63941ffba0a9d59b7ad1fd2765e57/tiktoken-0.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71", size = 920667, upload-time = "2025-10-06T20:22:34.444Z" },
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokenizers"
|
name = "tokenizers"
|
||||||
version = "0.22.0"
|
version = "0.22.0"
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue