diff --git a/CHANGELOG.md b/CHANGELOG.md index 3da7b98f..7ea10427 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ - `update_document` accepts a `uri` argument to change a document's URI. - docling-serve requests fail over to another instance on transport/5xx errors and skip instances whose circuit breaker is open; tune via `providers.docling_serve.max_attempts` and `providers.docling_serve.circuit_breaker`. +### Fixed + +- Concurrent ingestion of the same URI no longer creates duplicate documents; the URI is re-checked under the write lock and a colliding create becomes an update. + ## [0.63.2] - 2026-07-03 ### Changed diff --git a/haiku_rag_slim/haiku/rag/client/documents.py b/haiku_rag_slim/haiku/rag/client/documents.py index 8266ce85..f49effeb 100644 --- a/haiku_rag_slim/haiku/rag/client/documents.py +++ b/haiku_rag_slim/haiku/rag/client/documents.py @@ -123,26 +123,48 @@ async def _store_document_with_chunks( async with client.store._write_lock: versions = await client.store.current_table_versions() - created_doc = await client.document_repository.create(document) + # A concurrent ingestion of the same URI may have created the document + # while this one was converting/embedding outside the lock. LanceDB has + # no unique constraint on `uri`, so re-check under the lock and update in + # place rather than inserting a duplicate. + existing = ( + await client.get_document_by_uri(document.uri) + if document.uri is not None + else None + ) try: - assert created_doc.id is not None, ( - "Document ID should not be None after creation" + if existing is not None: + document.id = existing.id + document.created_at = existing.created_at + stored_doc = await client.document_repository.update(document) + else: + stored_doc = await client.document_repository.create(document) + + assert stored_doc.id is not None, ( + "Document ID should not be None after storing" ) for order, chunk in enumerate(chunks): - chunk.document_id = created_doc.id + chunk.document_id = stored_doc.id chunk.order = order - - await client.chunk_repository.create(chunks) - for item in items: - item.document_id = created_doc.id - await client.document_item_repository.create_items(created_doc.id, items) + item.document_id = stored_doc.id + + if existing is not None: + await client.chunk_repository.replace_for_document( + stored_doc.id, chunks + ) + await client.document_item_repository.replace_for_document( + stored_doc.id, items + ) + else: + await client.chunk_repository.create(chunks) + await client.document_item_repository.create_items(stored_doc.id, items) if client._config.storage.auto_vacuum: client._schedule_vacuum() - return created_doc + return stored_doc except Exception: await client.store.restore_table_versions(versions) raise diff --git a/tests/cassettes/test_duplicate_ingestion/test_concurrent_same_uri_ingestion_creates_single_document.yaml b/tests/cassettes/test_duplicate_ingestion/test_concurrent_same_uri_ingestion_creates_single_document.yaml new file mode 100644 index 00000000..cfbbc550 --- /dev/null +++ b/tests/cassettes/test_duplicate_ingestion/test_concurrent_same_uri_ingestion_creates_single_document.yaml @@ -0,0 +1,82 @@ +interactions: +- request: + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '109' + content-type: + - application/json + host: + - localhost:11434 + method: POST + parsed_body: + encoding_format: base64 + input: + - Duplicate ingestion regression content. + model: qwen3-embedding:4b + uri: http://localhost:11434/v1/embeddings + response: + headers: + content-type: + - application/json + transfer-encoding: + - chunked + parsed_body: + data: + - embedding: WGS+uZDAnruKOYa7BVIVvLgI4LqYS3c9wjEBPWxHgjsHUw88XzCNvJpiWLx12DQ9Le6pOhzL97zxI9A71NIkvRS3XbwUopa8TXFhPGcH1jhkgsO809/vPIX0hzziKDu6vyVJvZYCwjwxQ9e8MhIUvQvGXT3fCTu8uKQ3vP89Xr1lmKA8CsZhvO1ptzoV5Aa8fKJ9Ow8EBLwwbT48hQLtutTSf7svRw+9dgCWPCfqnbrH8xG8GETbPNB7ODwF9jW9Sgf0vDCagLwqZUo7czyCPM0Eb7xXYZm83KYAveNiUrxIFZg8HEBCu9L/Z72OTY+8jttWu6IEVjxLrgY8ZoIpvLoUp7uTP/289MFZvN0V3TtlcNc76PgsvE02zDqnblc619cuPO+6vDyROf48WrUovWoJX7xRRdY84hBSO5JMXDzA9QQ9yKSJO7kIkDr/dsQ8FwrbPD3P2by06sc5eKtLPIvbxbzhzVA8bMyBPEtP2TySNKQ8pWOpPDxQYbp90548rtGOu+yej7ztl5q8xUsLPDAjqbuuF7s7g31RPKCD+jsnoWE8KIoFvTKqn7wdShk7zB9NvOl9rjuLTpi8IgbBuw4NPDyuOLQ8HcGrvG3/ZDsXRMY8Z2qfPBqySzySebM8CFs9vKI14jw+MQ08F0leO9QQlTtwXni877SzuzkUarxKnUe8lOwHPNeg2juDOkq8BGQiPJXwD7tAKPe8dBcJu7YC87tjP0C8ppw8vK6UQDzmXmo7iXoIvFk1lTzcnQk8v7ZcvFRPADyaCrI7W+LgPIo1KTx70qc8CfefPCZzXDzi9Um8eNR5PK1tHDw2T9c82uR7vDKghjzu80o6SEVeOzd4mzvXGkQ84Yi3u2xBSruWdKO6unJiOw43pTsYbPq7MxoJvdlLFjxpz1G885qZvJR6+TuII3i8y+XSvMLzsbkt2Ni8OmUkvCbZm7zrvXA9y7KQu0ifrTwxUBu9OMgzvM0o3zw6Qyk8FsVTu9pr6zsvnmg8KjsrPBPqRbsc35s7tHmLO3gmEDzus8C5/L1HvBrDYLw89Qw9ICxzvLgpTDxecTQ7MlpwO5AdjTw5tjm8rrs6PDWYXbqen626u5PuuxzwnTvzPgG9oxzNPBveKLwMduO8ONU9verCuDtUgqM7WVmyvE1riLwATZU8XnwDPN57JjvbVMU7+BCLvGPVRDy3nha97lpdPDvCPrsdPJS6mIqNO8TkwryworE8nOrVPAYdqjqOnrC7o+3Auy4nZDt3ZQk8OyZEu88B/7voo0C9RackOuFDtrsNhfu6Ee/3O808ZTyPkL28fZUpuqoYqLoeKBU7PpxtvJLN3TtZVU08yWcCu1DnE7tlLy68wcUTvCYnz7t9xG+80bWmO83IzLugciw7JSCIu7606LsOkEG7gZSnuarhkLxT95288LUKuDv1tzv7/s+8Ku9PPYYxD7yz+uM71AKpO6np1LuSdKe87DUPvP8Atzsu0/A842YRPaSimLzqxDs8MxuqvP0EBLzewR88wB1xPClhizyAut47wtfTuoloabvHFqk8NaXdvJojmzzPnqW5HHMyOjFEwbviUds75gqNvKLMN7ypRaO7M+MBu2uRQrw1jXo671/KO1oVKrzuId08kHiMPMHl4jzkYT29JG61O9O/NLz8E1e8R/O+upv0Z7wIhlM837/8vLPgeLttpKS8vUKFvBLHsrxC8sQ7evwHvVkZkbwHIBS8b+r/OqszlDw/yZM8BAmXPHGL7btGgTc9+J+jvAjZLD3yv0i9bwocvCa6ezt5xYY8PgCWOQ5oYj0JkxK8+ZuzO2qsBTzNKAe81SCUPMUzXDoIQiW9wJVyvCqDvDzmxwc8tNUvvBWyGbt6LjO84UamvPdJHb04q6G8NucmPRdV5zw7TmS6gr6bPOI5wDyInbA8PmV1vMxcEzzQsy48AmOQPIDl5Lw6rOe6TVXevPfwCz1st4a6ros7vT2eiDxtNlg6Vd+JPI41VbtfsDK8YESBOpB9yDs9d9i7bEv3u9PxnDsiV8w8rXAOvDlEV7v0f7k8XHhwvJzesjtBFbe8hsh1uzGvR7x9saU8FEp/Oop9MLq+wik7D0GKPGDjT7uhTog8yCQ5PPOCAz2Ni7Q8L0ZivLoRET3d82Q7L/k8vUp3izw77Y67Xr3AvKe3CTyrrE89Cwk+PMl+XzyAPQs8JNA4PEqj0jsy1jO935EmvRpH7DyE/9u7MJYGvE1gML068yG8Xba8PBZUJ7xiA9O5YFRmu+zDIjylNLM7SzR2Ox2kIDxSKQo8nBEpOx5JTrvZhsM8FXMAPamyzTzorai8SQgmPUOotDrfWhE6Lat/vEJ5zbz6pde8SGvFvBvuXDzS78o8O+ypvB2BlTx1VRQ9SwG4PDyxFrzpJsG7QB8rvUZRjbzO9Ts84Z8eu32+4rxABa47eQVru1WFWjtRuxu9XapVPI162r1w6lU8/W2du0/Do7mx1xu87jkbvA963Dwg7UY8mtczPamIlTzYRki7SyPpu5R+wrzF9MY8ApbbvEXbT7v2ltM6QqSTO51w7Dz6Kz87C58XPSc+EbzLISw9MwUfPfTlDTxX8FQ8V18VPPIKsTzoH4w8+TANvPVOfzyRxx09cmOjO7Fpr7zMivS5ak8kPPj/ODzt+hi9MK6au8JRjLtoTH28w2SGu7x7HLwZCHw8c2dRvE0FArtWUJQ8CdbuvC2TpLySkI68FHLxPO761Lk6QZ28WKtPvZRi/rpDTeK6kNg8vO4amby/PVa6UhoDO9x4Ar0LhRA8XhE5vJ1g37zgmH48vnGFPLYcvzygdHu8xryoPDE9HzylZgC7Uoh8O1nOoDzg3hg8FmzAvLMYwLyQ7XC7r2qZOyneljySBKm7GTM3vOI8STwjUC+87qpGuySyajsFFJS5+9hkuygAmzy5ZR69d+z4PCuQpzs5U728IzuZu2tR/7vBdjY8NgBwPE7S0jyIEUe85RcvOyNE9TtfJuw8DJFCvCMkiLozIpY8Yw7PPH3RuLygXAO9vgrXPDQq4Lx7U8C8bZTrujRaIz3LQWY7diqDu3oaujxu1Hs8UXO7PJBLsTsYVbs7YBqGPNffDjxqzyE9AiX0vBWgPbzLTpI8JyGpuwN0VbxG2Rm8DRNDPPF70rsRqM+8bEgtOnCUljz03kM84MwXO7/TyryQIKi6GlBevU97ADz6+i89f5kXPVnwbLmSPCm8sKpqvGsBdLp6YFA8512SPKf04Dz6TSK9JQrqPCu8AbwZZDu7RqO3uw7/mzsFmQY9dvsqvZRibrwzz7a8ECDWu57dlDwV/Z+8kSv9urMh6zs8S7O8GZEoPAd8Cb2TgKs7mBrGvNUiF7w26tK8MTWTOQlb0bvDxzs9+CSPvChh5zwC4KK8DCjLvMMvzjy/oYE870/tvFW93TxaNno8RgYrPad8AT18xsk8YzPXvACs9byyt3i7zecpvPhPnTwVbwm99HgSu+yCmbxgyii7c41xPIkjDr3BIA494F/fvAn7Fz1+4Ce75J2PuljzIzv20Bk8GFQ8PGn5/bwWQ2g8MEpRvFlXwDqYEDs9+62dPIcRabxKuB49K0S7O149mrqwAt47amFFvZPZQjxNmPY7Rbvxu8Qvbjyb7cw8hoSnPOzJpTyjEbU8chSLPAsUNzyCE3A80iHaO4f/uLtTrgE9dw55OwzO3zxj5oc8q6dFPZ4NgDw7LBo8CPpNPJgfnruDzg29RomZvC2/FLzrtj07gyuePF97z7pn32M8QCrYO9YzNLz9ZoA87K8lPMxpoDxwz/Q8qkhGvYXt27rL7C68Nl/avJwrhbzvssu8TcM+u4cAHrw6ZBW8JLx5vG3LgjxVex69gAZQPM7CHTykxT68Uc/uu6aiJDyF4YS8SPO0O1BrZTwUXQu8VPc2O/Tm9Txip0C8unGLPMpx2Lts9aS8UV07PFrzgTxNLdM8ZbDTPFMfwzwNDyw83x0PvcxUizz6DFG81fE0vEZUcTyE6+y7MAMAPPlUwjuJsI48GxKqOu8Ozbyv7hI9Kn8wPCWkZjwnAEU8Dy/7O74t6LwPSbu8QHM/vBVNrbx4j2a8nrkPvHv2KT2Ca1S8HCCHO+VbrbsDC7c56Q0KPV6tMrwWrRM4MmXdPII2SjyvDBK9vtBJuTOQAz3z/We5REa+vCtk57sWEFO8AnQ/OyGKAL22jqG89mHGvLLSgrzgOGM8hufbO18fGbzHHNC8Y4dLvPHChjy+dOA7THg2vKuJlTxlsQ+89wCxvK5PU7wTOwG8Z3/4u1ohZLz+LKQ8nWEbO0+ZEzxqNaG75Q4RvAMgYDyEseI8BRTgPLTKT70/Y0u9dj1hPIDhVLxx5NW8LeTLPLxZJDvCWVa6FFH3O6FZATp7zH+7S/yOvKDNXbsRsN47ZCenunlOrTzPg2684zb3u9GbZT1zkgk9gmYuPByxQLu8bII85WK5OihDOLwRfXE7alaZvfmgOL0bfpK8n36IuT2mA73dMW07TACtvKHVz7zBbvW5EthpvNoVqTqyDcY6KfQavC5jgTsNlWo9btaTPPS6Zzxiz308JAqnPHX9dLyB3Yy79EOKPPCqKDtvZjo8556QvOUJc7w4VWG78SQ8vORXp7xlN8A6C5l/umVdIT1j5Cy9eDseO9l1/jxd1jY87v+HPKK9brfLAcc7Oo03vEAEGj3KOxi9xxCvul+5Tzx8Pr46fNAyPYNR0roPAY67XuGzO+I2ILpSvVs8AtWTO4rhzDyAyb27ESwGPZ/CZzsjqK27NKAAPUSrPjzwa+y6EJSbvPynhDyxDuO8eT9NvNwkNj0fTBS86mdOPO9QRz36Dzg7RsNCOuZUpTyPXny7OI81vPuCyLxO8Ya7WXvQPPiypDu3PyS9Y+5evDC6Er3ZWb88Hg23vOCV5Ly/J907HkkyPC7zibz8Klq7WKM7PcA1FLzVwB28y9dMuk8mQ72Yqc46Otqmu+MATrw9MJq8jUjaOidvAb3GWc08vgd7vO1CpLuABQ88j2w9u1j15rs5is68+NXqu2G5Uj2BDzw8+X7FvJeiNTwY8Zs8G+qkPBHqOj158j08U9NkvCZ5/Lyi+JW8spA/vCzZS7zyvd+7BL5GvIHAALxuYK+64m2WvMlFWrzjheG8eehrPDqiwjzO1Zc8h2ENvD77Xbyhxie7byrsvCTwSDvxwxO8PvWcPJN0j7v5shg8smMuOxubG70fDrc8JJ/su9lv7roq/kM7DHPFPPsNoLxip5S8yzngO3T6dLxNVEG57c+6vG3JSjrhrj69yNzJuxnIYDrtKPO7pMDzO+AdaDsxQTk9siAkvEE0D7yU9JI8UkOKOpW8arw/VEA8y8LfO/IT5bxxpWG8wHTvvKKp1TxUm/I7WabdPEdLj7wemrO7cP+tuuzgX7wYrU68zo8qPGQHkDw+rJ68EcGgPFjeFr0Jg7S6IO/xvMW9Czx1F2u8ESqjvB3yAr24mM+7H6xjPV78EDt1bGe85ZvQPILj0DxtmU07TxzIuyt3UTwEJPw8mrNQvUEF67tBFC89Vy2bPFFfWTxzAqi8X9+mPBIRo7zu6HC8GiLBuyLAXDygPOU8+JgIvFEKP72LW5076OAwvFsT9DsHhx07OXwDPbYRBrzifIE8HKAdPCZqB7yLo+w7FZ5gPJrZhzyTBXG8eCCMvPJzzbyhu/g7eXyqvI4AC716rcs8TvWHPGekAD1ydwG9zJGzO+T9rjqKAic7j6ObPHiMhztjAy68QledvB0uF72v54e8oPEJvbkco7yaNtu8dCFLvcfTTLwrzdm8L7UMPR3VBzzzQh47GNKXPNbS2rrIy2k8RlLhvMMB07zeOCg9II19PBJ1hrxrckG8scGiPFomgDvJjqA7jXovuwHg8DywtcE7qX2huxg+oLyptJ88UXa2O9QBKrwrjEA98MCovIN0FLuufMa8z78yu9H0lDsndQm8cnX7vJ8nOrxZjRe9VM9bvIY+5rwuv1s8iWIcvEvKwrxvWzI7GaMyvGJenbtoUue8Os8TvF7lVjxSonc7eCNkPA8IizyaIQU7rlzrPFYO1zvvOaa7jFMGPNJQ6Tx7lzW8lL97vFoNuTu0E8C8zEFVvNcAWTw25Sc9j36IvCn5eTx+9pW6fpLTO6CCfbxtQM88HtjwvHNeoTwKBwe8KGQcvLrKqTzYllU8+wSGuzrQRTykHmu7hkmAvP8lUTxDQvq6juu9vD9lCznTxYA7MdgYvRYP8rxGXw67ga7POn2VzLxauj67aNF0vDw9rDyef5w8LGZQPHshjryWlcM88nUqvGcFu7vsFas8v/34OpkS2bwswZK7IRPgPBcgkDpEaLy7RBwyujXX2zz7u0898omTu+Tb1jxnPDe8e2erPIRF87x9U6S86k2NvB8fGLy/bfy8r/P4u1JtT7xrJxe8SJu6vDbVUDxwpj081m2vPK3MsTzzl0e8R1gIPQyMubukkVO8TlWPvBgA5TyteQS9lanhvM/n4Tw1S9q8eqFCO4ukQTw4ks28MYUevDxujDyx/p68pFmuvCDoSLyGe5G71QNfPJ3CtTzaNWe7FXLLu6tIozzijdo8+IkcO26j07yyrpU7z365vE9voLy1PqI8eof7Or5GF7vHchM8HCmkPFjYNzoJ+kk9RuOWPJsSNb0KtIg7I5NWuxcokrvuR/C8mOzNOzG7WryjuSg8kq8LPP5QHLuyfB09WDBbO/QXDztriuU8zFAdvWqERjxstem89KqevDofvDuxnCK8bInoPM64kLv9uAc8i3itOS9VxjwLSAk83xyEu8urrzwV9Im8DIK3OW8blDzsW6C8Q5qlvLo6fTxMh9O8/AsZPEmXwzx3qKg7xj6butNrHTrFnaS81bKKOxlkcbyomd28xp63PBaEkTv6KVm9ieUBPepjh7zxKO675PgUPP6eQrxPZAu8cUPOu5ImILqVGgW8ScPKOkUk7LzRACS8oGfLvL9faLyz/pS8o3lMvAc2BDzLU488lvicvOv1KbygoQY8xltCOoialrxHN/W7DcGfvLfBFLxzuR88Vsh8vC6EnLwVdtq8mTnHPM0Np7tB1VK7l0bMPPnpcryY0N+7Ijo1vDAtxDoONKo8WoOKPHNhYDwhF1q8p6c2PCkZhzxtIr88spSGOz3E7jtDVWy86gzWvGrl37wcW9c8XLDHOw7UHT2dxCK97dcDO1Cq7zwVbxK9qekRPRmEQzxxmiQ6UtsmvI1k6zt3FwE8b+mkvNoPqTxE6TK8+UiZPOPMiT3O2dq8eWxsPCBbcLxRJkk80rAKPI2yCD3CNcg8WzmFvLJ6dTwz2jo6/YhHPct+uDs6zim7tyjaOxj21DsCWOC8HEYVO8Sllbv3quG8cTIHPLe1G71/8wE8R1aFOE5b9zxbYjg8gNw+vV98krpvnIs7KJzouvw8ILvLdVw8X+YzvUR2OjwDP6A8zTQcvXL83zxBIAO9/1whvTSfOjzDJj68pWh6OYQlML2+ToQ683+DuwYfeLzlIEK8neZDvVxC1buvDVi9J5PbO4fHRj17RLO8b+bXvPihHbwzDgC8I1EJPECX7ry6M9i8TrFoO+5PBj2V8R87Ady3O8a5GjyyaQm7B3/kvOmcAD008Mw75oSuuxZ4jLzPiK48vdlVvId3F7y6jtM77EhQuvSrCryV7js7h45eu5OwNz18y548OR0evWn+5DujVXu5xf6Su1k9O7xR1NS7jMCovHxkELyIOEE6oyeJO96h/roFHa66m7N5vNAPMLyvhXQ8FxYjPG5EC7wgpBy7AJswPEVsDjx8ErM8Hvr4PLe/lru70Qg9RSJCPT73ObxVHQu9WQe8Oo5Rx7se9N67ofLkvL2/JrxkX7c7OKTJPCwJFj3ab347cuniPP5k+rw2iKo8sj1XvBmxeruPGZM80xUtPZfhITxQlfy89KAkPHqBIr1x6Mm8s3tRvBHIdzsXGpu8vGfCOymcyLyuuhQ8iiikPHJuqbp5LJO80sYRvD73C7zWzTm92lxRvOGrc7sviz+8ieagPII/njy78y+9ZSIWvHzxVzwKJGe6QYvDvDxVx7t4EQs8AJSSPICJ4Dy8Qum74nY0u+ZXybw7Zl66DhJMuwkngzyXULw5Q7qlvAnmqTwEvno8XZKmujSBI7wtP4Q8At29O0Jo27oh30W8TzpMvGFOnrwYtCk8nZHEvEmnGbxkf4Y631gOvR17tDzLMW88J0jpu3P/uzy1ij88yxrquxKM97stN8c8c0+4uj4JiTu/y/I8oA3mvMBsDzv4HE+8u4ekvCQpGzxACIE8GM7pvL6A6DwCRec80dX/u7NaUrxjX2I8yMXFu32bvTyVVty8en6JuxLooTpxa/m71tWTuweEmTxUyBy9p6GUvJFv7Lxju7M8vvFnPGZ7MT2lufK4oOJuPFibezycybE6DEpcvNHp6DzENL+7KRgrPLEfNrz66LW8k0YePCfN7DvGzt88W05FPC9gDjxsQTC7MbqYvHtXyjxk7Qo8Lgsxu3zVubtyodU6dzABvKCD/zs6ejM8OsoSPaE6AL2KVou8lUyRu5CtsDzmvM28lCzvu0VZBr14FI68jC6DvClSzruBR/I5aU3lO57Vrrv41uG8eoe3vCmP3jvsV+u8CigfPeck9DzCfZE8W8QRPUblfDwMmEE7ye4DvfMP4Dwf5o889cEsu8p8qDsS2ca8InUsvC8wDT3rZsy8+1lDvM+nhbwqiyO9uo0hvf+jzToMRvA8COwqPKGJQjzr0M68K9PDvAa0Czw2acq8UW4kPHMxZbwX6Za6qbCNPCJEHDxsYse7kM/ePF16QbyUGCS7vV/qvPy1vzqWg9+8FCL+O6kTOzxYtQM8OAhiuyXfXjuMaY+7XRuzPI02hzzFldo7x486vN/ywDyPKec6v2vqum9ukTym9Qw8ynvEvLGYnbtupHs7vm+fvO0MA73C3fC8u/0jParppbxaWHG88uo3vK77TLxJNSM7ZS4wPT0xcjtL62i90LYAPdJn4Tx6JB28TzG9uiZv7TtMhn48MMO1u3xDID0zaEq8yeDsPP496TtUdaA8hcQ0veCCKDvGGbS8w1kCPdH17TsxXSy8uIiePA1MBrzWYx29rJ8jvB31t7ycoOo63w3aOy6ezTviGDK992v6vH45dLvP7vk82WCivFhbtbvQWq48fN+UvFurhTw8P+o8ZVVZu+o407zyY5c8t+hDvEzOzTz9Ldw7cRt+PHQA7jwjTrk50ZTHuyBiLj1OiA28bd8KPXOwBrzMeRO8F9ZQvJqkCL0hcaw75sRLPNYZGDzhTZg7Tg23O0G7gDuMKa+6adW5u7Xh3jw2KGc7zHkhvKPpXDu9ax26b/sQvRGigDy/Kg68SbrgPC8mijwqCvA8PtKFvD1HCrtTQqq7zWV9O6G5GrxG5Ua8phfJuwlPJTw7s7e8tDSUPLY3i7xGibg7RlKDvISfDD2GuZ48UmMYPQR9eDlTlsY8cakduSEIBz3Ymj67NKYlu0d6hbymg+y8PfWjvGNdAjwZAnI872Pvu9/u2rxuKxY7Wa/CPKTRtDxIQr+8MR2LPFfRHrwd8vG8K/XsuiP/WzxthNe7QXPMOWkojbzotuM8wevsvHje4ryDEoC78njDvDAtyzx3KO088lVavGtZKDxVsxE9q/pBu6hMfz2jvIw80i3eOmMkjTyG/wg8JD/Fu7YJJDs3NKc8aTm7O3x3y7vhHnG8vkslvBM1Kb3T9rI8lWZhvHuePL2ZD608F6OFvNZTIbvs4MI8zJS2vKcLID3LvzO9UwwHvXrEwjxqhKe8cWSLulwiKb0riws9Qn0QvTTh/LuHz4Y8ohY0vIXKvzzhSRG8EHlBvF5HDjmpqbw8GF+5vCbdsjw5uB88LljVvGslaLybZg68E8KPvOymPrzWjGk8/AE7PLvcPzxxeCi76q1HvSA8gL2FS8W6WxPsu/seKjz9P6I8mjWRu+b9ZTyqemk69a6bvPRc1TvrJMm8QoV5vFoZCDy02t47HrVaPNsZBz1lXRU8IMeQuzyGHzyWHRs8z1olPS9VizxtTIY8sqbpPBJmaDxkMmO8vvrVO4h2AD0+Bsm8jh6RvPCcrDxab4G8fRygPOyYczu5VMg8z32uu8AHbDurcuO601qCPJflJbzTnbK8pb2jvDOkeTnwoC08IxnDPG1knryztOu7GrMfvZVs3zxA7D26s83LPO2shryOSDq79I1mPJ8PTzxMbiQ8X2zIuyqonrtV0PW8mMu8PKk77juIQp48y7vxuQv9nDxnYZE8cL5uvJlEiTxot7k7483HvNnL8rltpl+72aAjPHEVqLojsda8Gj3fPLPO9Dz9tn66z5/zO4Vb37uYAaI81cy/O9gKfTgx1Vc8YUCquzy/+byNQC08NHd+Ou7q2ruD8OI7wsSIvCaEJ7z4BPi7DoAsvF/cf7yKjbU8crHYunbyjbyysam8nOIYPA+/xTt0g4g8GxGFvX3e4DjoZsa8sc7WPIghIT1G9S+9PaH+upunQbylnaW7LDKXOuqUijxHOLm8wJ6mPHbkXbr8zAW9wC1Du2mYyLwLGzW9VPAJPaoknrz6Ssy7f8gFPMPwYDwgWrG7oRjNPD3nfLz7YYK8kQZjPED3QbwmxnS8c1jnvGHPkjsttMQ8e7rivOlhdjyspQ09xzSDPM01Gz2qtlG8TbHIOX3DOz2SqBq8J5W5PARSCb2sxoS7gxp/PDAfYrxyrpe8hIrDvNZpDb3FWgI8W71OvaNrXbyzj0O8ruD3O3k84rsPWRy80E5TugqEk7tVQig8st+xu/uTsjt/MKy89l8EvelrujzAIaY69rCLvHcDcLoKXum7f7dwO1vEj7wZwE883QU3PFHggLyiqxK9jfTsuwVNkjo6bYU7gLE5PA1f4Lx+K1i8YcRdO1XQc7pH/wy8jWWXvO/hHbx1hhC9uyKYPA7SL7ztJBI85lOBOrd9y7oFlwW9g96bvOquIj3GwsQ7+5P/Ol5omjspY627IuBAPYiKi7u2Nby86JvEu5Irrjt/1US7BVGIvOEXiTyin8+8rjAcvFTbcLpMNrm8hRirPOPkErx+pdg7nClQvACthLyxb/u8zIn9OsUH2bxez7K8g8f3O8RK1TyChCG92c4PvA4PpzxOpvw7savwuoqhY7xiFhG8sL3XPIJSpLsbmFg8SgigPDrptDydwPg8o0+POo+65jwG2U68P0DsPIUSLrv9eBG7DsEHPR8phbwEVwE8pdMQPdcgdbwoKpK8gfkrPDAelLzsVTk6EXZwvEL4Uj1HywC7zBSwuyKgQTz+4ke8XXABPL/mBrzgnZU5cQUwva0bXbx1Q4M8OBoAPV67fTwibIq8eQC9vKouxbzW5gq92HDUvLHs2rzM85k5OGsOvF4sDLx5beW8vvCXuwfcCjy3tgm8aoDUOaJNnrpwoiC7HwCiPHiRhzwfUhs80RqtOcXflDx/iuO6ts+hPMA5vDzdBQG9YKgSPNH7Rb2F/+g7zzzFu4QP1bwF+pc8vsvYPPFLFL0HPua8YeGJO2Q+KbwK6yK83nfDvJb5IbxBroS8iLXmvFW7VL2TyBq7L2tXux43vrt5HIS82UOTPCH4qzyxUQQ8HRDIvD7XYrwLhsS7UrzOPMWsN7s6uwu9wZc+PBHkl7yLsOG86QQYvKNdDb1Q9QI7H4I7PEfyiD1/haE8uYaDPIx1h7zkG+I8+tT2u6IFYjy/mxM9FJvQPI1QNLy1WRa9yAJNvIM0ybsMS9U84NKjvEaqzDxdasI8r/dgPAC1UzyLMRS90hlDPJX9sztj5fi69PStPCVY67wHr/48YcKQPG2aQzv0vjA9fa//u4LRwbyOS388HgF1PL7nxrw7UsI8UDU6vOu6jTz61om8qt8wOwKrHT02Rds8esHNPAbRDbwSUSE8IuagPODl6zutSYk8P/4PPavqN7zGuGm82msSvPzDAT17sae7e3XBPOsb6bwilJG8BCeBu6h8Fr0wGcC6ik68PLevC7xP1I47GSsbvVSMLLy2Zog7qoqlu9Fj2LusPJc6nMexvKtXaTtTF+E720pHvJKIHjv+CTS8cXg5PBvzpTxl39O8ynSMPKt8qjvzdcK8hZv0u/SKD720nIq7zo7RO0hSRrt0c0g8D3YTPAJIyzychAy8BRxJvD51QjzUDB48Nam0PJx++rvaEZ88xwkNO/mzMbzepCW9XPaXvO6Sxrnsnsy7kvbvOxw97bsreo8813uZPCbGXTx+jac6OFB0vP28YjxEqtK7hRFHvE+bOTymtHu8K4AkugWOojwxB/W8+GcPPRH0Czx0NQ691j3oufCKCL0mtAw81aWvvP4PQ7tXWOw7ZV6HvFKV/7xNZyg8oRrpOV6THTzCuEY8qOxGvBJnP7xu6oG7BnG3vMZVljyQNy+9xcq8u8nrsjtqId08FEvbvIIMTzv94ZK8tVswvAdDDDyUKZa8Dtn0vO3ZmjwPWgO86FSIvA67XDwPMYU7GOmPO9ptMz0m1dI6FiwWurG+vLzOBl28xraKO5q+hTz6Yz68aoNTvDlUgTtEQvi8MxaZvH0Mw7z4PVu8DaZ+vDIoKDyto4+7xXBpu8FrBbusn5E8ljTdu2JOKz3GCOk8dM8aO7c+UDoQFX48rlFoPBumTrzS2/k7UFpBPCQdizxCWeS8dLG7PJO2R7sJIo68myotuQAyurjCMYU8oqNPPIyNJjyDoKU8noKePOmfgrxfVzC9J7cjvdQLQTzgEV+8jL2UO6lSzTr59iM95CrPPAiyOjzxD8M83y8LPFE9d7sbw3Q46rNPvLxTuDscFEs6YgwwPCp+vDwvAHi7aQRau+fWGLxdK+y8YvYjvX0ogbvM9F28YcMovFNW2DvgOCu8LtChvKJMeTswB007BrE6OubmLrtE6cY7Hx2PvKPx2rxMmTe839zEO8QvrbvINKw7NtKBvHPyOr1U44O8tA4RvGC3RLxwtFC83ss9vK0SeLsQ+k48CR3nvHB+ibws2g+8VhB+PFJkPLxs97k6jqsCvH9vkDwRucM7MG3KOztCmTwBDYU82+cVPYjL0rtE6ac8mD5xPDBt6jwomiW8zZlIvJLTBb24vFA8Gq7wu92XgbwiBjo6haKHPNmYMzzm/gc8EQhdvPAnobvCOoo748o3PG5nFbws4DM8ChUsvJQrDz1U89u71aE5vNZTRjtA7NU65iuau3CFWLw6RCi9ojwwO+xQsTwmRWk5aHqyvLB8HD0vg6g8KE6oPDrIdDsSmky6iNySu9L11rwWYVi8wv7DuIDiszyp9+28+bjaO8JSjDvxbqM8MeXYPFrZkbz3HLi8UL6VvCzBVTsz23C79vkuPFd7dzwGZ7284m8ePEjI7rvFUcK8toxjOyisYTxOJpg8omfnu9rRmzwVYqC8fCAFPf+cBD0i81G7++l5OsJzlLu49am7uu78u8F6BTw7eVC8nF22vHC6qroLmqM8Pr8LvAbNnLy96KS635MCOXwwiTp7PxM88mFDvAgHT7vHPg29lKtau4cyszsCWxS9PRQjO7p8O7i7tLk7kfCFO+e4V7yZN0w80tmAuXPUBjz0qgs8UMC1vN4zd7ySnAy7dlWyvKi1JbzzIf47Nl/TvNuTBb2i25U7UJ63u5Mzw7q/Ors7f91wvKn1Fb11aDw7ZK4fvA== + index: 0 + object: embedding + model: qwen3-embedding:4b + object: list + usage: + prompt_tokens: 6 + total_tokens: 6 + status: + code: 200 + message: OK +- request: + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate, zstd + connection: + - keep-alive + content-length: + - '109' + content-type: + - application/json + host: + - localhost:11434 + method: POST + parsed_body: + encoding_format: base64 + input: + - Duplicate ingestion regression content. + model: qwen3-embedding:4b + uri: http://localhost:11434/v1/embeddings + response: + headers: + content-type: + - application/json + transfer-encoding: + - chunked + parsed_body: + data: + - embedding: s/K+uUf8m7vU5Ya7ciUVvKDc4LoxfHc9m04APT7eaTsxKw88V2CNvNvgVrzUmDU9LX2vOsVd+LyiL9U7kugkvYIpXbxxZ5a8S2JiPJsLBDmqOsO8hOfvPLRgiDylBTi6PB5Jvef0wDxc1ta8kqITvbTZXT19Dzi8yh06vHv4Xb1NxZ48NxhgvGAOrzplKAi8NqZ7O827A7yjvz08BpLOushSfruDMg+9YeKUPEwknrojyxO8QhLcPI9HOTw30DW9anf1vKVggLw8Bko79dCBPBSccbxy45i88A8CvQQcU7yVIpg8txpBu4hTaL3aR5G8zQlZu/2aVjx1BQM837UpvHHLpruxo/y8kplYvBk43TsvP9s7p/MpvNmvvzpuWRc6cGkuPCIuvjy2H/88Y+IovY80YLyKodU85cNYO0zYWjzyuQQ9JvKMO/8QjjpFpsU8SkfbPCPp2LyOi105urFLPPbrw7zxc1E8l6CBPFJ12jxCr6M8yACpPEkKW7qcKZ88xvKOuyH/j7wpQ5u8ycwKPDcxq7tpmrs7kTdSPJNDADxNKVw8ODsFvcH8n7zXNBA7nExNvL/NqjuflZi8shjBu8L2OzwwRbM80PSrvDN6ZTv2ocU8QxufPPPdSjyKb7E8fBw+vAxs4jy8RA88O3RhO2HtlDvGc3e81Zyyu3qlarzXoEi8vSAHPFOl1TujdEy8hpMmPMYoEbu1rvW8PsABu9oF77t9L0C8rQI+vJ9HQDzz+W0731MHvM6wlDwtYwo8SEBcvK/nAjzRULA7F2zhPFBoKTwdhac8aYeePKxlXTyNJ0u8xEl3POfgHTy5VNY8iCh8vFyChjx0X3E6DIxgO8bknju0+EM8K4u2uxIgWbvp6o+6bn9eO4dWpjtMFwC8BOkIvYNJEzwkQlK8PsSZvBcC/TsLr3i8317TvHkSA7q5Rtm8FFglvP8Ym7zRZ3A9K+2Pu+unrDzE1hm9e2k0vNaF3zxyHSo8XdlUu9N07ztTNGg8QHYtPNDGNbtlPp470ySMO3TTDzyN+sK5Kj9FvARrXrx9eAw9rcB0vLqZSzzwKTA71PxzO4vWjTzlKTu8EUU7PF/vPbq8Yqu650/wu+U1nDuZIgG9abzMPEirKLwU5+K8yT8+vaVMuDvKBaA7dPyxvF6KiLwXPJU8GRQEPCrDJzt3DcU7JvCJvG+LRTzgVRa9lhRePE0LOrsZB4y6IeKNOzeZwrwqe7E8U6/VPDA+qDqtG7G7LXrBu/oCYTtcIA08bD9AuzBSA7yID0G9U5VJOr1etLtPNwa7h/P2O8TTZDzSMr28cJ8huk6vo7oApxw7Z2lsvLL23Dv8yU48BZQDu04WD7sKHC28rVIRvAqnyLvvV2686d2qO8Ywzbs8WS07LtiIu/Ce6bvrTz27MSLaufQEkbyUq568QHasuAmfvDuaKNC8odFQPQZKD7ya5eM774GsOzLC1bsbX6e8B1wMvDT/tTvQBPE86EARPe8hmbyc3Dk8pZGpvJ+PBLwS8h885mFwPOGsizwnNOY7jBXXuinVY7sjSqk8BkfdvNkFmzwuK0m5jTmOOmYMu7uk8Nk7mNONvNPON7yP2527rqUUu+rmQrzIUGg69kXGO/MeKbw2I908L6WMPMu14zxlFj29hNa5O1A0Nbx9nFW8aGLBuu2uZrylmVI8q9z8vAkSdrvCC6W8D/+FvJv4s7zzMcE721gHveoAkrxS4xK83EUKOzoNlDxlZpM8Uf+VPJ035rusCDc97qKkvIntLD2bnUi9b0QYvPhUdzs6QIc8e66nOV/xYT3dWhK8U5KyO1LABDx4QAm85n6TPAu8QDrnviW9yaxzvBDQvDywSgc8Cx0vvGzMGrvtWDG8CXGkvIBEHb1y86K8q38nPSM15zzkWUa6fT+bPA6YvjznGLA8EGJ1vMMTFTz2yjA8kX2QPGmm5Lz/KvC6Oj3evB1PDD0qaHG6TIg7vZTZiDwBimk6dBqKPN/STruXiDK8iJSIOrWfyDsegte7UEz4ux5onTvyO8w8b74RvNPEU7v5pbk88PJuvER0rzsLBra8bG55u498RbyWzKU8MkhlOoOMLLri6Cw7mOyJPKepQbuVWYg8OKQ5PPtoAz2rY7Q8SxlivNCKED0vTXI75ug8vW62izzI6pS7gBDBvFeLCzyjXU89cyNBPPegYDwJugw890I6PGQy0zsQCjS9ru8mvbZ56zy22d27EOoDvMSiML2KHyK8APW8PPbtJby3OgC6zadmu4rQHzwHTLQ7JxWAO4E/ITydOw484182O5MlT7sjhcI8kG0APZBVzTyKs6m8MFomPeXwrzrPLxM6E3d+vILkzryNSde8M+jFvKOZWzwEwMo8rOmpvHOZlDzZBxQ9WLy4PIjvF7xdhry7/H4rvfWojry6BT08Ulsgu0Fs4rzFCrA7lll4u4YUZDtOphu9t/hWPDKX2r2MW1U8/GGcu484wLm5jhi8L4IcvBDx3Dzes0c8yBo0PT2OlTzUnD+7k7fkuxWBw7ysisc8gY3bvFVhU7tBods6bOiVOyJE7DzaIzo7cwgYPe+6ELyvxis9YwsfPf6nDjyyg1Y8iE8UPMVTsDxY3Yw8cwcNvPdofzzW+B09LJmnO9gIsLzLp865pIMjPBAnODzBTRi9plaZu8Myk7sFbH28XP2Fu/ZpGryi5Ho8JcxQvHSUALuJ9ZQ8LkLuvPqcpbwfX46817TxPHLuAbrjL568yfhPvePPB7uM5vi6gGY7vNajmbzStWu6e0YFO87GAr1OVRA8Y0M7vHhQ3bzsuoA8wD2FPPZivzwMm3m8o96oPAUTHjy48Qy7hKx/O1aHoDzQ7hc80KXAvKdnwbxZx2u7okOWO+XyljxJPay7Vbs2vFiYSDwxty68JaZUu9TEZTtt61u53Rdou+3BmzyXrB29ENz4PFejozvFlr28cVSdu45oAbxShTQ8PT9vPLKS0jwfake8iY0pO76H9jv0Mew8XzFGvJtkgbqjOZg8fjfPPJwTuLyz4QK9NYDXPAZn4LwLSsG8D6Xmupc4Iz1uoWg7w8qJu7Ewujy11H48aS+7PLTbrzvGxMI7dxqGPE2+DzzkySE9kFP0vDJ4PrxgQZM8AeCju4JEVrxpohm81zpFPLfT1LuCa8+8lDtKOrF2ljxYBkM8ISISO2qvyrycTJq6pmtevSksAjwpNjA9gD4XPdRKdrlPZyq8Gr9qvLkJZLrNw008ZrKTPALK4DzuNyG9fGvpPGidAbzfNTy7zqW5u0Tanzvd2QY9wRkrvbQlbLxIn7W8XBvXu1r/lDw0ZZ+8eHr0ujlv5Tv4mbK8yvAmPElxCb26W7A7ChnFvOzdFby0gtC8lPDAOdWJ0bsa3zs9whaPvIol6Tx3CKK8mrTKvMq6zTw7MoE8NRrsvC2y3jy+zHc864IrPTLlAT03xck846DWvMjO9Ly4IoC7JGsrvERXnTxKaQm9+fYSu0A1mbx9hyW7GRtzPOUuDr0juA099p3fvDr9Fz3XqR+7gpSNuhBLJDsNDhk8vB46PLM9/bzlOWg87V9OvFyWtDqg/To9bWGcPIgTabw/Vh89bWK2O6M0lbrg1N07f3pFvTNkRDyzyPk7p532u6MdbzwVrM08H1GoPEn5pTy2CLU8SUOLPJbsNjy533I8SlXcO8++ubt9CAE9HKWEO5or4TzgNYc8XG1FPd9OgDznoxc802ZRPGdfnbuB8g29diuavKJJFLzoiTk71zSePDtj2LoHDWc8wFLWO+gLN7xIoIA8ldEjPLLqoDxg8vQ8bg1GvX2czbrDxSu8RqjZvI4zhrwlisy8OkJDu/ePHryznRS8fiF7vCHSgTxllh69dapNPKMWHjzH0D68+eLtu/8vJTxnqIW81Y+wO5F9Yzy+bAy8loY0Ozjj9jyZtj+8ofmKPDpG2rvOYKS8Z7o8PDPMgjyUhdI8mLrTPIDWwzynFys8UZAPvQXnijz5ME+8xnw0vHKHbzyvJOu7owwBPHYuxTu1Ro48vgOmOrjwyryQ/BI99V8yPGR4ZjzHgUM8JNL/O52d6bzmabu87RVAvOJsrbzqYma8kB4PvDerKT3BZ1W8ZZ+EO/ErsLuOStk5j9wJPZPHLrzfvKc1JnDePPwxSTwOYBK9y3uSuV8cAz1Z1B25xb++vP9F3bsGs0+8QJs6OyhUAb2+VKG81CHGvIdfgrxMh2M8N7reO1QJGLyyys68v8JGvJfThTzxeNw7+Yo1vM68ljwKPQ+8GemwvB6pVLyADgS8eaP6u1ChZLxRrqQ8/3EdO5UsFDyduJq7sSoTvJoCYDwlfOM8JcnfPL9aT71KDUy9RIhiPAQjVLxJGda8MB7MPBpOKjsq80G6Sob0O9AsAzqmBYW7wbeOvBUAXbsv7t47oV2lupqcrDyXF268pPPyuyrhZT2fiAk9feQuPLxuNLtERoI8akDQOgsvOLzEvng7zwWZvczlOL1X/pK8wlKzufzhA71WUFI7uHauvO8L0byqKAW6RsNpvBxInzraubg6WF8dvIQOgjvFvWo9dfuUPDXvaDwDB3s8dRioPM1adbya+Yy7kGCJPDh2JjsxVTg8i7mQvGWUdLxpCGW7ZrE9vMEfqLz3x646DhuWuvqDIT1Kmiy9bJggO19p/TxUXDM88lWHPNvT8bgucsc7YRc1vDs7Gj1Xnhe9wrO4uiD1TzxZpM86EcAyPaY91brtNou7o0O1O3eUPboOu108J5uVOyrRzDykr7+709UGPVeMZTs8N6+7MYAAPctUQDyBFu66KD+dvEY2hDynNuG8cihMvJT7NT0WARW8+QlOPN1CRz3tjjs7kho2OuO6pTycFIK7fJEzvHrpyLzXpoO7QHbRPBPuoDsyxCS9MNJevNUJE73eAb08KKq1vFko5rwdudo7TXUxPLe5ibwRrWC7sG87PUqzE7wEDx682z41urXsQr1iL7w6dJymu6qXTbyV55i8fe3YOviRAb2Fycw807B8vEh+o7uhXxA89axDu/1P57uvHc68nPznuwSOUj2jVz08c6nFvC8SMzwK6Zw890+lPKrUOj0ZeTs82oNmvCHH/Lx7LJa85WJAvLJXTLzFz+K7ppNEvD8r/LsbeLO6rpSWvJrBWbzX3uG8qgdsPCpRwzzQUJg8RNcOvNnAXLx6ZTW7zJvqvFh/RjtfOhS85UOcPEQfirvZ0Bg8v7MvO3RkG707S7c8DJLxu7TP87pj7D07MXLFPIYUn7xXZ5W80k7iO+djc7w4VkC5ksG7vLaVFzqK6T69eWfPu2rwdzr36fG7onP0O5DdbjtImzk98QgkvEJwDbxoIZM8TCF0Ot52bryMu0E80bvdO1bL5LxP52O8jaXvvJiu1jwc2fM71s/dPM5bj7xdU7G7jiauurjsX7zAG0+8wksrPFHjkDzdDJ+8XWuhPFwcF71V7rO6vOvwvKg1DDx9nmm846OhvFPGAr2uXcu73zRkPX75DTs+bWi8tWLPPN5g0TyJeE07kp7Ju+HGUTyWGP08ALlQvdQ277vYMy89QA2bPGjeXDxFMKe8ITGnPM+6orx/kHC8D5K/uzKEYDxdVuQ8IfUGvJs6P72sBZ8701cwvBxY8TvQzyA7sj8EPZEbBryFg4E8TgEcPC3gB7wFtu47N0xfPMFGhzwSem+87q2LvHLPzbx+jfk7FQWrvPQZC72DTcw8bg6JPJbTAD0oUAG9+wWyO1aBwDo1Fyk79BKbPK4+iTsRSyy8jNCcvHL7F72SsYm8QcgJvfcno7w66tq8sHVKvZsqTbzEiNi8QlIMPWEiBjw4UyY7MsOZPPct27p+A2o8iyvhvOe70bxJVSg9moR+PNUgh7zemD+8r7KiPKeLdjvujZ87UNcvu10n8DwAHcQ7DD6hu0a1n7wFEKE8dSK1OzTGKbyD0kA9/vemvHxUF7uktMW8Op44u+drlTtO/gm8n/z7vCKfOLygXxe9dsRcvNY35rzIpVg8DaobvAzNw7xKgDY7X3IwvDdSortomei8EeoTvE6FVTwuQ4E7nJFjPG76ijyuaBI7d8bpPMzh0Tu+EaW7vfoFPHnB6jwh8zW8v794vPmatDtEsMC8FhtWvKFrXDxjeic9KKeIvG+geTwVOqK6v0nUO/Jif7zAadA8w8zwvCWeoTyOege85LcbvAlSqjyyIlU8hSuJu46NRDy0UWy7ly9/vMpbUzw6cgu7oD++vCPTKjmlk4M790gZvYmo8bzhVgy7C5jVOrE8zLzaIkC7aE92vG5hqzxFMZo82BRNPO02jrwCWcM8434pvD2gvbsu76g8VFD8OmSI2rypc5S7LHPgPAPsizrF+cC78rk0uhiG3Dy/Uk89rTGUu+Lo1zzAoDS8dV2sPIrx87zw5aa884iNvFwMFryoJvy8eGz6uy1EULwnWRa80d65vFaDUTwuED08MPWvPL95sjwFrEW8nOsHPbyUuLs59lG8faqPvMq35DxeEwS93YPjvPpH4TzWetq89HxOOzHPQTw5Uc+80SwcvN2mizwdqZ68W6+uvAU0SLyrx4+7OMdePERltzyjH127nE/Gu63MozzrR9o8gq8gO/2V1Lx95pY75em5vJlbobz8vaI8ae75OrAVDrsJ5RI8Q8KiPDUGHjoBPUk9axKXPJWiNb2o8Yk7q0FPu4ZvkLsUye68fhXKO7uBWLwdsio8vjYLPOwwIruEqBw9yG5PO+shBju7M+Q8eXQdvU2RRzyT5um8rDuevNc0tjvudyG87SbpPEwQjrvH5wk8PBSWOVPwxTxkRQg8WvyAu8IErzyqIoq8FBigOWALlDxf06C8WWalvOAzejyNwtO8xWsaPBVFxDz7tKw7C4Wduiy3STprV6S8qmyKOxITcLzxidy8Yse4PLbYmDvdZFm90N0BPc2aiLw/xPC7jKgXPBlyRbwTBAy87I/Xu+Ez6bnhfAS8norOOkji7LxPayO85obLvNeWaLzFu5S8eklMvLBTBTzTGo88g2ScvK1HLLxfGgc8wGcsOpWvlbxyW/S7JSOfvAiMFLzqzB48o7l8vIZdm7wEPdq8ey7HPHZuqbulnl27bmPLPBQsc7ziEdu7LIQ0vKVkzDqmlak8KCCLPLedXjz+XVy8M3s2PH8ehzyuWL48aPOEO/6R9Tu2smu8VVzWvAfE37wjCdc8BaTJO3ISHj0m7iK90BECOyfn7jx85hK9zkISPcS4QDz5KSE6CaomvJ3a7DuGcP07yKukvLVoqzwSUzO8xFOZPAS4iT3/7Nm8McFrPLWJcLx+Y0g8S9kLPHQyCT3Bfsg8SvuEvJFYdjzkp206l1NHPZbBvjsSgyi73grUO7Jr2Dsf7OC8/pwROxHUkrtXuuG8d+oEPIwDHL3aav87Vzn2uFeV9jwDRzo8JyE/vQn1lroZH4c7FRgDu4rHH7vZ0Fw8NfEzvRQnOzyPu588/1scvZFS3zzc7wK9VG0hvYp5OTw2+D68Z6zOOQIbML1qy346FL6Eu1FBebwTw0O8as5DvSNZ2LvrOFi91OraO+BbRj0d0bK85aTVvKdRHryAQf67mPcIPD4h77yBqti8rj9jO8pgBj0XMyg7/nWyO/hqGDweFA27hGHlvCpbAD3pRsw7/EKqu02LjLyf7q48UyBWvEc+FrzCYtg7JGVGuuX8C7wOczs7aP5cu5xuNz207p4808gdvZ1T5Dun/uK5AZCWu4z3Pby9ANC7TVypvHd4Drxl0EU6kAiJO7EFCbsILb66hKp5vO/CL7xUiHY88r8hPE0bCbyl8RS76tcwPAqCDzznjbM8Lc75PCtfkbtDdQk9p4FCPbRWOLxvRwu9umm3Ornzxbs5ktu7SVjlvBOzJrzCGLU7Xd/JPCqhFj35hYA7iPrkPCXw+rydu6o87HNZvMyFd7uqiJI8HtwtPWVpHzxtu/y8zU0mPM6cIb3b/sq8TNpRvKG8dDtdi5u8QcjCOzgBybyDPBM8BkqjPACAlbrPLpO8oK8TvB0ZELxdjjm9z+hTvJQceLtjk0C8yVWgPA5mnjyPEjC9NWEUvN4CWTzjMn+6elvDvLILx7sC4Qo8AdKQPNSm4Dxf/ee71ls3u8MuybyS12i6bU5Fu/FigzyxlII5Q/qmvLVEqjz5NHs8BbmSuhlHJLztuYQ865G4O2Tp2boCYka8Aa9KvApbn7zHyCs8kGnEvH10GLzR+mE6SIcOvSJatTz17m08uGHnu+IruzwSj0A8N2zsu1fW+bt3usc8B8+9ulO4jjsVCPM8nw7nvBVcCTsgQFO8Ya+lvJvaGDyl64A86hfqvNRQ5zz6dec8lL4DvMiiUbzUxmI8pNDFu6ycuzy0ztu8tJqIu1SdmToeLPy7zC6Wuz8SmTxWQB29sFGVvMIC7Lz5kLM8rKBnPB2WMT02Wx65Qf5vPLTWejxwXc06q+hcvLeO6TzLysS73wEsPK0GOLx1LrS880McPLdR7jsag9886otGPGgZETxYIC67OeOXvA60yTxDyAs89RM+u1lFubsU6tM6708AvOl/9zuSLTQ8mUcSPRp3AL0Ei4y8GiCSu6+irzyrXs28E4zsuxpZBr2+AY+8e/qCvGg20LtatRI6VEnjO6RtsbuEvuG8QZ+3vFRI4DsR7eq8DWQfPQVr9DxUWJA8fBESPbKDezyv6ks7+UwDvayq4DylkJA8gF0suwWFrDvod8e8rT8rvBteDT1wxcy823xDvBm6hbwEtyK9RvkhvSEu0Trepu48AukqPLLaQTymZM+8PNnDvFH2CjzSfsm8fhYmPDphZLxJz5K6t0CNPPVRHDzxN8O7k+vcPO5aQbwI7yG7fYbpvGfF0zqvgt+8zYL5O1HBPTz2PwQ8Kdlgu1vpYTul/Yq79FqyPF90hzxgp9w79kc5vEEOwjza1906/HXvukD9kTx9JQ08nsrDvJkSortTRng7W3igvNjbAr36oPC8XP8jPS5upbxGMXK8+so1vOfnS7x13yU7wKYvPRuPdDu+eWm9u40APUxO4jzjzBy8SL6ruoB66jswboA8knm2u67oHz3RY0q8RrzrPNmZ7DsTGqI8VPE0vYjPLDv8IrO8qAkDPYoB8ju9byu8D1ifPH3UBby5VR29hHMivBeQt7zoh9I6oTHXO7rYzTuQ6DG9G4v6vNenebvrrPk8mL6ivH9Ytru08q081sOVvHW8hTzot+o8UydTuxaW0rwOf5c8XrZEvCITzjym59w7iGp/PFUV7jw9gLg5hxDEu67RLT07LQ+80d0KPfwYBrwjAhK8vEhUvGYeCb06has7FbJMPKM7GDyVcZs7Jyu6O9+7gDv5xqK6SWq4u3pV3zylslU7FyskvIKWVDuBCja6DyIRvUI+gTyqPg28JZXfPPq3ijzuivE8tw6GvPecELuB66i7bgSBO1xlGrxs6ke88jzEu0ovJTz/QLi8OvCSPCaLi7wIIb07ca+CvI0dDD2ZxZ486fcXPXbvnjk8AMc8Eox3uTaLBz31rjm7rowhu+/DhLycney8J96kvDv5ATxCk3I8wb3vu2Aq3Lz0hxc73HvCPKjWszzql7+8lgGNPEveILw9gPK8DMrzurZDXjz5q9a7gNbcOeAYjbzaXeQ8akHsvJ664byXy327gsfDvNMLzDyfee08HEhYvPKMLDyvuxE9MaVHuwzKfj0rE408Sq/sOidPjTxzXgk8mTDEu0tNIzv4NqY8eAG7OxmiyLuDyXG8rWcjvN6gKb2VT7M8MpRivPDVPL1HNa08twKGvInBJLuvr8I8wdi1vI2TID1kZzO9CV8Hvaz4wjzlwaa8HmOMuvfUKL0YDww9BiYQvfbA/rtCb4c891kzvBYdvzy5uxG8rKdAvMp9iTniSb086Mu3vI4ptDzcWR080qnVvMspabxQPRC8ceCPvO0mP7yIHmo89FA8PGBnQDyPSSO7N9hHvehUgL1Si726B6vwu/lwLDwCuaI8g9GMu18LaDxEdFc6VZObvCnLzjuQn8i8rF15vLECBzwo4d47sQpbPCsrBz2LhRM8WxiOu8SlHDwx+Ro84WwlPQrsijwXc4c8z2/pPNbuZjxPRGK8jQHVO2ttAD2KL8i8l2mRvPs7rTycn4G8W0ugPKPgfDtJicg8in2yu0hjaDvRbeS6fCOCPLYsJbwRD7K8nhykvII2lzkmwCw8xc7DPKY1n7zhyu270lcgvdJ43jwRRju6dMnKPN5Qh7y1XDq78rhoPDanSjwdQiI8vT/Ru83qn7t9UPW8Ao+9PGS66jsJkp48nbA7ut2XnDyXu5A8SYJvvNMcijwD/bY7B2nHvA8vI7pGA1y7lxQkPP8ot7p8W9W8AnfdPCDO9DyRLoK6WuntO4tY3LvN0aE8kLXBO3I6kzgG2FY86G2mu/EF+rzpYS88nyRuOgXT3LulmuY7kvWIvJbPJ7zRwvG7JQAwvMa1f7yOmLU8dgS6utvfjbwJKaq84CAaPOzrxTvp9og8D8yEvfpKWjmRh8a82XPYPBwuIT1gljC9Gfn0unQ9P7y/oaa7PHygOkawijxf9Lm8UiWnPHpVabrRBgW994RHu3g2x7yHXjW9NccJPfuGnbwns827Lz4GPFK8YDz2k627e3/MPPrXe7yMH4O8NkNiPK2xQ7weCXa8ipnnvD9vlDv+RMQ8Q8TjvLcMeTziLA095n+BPLtWGz3VwlO8jFF5OR7iOz0ELB68UkC5PIleCb3psIe7THN/PHlQYLzun5i8JPfDvI0WDb0q/P07h0dOvb89X7yz0UC8pxn3OwSD37vxuh28tn9Wuik4mLvJ6Cg8bV6wu/63tju016u8f3kEvTlcuTwSraM6CSKNvJ65e7qfDeS7CsBqOwhWj7wmqU48lH43PA7WgLx7rRK9wwHsuxxgczqxYok73C05PNBI4LxNvlm8TitdO1caZLoI5Q68oQGYvMgOHrwF3Q+98j2XPDnCLryaBhE8aZiFOlT/xbq8TgW9UlycvHanIj3Vacg7XigKO/xamjvA2Kq7cs5APTV6hLtfe7u8+J7CuyHaqjuIlUS7uA+HvCqmiDz1g8+8ZpYXvOR3RbqmV7q80vmqPFBuEbwPaNg7jhZPvIcmhbzJqfu8Ukj9Ov5017ytMLG8syn3O9zu1TxtmSG9hu0PvCFNpzxhwP47oZviumFGYby8FhC8jL7WPAm9orvgKlc8fcSfPHtsszwhu/g8Df6cOuVb5jxmUVC8Xk3sPBJgJruEBwm7od0HPXx4hLz1lQA8BTARPWXLc7zH6JG8NbIsPKrqk7znl186HE5wvCI1Uz195wa7at+wu/WWQDxcLka8moMBPFsUB7wgh68557wvvbWZX7z9pYM8AAwAPUhyfTw60Yq8zgi+vIvhxbwS5gq9TCHUvCg32ryk1tE5oswMvIiLDLzhE+W84mKWu+8vCzzwbwq8Px2SOS/uqLqJPB2740CiPEuwhzx1ARo8gJKGOf9slDxkitm6fwOhPNhRvDwrYgG9nG0TPDipRb0AO+Q7BSPIu5YX1bz0eZc8K4vYPG1cFL3RD+i8iRKIO1g6JrwLBCO8GxzDvCmAIbz2mYS8Fu7mvL3jVL1zyhi7KBBPu6atu7vP1YS8ZwSSPDW2qjzDqAI8wFLIvH1GYrwdgMG7xv3MPM3YLbutqAq9cYU/PEiNl7xGE+K8HVIYvHjnDL0/7f86mlw+PEU3iT0Dv6A8scaEPCvTh7xwDeE8H730u0PjYzwAfRM9l23QPHSCN7x0qRa9KoxLvE9rw7t5KNc8IsWjvB7azTwMHsM84L1hPKSKVTy3hBO9T0dBPK1Xsjvk0fS6eHWsPHBx67z+GwA9nryRPD1cOjvj/TA9IX0AvOWGwbywmH48Mu10PBsxxrw6/8E8stY3vCTpjDz/74i80tsnOyGVHT3Rvds85ibPPFNYDbyNfSE8BjOgPOSd7Ts5w4k8iLcQPaQ8ObwzgGq8IhQUvNy7AT1On6a7eKXCPBlw6LzToZC8xACHu8cYF73h97i6XEa8PEVnCrw0cYw7UngbvRnoLLzy+Yc7kWinuzEh3rvxkpE6lw+yvDB0aDuw6OU7rJ5GvEW2Ijt/OzO8cbY6PNzupDzEstO8RGWNPOfVpzukXcG8YVLtu7ShD71nPIm7IoTOO3PySrvshUg8d6gUPB8qyzwKJwm8l4pIvPE9RDy+gB48huC1PDmG+Ls+bJ48zZISO1AQMLzrEya9unWXvL0evbnEa9C77uXsO1sX8LtH9Y48yA+ZPLq8Wzyrurg6ixV2vOQzYjwOAs+7rkBHvDGwODwPl3q8gvsUutrGoTwyKfW8WPcPPdLfCjyNeQ69d5MRutKgCL3UOgw8nYWvvPHOQ7tNbvU746SGvM3V/rx4ESo86TvCOX5UHjw8c0U8bIxGvAXmPbw7w4K7jUe3vBccljzx4S697e67u82utjummNs8bN7ZvMMuSjsAspK8GecwvNwlDDxWB5i8o8n0vOURmjwTKAO8J+uGvFS/WjwUSYU7D32VO5lTND1bWtg6vh7Vud9JvLz0dFu8qLSQO9C1hjwKQkG8VflSvJkWejsFEvm8AP6ZvHFDw7zPc1q8LQF+vLKuJzyerZG7WiNouzQBCrtm9pE8/GzYu2xOKz0DTek8QXEkO8njbTpsWYA87wppPCuOTLwv8vo7JxVFPM0Gizw1fOW80Hy8PImRUbuTqo28e27wuECYH7jawIU8FzRRPHhGIjxTtKU8BNifPCK/gbx2ajC98AgjvYheQjzTTVy8JFuWO7DPyTod/CM912zNPJWKOzxdisI8+UAKPMrJfLspEUA5P+NOvBZAvDsewVs6fgExPPIFuzxEsni7ezhXu/QqGrxYa+u8efEjvRTcg7tCcGC889kmvNzi2TsDWyy8hHOgvL1efDsil007IH8jOrsKN7sX0sY7WwWPvGi92rwQjTm8zce/Oz/ws7u7SK07/k6CvG94Or3vZYK8VaQSvI9GRLy+NFG87OQ+vHKUe7vOV088IWHnvKGXibyVFQ+8IlB+PKFLQLzLS7s6tD4DvBQQjjx9y7w7sD/HO6V5mDzjMoc8lmwVPTcZyruKMaY8YfdyPDO06Tw5QiW8cW1HvE/LBb0hMVA83Njtu7qdgrx5SSQ6+QOIPCw5MzxGags8glJcvMmZn7sf2Ys7P4I1PMqdFbyQdjI8XAoqvCtaDz19b9W7Xes5vCeASzv8l+M62oOWuzpHXLxEWCi9xyE3O7D3rzwaEPw40CuyvEHsHD3nCqk8kY6oPJx/eDvW/1i6A6GUu3Kn17z33Vi8yCF4OIortDzaZ+68fM3WO5PIjjsS06E8sfnXPCu0kbxNu7i8oPuUvFt2XDsXeHW7/k4tPE8BeDzewr686SQfPCxa7btis8C8uzxlO84GYjxYBJc84sXqu61mmzztwaC87LYFPd78BD2RslG72X5/OhlDlbuFDKm7xCH3u4CvBjxkaVC8a0+3vKkcqLqUsaQ8qCILvF83nbwxvpu6kob3NsflYjp6JA88hhZDvJ/GULsciA29GTNgu8ZGsDucQRS9DjIhO8D6lzgQw7g75iCHO5hWVry6vko8tDaTudtJBjyRxg48G9q1vNUhd7yRMwW7h4CxvDTXKLxJW/47s67TvCi+Bb2eY5I74460u2iJzLqWHME7WV5tvMqhFb0j0Uc7SaodvA== + index: 0 + object: embedding + model: qwen3-embedding:4b + object: list + usage: + prompt_tokens: 6 + total_tokens: 6 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/test_duplicate_ingestion.py b/tests/test_duplicate_ingestion.py new file mode 100644 index 00000000..c8c675db --- /dev/null +++ b/tests/test_duplicate_ingestion.py @@ -0,0 +1,42 @@ +import asyncio +import tempfile +from pathlib import Path + +import pytest + +from haiku.rag.client import HaikuRAG +from haiku.rag.store.models.document import Document + +pytestmark = pytest.mark.asyncio + + +@pytest.mark.vcr() +async def test_concurrent_same_uri_ingestion_creates_single_document(temp_db_path): + """Two concurrent ingestions of the same URI must not create duplicates. + + Both calls read `existing_doc=None` before either acquires the write lock; + the atomic re-check under the lock turns the loser into an update instead of + a second insert (LanceDB has no unique constraint on `uri`). + """ + async with HaikuRAG(temp_db_path, create=True) as client: + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) / "dup.txt" + temp_path.write_text("Duplicate ingestion regression content.") + + results = await asyncio.gather( + client.create_document_from_source(temp_path), + client.create_document_from_source(temp_path), + ) + + for doc in results: + assert isinstance(doc, Document) + # Both concurrent calls resolve to one document, not two. + assert results[0].id == results[1].id + assert await client.count_documents() == 1 + + # The surviving document owns exactly one ingestion's chunks; the + # loser's create was collapsed into an update, leaving no orphans. + surviving = await client.get_document_by_uri(temp_path.as_uri()) + assert surviving is not None + chunks = await client.chunk_repository.get_by_document_id(surviving.id) + assert len(chunks) >= 1