When populating the eval db, check if chunks for the document have been created. Protects against interrupts
This commit is contained in:
parent
55dc4bbdd2
commit
eb2fc67e1c
1 changed files with 6 additions and 2 deletions
|
|
@ -47,8 +47,12 @@ async def populate_db(spec: DatasetSpec) -> None:
|
||||||
|
|
||||||
existing = await rag.get_document_by_uri(payload.uri)
|
existing = await rag.get_document_by_uri(payload.uri)
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
progress.advance(task)
|
assert existing.id
|
||||||
continue
|
chunks = await rag.chunk_repository.get_by_document_id(existing.id)
|
||||||
|
if chunks:
|
||||||
|
progress.advance(task)
|
||||||
|
continue
|
||||||
|
await rag.document_repository.delete(existing.id)
|
||||||
|
|
||||||
await rag.create_document(
|
await rag.create_document(
|
||||||
content=payload.content,
|
content=payload.content,
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue