Rebuild now re-adds files.
This commit is contained in:
parent
22ed9ca51e
commit
0b3477c46c
1 changed files with 40 additions and 6 deletions
|
|
@ -125,9 +125,12 @@ class HaikuRAG:
|
||||||
parsed_url = urlparse(source_str)
|
parsed_url = urlparse(source_str)
|
||||||
if parsed_url.scheme in ("http", "https"):
|
if parsed_url.scheme in ("http", "https"):
|
||||||
return await self._create_or_update_document_from_url(source_str, metadata)
|
return await self._create_or_update_document_from_url(source_str, metadata)
|
||||||
|
elif parsed_url.scheme == "file":
|
||||||
# Handle as file path
|
# Handle file:// URI by converting to path
|
||||||
source_path = Path(source) if isinstance(source, str) else source
|
source_path = Path(parsed_url.path)
|
||||||
|
else:
|
||||||
|
# Handle as regular file path
|
||||||
|
source_path = Path(source) if isinstance(source, str) else source
|
||||||
if source_path.suffix.lower() not in FileReader.extensions:
|
if source_path.suffix.lower() not in FileReader.extensions:
|
||||||
raise ValueError(f"Unsupported file extension: {source_path.suffix}")
|
raise ValueError(f"Unsupported file extension: {source_path.suffix}")
|
||||||
|
|
||||||
|
|
@ -361,6 +364,13 @@ class HaikuRAG:
|
||||||
async def rebuild_database(self) -> AsyncGenerator[int, None]:
|
async def rebuild_database(self) -> AsyncGenerator[int, None]:
|
||||||
"""Rebuild the database by deleting all chunks and re-indexing all documents.
|
"""Rebuild the database by deleting all chunks and re-indexing all documents.
|
||||||
|
|
||||||
|
For documents with URIs:
|
||||||
|
- Deletes the document and re-adds it from source if source exists
|
||||||
|
- Skips documents where source no longer exists
|
||||||
|
|
||||||
|
For documents without URIs:
|
||||||
|
- Re-creates chunks from existing content
|
||||||
|
|
||||||
Yields:
|
Yields:
|
||||||
int: The ID of the document currently being processed
|
int: The ID of the document currently being processed
|
||||||
"""
|
"""
|
||||||
|
|
@ -376,10 +386,34 @@ class HaikuRAG:
|
||||||
documents = await self.list_documents()
|
documents = await self.list_documents()
|
||||||
|
|
||||||
for doc in documents:
|
for doc in documents:
|
||||||
if doc.id is not None:
|
assert doc.id is not None, "Document ID should not be None"
|
||||||
# Convert content to DoclingDocument for rebuild
|
if doc.uri:
|
||||||
docling_document = text_to_docling_document(doc.content)
|
# Document has a URI - delete and try to re-add from source
|
||||||
|
try:
|
||||||
|
# Delete the old document first
|
||||||
|
await self.delete_document(doc.id)
|
||||||
|
|
||||||
|
# Try to re-create from source (this creates the document with chunks)
|
||||||
|
new_doc = await self.create_document_from_source(
|
||||||
|
doc.uri, doc.metadata or {}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert new_doc.id is not None, "New document ID should not be None"
|
||||||
|
yield new_doc.id
|
||||||
|
|
||||||
|
except (FileNotFoundError, ValueError, OSError) as e:
|
||||||
|
# Source doesn't exist or can't be accessed - document already deleted, skip
|
||||||
|
print(f"Skipping document with URI {doc.uri}: {e}")
|
||||||
|
continue
|
||||||
|
except Exception as e:
|
||||||
|
# Unexpected error - log it and skip
|
||||||
|
print(
|
||||||
|
f"Unexpected error processing document with URI {doc.uri}: {e}"
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
# Document without URI - re-create chunks from existing content
|
||||||
|
docling_document = text_to_docling_document(doc.content)
|
||||||
await self.chunk_repository.create_chunks_for_document(
|
await self.chunk_repository.create_chunks_for_document(
|
||||||
doc.id, docling_document, commit=False
|
doc.id, docling_document, commit=False
|
||||||
)
|
)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue