feat(#78): full-text search in indexed chunks
Backend: GET /api/ingestion/search?q=…&doc_id=… endpoint with SearchResponse schema. Frontend: search bar in Documents page, results with filename, page, chunk index, relevance score. 3 new API tests.
This commit is contained in:
parent
efabe84d66
commit
830184b12e
6 changed files with 341 additions and 5 deletions
|
|
@ -5,9 +5,14 @@ from __future__ import annotations
|
|||
import logging
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
|
||||
from api.schemas import IngestionResponse, IngestionStatusResponse
|
||||
from api.schemas import (
|
||||
IngestionResponse,
|
||||
IngestionStatusResponse,
|
||||
SearchResponse,
|
||||
SearchResultItem,
|
||||
)
|
||||
from services.analysis_service import AnalysisService
|
||||
from services.ingestion_service import IngestionService
|
||||
|
||||
|
|
@ -77,6 +82,38 @@ async def delete_ingested_document(doc_id: str, ingestion: IngestionDep) -> None
|
|||
|
||||
@router.get("/status", response_model=IngestionStatusResponse)
|
||||
async def ingestion_status(request: Request) -> IngestionStatusResponse:
|
||||
"""Check if the ingestion pipeline is available."""
|
||||
available = request.app.state.ingestion_service is not None
|
||||
return IngestionStatusResponse(available=available)
|
||||
"""Check if the ingestion pipeline is available and OpenSearch is connected."""
|
||||
svc = request.app.state.ingestion_service
|
||||
if svc is None:
|
||||
return IngestionStatusResponse(available=False, opensearch_connected=False)
|
||||
|
||||
connected = await svc.ping()
|
||||
return IngestionStatusResponse(available=True, opensearch_connected=connected)
|
||||
|
||||
|
||||
@router.get("/search", response_model=SearchResponse)
|
||||
async def search_chunks(
|
||||
ingestion: IngestionDep,
|
||||
q: str = Query(..., min_length=1, description="Search query"),
|
||||
doc_id: str | None = Query(None, description="Filter by document ID"),
|
||||
k: int = Query(20, ge=1, le=100, description="Max results"),
|
||||
) -> SearchResponse:
|
||||
"""Full-text search across indexed chunks.
|
||||
|
||||
Returns matching chunks with content and metadata.
|
||||
Optionally filter by document ID.
|
||||
"""
|
||||
results = await ingestion.search_fulltext(q, k=k, doc_id=doc_id)
|
||||
items = [
|
||||
SearchResultItem(
|
||||
doc_id=r.chunk.doc_id,
|
||||
filename=r.chunk.filename,
|
||||
content=r.chunk.content,
|
||||
chunk_index=r.chunk.chunk_index,
|
||||
page_number=r.chunk.page_number,
|
||||
score=r.score,
|
||||
headings=r.chunk.headings,
|
||||
)
|
||||
for r in results
|
||||
]
|
||||
return SearchResponse(results=items, total=len(items), query=q)
|
||||
|
|
|
|||
|
|
@ -190,3 +190,23 @@ class IngestionResponse(_CamelModel):
|
|||
|
||||
class IngestionStatusResponse(_CamelModel):
|
||||
available: bool
|
||||
opensearch_connected: bool = False
|
||||
|
||||
|
||||
class SearchResultItem(_CamelModel):
|
||||
"""A single search result with content and metadata."""
|
||||
|
||||
doc_id: str
|
||||
filename: str
|
||||
content: str
|
||||
chunk_index: int
|
||||
page_number: int
|
||||
score: float
|
||||
headings: list[str] = []
|
||||
highlights: list[str] = []
|
||||
|
||||
|
||||
class SearchResponse(_CamelModel):
|
||||
results: list[SearchResultItem]
|
||||
total: int
|
||||
query: str
|
||||
|
|
|
|||
|
|
@ -112,3 +112,65 @@ class TestIngestionDisabled:
|
|||
tc = TestClient(app)
|
||||
resp = tc.post("/api/ingestion/job-1")
|
||||
assert resp.status_code == 503
|
||||
|
||||
|
||||
class TestStatusOpenSearch:
|
||||
def test_status_with_opensearch_connected(
|
||||
self, client: TestClient, mock_ingestion_service: AsyncMock
|
||||
) -> None:
|
||||
mock_ingestion_service.ping.return_value = True
|
||||
resp = client.get("/api/ingestion/status")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["available"] is True
|
||||
assert data["opensearchConnected"] is True
|
||||
|
||||
def test_status_with_opensearch_disconnected(
|
||||
self, client: TestClient, mock_ingestion_service: AsyncMock
|
||||
) -> None:
|
||||
mock_ingestion_service.ping.return_value = False
|
||||
resp = client.get("/api/ingestion/status")
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["available"] is True
|
||||
assert data["opensearchConnected"] is False
|
||||
|
||||
|
||||
class TestSearchEndpoint:
|
||||
def test_search_success(self, client: TestClient, mock_ingestion_service: AsyncMock) -> None:
|
||||
from domain.vector_schema import IndexedChunk, SearchResult
|
||||
|
||||
chunk = IndexedChunk(
|
||||
doc_id="doc-1",
|
||||
filename="test.pdf",
|
||||
content="hello world",
|
||||
embedding=[],
|
||||
chunk_index=0,
|
||||
chunk_type="text",
|
||||
page_number=1,
|
||||
headings=["Intro"],
|
||||
)
|
||||
mock_ingestion_service.search_fulltext.return_value = [
|
||||
SearchResult(chunk=chunk, score=0.95)
|
||||
]
|
||||
resp = client.get("/api/ingestion/search", params={"q": "hello"})
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["total"] == 1
|
||||
assert data["query"] == "hello"
|
||||
assert data["results"][0]["content"] == "hello world"
|
||||
assert data["results"][0]["score"] == 0.95
|
||||
|
||||
def test_search_empty_query(self, client: TestClient) -> None:
|
||||
resp = client.get("/api/ingestion/search", params={"q": ""})
|
||||
assert resp.status_code == 422
|
||||
|
||||
def test_search_with_doc_filter(
|
||||
self, client: TestClient, mock_ingestion_service: AsyncMock
|
||||
) -> None:
|
||||
mock_ingestion_service.search_fulltext.return_value = []
|
||||
resp = client.get("/api/ingestion/search", params={"q": "test", "doc_id": "doc-1"})
|
||||
assert resp.status_code == 200
|
||||
mock_ingestion_service.search_fulltext.assert_awaited_once_with(
|
||||
"test", k=20, doc_id="doc-1"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -140,6 +140,44 @@ class TestSearch:
|
|||
mock_vector_store.search_similar.assert_awaited_once()
|
||||
|
||||
|
||||
class TestSearchFulltext:
|
||||
async def test_delegates_to_vector_store(
|
||||
self, service: IngestionService, mock_vector_store: AsyncMock
|
||||
) -> None:
|
||||
mock_vector_store.search_fulltext.return_value = []
|
||||
await service.search_fulltext("hello world", k=5)
|
||||
mock_vector_store.search_fulltext.assert_awaited_once_with(
|
||||
"test-idx", "hello world", k=5, doc_id=None
|
||||
)
|
||||
|
||||
async def test_filters_by_doc_id(
|
||||
self, service: IngestionService, mock_vector_store: AsyncMock
|
||||
) -> None:
|
||||
mock_vector_store.search_fulltext.return_value = []
|
||||
await service.search_fulltext("hello", doc_id="doc-1")
|
||||
mock_vector_store.search_fulltext.assert_awaited_once_with(
|
||||
"test-idx", "hello", k=20, doc_id="doc-1"
|
||||
)
|
||||
|
||||
|
||||
class TestPing:
|
||||
async def test_ping_success(
|
||||
self, service: IngestionService, mock_vector_store: AsyncMock
|
||||
) -> None:
|
||||
mock_vector_store._client = AsyncMock()
|
||||
mock_vector_store._client.info.return_value = {"cluster_name": "test"}
|
||||
result = await service.ping()
|
||||
assert result is True
|
||||
|
||||
async def test_ping_failure(
|
||||
self, service: IngestionService, mock_vector_store: AsyncMock
|
||||
) -> None:
|
||||
mock_vector_store._client = AsyncMock()
|
||||
mock_vector_store._client.info.side_effect = ConnectionError("down")
|
||||
result = await service.ping()
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestEnsureIndex:
|
||||
async def test_calls_vector_store(
|
||||
self, service: IngestionService, mock_vector_store: AsyncMock
|
||||
|
|
|
|||
|
|
@ -8,6 +8,23 @@ export interface IngestionResult {
|
|||
|
||||
export interface IngestionStatus {
|
||||
available: boolean
|
||||
opensearchConnected: boolean
|
||||
}
|
||||
|
||||
export interface SearchResultItem {
|
||||
docId: string
|
||||
filename: string
|
||||
content: string
|
||||
chunkIndex: number
|
||||
pageNumber: number
|
||||
score: number
|
||||
headings: string[]
|
||||
}
|
||||
|
||||
export interface SearchResponse {
|
||||
results: SearchResultItem[]
|
||||
total: number
|
||||
query: string
|
||||
}
|
||||
|
||||
export function ingestAnalysis(jobId: string): Promise<IngestionResult> {
|
||||
|
|
@ -23,3 +40,13 @@ export function deleteIngested(docId: string): Promise<unknown> {
|
|||
export function fetchIngestionStatus(): Promise<IngestionStatus> {
|
||||
return apiFetch<IngestionStatus>('/api/ingestion/status')
|
||||
}
|
||||
|
||||
export function searchChunks(
|
||||
query: string,
|
||||
options: { docId?: string; k?: number } = {},
|
||||
): Promise<SearchResponse> {
|
||||
const params = new URLSearchParams({ q: query })
|
||||
if (options.docId) params.set('doc_id', options.docId)
|
||||
if (options.k) params.set('k', String(options.k))
|
||||
return apiFetch<SearchResponse>(`/api/ingestion/search?${params}`)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,6 +30,46 @@
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- Full-text chunk search -->
|
||||
<div v-if="ingestionStore.available" class="chunk-search-bar">
|
||||
<input
|
||||
v-model="chunkSearchQuery"
|
||||
type="text"
|
||||
class="search-input chunk-search"
|
||||
:placeholder="t('ingestion.searchChunks')"
|
||||
@keyup.enter="runChunkSearch"
|
||||
/>
|
||||
<div v-if="ingestionStore.searching" class="spinner-xs" />
|
||||
</div>
|
||||
<div v-if="ingestionStore.searchResults.length > 0" class="search-results">
|
||||
<div
|
||||
v-for="(result, idx) in ingestionStore.searchResults"
|
||||
:key="idx"
|
||||
class="search-result-item"
|
||||
>
|
||||
<div class="result-header">
|
||||
<span class="result-filename">{{ result.filename }}</span>
|
||||
<span class="result-meta"
|
||||
>p.{{ result.pageNumber }} — chunk #{{ result.chunkIndex }}</span
|
||||
>
|
||||
<span class="result-score">{{ (result.score * 100).toFixed(0) }}%</span>
|
||||
</div>
|
||||
<p class="result-content">
|
||||
{{ result.content.slice(0, 200) }}{{ result.content.length > 200 ? '…' : '' }}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div
|
||||
v-if="
|
||||
ingestionStore.searchQuery &&
|
||||
!ingestionStore.searching &&
|
||||
ingestionStore.searchResults.length === 0
|
||||
"
|
||||
class="tab-empty"
|
||||
>
|
||||
{{ t('ingestion.noResults', { q: ingestionStore.searchQuery }) }}
|
||||
</div>
|
||||
|
||||
<div class="page-content">
|
||||
<div v-if="filteredDocs.length === 0" class="tab-empty">
|
||||
{{ t('history.emptyDocs') }}
|
||||
|
|
@ -76,6 +116,20 @@
|
|||
/>
|
||||
</svg>
|
||||
</button>
|
||||
<button
|
||||
v-if="ingestionStore.ingestedDocs[doc.id]"
|
||||
class="action-btn unindex"
|
||||
:title="t('ingestion.deleteIndex')"
|
||||
@click="confirmRemoveFromIndex(doc.id)"
|
||||
>
|
||||
<svg viewBox="0 0 20 20" fill="currentColor">
|
||||
<path
|
||||
fill-rule="evenodd"
|
||||
d="M3 17a1 1 0 011-1h12a1 1 0 110 2H4a1 1 0 01-1-1zM6.293 6.707a1 1 0 010-1.414l3-3a1 1 0 011.414 0l3 3a1 1 0 01-1.414 1.414L11 5.414V13a1 1 0 11-2 0V5.414L7.707 6.707a1 1 0 01-1.414 0z"
|
||||
clip-rule="evenodd"
|
||||
/>
|
||||
</svg>
|
||||
</button>
|
||||
<button class="action-btn delete" @click="handleDelete(doc.id)">
|
||||
<svg viewBox="0 0 20 20" fill="currentColor">
|
||||
<path
|
||||
|
|
@ -107,6 +161,7 @@ const router = useRouter()
|
|||
const { t } = useI18n()
|
||||
|
||||
const searchQuery = ref('')
|
||||
const chunkSearchQuery = ref('')
|
||||
const activeFilter = ref<'all' | 'indexed' | 'not-indexed'>('all')
|
||||
const sortBy = ref<'name' | 'date'>('date')
|
||||
|
||||
|
|
@ -152,6 +207,12 @@ function openInStudio(doc: Document) {
|
|||
router.push('/studio')
|
||||
}
|
||||
|
||||
function confirmRemoveFromIndex(docId: string) {
|
||||
if (confirm(t('ingestion.deleteConfirm'))) {
|
||||
ingestionStore.deleteIngested(docId)
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDelete(docId: string) {
|
||||
if (ingestionStore.ingestedDocs[docId]) {
|
||||
await ingestionStore.deleteIngested(docId)
|
||||
|
|
@ -159,6 +220,14 @@ async function handleDelete(docId: string) {
|
|||
await docStore.remove(docId)
|
||||
}
|
||||
|
||||
function runChunkSearch() {
|
||||
if (chunkSearchQuery.value.trim()) {
|
||||
ingestionStore.search(chunkSearchQuery.value)
|
||||
} else {
|
||||
ingestionStore.clearSearch()
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
docStore.load()
|
||||
ingestionStore.checkAvailability()
|
||||
|
|
@ -373,8 +442,91 @@ onMounted(() => {
|
|||
background: rgba(239, 68, 68, 0.1);
|
||||
}
|
||||
|
||||
.action-btn.unindex:hover {
|
||||
color: var(--warning, #f59e0b);
|
||||
background: rgba(245, 158, 11, 0.1);
|
||||
}
|
||||
|
||||
.action-btn svg {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
}
|
||||
|
||||
/* Chunk search */
|
||||
.chunk-search-bar {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 8px 24px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.chunk-search {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.spinner-xs {
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
border: 2px solid var(--border);
|
||||
border-top-color: var(--accent);
|
||||
border-radius: 50%;
|
||||
animation: spin 0.6s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
|
||||
/* Search results */
|
||||
.search-results {
|
||||
max-height: 300px;
|
||||
overflow-y: auto;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.search-result-item {
|
||||
padding: 10px 24px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.search-result-item:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.result-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
|
||||
.result-filename {
|
||||
font-weight: 600;
|
||||
font-size: 13px;
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
.result-meta {
|
||||
font-size: 11px;
|
||||
color: var(--text-muted);
|
||||
font-family: 'IBM Plex Mono', monospace;
|
||||
}
|
||||
|
||||
.result-score {
|
||||
margin-left: auto;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
color: var(--accent);
|
||||
font-family: 'IBM Plex Mono', monospace;
|
||||
}
|
||||
|
||||
.result-content {
|
||||
font-size: 13px;
|
||||
color: var(--text-muted);
|
||||
line-height: 1.5;
|
||||
margin: 0;
|
||||
}
|
||||
</style>
|
||||
|
|
|
|||
Loading…
Reference in a new issue