Merge pull request #158 from scub-france/feature/ingestion-pipeline
feat: ingestion pipeline #77-#81 — progress, search, delete, status, auto-ingest
This commit is contained in:
commit
07f9568e14
11 changed files with 632 additions and 7 deletions
|
|
@ -5,9 +5,14 @@ from __future__ import annotations
|
||||||
import logging
|
import logging
|
||||||
from typing import Annotated
|
from typing import Annotated
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||||
|
|
||||||
from api.schemas import IngestionResponse, IngestionStatusResponse
|
from api.schemas import (
|
||||||
|
IngestionResponse,
|
||||||
|
IngestionStatusResponse,
|
||||||
|
SearchResponse,
|
||||||
|
SearchResultItem,
|
||||||
|
)
|
||||||
from services.analysis_service import AnalysisService
|
from services.analysis_service import AnalysisService
|
||||||
from services.ingestion_service import IngestionService
|
from services.ingestion_service import IngestionService
|
||||||
|
|
||||||
|
|
@ -77,6 +82,38 @@ async def delete_ingested_document(doc_id: str, ingestion: IngestionDep) -> None
|
||||||
|
|
||||||
@router.get("/status", response_model=IngestionStatusResponse)
|
@router.get("/status", response_model=IngestionStatusResponse)
|
||||||
async def ingestion_status(request: Request) -> IngestionStatusResponse:
|
async def ingestion_status(request: Request) -> IngestionStatusResponse:
|
||||||
"""Check if the ingestion pipeline is available."""
|
"""Check if the ingestion pipeline is available and OpenSearch is connected."""
|
||||||
available = request.app.state.ingestion_service is not None
|
svc = request.app.state.ingestion_service
|
||||||
return IngestionStatusResponse(available=available)
|
if svc is None:
|
||||||
|
return IngestionStatusResponse(available=False, opensearch_connected=False)
|
||||||
|
|
||||||
|
connected = await svc.ping()
|
||||||
|
return IngestionStatusResponse(available=True, opensearch_connected=connected)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/search", response_model=SearchResponse)
|
||||||
|
async def search_chunks(
|
||||||
|
ingestion: IngestionDep,
|
||||||
|
q: str = Query(..., min_length=1, description="Search query"),
|
||||||
|
doc_id: str | None = Query(None, description="Filter by document ID"),
|
||||||
|
k: int = Query(20, ge=1, le=100, description="Max results"),
|
||||||
|
) -> SearchResponse:
|
||||||
|
"""Full-text search across indexed chunks.
|
||||||
|
|
||||||
|
Returns matching chunks with content and metadata.
|
||||||
|
Optionally filter by document ID.
|
||||||
|
"""
|
||||||
|
results = await ingestion.search_fulltext(q, k=k, doc_id=doc_id)
|
||||||
|
items = [
|
||||||
|
SearchResultItem(
|
||||||
|
doc_id=r.chunk.doc_id,
|
||||||
|
filename=r.chunk.filename,
|
||||||
|
content=r.chunk.content,
|
||||||
|
chunk_index=r.chunk.chunk_index,
|
||||||
|
page_number=r.chunk.page_number,
|
||||||
|
score=r.score,
|
||||||
|
headings=r.chunk.headings,
|
||||||
|
)
|
||||||
|
for r in results
|
||||||
|
]
|
||||||
|
return SearchResponse(results=items, total=len(items), query=q)
|
||||||
|
|
|
||||||
|
|
@ -190,3 +190,23 @@ class IngestionResponse(_CamelModel):
|
||||||
|
|
||||||
class IngestionStatusResponse(_CamelModel):
|
class IngestionStatusResponse(_CamelModel):
|
||||||
available: bool
|
available: bool
|
||||||
|
opensearch_connected: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class SearchResultItem(_CamelModel):
|
||||||
|
"""A single search result with content and metadata."""
|
||||||
|
|
||||||
|
doc_id: str
|
||||||
|
filename: str
|
||||||
|
content: str
|
||||||
|
chunk_index: int
|
||||||
|
page_number: int
|
||||||
|
score: float
|
||||||
|
headings: list[str] = []
|
||||||
|
highlights: list[str] = []
|
||||||
|
|
||||||
|
|
||||||
|
class SearchResponse(_CamelModel):
|
||||||
|
results: list[SearchResultItem]
|
||||||
|
total: int
|
||||||
|
query: str
|
||||||
|
|
|
||||||
|
|
@ -164,3 +164,27 @@ class IngestionService:
|
||||||
k=k,
|
k=k,
|
||||||
doc_id=doc_id,
|
doc_id=doc_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
async def search_fulltext(
|
||||||
|
self,
|
||||||
|
query: str,
|
||||||
|
*,
|
||||||
|
k: int = 20,
|
||||||
|
doc_id: str | None = None,
|
||||||
|
) -> list:
|
||||||
|
"""Full-text keyword search in indexed chunks."""
|
||||||
|
return await self._vector_store.search_fulltext(
|
||||||
|
self._config.index_name,
|
||||||
|
query,
|
||||||
|
k=k,
|
||||||
|
doc_id=doc_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
async def ping(self) -> bool:
|
||||||
|
"""Check if the OpenSearch cluster is reachable."""
|
||||||
|
try:
|
||||||
|
info = await self._vector_store._client.info()
|
||||||
|
return bool(info)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("OpenSearch ping failed", exc_info=True)
|
||||||
|
return False
|
||||||
|
|
|
||||||
|
|
@ -112,3 +112,65 @@ class TestIngestionDisabled:
|
||||||
tc = TestClient(app)
|
tc = TestClient(app)
|
||||||
resp = tc.post("/api/ingestion/job-1")
|
resp = tc.post("/api/ingestion/job-1")
|
||||||
assert resp.status_code == 503
|
assert resp.status_code == 503
|
||||||
|
|
||||||
|
|
||||||
|
class TestStatusOpenSearch:
|
||||||
|
def test_status_with_opensearch_connected(
|
||||||
|
self, client: TestClient, mock_ingestion_service: AsyncMock
|
||||||
|
) -> None:
|
||||||
|
mock_ingestion_service.ping.return_value = True
|
||||||
|
resp = client.get("/api/ingestion/status")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
data = resp.json()
|
||||||
|
assert data["available"] is True
|
||||||
|
assert data["opensearchConnected"] is True
|
||||||
|
|
||||||
|
def test_status_with_opensearch_disconnected(
|
||||||
|
self, client: TestClient, mock_ingestion_service: AsyncMock
|
||||||
|
) -> None:
|
||||||
|
mock_ingestion_service.ping.return_value = False
|
||||||
|
resp = client.get("/api/ingestion/status")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
data = resp.json()
|
||||||
|
assert data["available"] is True
|
||||||
|
assert data["opensearchConnected"] is False
|
||||||
|
|
||||||
|
|
||||||
|
class TestSearchEndpoint:
|
||||||
|
def test_search_success(self, client: TestClient, mock_ingestion_service: AsyncMock) -> None:
|
||||||
|
from domain.vector_schema import IndexedChunk, SearchResult
|
||||||
|
|
||||||
|
chunk = IndexedChunk(
|
||||||
|
doc_id="doc-1",
|
||||||
|
filename="test.pdf",
|
||||||
|
content="hello world",
|
||||||
|
embedding=[],
|
||||||
|
chunk_index=0,
|
||||||
|
chunk_type="text",
|
||||||
|
page_number=1,
|
||||||
|
headings=["Intro"],
|
||||||
|
)
|
||||||
|
mock_ingestion_service.search_fulltext.return_value = [
|
||||||
|
SearchResult(chunk=chunk, score=0.95)
|
||||||
|
]
|
||||||
|
resp = client.get("/api/ingestion/search", params={"q": "hello"})
|
||||||
|
assert resp.status_code == 200
|
||||||
|
data = resp.json()
|
||||||
|
assert data["total"] == 1
|
||||||
|
assert data["query"] == "hello"
|
||||||
|
assert data["results"][0]["content"] == "hello world"
|
||||||
|
assert data["results"][0]["score"] == 0.95
|
||||||
|
|
||||||
|
def test_search_empty_query(self, client: TestClient) -> None:
|
||||||
|
resp = client.get("/api/ingestion/search", params={"q": ""})
|
||||||
|
assert resp.status_code == 422
|
||||||
|
|
||||||
|
def test_search_with_doc_filter(
|
||||||
|
self, client: TestClient, mock_ingestion_service: AsyncMock
|
||||||
|
) -> None:
|
||||||
|
mock_ingestion_service.search_fulltext.return_value = []
|
||||||
|
resp = client.get("/api/ingestion/search", params={"q": "test", "doc_id": "doc-1"})
|
||||||
|
assert resp.status_code == 200
|
||||||
|
mock_ingestion_service.search_fulltext.assert_awaited_once_with(
|
||||||
|
"test", k=20, doc_id="doc-1"
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -140,6 +140,44 @@ class TestSearch:
|
||||||
mock_vector_store.search_similar.assert_awaited_once()
|
mock_vector_store.search_similar.assert_awaited_once()
|
||||||
|
|
||||||
|
|
||||||
|
class TestSearchFulltext:
|
||||||
|
async def test_delegates_to_vector_store(
|
||||||
|
self, service: IngestionService, mock_vector_store: AsyncMock
|
||||||
|
) -> None:
|
||||||
|
mock_vector_store.search_fulltext.return_value = []
|
||||||
|
await service.search_fulltext("hello world", k=5)
|
||||||
|
mock_vector_store.search_fulltext.assert_awaited_once_with(
|
||||||
|
"test-idx", "hello world", k=5, doc_id=None
|
||||||
|
)
|
||||||
|
|
||||||
|
async def test_filters_by_doc_id(
|
||||||
|
self, service: IngestionService, mock_vector_store: AsyncMock
|
||||||
|
) -> None:
|
||||||
|
mock_vector_store.search_fulltext.return_value = []
|
||||||
|
await service.search_fulltext("hello", doc_id="doc-1")
|
||||||
|
mock_vector_store.search_fulltext.assert_awaited_once_with(
|
||||||
|
"test-idx", "hello", k=20, doc_id="doc-1"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestPing:
|
||||||
|
async def test_ping_success(
|
||||||
|
self, service: IngestionService, mock_vector_store: AsyncMock
|
||||||
|
) -> None:
|
||||||
|
mock_vector_store._client = AsyncMock()
|
||||||
|
mock_vector_store._client.info.return_value = {"cluster_name": "test"}
|
||||||
|
result = await service.ping()
|
||||||
|
assert result is True
|
||||||
|
|
||||||
|
async def test_ping_failure(
|
||||||
|
self, service: IngestionService, mock_vector_store: AsyncMock
|
||||||
|
) -> None:
|
||||||
|
mock_vector_store._client = AsyncMock()
|
||||||
|
mock_vector_store._client.info.side_effect = ConnectionError("down")
|
||||||
|
result = await service.ping()
|
||||||
|
assert result is False
|
||||||
|
|
||||||
|
|
||||||
class TestEnsureIndex:
|
class TestEnsureIndex:
|
||||||
async def test_calls_vector_store(
|
async def test_calls_vector_store(
|
||||||
self, service: IngestionService, mock_vector_store: AsyncMock
|
self, service: IngestionService, mock_vector_store: AsyncMock
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,23 @@ export interface IngestionResult {
|
||||||
|
|
||||||
export interface IngestionStatus {
|
export interface IngestionStatus {
|
||||||
available: boolean
|
available: boolean
|
||||||
|
opensearchConnected: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SearchResultItem {
|
||||||
|
docId: string
|
||||||
|
filename: string
|
||||||
|
content: string
|
||||||
|
chunkIndex: number
|
||||||
|
pageNumber: number
|
||||||
|
score: number
|
||||||
|
headings: string[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SearchResponse {
|
||||||
|
results: SearchResultItem[]
|
||||||
|
total: number
|
||||||
|
query: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export function ingestAnalysis(jobId: string): Promise<IngestionResult> {
|
export function ingestAnalysis(jobId: string): Promise<IngestionResult> {
|
||||||
|
|
@ -23,3 +40,13 @@ export function deleteIngested(docId: string): Promise<unknown> {
|
||||||
export function fetchIngestionStatus(): Promise<IngestionStatus> {
|
export function fetchIngestionStatus(): Promise<IngestionStatus> {
|
||||||
return apiFetch<IngestionStatus>('/api/ingestion/status')
|
return apiFetch<IngestionStatus>('/api/ingestion/status')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function searchChunks(
|
||||||
|
query: string,
|
||||||
|
options: { docId?: string; k?: number } = {},
|
||||||
|
): Promise<SearchResponse> {
|
||||||
|
const params = new URLSearchParams({ q: query })
|
||||||
|
if (options.docId) params.set('doc_id', options.docId)
|
||||||
|
if (options.k) params.set('k', String(options.k))
|
||||||
|
return apiFetch<SearchResponse>(`/api/ingestion/search?${params}`)
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,35 +2,68 @@ import { defineStore } from 'pinia'
|
||||||
import { ref } from 'vue'
|
import { ref } from 'vue'
|
||||||
import * as api from './api'
|
import * as api from './api'
|
||||||
|
|
||||||
|
export type IngestionStep = 'embedding' | 'indexing' | 'done'
|
||||||
|
|
||||||
export const useIngestionStore = defineStore('ingestion', () => {
|
export const useIngestionStore = defineStore('ingestion', () => {
|
||||||
const available = ref(false)
|
const available = ref(false)
|
||||||
|
const opensearchConnected = ref(false)
|
||||||
const ingesting = ref(false)
|
const ingesting = ref(false)
|
||||||
const error = ref<string | null>(null)
|
const error = ref<string | null>(null)
|
||||||
/** Map of docId → chunks indexed count (tracks which docs are ingested) */
|
/** Map of docId → chunks indexed count (tracks which docs are ingested) */
|
||||||
const ingestedDocs = ref<Record<string, number>>({})
|
const ingestedDocs = ref<Record<string, number>>({})
|
||||||
|
/** Current step of the ingestion pipeline (null when idle) */
|
||||||
|
const currentStep = ref<IngestionStep | null>(null)
|
||||||
|
/** Search results */
|
||||||
|
const searchResults = ref<api.SearchResultItem[]>([])
|
||||||
|
const searchQuery = ref('')
|
||||||
|
const searching = ref(false)
|
||||||
|
|
||||||
|
let _pollTimer: ReturnType<typeof setInterval> | null = null
|
||||||
|
|
||||||
async function checkAvailability(): Promise<void> {
|
async function checkAvailability(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
const status = await api.fetchIngestionStatus()
|
const status = await api.fetchIngestionStatus()
|
||||||
available.value = status.available
|
available.value = status.available
|
||||||
|
opensearchConnected.value = status.opensearchConnected
|
||||||
} catch {
|
} catch {
|
||||||
available.value = false
|
available.value = false
|
||||||
|
opensearchConnected.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function startPolling(intervalMs = 30_000): void {
|
||||||
|
stopPolling()
|
||||||
|
_pollTimer = setInterval(checkAvailability, intervalMs)
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopPolling(): void {
|
||||||
|
if (_pollTimer) {
|
||||||
|
clearInterval(_pollTimer)
|
||||||
|
_pollTimer = null
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function ingest(jobId: string): Promise<api.IngestionResult | null> {
|
async function ingest(jobId: string): Promise<api.IngestionResult | null> {
|
||||||
ingesting.value = true
|
ingesting.value = true
|
||||||
error.value = null
|
error.value = null
|
||||||
|
currentStep.value = 'embedding'
|
||||||
try {
|
try {
|
||||||
|
currentStep.value = 'indexing'
|
||||||
const result = await api.ingestAnalysis(jobId)
|
const result = await api.ingestAnalysis(jobId)
|
||||||
|
currentStep.value = 'done'
|
||||||
ingestedDocs.value[result.docId] = result.chunksIndexed
|
ingestedDocs.value[result.docId] = result.chunksIndexed
|
||||||
return result
|
return result
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
error.value = (e as Error).message || 'Ingestion failed'
|
error.value = (e as Error).message || 'Ingestion failed'
|
||||||
console.error('Ingestion failed', e)
|
console.error('Ingestion failed', e)
|
||||||
|
currentStep.value = null
|
||||||
return null
|
return null
|
||||||
} finally {
|
} finally {
|
||||||
ingesting.value = false
|
ingesting.value = false
|
||||||
|
// Reset step after a short delay so the user sees the "done" state
|
||||||
|
setTimeout(() => {
|
||||||
|
currentStep.value = null
|
||||||
|
}, 2000)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -44,13 +77,46 @@ export const useIngestionStore = defineStore('ingestion', () => {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function search(query: string, docId?: string): Promise<void> {
|
||||||
|
if (!query.trim()) {
|
||||||
|
searchResults.value = []
|
||||||
|
searchQuery.value = ''
|
||||||
|
return
|
||||||
|
}
|
||||||
|
searching.value = true
|
||||||
|
searchQuery.value = query
|
||||||
|
try {
|
||||||
|
const resp = await api.searchChunks(query, { docId })
|
||||||
|
searchResults.value = resp.results
|
||||||
|
} catch (e) {
|
||||||
|
console.error('Search failed', e)
|
||||||
|
searchResults.value = []
|
||||||
|
} finally {
|
||||||
|
searching.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function clearSearch(): void {
|
||||||
|
searchResults.value = []
|
||||||
|
searchQuery.value = ''
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
available,
|
available,
|
||||||
|
opensearchConnected,
|
||||||
ingesting,
|
ingesting,
|
||||||
error,
|
error,
|
||||||
ingestedDocs,
|
ingestedDocs,
|
||||||
|
currentStep,
|
||||||
|
searchResults,
|
||||||
|
searchQuery,
|
||||||
|
searching,
|
||||||
checkAvailability,
|
checkAvailability,
|
||||||
|
startPolling,
|
||||||
|
stopPolling,
|
||||||
ingest,
|
ingest,
|
||||||
deleteIngested,
|
deleteIngested,
|
||||||
|
search,
|
||||||
|
clearSearch,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,46 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<!-- Full-text chunk search -->
|
||||||
|
<div v-if="ingestionStore.available" class="chunk-search-bar">
|
||||||
|
<input
|
||||||
|
v-model="chunkSearchQuery"
|
||||||
|
type="text"
|
||||||
|
class="search-input chunk-search"
|
||||||
|
:placeholder="t('ingestion.searchChunks')"
|
||||||
|
@keyup.enter="runChunkSearch"
|
||||||
|
/>
|
||||||
|
<div v-if="ingestionStore.searching" class="spinner-xs" />
|
||||||
|
</div>
|
||||||
|
<div v-if="ingestionStore.searchResults.length > 0" class="search-results">
|
||||||
|
<div
|
||||||
|
v-for="(result, idx) in ingestionStore.searchResults"
|
||||||
|
:key="idx"
|
||||||
|
class="search-result-item"
|
||||||
|
>
|
||||||
|
<div class="result-header">
|
||||||
|
<span class="result-filename">{{ result.filename }}</span>
|
||||||
|
<span class="result-meta"
|
||||||
|
>p.{{ result.pageNumber }} — chunk #{{ result.chunkIndex }}</span
|
||||||
|
>
|
||||||
|
<span class="result-score">{{ (result.score * 100).toFixed(0) }}%</span>
|
||||||
|
</div>
|
||||||
|
<p class="result-content">
|
||||||
|
{{ result.content.slice(0, 200) }}{{ result.content.length > 200 ? '…' : '' }}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
v-if="
|
||||||
|
ingestionStore.searchQuery &&
|
||||||
|
!ingestionStore.searching &&
|
||||||
|
ingestionStore.searchResults.length === 0
|
||||||
|
"
|
||||||
|
class="tab-empty"
|
||||||
|
>
|
||||||
|
{{ t('ingestion.noResults', { q: ingestionStore.searchQuery }) }}
|
||||||
|
</div>
|
||||||
|
|
||||||
<div class="page-content">
|
<div class="page-content">
|
||||||
<div v-if="filteredDocs.length === 0" class="tab-empty">
|
<div v-if="filteredDocs.length === 0" class="tab-empty">
|
||||||
{{ t('history.emptyDocs') }}
|
{{ t('history.emptyDocs') }}
|
||||||
|
|
@ -76,6 +116,20 @@
|
||||||
/>
|
/>
|
||||||
</svg>
|
</svg>
|
||||||
</button>
|
</button>
|
||||||
|
<button
|
||||||
|
v-if="ingestionStore.ingestedDocs[doc.id]"
|
||||||
|
class="action-btn unindex"
|
||||||
|
:title="t('ingestion.deleteIndex')"
|
||||||
|
@click="confirmRemoveFromIndex(doc.id)"
|
||||||
|
>
|
||||||
|
<svg viewBox="0 0 20 20" fill="currentColor">
|
||||||
|
<path
|
||||||
|
fill-rule="evenodd"
|
||||||
|
d="M3 17a1 1 0 011-1h12a1 1 0 110 2H4a1 1 0 01-1-1zM6.293 6.707a1 1 0 010-1.414l3-3a1 1 0 011.414 0l3 3a1 1 0 01-1.414 1.414L11 5.414V13a1 1 0 11-2 0V5.414L7.707 6.707a1 1 0 01-1.414 0z"
|
||||||
|
clip-rule="evenodd"
|
||||||
|
/>
|
||||||
|
</svg>
|
||||||
|
</button>
|
||||||
<button class="action-btn delete" @click="handleDelete(doc.id)">
|
<button class="action-btn delete" @click="handleDelete(doc.id)">
|
||||||
<svg viewBox="0 0 20 20" fill="currentColor">
|
<svg viewBox="0 0 20 20" fill="currentColor">
|
||||||
<path
|
<path
|
||||||
|
|
@ -107,6 +161,7 @@ const router = useRouter()
|
||||||
const { t } = useI18n()
|
const { t } = useI18n()
|
||||||
|
|
||||||
const searchQuery = ref('')
|
const searchQuery = ref('')
|
||||||
|
const chunkSearchQuery = ref('')
|
||||||
const activeFilter = ref<'all' | 'indexed' | 'not-indexed'>('all')
|
const activeFilter = ref<'all' | 'indexed' | 'not-indexed'>('all')
|
||||||
const sortBy = ref<'name' | 'date'>('date')
|
const sortBy = ref<'name' | 'date'>('date')
|
||||||
|
|
||||||
|
|
@ -152,6 +207,12 @@ function openInStudio(doc: Document) {
|
||||||
router.push('/studio')
|
router.push('/studio')
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function confirmRemoveFromIndex(docId: string) {
|
||||||
|
if (confirm(t('ingestion.deleteConfirm'))) {
|
||||||
|
ingestionStore.deleteIngested(docId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function handleDelete(docId: string) {
|
async function handleDelete(docId: string) {
|
||||||
if (ingestionStore.ingestedDocs[docId]) {
|
if (ingestionStore.ingestedDocs[docId]) {
|
||||||
await ingestionStore.deleteIngested(docId)
|
await ingestionStore.deleteIngested(docId)
|
||||||
|
|
@ -159,6 +220,14 @@ async function handleDelete(docId: string) {
|
||||||
await docStore.remove(docId)
|
await docStore.remove(docId)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function runChunkSearch() {
|
||||||
|
if (chunkSearchQuery.value.trim()) {
|
||||||
|
ingestionStore.search(chunkSearchQuery.value)
|
||||||
|
} else {
|
||||||
|
ingestionStore.clearSearch()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
onMounted(() => {
|
onMounted(() => {
|
||||||
docStore.load()
|
docStore.load()
|
||||||
ingestionStore.checkAvailability()
|
ingestionStore.checkAvailability()
|
||||||
|
|
@ -373,8 +442,91 @@ onMounted(() => {
|
||||||
background: rgba(239, 68, 68, 0.1);
|
background: rgba(239, 68, 68, 0.1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.action-btn.unindex:hover {
|
||||||
|
color: var(--warning, #f59e0b);
|
||||||
|
background: rgba(245, 158, 11, 0.1);
|
||||||
|
}
|
||||||
|
|
||||||
.action-btn svg {
|
.action-btn svg {
|
||||||
width: 16px;
|
width: 16px;
|
||||||
height: 16px;
|
height: 16px;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Chunk search */
|
||||||
|
.chunk-search-bar {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
padding: 8px 24px;
|
||||||
|
border-bottom: 1px solid var(--border);
|
||||||
|
}
|
||||||
|
|
||||||
|
.chunk-search {
|
||||||
|
flex: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.spinner-xs {
|
||||||
|
width: 14px;
|
||||||
|
height: 14px;
|
||||||
|
border: 2px solid var(--border);
|
||||||
|
border-top-color: var(--accent);
|
||||||
|
border-radius: 50%;
|
||||||
|
animation: spin 0.6s linear infinite;
|
||||||
|
}
|
||||||
|
|
||||||
|
@keyframes spin {
|
||||||
|
to {
|
||||||
|
transform: rotate(360deg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Search results */
|
||||||
|
.search-results {
|
||||||
|
max-height: 300px;
|
||||||
|
overflow-y: auto;
|
||||||
|
border-bottom: 1px solid var(--border);
|
||||||
|
}
|
||||||
|
|
||||||
|
.search-result-item {
|
||||||
|
padding: 10px 24px;
|
||||||
|
border-bottom: 1px solid var(--border);
|
||||||
|
}
|
||||||
|
|
||||||
|
.search-result-item:last-child {
|
||||||
|
border-bottom: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-header {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
margin-bottom: 4px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-filename {
|
||||||
|
font-weight: 600;
|
||||||
|
font-size: 13px;
|
||||||
|
color: var(--text);
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-meta {
|
||||||
|
font-size: 11px;
|
||||||
|
color: var(--text-muted);
|
||||||
|
font-family: 'IBM Plex Mono', monospace;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-score {
|
||||||
|
margin-left: auto;
|
||||||
|
font-size: 11px;
|
||||||
|
font-weight: 600;
|
||||||
|
color: var(--accent);
|
||||||
|
font-family: 'IBM Plex Mono', monospace;
|
||||||
|
}
|
||||||
|
|
||||||
|
.result-content {
|
||||||
|
font-size: 13px;
|
||||||
|
color: var(--text-muted);
|
||||||
|
line-height: 1.5;
|
||||||
|
margin: 0;
|
||||||
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|
|
||||||
|
|
@ -114,6 +114,47 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Ingestion progress stepper -->
|
||||||
|
<div v-if="ingestionStore.currentStep" class="ingestion-stepper">
|
||||||
|
<div
|
||||||
|
class="step"
|
||||||
|
:class="{
|
||||||
|
active: ingestionStore.currentStep === 'embedding',
|
||||||
|
done: ingestionStore.currentStep === 'indexing' || ingestionStore.currentStep === 'done',
|
||||||
|
}"
|
||||||
|
>
|
||||||
|
<span class="step-dot" />
|
||||||
|
<span class="step-label">{{ t('ingestion.stepEmbedding') }}</span>
|
||||||
|
</div>
|
||||||
|
<div
|
||||||
|
class="step-line"
|
||||||
|
:class="{
|
||||||
|
done: ingestionStore.currentStep === 'indexing' || ingestionStore.currentStep === 'done',
|
||||||
|
}"
|
||||||
|
/>
|
||||||
|
<div
|
||||||
|
class="step"
|
||||||
|
:class="{
|
||||||
|
active: ingestionStore.currentStep === 'indexing',
|
||||||
|
done: ingestionStore.currentStep === 'done',
|
||||||
|
}"
|
||||||
|
>
|
||||||
|
<span class="step-dot" />
|
||||||
|
<span class="step-label">{{ t('ingestion.stepIndexing') }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="step-line" :class="{ done: ingestionStore.currentStep === 'done' }" />
|
||||||
|
<div
|
||||||
|
class="step"
|
||||||
|
:class="{
|
||||||
|
active: ingestionStore.currentStep === 'done',
|
||||||
|
done: ingestionStore.currentStep === 'done',
|
||||||
|
}"
|
||||||
|
>
|
||||||
|
<span class="step-dot" />
|
||||||
|
<span class="step-label">{{ t('ingestion.stepDone') }}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Document info bar -->
|
<!-- Document info bar -->
|
||||||
<div class="doc-infobar">
|
<div class="doc-infobar">
|
||||||
<div class="doc-info-left">
|
<div class="doc-info-left">
|
||||||
|
|
@ -617,12 +658,22 @@ watch(currentPage, () => {
|
||||||
})
|
})
|
||||||
|
|
||||||
// Auto-switch to verify when analysis completes + refresh document data (pageCount)
|
// Auto-switch to verify when analysis completes + refresh document data (pageCount)
|
||||||
|
// Auto-trigger ingestion if pipeline is available (#81)
|
||||||
watch(
|
watch(
|
||||||
() => analysisStore.currentAnalysis?.status,
|
() => analysisStore.currentAnalysis?.status,
|
||||||
(status) => {
|
async (status) => {
|
||||||
if (status === 'COMPLETED') {
|
if (status === 'COMPLETED') {
|
||||||
mode.value = 'verify'
|
mode.value = 'verify'
|
||||||
documentStore.load()
|
documentStore.load()
|
||||||
|
|
||||||
|
// Auto-ingest if chunks are available and pipeline is configured
|
||||||
|
if (
|
||||||
|
ingestionStore.available &&
|
||||||
|
analysisStore.currentAnalysis?.chunksJson &&
|
||||||
|
analysisStore.currentAnalysis?.id
|
||||||
|
) {
|
||||||
|
await ingestionStore.ingest(analysisStore.currentAnalysis.id)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
@ -873,6 +924,79 @@ onBeforeUnmount(() => {
|
||||||
animation: spin 0.6s linear infinite;
|
animation: spin 0.6s linear infinite;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Ingestion stepper */
|
||||||
|
.ingestion-stepper {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
gap: 0;
|
||||||
|
padding: 8px 20px;
|
||||||
|
background: var(--bg-surface);
|
||||||
|
border-bottom: 1px solid var(--border);
|
||||||
|
}
|
||||||
|
|
||||||
|
.step {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
padding: 0 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.step-dot {
|
||||||
|
width: 10px;
|
||||||
|
height: 10px;
|
||||||
|
border-radius: 50%;
|
||||||
|
background: var(--border);
|
||||||
|
transition: all 0.3s ease;
|
||||||
|
}
|
||||||
|
|
||||||
|
.step.active .step-dot {
|
||||||
|
background: var(--accent);
|
||||||
|
box-shadow: 0 0 6px var(--accent);
|
||||||
|
animation: pulse-dot 1s ease-in-out infinite;
|
||||||
|
}
|
||||||
|
|
||||||
|
.step.done .step-dot {
|
||||||
|
background: var(--success, #22c55e);
|
||||||
|
}
|
||||||
|
|
||||||
|
.step-label {
|
||||||
|
font-size: 12px;
|
||||||
|
color: var(--text-muted);
|
||||||
|
font-weight: 500;
|
||||||
|
}
|
||||||
|
|
||||||
|
.step.active .step-label {
|
||||||
|
color: var(--accent);
|
||||||
|
}
|
||||||
|
|
||||||
|
.step.done .step-label {
|
||||||
|
color: var(--success, #22c55e);
|
||||||
|
}
|
||||||
|
|
||||||
|
.step-line {
|
||||||
|
width: 40px;
|
||||||
|
height: 2px;
|
||||||
|
background: var(--border);
|
||||||
|
transition: background 0.3s ease;
|
||||||
|
}
|
||||||
|
|
||||||
|
.step-line.done {
|
||||||
|
background: var(--success, #22c55e);
|
||||||
|
}
|
||||||
|
|
||||||
|
@keyframes pulse-dot {
|
||||||
|
0%,
|
||||||
|
100% {
|
||||||
|
transform: scale(1);
|
||||||
|
opacity: 1;
|
||||||
|
}
|
||||||
|
50% {
|
||||||
|
transform: scale(1.3);
|
||||||
|
opacity: 0.7;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* Doc info bar */
|
/* Doc info bar */
|
||||||
.doc-infobar {
|
.doc-infobar {
|
||||||
display: flex;
|
display: flex;
|
||||||
|
|
|
||||||
|
|
@ -140,6 +140,8 @@ const messages: Messages = {
|
||||||
'ingestion.chunksIndexed': '{n} chunks indexés',
|
'ingestion.chunksIndexed': '{n} chunks indexés',
|
||||||
'ingestion.openInStudio': 'Ouvrir dans le Studio',
|
'ingestion.openInStudio': 'Ouvrir dans le Studio',
|
||||||
'ingestion.deleteIndex': "Supprimer de l'index",
|
'ingestion.deleteIndex': "Supprimer de l'index",
|
||||||
|
'ingestion.deleteConfirm':
|
||||||
|
'Retirer ce document de l\u2019index ? Les chunks seront supprimés mais le document source restera.',
|
||||||
'ingestion.unavailable': 'Ingestion non disponible',
|
'ingestion.unavailable': 'Ingestion non disponible',
|
||||||
'ingestion.filterAll': 'Tous',
|
'ingestion.filterAll': 'Tous',
|
||||||
'ingestion.filterIndexed': 'Indexés',
|
'ingestion.filterIndexed': 'Indexés',
|
||||||
|
|
@ -147,6 +149,13 @@ const messages: Messages = {
|
||||||
'ingestion.sortName': 'Nom',
|
'ingestion.sortName': 'Nom',
|
||||||
'ingestion.sortDate': 'Date',
|
'ingestion.sortDate': 'Date',
|
||||||
'ingestion.search': 'Rechercher...',
|
'ingestion.search': 'Rechercher...',
|
||||||
|
'ingestion.searchChunks': 'Rechercher dans les chunks…',
|
||||||
|
'ingestion.noResults': 'Aucun résultat pour « {q} ».',
|
||||||
|
'ingestion.stepEmbedding': 'Embedding…',
|
||||||
|
'ingestion.stepIndexing': 'Indexation…',
|
||||||
|
'ingestion.stepDone': 'Terminé',
|
||||||
|
'ingestion.opensearchConnected': 'OpenSearch connecté',
|
||||||
|
'ingestion.opensearchDisconnected': 'OpenSearch déconnecté',
|
||||||
|
|
||||||
// Pagination
|
// Pagination
|
||||||
'pagination.pageOf': 'Page {current} sur {total}',
|
'pagination.pageOf': 'Page {current} sur {total}',
|
||||||
|
|
@ -291,6 +300,8 @@ const messages: Messages = {
|
||||||
'ingestion.chunksIndexed': '{n} chunks indexed',
|
'ingestion.chunksIndexed': '{n} chunks indexed',
|
||||||
'ingestion.openInStudio': 'Open in Studio',
|
'ingestion.openInStudio': 'Open in Studio',
|
||||||
'ingestion.deleteIndex': 'Remove from index',
|
'ingestion.deleteIndex': 'Remove from index',
|
||||||
|
'ingestion.deleteConfirm':
|
||||||
|
'Remove this document from the index? Chunks will be deleted but the source document will remain.',
|
||||||
'ingestion.unavailable': 'Ingestion unavailable',
|
'ingestion.unavailable': 'Ingestion unavailable',
|
||||||
'ingestion.filterAll': 'All',
|
'ingestion.filterAll': 'All',
|
||||||
'ingestion.filterIndexed': 'Indexed',
|
'ingestion.filterIndexed': 'Indexed',
|
||||||
|
|
@ -298,6 +309,13 @@ const messages: Messages = {
|
||||||
'ingestion.sortName': 'Name',
|
'ingestion.sortName': 'Name',
|
||||||
'ingestion.sortDate': 'Date',
|
'ingestion.sortDate': 'Date',
|
||||||
'ingestion.search': 'Search...',
|
'ingestion.search': 'Search...',
|
||||||
|
'ingestion.searchChunks': 'Search indexed chunks…',
|
||||||
|
'ingestion.noResults': 'No results for "{q}".',
|
||||||
|
'ingestion.stepEmbedding': 'Embedding…',
|
||||||
|
'ingestion.stepIndexing': 'Indexing…',
|
||||||
|
'ingestion.stepDone': 'Done',
|
||||||
|
'ingestion.opensearchConnected': 'OpenSearch connected',
|
||||||
|
'ingestion.opensearchDisconnected': 'OpenSearch unreachable',
|
||||||
|
|
||||||
'pagination.pageOf': 'Page {current} of {total}',
|
'pagination.pageOf': 'Page {current} of {total}',
|
||||||
'pagination.perPage': '/ page',
|
'pagination.perPage': '/ page',
|
||||||
|
|
|
||||||
|
|
@ -79,6 +79,21 @@
|
||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
<div class="sidebar-footer">
|
<div class="sidebar-footer">
|
||||||
|
<div
|
||||||
|
v-if="ingestionStore.available"
|
||||||
|
class="opensearch-status"
|
||||||
|
:title="
|
||||||
|
ingestionStore.opensearchConnected
|
||||||
|
? t('ingestion.opensearchConnected')
|
||||||
|
: t('ingestion.opensearchDisconnected')
|
||||||
|
"
|
||||||
|
>
|
||||||
|
<span
|
||||||
|
class="status-dot"
|
||||||
|
:class="ingestionStore.opensearchConnected ? 'connected' : 'disconnected'"
|
||||||
|
/>
|
||||||
|
<span class="status-label">OpenSearch</span>
|
||||||
|
</div>
|
||||||
<a
|
<a
|
||||||
class="github-badge"
|
class="github-badge"
|
||||||
href="https://github.com/scub-france/Docling-Studio"
|
href="https://github.com/scub-france/Docling-Studio"
|
||||||
|
|
@ -97,12 +112,14 @@
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { computed } from 'vue'
|
import { computed, onMounted, onBeforeUnmount } from 'vue'
|
||||||
import { RouterLink, useRoute } from 'vue-router'
|
import { RouterLink, useRoute } from 'vue-router'
|
||||||
import { useI18n } from '../i18n'
|
import { useI18n } from '../i18n'
|
||||||
import { useFeatureFlagStore } from '../../features/feature-flags/store'
|
import { useFeatureFlagStore } from '../../features/feature-flags/store'
|
||||||
|
import { useIngestionStore } from '../../features/ingestion/store'
|
||||||
|
|
||||||
const featureStore = useFeatureFlagStore()
|
const featureStore = useFeatureFlagStore()
|
||||||
|
const ingestionStore = useIngestionStore()
|
||||||
const version = computed(() => featureStore.appVersion)
|
const version = computed(() => featureStore.appVersion)
|
||||||
const route = useRoute()
|
const route = useRoute()
|
||||||
const { t } = useI18n()
|
const { t } = useI18n()
|
||||||
|
|
@ -110,6 +127,15 @@ const { t } = useI18n()
|
||||||
defineProps({
|
defineProps({
|
||||||
open: { type: Boolean, default: false },
|
open: { type: Boolean, default: false },
|
||||||
})
|
})
|
||||||
|
|
||||||
|
onMounted(() => {
|
||||||
|
ingestionStore.checkAvailability()
|
||||||
|
ingestionStore.startPolling(30_000)
|
||||||
|
})
|
||||||
|
|
||||||
|
onBeforeUnmount(() => {
|
||||||
|
ingestionStore.stopPolling()
|
||||||
|
})
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
|
@ -196,4 +222,35 @@ defineProps({
|
||||||
color: var(--text-muted);
|
color: var(--text-muted);
|
||||||
font-family: 'IBM Plex Mono', monospace;
|
font-family: 'IBM Plex Mono', monospace;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.opensearch-status {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
margin-bottom: 10px;
|
||||||
|
cursor: default;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-dot {
|
||||||
|
width: 8px;
|
||||||
|
height: 8px;
|
||||||
|
border-radius: 50%;
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-dot.connected {
|
||||||
|
background: var(--success, #22c55e);
|
||||||
|
box-shadow: 0 0 4px var(--success, #22c55e);
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-dot.disconnected {
|
||||||
|
background: var(--error, #ef4444);
|
||||||
|
box-shadow: 0 0 4px var(--error, #ef4444);
|
||||||
|
}
|
||||||
|
|
||||||
|
.status-label {
|
||||||
|
font-size: 11px;
|
||||||
|
color: var(--text-muted);
|
||||||
|
font-family: 'IBM Plex Mono', monospace;
|
||||||
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue