"""Build a Cytoscape-shaped graph payload straight from a serialized `DoclingDocument` (i.e. the `document_json` blob stored in SQLite). Mirrors `infra.neo4j.queries.fetch_graph` so the frontend can reuse the same `GraphView` component — the only intentional difference is the absence of Chunk nodes / HAS_CHUNK / DERIVED_FROM edges, since chunks are a product of the Maintain step and don't exist in `document_json` alone. Used by the reasoning-trace viewer, which needs the structural graph to overlay iterations onto but does NOT need (and should not require) Neo4j. """ from __future__ import annotations import json from itertools import pairwise from typing import Any from infra.docling_tree import ( build_collapse_index, dfs_order, element_label, is_inline_group, iter_items, iter_pages, iter_provs, parent_ref, ) from infra.neo4j.queries import GraphPayload def _element_node( doc_id: str, item: dict[str, Any], provs: list[dict[str, Any]], *, text_override: str | None = None, ) -> dict[str, Any]: first_page = provs[0].get("page_no") if provs else None raw_text = text_override if text_override is not None else (item.get("text") or "") return { "id": f"elem::{item.get('self_ref')}", "group": "element", "label": element_label(item.get("label") or ""), "docling_label": (item.get("label") or "").lower(), "self_ref": item.get("self_ref"), "text": raw_text[:200], "prov_page": first_page, "provs": provs, "level": item.get("level"), "doc_id": doc_id, } def _page_node(doc_id: str, page: dict[str, Any]) -> dict[str, Any]: return { "id": f"page::{page.get('page_no')}", "group": "page", "page_no": page.get("page_no"), "width": page.get("width"), "height": page.get("height"), "doc_id": doc_id, } def _edge(source: str, target: str, edge_type: str, *, order: int | None = None) -> dict[str, Any]: return { "id": f"{edge_type}::{source}::{target}", "source": source, "target": target, "type": edge_type, "order": order, } def build_graph_payload( document_json: str, *, doc_id: str, title: str | None = None, max_pages: int = 200, ) -> GraphPayload: """Build a `GraphPayload` equivalent to `fetch_graph(neo4j, doc_id)` from the raw `DoclingDocument` JSON. Returns `truncated=True` with empty node/edge lists beyond `max_pages`, so the caller can mirror the Neo4j endpoint's 413 behavior. """ doc_data = json.loads(document_json) pages_raw = list(iter_pages(doc_data)) page_count = len(pages_raw) if page_count > max_pages: return GraphPayload( doc_id=doc_id, nodes=[], edges=[], node_count=0, edge_count=0, truncated=True, page_count=page_count, ) nodes: list[dict[str, Any]] = [] edges: list[dict[str, Any]] = [] doc_node_id = f"doc::{doc_id}" nodes.append( { "id": doc_node_id, "group": "document", "doc_id": doc_id, "title": title, # `stages_applied` is a Neo4j-only artifact; keep the key present # for shape parity but leave it empty since SQLite doesn't track it. "stages_applied": [], } ) # Page nodes. for p in pages_raw: nodes.append(_page_node(doc_id, p)) # Issue #197: collapse Docling noise — InlineGroup style runs and the # internal text labels Docling extracts from pictures/charts. skip_refs, inline_meta = build_collapse_index(doc_data) # Element nodes + collect parent/body metadata for edges below. The # `element_idx` mirrors TreeWriter's `enumerate(elements)` so PARENT_OF # carries the same `order` the Neo4j projection does. by_ref: dict[str, dict[str, Any]] = {} element_idx = 0 for _, item in iter_items(doc_data): ref = item.get("self_ref") if not ref or ref in skip_refs: continue by_ref[ref] = item if is_inline_group(item): meta = inline_meta.get(ref, {"text": "", "provs": []}) provs = meta["provs"] text_override: str | None = meta["text"] else: provs = iter_provs(item) text_override = None nodes.append(_element_node(doc_id, item, provs, text_override=text_override)) pref = parent_ref(item) if pref == "#/body": edges.append(_edge(doc_node_id, f"elem::{ref}", "HAS_ROOT")) elif pref: edges.append(_edge(f"elem::{pref}", f"elem::{ref}", "PARENT_OF", order=element_idx)) # ON_PAGE, dedup'd per (element, page) — matches the Neo4j query's # DISTINCT projection through Provenance. seen_pages: set[int] = set() for prov in provs: page_no = prov.get("page_no") if page_no is None or page_no in seen_pages: continue seen_pages.add(page_no) edges.append(_edge(f"elem::{ref}", f"page::{page_no}", "ON_PAGE")) element_idx += 1 # NEXT chain (DFS pre-order from body), inline-group children skipped. for a, b in pairwise(dfs_order(doc_data, skip_refs)): if a in by_ref and b in by_ref: edges.append(_edge(f"elem::{a}", f"elem::{b}", "NEXT")) return GraphPayload( doc_id=doc_id, nodes=nodes, edges=edges, node_count=len(nodes), edge_count=len(edges), truncated=False, page_count=page_count, )