fix: batch metadata cache entity lookups

MetadataCache.get_search_results previously looped over each cached
entity ID and issued one SELECT per ID, producing N extra queries per
cached search hit. It now resolves all entities in a single batched
IN query (chunked at 500 to stay under the SQLite variable limit),
then reconstructs the result list in the original result_ids order
using an in-memory dict lookup.
This commit is contained in:
JohnBaumb 2026-04-19 14:33:07 -07:00
parent 4cfe6c6bf8
commit d3d648d9fd

View file

@ -326,20 +326,28 @@ class MetadataCache:
""", (row['id'],)) """, (row['id'],))
conn.commit() conn.commit()
# Resolve entity IDs to full data # Resolve entity IDs to full data via a single batched query
# (chunked to stay below SQLite's default variable limit).
result_ids = json.loads(row['result_ids']) result_ids = json.loads(row['result_ids'])
if not result_ids: if not result_ids:
return [] return []
results = [] raw_by_id: Dict[str, dict] = {}
for eid in result_ids: for i in range(0, len(result_ids), 500):
cursor.execute(""" chunk = result_ids[i:i + 500]
SELECT raw_json FROM metadata_cache_entities placeholders = ','.join('?' * len(chunk))
WHERE source = ? AND entity_type = ? AND entity_id = ? cursor.execute(f"""
""", (source, search_type, eid)) SELECT entity_id, raw_json FROM metadata_cache_entities
erow = cursor.fetchone() WHERE source = ? AND entity_type = ? AND entity_id IN ({placeholders})
if erow: """, [source, search_type, *chunk])
results.append(json.loads(erow['raw_json'])) for erow in cursor.fetchall():
try:
raw_by_id[erow['entity_id']] = json.loads(erow['raw_json'])
except (ValueError, TypeError):
continue
# Preserve the original result_ids ordering.
results = [raw_by_id[eid] for eid in result_ids if eid in raw_by_id]
# Only return if we found all (or most) entries — partial results are unreliable # Only return if we found all (or most) entries — partial results are unreliable
if len(results) >= len(result_ids) * 0.8: if len(results) >= len(result_ids) * 0.8: