M3U export: resolve paths via one bulk read instead of a per-artist search loop (fixes 'Export M3U hangs forever' under active enrichment/scan DB writes)
This commit is contained in:
BoulderBadgeDad
parent
608efb1d85
commit
6e7fd3ff5c
3 changed files with 115 additions and 32 deletions
|
|
@ -6926,6 +6926,31 @@ class MusicDatabase:
|
|||
logger.error(f"API: Error searching tracks with title='{title}', artist='{artist}': {e}")
|
||||
return []
|
||||
|
||||
def get_tracks_for_m3u_resolution(self, server_source: Optional[str] = None) -> List[Dict[str, str]]:
|
||||
"""Bulk-load (artist, title, file_path) for in-memory M3U path resolution.
|
||||
|
||||
ONE indexed read instead of a per-artist search loop. SQLite WAL allows it
|
||||
to run concurrently with the enrichment/scan writers, so M3U export no
|
||||
longer blocks behind them (the 'Export M3U hangs forever' report). Only
|
||||
rows that actually have a file_path are returned (the rest can't go in an
|
||||
M3U anyway)."""
|
||||
try:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
sql = ("SELECT tracks.title AS title, artists.name AS artist_name, tracks.file_path AS file_path "
|
||||
"FROM tracks JOIN artists ON tracks.artist_id = artists.id "
|
||||
"WHERE tracks.file_path IS NOT NULL AND tracks.file_path != ''")
|
||||
params: list = []
|
||||
if server_source:
|
||||
sql += " AND tracks.server_source = ?"
|
||||
params.append(server_source)
|
||||
cursor.execute(sql, params)
|
||||
return [{'title': r['title'] or '', 'artist': r['artist_name'] or '', 'file_path': r['file_path']}
|
||||
for r in cursor.fetchall()]
|
||||
except Exception as e:
|
||||
logger.error(f"Error bulk-loading tracks for M3U resolution: {e}")
|
||||
return []
|
||||
|
||||
def _search_tracks_basic(self, cursor, title: str, artist: str, limit: int, server_source: str = None,
|
||||
rank_artist: str = None) -> List[DatabaseTrack]:
|
||||
"""Basic SQL LIKE search - fastest method"""
|
||||
|
|
|
|||
60
tests/test_m3u_track_resolution.py
Normal file
60
tests/test_m3u_track_resolution.py
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
"""Bulk track-load used by M3U export path resolution.
|
||||
|
||||
M3U export used to resolve each track with a per-artist search_tracks() loop,
|
||||
which could block for a long time behind the enrichment/scan writers (the
|
||||
"Export M3U hangs forever" report). It now bulk-loads (artist, title, file_path)
|
||||
in one WAL-concurrent read; this pins that method's contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from database.music_database import MusicDatabase
|
||||
|
||||
|
||||
def _db_with_track(tmp_path, *, title, artist, file_path, server='jellyfin'):
|
||||
db = MusicDatabase(str(tmp_path / 'm.db'))
|
||||
with db._get_connection() as c:
|
||||
c.execute("INSERT INTO artists (id, name) VALUES (1, ?)", (artist,))
|
||||
c.execute("INSERT INTO albums (id, title, artist_id) VALUES (1, 'Album', 1)")
|
||||
c.execute(
|
||||
"INSERT INTO tracks (id, title, artist_id, album_id, file_path, server_source) "
|
||||
"VALUES (1, ?, 1, 1, ?, ?)",
|
||||
(title, file_path, server),
|
||||
)
|
||||
c.commit()
|
||||
return db
|
||||
|
||||
|
||||
def test_returns_artist_title_path(tmp_path):
|
||||
db = _db_with_track(tmp_path, title='How You Remind Me', artist='Nickelback',
|
||||
file_path='/music/nb/how.flac')
|
||||
rows = db.get_tracks_for_m3u_resolution(server_source='jellyfin')
|
||||
assert rows == [{'title': 'How You Remind Me', 'artist': 'Nickelback',
|
||||
'file_path': '/music/nb/how.flac'}]
|
||||
|
||||
|
||||
def test_filters_by_server_source(tmp_path):
|
||||
db = _db_with_track(tmp_path, title='X', artist='Y', file_path='/m/x.flac', server='jellyfin')
|
||||
assert db.get_tracks_for_m3u_resolution(server_source='jellyfin') # match
|
||||
assert db.get_tracks_for_m3u_resolution(server_source='plex') == [] # other server
|
||||
|
||||
|
||||
def test_excludes_rows_without_file_path(tmp_path):
|
||||
db = MusicDatabase(str(tmp_path / 'm.db'))
|
||||
with db._get_connection() as c:
|
||||
c.execute("INSERT INTO artists (id, name) VALUES (1, 'A')")
|
||||
c.execute("INSERT INTO albums (id, title, artist_id) VALUES (1, 'Al', 1)")
|
||||
# one with a path, one without — only the first should come back.
|
||||
c.execute("INSERT INTO tracks (id, title, artist_id, album_id, file_path, server_source) "
|
||||
"VALUES (1, 'Has Path', 1, 1, '/m/a.flac', 'jellyfin')")
|
||||
c.execute("INSERT INTO tracks (id, title, artist_id, album_id, file_path, server_source) "
|
||||
"VALUES (2, 'No Path', 1, 1, NULL, 'jellyfin')")
|
||||
c.commit()
|
||||
rows = db.get_tracks_for_m3u_resolution()
|
||||
titles = {r['title'] for r in rows}
|
||||
assert titles == {'Has Path'}
|
||||
|
||||
|
||||
def test_empty_db_safe(tmp_path):
|
||||
db = MusicDatabase(str(tmp_path / 'm.db'))
|
||||
assert db.get_tracks_for_m3u_resolution() == []
|
||||
|
|
@ -2736,43 +2736,41 @@ def generate_playlist_m3u():
|
|||
s = _re.sub(r'\s*remaster(ed)?.*', '', s)
|
||||
return _re.sub(r'\s+', ' ', s).strip()
|
||||
|
||||
# Group tracks by primary artist to minimise DB queries
|
||||
# Resolve each track's library file path. We bulk-load the library ONCE and
|
||||
# match in memory, keyed by cleaned artist, instead of issuing a
|
||||
# search_tracks() query per distinct artist. The per-artist loop could
|
||||
# block for a long time behind the enrichment/scan writers (SQLite lock
|
||||
# contention) — which is exactly why "Export M3U" hung with nothing in the
|
||||
# logs. One WAL-concurrent read can't be starved that way.
|
||||
from collections import defaultdict
|
||||
artist_groups = defaultdict(list)
|
||||
for idx, t in enumerate(tracks):
|
||||
artist_groups[t.get('artist', '') or ''].append((idx, t))
|
||||
lib_by_artist = defaultdict(list)
|
||||
for row in db.get_tracks_for_m3u_resolution(server_source=active_server):
|
||||
lib_by_artist[_clean(row['artist'])].append(
|
||||
(_norm(row['title']), _clean(row['title']), row['file_path'])
|
||||
)
|
||||
|
||||
file_path_map = {}
|
||||
for artist, group in artist_groups.items():
|
||||
if not artist:
|
||||
for idx, _ in group:
|
||||
file_path_map[idx] = None
|
||||
for idx, track in enumerate(tracks):
|
||||
name = track.get('name', '') or ''
|
||||
artist = track.get('artist', '') or ''
|
||||
if not name or not artist:
|
||||
file_path_map[idx] = None
|
||||
continue
|
||||
|
||||
db_tracks = db.search_tracks(artist=artist, limit=500, server_source=active_server)
|
||||
if not db_tracks:
|
||||
for idx, _ in group:
|
||||
file_path_map[idx] = None
|
||||
candidates = lib_by_artist.get(_clean(artist))
|
||||
if not candidates:
|
||||
file_path_map[idx] = None
|
||||
continue
|
||||
|
||||
db_entries = [(_norm(t.title), _clean(t.title), t) for t in db_tracks]
|
||||
|
||||
for idx, track in group:
|
||||
name = track.get('name', '')
|
||||
if not name:
|
||||
file_path_map[idx] = None
|
||||
continue
|
||||
s_norm, s_clean = _norm(name), _clean(name)
|
||||
matched = None
|
||||
for db_n, db_c, db_t in db_entries:
|
||||
if s_norm == db_n or s_clean == db_c:
|
||||
matched = db_t
|
||||
break
|
||||
if max(SequenceMatcher(None, s_norm, db_n).ratio(),
|
||||
SequenceMatcher(None, s_clean, db_c).ratio()) >= 0.7:
|
||||
matched = db_t
|
||||
break
|
||||
file_path_map[idx] = matched.file_path if matched else None
|
||||
s_norm, s_clean = _norm(name), _clean(name)
|
||||
matched_path = None
|
||||
for db_n, db_c, fp in candidates:
|
||||
if s_norm == db_n or s_clean == db_c:
|
||||
matched_path = fp
|
||||
break
|
||||
if max(SequenceMatcher(None, s_norm, db_n).ratio(),
|
||||
SequenceMatcher(None, s_clean, db_c).ratio()) >= 0.7:
|
||||
matched_path = fp
|
||||
break
|
||||
file_path_map[idx] = matched_path
|
||||
|
||||
# --- build M3U content ---
|
||||
import datetime as _dt
|
||||
|
|
|
|||
Loading…
Reference in a new issue