Library export: export the whole library roster too (corruption's request)

Extends the watchlist export to the full library. The exporter is now general
(core/exports/artist_export.py, renamed from watchlist_export) — adds tidal/qobuz
links and an extra_fields passthrough, so the library export also carries
lastfm/genius URLs + soul_id, and an optional "library counts" toggle adds owned
album/track counts per artist.

- GET /api/library/artists/export?format=&links=&contents= — pulls every artists
  row, normalizes onto the canonical *_artist_id keys, optionally GROUP-BY counts
  for album/track totals.
- The export modal is now openArtistExportModal(scope): "Export Library" button in
  the library header + the existing "Export" on the watchlist bar (a thin wrapper).
  Library mode shows the extra "library counts" toggle.

Tests (11): builder across formats + the new tidal/qobuz links + extra_fields
columns; watchlist + library endpoint wiring. 64 integrity green; ruff clean.

core/exports/artist_export.py

@@ -1,10 +1,12 @@
-"""Export a watchlist (or library) artist roster to JSON / CSV / plain text
+"""Export an artist roster — watchlist OR library — to JSON / CSV / plain text
 (corruption's request).
 
 Pure shaping + formatting so it's the single source of truth and unit-testable —
-web_server fetches the artists and hands them here; the UI just picks options and
-downloads. Always exports the name + whatever source IDs each artist has; the
-``include_links`` option adds the external discography URLs built from those IDs.
+web_server fetches the artists (normalizing each source's fields onto the canonical
+``*_artist_id`` keys below) and hands them here; the UI just picks options and
+downloads. Always exports the name + whatever source IDs each artist has;
+``include_links`` adds external discography URLs; ``extra_fields`` passes through
+source-specific extras (e.g. library album/track counts) in a stable order.
 """
 
 from __future__ import annotations
@@ -14,17 +16,21 @@ import io
 import json
 from typing import Any, Dict, List, Optional
 
-# Source id field → external URL builder.
+# Canonical id field → external URL builder.
 _LINKS = {
     'spotify_artist_id':     lambda i: f'https://open.spotify.com/artist/{i}',
     'musicbrainz_artist_id': lambda i: f'https://musicbrainz.org/artist/{i}',
     'deezer_artist_id':      lambda i: f'https://www.deezer.com/artist/{i}',
     'discogs_artist_id':     lambda i: f'https://www.discogs.com/artist/{i}',
     'itunes_artist_id':      lambda i: f'https://music.apple.com/artist/{i}',
+    'tidal_artist_id':       lambda i: f'https://tidal.com/artist/{i}',
+    'qobuz_artist_id':       lambda i: f'https://www.qobuz.com/artist/{i}',
 }
-# Order is stable so CSV columns + JSON keys are deterministic.
+# Stable order so CSV columns + JSON keys are deterministic. amazon carries an id
+# but no clean public URL.
 _ID_FIELDS = ['spotify_artist_id', 'musicbrainz_artist_id', 'deezer_artist_id',
-              'discogs_artist_id', 'itunes_artist_id', 'amazon_artist_id']
+              'discogs_artist_id', 'itunes_artist_id', 'tidal_artist_id',
+              'qobuz_artist_id', 'amazon_artist_id']
 
 VALID_FORMATS = ('json', 'csv', 'txt')
 
@@ -37,11 +43,14 @@ def _short(field: str) -> str:
     return field.replace('_artist_id', '')
 
 
-def _row(a: Dict[str, Any], include_links: bool) -> Dict[str, Any]:
+def _row(a: Dict[str, Any], include_links: bool, extra_fields: List[str]) -> Dict[str, Any]:
     row: Dict[str, Any] = {'name': _name(a)}
     for f in _ID_FIELDS:
         if a.get(f):
             row[f] = str(a[f])
+    for f in extra_fields:
+        if a.get(f) not in (None, ''):
+            row[f] = a[f]
     if include_links:
         links = {_short(f): b(a[f]) for f, b in _LINKS.items() if a.get(f)}
         if links:
@@ -49,17 +58,19 @@ def _row(a: Dict[str, Any], include_links: bool) -> Dict[str, Any]:
     return row
 
 
-def build_watchlist_export(artists: Optional[List[Dict[str, Any]]],
-                           fmt: str = 'json', include_links: bool = False) -> str:
+def build_artist_export(artists: Optional[List[Dict[str, Any]]],
+                        fmt: str = 'json', include_links: bool = False,
+                        extra_fields: Optional[List[str]] = None) -> str:
     """Return the roster serialized in ``fmt`` (json | csv | txt).
 
     - ``txt``  → one artist name per line.
-    - ``csv``  → name + each source-id column (+ a *_url column per service when
-      ``include_links``).
-    - ``json`` → a list of objects: name, present source ids, and a ``links`` map
-      when ``include_links``.
+    - ``csv``  → name + each source-id column + ``extra_fields`` columns (+ a
+      *_url column per service when ``include_links``).
+    - ``json`` → a list of objects: name, present source ids, present extras, and
+      a ``links`` map when ``include_links``.
     """
     artists = artists or []
+    extra_fields = list(extra_fields or [])
     fmt = (fmt or 'json').lower()
     if fmt not in VALID_FORMATS:
         fmt = 'json'
@@ -68,7 +79,7 @@ def build_watchlist_export(artists: Optional[List[Dict[str, Any]]],
         return '\n'.join(n for n in (_name(a) for a in artists) if n)
 
     if fmt == 'csv':
-        cols = ['name'] + _ID_FIELDS
+        cols = ['name'] + _ID_FIELDS + extra_fields
         if include_links:
             cols += [f'{_short(f)}_url' for f in _LINKS]
         out = io.StringIO()
@@ -76,12 +87,14 @@ def build_watchlist_export(artists: Optional[List[Dict[str, Any]]],
         w.writerow(cols)
         for a in artists:
             line = [_name(a)] + [str(a.get(f) or '') for f in _ID_FIELDS]
+            line += [str(a.get(f) if a.get(f) is not None else '') for f in extra_fields]
             if include_links:
                 line += [_LINKS[f](a[f]) if a.get(f) else '' for f in _LINKS]
             w.writerow(line)
         return out.getvalue()
 
-    return json.dumps([_row(a, include_links) for a in artists], indent=2, ensure_ascii=False)
+    return json.dumps([_row(a, include_links, extra_fields) for a in artists],
+                      indent=2, ensure_ascii=False)
 
 
 def export_mime_and_ext(fmt: str):
@@ -93,4 +106,4 @@ def export_mime_and_ext(fmt: str):
     }.get((fmt or 'json').lower(), ('application/json', 'json'))
 
 
-__all__ = ['build_watchlist_export', 'export_mime_and_ext', 'VALID_FORMATS']
+__all__ = ['build_artist_export', 'export_mime_and_ext', 'VALID_FORMATS']

tests/test_artist_export.py

@@ -7,7 +7,7 @@ import csv
 import io
 import json
 
-from core.exports.watchlist_export import build_watchlist_export, export_mime_and_ext
+from core.exports.artist_export import build_artist_export, export_mime_and_ext
 
 
 _ARTISTS = [
@@ -18,12 +18,12 @@ _ARTISTS = [
 
 
 def test_txt_is_names_one_per_line():
-    out = build_watchlist_export(_ARTISTS, fmt='txt')
+    out = build_artist_export(_ARTISTS, fmt='txt')
     assert out == 'Rob Zombie\nNobody IDs'
 
 
 def test_json_includes_present_ids_only():
-    out = json.loads(build_watchlist_export(_ARTISTS, fmt='json'))
+    out = json.loads(build_artist_export(_ARTISTS, fmt='json'))
     assert out[0]['name'] == 'Rob Zombie'
     assert out[0]['spotify_artist_id'] == 'sp1' and out[0]['musicbrainz_artist_id'] == 'mb1'
     assert 'deezer_artist_id' in out[0]
@@ -31,14 +31,14 @@ def test_json_includes_present_ids_only():
 
 
 def test_json_links_when_requested():
-    out = json.loads(build_watchlist_export(_ARTISTS, fmt='json', include_links=True))
+    out = json.loads(build_artist_export(_ARTISTS, fmt='json', include_links=True))
     assert out[0]['links']['spotify'] == 'https://open.spotify.com/artist/sp1'
     assert out[0]['links']['musicbrainz'] == 'https://musicbrainz.org/artist/mb1'
     assert 'links' not in out[1]                     # no ids → no links
 
 
 def test_csv_header_and_rows():
-    out = build_watchlist_export(_ARTISTS, fmt='csv')
+    out = build_artist_export(_ARTISTS, fmt='csv')
     rows = list(csv.reader(io.StringIO(out)))
     assert rows[0][0] == 'name' and 'spotify_artist_id' in rows[0]
     assert rows[1][0] == 'Rob Zombie'
@@ -46,15 +46,15 @@ def test_csv_header_and_rows():
 
 
 def test_csv_adds_url_columns_with_links():
-    out = build_watchlist_export(_ARTISTS, fmt='csv', include_links=True)
+    out = build_artist_export(_ARTISTS, fmt='csv', include_links=True)
     header = next(csv.reader(io.StringIO(out)))
     assert 'spotify_url' in header and 'discogs_url' in header
 
 
 def test_empty_and_bad_format():
-    assert build_watchlist_export([], fmt='txt') == ''
-    assert build_watchlist_export(None, fmt='json') == '[]'
-    assert build_watchlist_export(_ARTISTS, fmt='nonsense').startswith('[')  # falls back to json
+    assert build_artist_export([], fmt='txt') == ''
+    assert build_artist_export(None, fmt='json') == '[]'
+    assert build_artist_export(_ARTISTS, fmt='nonsense').startswith('[')  # falls back to json
 
 
 def test_mime_and_ext():
@@ -86,3 +86,39 @@ def test_export_endpoint_wiring(client):
     r2 = client.get('/api/watchlist/export?format=csv&links=1')
     assert r2.status_code == 200 and r2.headers.get('X-Export-Ext') == 'csv'
     assert 'spotify_url' in r2.data.decode().splitlines()[0]   # header row with links
+
+
+# ── library-side: extra services + extra_fields passthrough ─────────────────
+_LIB = [{
+    'name': 'Rob Zombie', 'spotify_artist_id': 'sp1', 'tidal_artist_id': 'td1',
+    'qobuz_artist_id': 'qz1', 'lastfm_url': 'https://last.fm/x', 'soul_id': 'soul_abc',
+    'album_count': 10, 'track_count': 159,
+}]
+
+
+def test_tidal_qobuz_links_and_extra_fields_json():
+    out = json.loads(build_artist_export(_LIB, fmt='json', include_links=True,
+                                         extra_fields=['lastfm_url', 'soul_id', 'album_count', 'track_count']))
+    a = out[0]
+    assert a['tidal_artist_id'] == 'td1' and a['qobuz_artist_id'] == 'qz1'
+    assert a['links']['tidal'] == 'https://tidal.com/artist/td1'
+    assert a['links']['qobuz'] == 'https://www.qobuz.com/artist/qz1'
+    assert a['lastfm_url'] == 'https://last.fm/x' and a['soul_id'] == 'soul_abc'
+    assert a['album_count'] == 10 and a['track_count'] == 159
+
+
+def test_extra_fields_become_csv_columns():
+    out = build_artist_export(_LIB, fmt='csv', extra_fields=['album_count', 'track_count'])
+    header = next(csv.reader(io.StringIO(out)))
+    assert 'album_count' in header and 'track_count' in header
+    assert 'tidal_artist_id' in header        # new service column present
+
+
+def test_library_export_endpoint_wiring(client):
+    r = client.get('/api/library/artists/export?format=json&contents=1&links=1')
+    assert r.status_code == 200
+    assert r.data.decode().strip() == '[]'      # empty test DB
+    assert r.headers.get('X-Export-Count') == '0'
+    r2 = client.get('/api/library/artists/export?format=csv&contents=1')
+    header = r2.data.decode().splitlines()[0]
+    assert 'album_count' in header and 'track_count' in header

web_server.py

@@ -26729,7 +26729,7 @@ def export_watchlist():
     as json / csv / txt. Returns the content for the export modal to preview +
     download (X-Export-Count / X-Export-Ext headers carry the metadata)."""
     try:
-        from core.exports.watchlist_export import build_watchlist_export, export_mime_and_ext
+        from core.exports.artist_export import build_artist_export, export_mime_and_ext
         fmt = (request.args.get('format', 'json') or 'json').lower()
         include_links = request.args.get('links', '') in ('1', 'true', 'yes')
         database = get_database()
@@ -26742,7 +26742,7 @@ def export_watchlist():
             'musicbrainz_artist_id': getattr(a, 'musicbrainz_artist_id', None),
             'amazon_artist_id': getattr(a, 'amazon_artist_id', None),
         } for a in database.get_watchlist_artists(profile_id=get_current_profile_id())]
-        content = build_watchlist_export(artists, fmt=fmt, include_links=include_links)
+        content = build_artist_export(artists, fmt=fmt, include_links=include_links)
         mime, ext = export_mime_and_ext(fmt)
         return Response(content, mimetype=mime,
                         headers={'X-Export-Count': str(len(artists)), 'X-Export-Ext': ext})
@@ -26751,6 +26751,72 @@ def export_watchlist():
         return jsonify({"success": False, "error": str(e)}), 500
 
 
+@app.route('/api/library/artists/export', methods=['GET'])
+def export_library_artists():
+    """Export the WHOLE library artist roster (name + every source id/url we have,
+    optional external links, optional owned album/track counts) as json/csv/txt."""
+    try:
+        from core.exports.artist_export import build_artist_export, export_mime_and_ext
+        fmt = (request.args.get('format', 'json') or 'json').lower()
+        include_links = request.args.get('links', '') in ('1', 'true', 'yes')
+        include_contents = request.args.get('contents', '') in ('1', 'true', 'yes')
+        database = get_database()
+        conn = database._get_connection()
+        try:
+            cur = conn.cursor()
+            cur.execute("""
+                SELECT id, name, spotify_artist_id, musicbrainz_id, deezer_id,
+                       discogs_id, itunes_artist_id, tidal_id, qobuz_id, amazon_id,
+                       lastfm_url, genius_url, soul_id
+                FROM artists ORDER BY name COLLATE NOCASE
+            """)
+            cols = [d[0] for d in cur.description]
+            rows = [dict(zip(cols, r, strict=False)) for r in cur.fetchall()]
+
+            counts = {}
+            if include_contents:
+                for table, key in (('albums', 'album_count'), ('tracks', 'track_count')):
+                    try:
+                        for aid, n in cur.execute(
+                                f"SELECT artist_id, COUNT(*) FROM {table} GROUP BY artist_id"):
+                            counts.setdefault(str(aid), {})[key] = n
+                    except Exception:  # noqa: S110 — counts are best-effort
+                        pass
+        finally:
+            conn.close()
+
+        # Normalize onto the canonical *_artist_id keys the exporter expects.
+        artists = []
+        for r in rows:
+            c = counts.get(str(r['id']), {})
+            artists.append({
+                'name': r['name'],
+                'spotify_artist_id': r['spotify_artist_id'],
+                'musicbrainz_artist_id': r['musicbrainz_id'],
+                'deezer_artist_id': r['deezer_id'],
+                'discogs_artist_id': r['discogs_id'],
+                'itunes_artist_id': r['itunes_artist_id'],
+                'tidal_artist_id': r['tidal_id'],
+                'qobuz_artist_id': r['qobuz_id'],
+                'amazon_artist_id': r['amazon_id'],
+                'lastfm_url': r['lastfm_url'],
+                'genius_url': r['genius_url'],
+                'soul_id': r['soul_id'],
+                'album_count': c.get('album_count'),
+                'track_count': c.get('track_count'),
+            })
+        extra = ['lastfm_url', 'genius_url', 'soul_id']
+        if include_contents:
+            extra += ['album_count', 'track_count']
+        content = build_artist_export(artists, fmt=fmt, include_links=include_links, extra_fields=extra)
+        mime, ext = export_mime_and_ext(fmt)
+        return Response(content, mimetype=mime,
+                        headers={'X-Export-Count': str(len(artists)), 'X-Export-Ext': ext})
+    except Exception as e:
+        logger.error(f"Library artist export failed: {e}")
+        return jsonify({"success": False, "error": str(e)}), 500
+
+
 @app.route('/api/watchlist/add', methods=['POST'])
 def add_to_watchlist():
     """Add an artist to the watchlist"""

webui/index.html

@@ -2488,6 +2488,10 @@
                                 <span class="stat-label">Artists</span>
                             </span>
                         </div>
+                        <button class="library-watchlist-all-btn library-export-btn" id="library-export-btn" onclick="openArtistExportModal('library')" title="Export your whole library — artists, IDs, links, optional album/track counts">
+                            <span class="watchlist-all-icon">⬇</span>
+                            <span class="watchlist-all-text">Export Library</span>
+                        </button>
                     </div>
 
                     <!-- Search and Filters -->

webui/static/library.js

@@ -9017,7 +9017,14 @@ function _arecEscAttr(s) {
 // optional external discography links. Reuses the DB-record modal aesthetic +
 // helpers (_jsonSyntaxHighlight / _arecCopy / _arecEsc). #export-request
 // ════════════════════════════════════════════════════════════════════════════
-async function openWatchlistExportModal() {
+function openWatchlistExportModal() { return openArtistExportModal('watchlist'); }
+
+async function openArtistExportModal(scope) {
+    scope = scope || 'watchlist';
+    const isLib = scope === 'library';
+    const endpoint = isLib ? '/api/library/artists/export' : '/api/watchlist/export';
+    const fileBase = isLib ? 'library_artists' : 'watchlist';
+
     const existing = document.getElementById('wl-export-overlay');
     if (existing) existing.remove();
 
@@ -9025,11 +9032,13 @@ async function openWatchlistExportModal() {
     overlay.id = 'wl-export-overlay';
     overlay.className = 'arec-overlay';
     overlay.innerHTML =
-        '<div class="arec-card" role="dialog" aria-label="Export watchlist">' +
+        '<div class="arec-card" role="dialog" aria-label="Export artists">' +
             '<div class="arec-header">' +
                 '<div class="arec-title-wrap">' +
-                    '<div class="arec-title"><span class="arec-dot"></span>Export Watchlist</div>' +
-                    '<div class="arec-sub">your watchlist artists — name, source IDs, optional links</div>' +
+                    '<div class="arec-title"><span class="arec-dot"></span>' + (isLib ? 'Export Library' : 'Export Watchlist') + '</div>' +
+                    '<div class="arec-sub">' + (isLib
+                        ? 'every library artist — name, source IDs, optional links + owned counts'
+                        : 'your watchlist artists — name, source IDs, optional links') + '</div>' +
                 '</div>' +
                 '<button class="arec-close" id="wlx-close" title="Close (Esc)">&times;</button>' +
             '</div>' +
@@ -9040,6 +9049,7 @@ async function openWatchlistExportModal() {
                     '<button class="arec-tab" data-fmt="txt">Text</button>' +
                 '</div>' +
                 '<label class="wlx-opt"><input type="checkbox" id="wlx-links"> external links</label>' +
+                (isLib ? '<label class="wlx-opt"><input type="checkbox" id="wlx-contents"> library counts</label>' : '') +
                 '<div class="arec-actions">' +
                     '<button class="arec-btn" id="wlx-copy">Copy</button>' +
                     '<button class="arec-btn" id="wlx-download">Download</button>' +
@@ -9051,7 +9061,7 @@ async function openWatchlistExportModal() {
     document.body.appendChild(overlay);
     requestAnimationFrame(() => overlay.classList.add('visible'));
 
-    let fmt = 'json', links = false, content = '';
+    let fmt = 'json', links = false, contents = false, content = '';
 
     const close = () => {
         overlay.classList.remove('visible');
@@ -9067,7 +9077,8 @@ async function openWatchlistExportModal() {
         const body = document.getElementById('wlx-body');
         body.innerHTML = '<div class="arec-loading">Building export…</div>';
         try {
-            const res = await fetch('/api/watchlist/export?format=' + fmt + '&links=' + (links ? '1' : '0'));
+            const res = await fetch(endpoint + '?format=' + fmt + '&links=' + (links ? '1' : '0')
+                + (isLib && contents ? '&contents=1' : ''));
             content = await res.text();
             const count = res.headers.get('X-Export-Count') || '?';
             document.getElementById('wlx-footer').innerHTML =
@@ -9092,6 +9103,8 @@ async function openWatchlistExportModal() {
         };
     });
     document.getElementById('wlx-links').addEventListener('change', (e) => { links = e.target.checked; refresh(); });
+    const _contentsEl = document.getElementById('wlx-contents');
+    if (_contentsEl) _contentsEl.addEventListener('change', (e) => { contents = e.target.checked; refresh(); });
     document.getElementById('wlx-copy').onclick = () => _arecCopy(content, 'Export copied');
     document.getElementById('wlx-download').onclick = () => {
         const ext = fmt;
@@ -9099,10 +9112,10 @@ async function openWatchlistExportModal() {
         const blob = new Blob([content || ''], { type: mime });
         const url = URL.createObjectURL(blob);
         const a = document.createElement('a');
-        a.href = url; a.download = 'watchlist_export.' + ext;
+        a.href = url; a.download = fileBase + '_export.' + ext;
         document.body.appendChild(a); a.click(); a.remove();
         setTimeout(() => URL.revokeObjectURL(url), 1000);
-        if (typeof showToast === 'function') showToast('Saved watchlist_export.' + ext, 'success');
+        if (typeof showToast === 'function') showToast('Saved ' + fileBase + '_export.' + ext, 'success');
     };
 
     refresh();