From 85d59048460f0f48c53fe3d21050e66e3b48c45a Mon Sep 17 00:00:00 2001 From: ragnarlotus Date: Sat, 27 Jun 2026 00:04:06 +0200 Subject: [PATCH] fix album completeness fragmented rows --- core/repair_jobs/album_completeness.py | 1239 ++++++++++++++--- ...test_album_completeness_fragmented_rows.py | 496 +++++++ 2 files changed, 1574 insertions(+), 161 deletions(-) create mode 100644 tests/test_album_completeness_fragmented_rows.py diff --git a/core/repair_jobs/album_completeness.py b/core/repair_jobs/album_completeness.py index 5744a5bf..98c68d80 100644 --- a/core/repair_jobs/album_completeness.py +++ b/core/repair_jobs/album_completeness.py @@ -1,5 +1,10 @@ """Album Completeness Checker Job — finds albums missing tracks.""" +import re +import unicodedata +from collections import defaultdict +from difflib import SequenceMatcher + from core.metadata_service import ( get_album_tracks_for_source, get_primary_source, @@ -23,8 +28,10 @@ class AlbumCompletenessJob(RepairJob): 'from your configured metadata sources. Counts cached during normal enrichment are ' 'used when no canonical edition is pinned; otherwise the exact canonical source and ' 'album ID are queried directly. The same canonical tracklist is used for both the ' - 'expected total and the missing-track calculation. Albums where tracks are missing ' - 'get flagged as findings with details about which tracks are absent.\n\n' + 'expected total and the missing-track calculation. Fragmented library rows are ' + 'combined only when they safely match that same canonical edition. Albums where ' + 'tracks are missing get flagged as findings with details about which tracks are ' + 'absent.\n\n' 'Useful for catching partial downloads or albums where some tracks failed to download. ' 'You can use the Download Missing feature from the album page to fill gaps.\n\n' 'Settings:\n' @@ -51,24 +58,28 @@ class AlbumCompletenessJob(RepairJob): min_completion_pct = settings.get('min_completion_pct', 0) primary_source = self._get_primary_source() - # Fetch all albums with ANY external source ID — not just Spotify + # Fetch all albums with ANY external source ID — not just Spotify. albums = [] conn = None has_itunes = False has_deezer = False + has_discogs = False + has_hydrabase = False + has_musicbrainz = False has_api_track_count = False has_canonical = False try: conn = context.db._get_connection() cursor = conn.cursor() - # Check which source columns exist (older DBs may lack some) + # Check which source columns exist (older DBs may lack some). cursor.execute("PRAGMA table_info(albums)") columns = {row[1] for row in cursor.fetchall()} has_itunes = 'itunes_album_id' in columns has_deezer = 'deezer_id' in columns has_discogs = 'discogs_id' in columns has_hydrabase = 'soul_id' in columns + has_musicbrainz = 'musicbrainz_release_id' in columns has_canonical = ( 'canonical_source' in columns and 'canonical_album_id' in columns @@ -90,6 +101,7 @@ class AlbumCompletenessJob(RepairJob): # from metadata-source enrichment) or a live API lookup. select_cols = [ ('al.id', 'album_id'), + ('al.artist_id', 'artist_id'), ('al.title', 'album_title'), ('ar.name', 'artist_name'), ('al.spotify_album_id', 'spotify_album_id'), @@ -107,22 +119,41 @@ class AlbumCompletenessJob(RepairJob): select_cols.append(('al.discogs_id', 'discogs_album_id')) if has_hydrabase: select_cols.append(('al.soul_id', 'hydrabase_album_id')) + if has_musicbrainz: + select_cols.append( + ('al.musicbrainz_release_id', 'musicbrainz_album_id') + ) if has_canonical: select_cols.extend([ ('al.canonical_source', 'canonical_source'), ('al.canonical_album_id', 'canonical_album_id'), ]) - # WHERE: album has at least one source ID or a complete canonical pair - where_parts = ["(al.spotify_album_id IS NOT NULL AND al.spotify_album_id != '')"] + # WHERE: album has at least one source ID or a complete canonical pair. + where_parts = [ + "(al.spotify_album_id IS NOT NULL AND al.spotify_album_id != '')", + ] if has_itunes: - where_parts.append("(al.itunes_album_id IS NOT NULL AND al.itunes_album_id != '')") + where_parts.append( + "(al.itunes_album_id IS NOT NULL AND al.itunes_album_id != '')" + ) if has_deezer: - where_parts.append("(al.deezer_id IS NOT NULL AND al.deezer_id != '')") + where_parts.append( + "(al.deezer_id IS NOT NULL AND al.deezer_id != '')" + ) if has_discogs: - where_parts.append("(al.discogs_id IS NOT NULL AND al.discogs_id != '')") + where_parts.append( + "(al.discogs_id IS NOT NULL AND al.discogs_id != '')" + ) if has_hydrabase: - where_parts.append("(al.soul_id IS NOT NULL AND al.soul_id != '')") + where_parts.append( + "(al.soul_id IS NOT NULL AND al.soul_id != '')" + ) + if has_musicbrainz: + where_parts.append( + "(al.musicbrainz_release_id IS NOT NULL " + "AND al.musicbrainz_release_id != '')" + ) if has_canonical: where_parts.append( "(al.canonical_source IS NOT NULL AND al.canonical_source != '' " @@ -130,7 +161,10 @@ class AlbumCompletenessJob(RepairJob): ) where_clause = ' OR '.join(where_parts) - select_sql = ', '.join(f'{expr} AS {alias}' for expr, alias in select_cols) + select_sql = ', '.join( + f'{expr} AS {alias}' + for expr, alias in select_cols + ) cursor.execute(f""" SELECT {select_sql} FROM albums al @@ -139,8 +173,20 @@ class AlbumCompletenessJob(RepairJob): WHERE {where_clause} GROUP BY al.id """) - albums = cursor.fetchall() - column_index = {alias: idx for idx, (_, alias) in enumerate(select_cols)} + raw_rows = cursor.fetchall() + column_index = { + alias: idx + for idx, (_, alias) in enumerate(select_cols) + } + albums = [ + { + alias: row[idx] + for alias, idx in column_index.items() + } + for row in raw_rows + ] + for order, album in enumerate(albums): + album['_scan_order'] = order except Exception as e: logger.error("Error fetching albums: %s", e, exc_info=True) result.errors += 1 @@ -149,76 +195,105 @@ class AlbumCompletenessJob(RepairJob): if conn: conn.close() - total = len(albums) + # Rows that share a source ID with a canonical album are candidates for + # the same release. A sibling joins the logical album only when at least + # one of its local tracks strictly matches the canonical tracklist. + work_items = self._prepare_work_items(context, albums) + + total = len(work_items) if context.update_progress: context.update_progress(0, total) - logger.info("Checking completeness of %d albums", total) + logger.info("Checking completeness of %d logical albums", total) if context.report_progress: - context.report_progress(phase=f'Checking {total} albums...', total=total) + context.report_progress( + phase=f'Checking {total} logical albums...', + total=total, + ) - for i, row in enumerate(albums): + for i, work_item in enumerate(work_items): if context.check_stop(): return result if i % 10 == 0 and context.wait_if_paused(): return result - album_id = row[column_index['album_id']] - title = row[column_index['album_title']] - artist_name = row[column_index['artist_name']] - spotify_album_id = row[column_index['spotify_album_id']] - actual_count = row[column_index['actual_count']] - album_thumb = row[column_index['album_thumb_url']] - artist_thumb = row[column_index['artist_thumb_url']] - itunes_album_id = row[column_index['itunes_album_id']] if 'itunes_album_id' in column_index else None - deezer_album_id = row[column_index['deezer_album_id']] if 'deezer_album_id' in column_index else None - discogs_album_id = row[column_index['discogs_album_id']] if 'discogs_album_id' in column_index else None - hydrabase_album_id = row[column_index['hydrabase_album_id']] if 'hydrabase_album_id' in column_index else None - canonical_source = row[column_index['canonical_source']] if 'canonical_source' in column_index else None - canonical_album_id = row[column_index['canonical_album_id']] if 'canonical_album_id' in column_index else None - # Cached authoritative track count from a prior API lookup (NULL - # on unscanned albums and on DBs predating the column migration). - cached_api_count = row[column_index['api_track_count']] if 'api_track_count' in column_index else None + row = work_item['row'] + album_id = row['album_id'] + title = row['album_title'] + artist_name = row['artist_name'] + spotify_album_id = row['spotify_album_id'] + actual_count = int(row['actual_count'] or 0) + album_thumb = row['album_thumb_url'] + artist_thumb = row['artist_thumb_url'] + itunes_album_id = row.get('itunes_album_id') + deezer_album_id = row.get('deezer_album_id') + discogs_album_id = row.get('discogs_album_id') + hydrabase_album_id = row.get('hydrabase_album_id') + musicbrainz_album_id = row.get('musicbrainz_album_id') + canonical_source = row.get('canonical_source') + canonical_album_id = row.get('canonical_album_id') + cached_api_count = row.get('api_track_count') result.scanned += 1 if context.report_progress: context.report_progress( - scanned=i + 1, total=total, + scanned=i + 1, + total=total, phase=f'Checking {i + 1} / {total}', - log_line=f'Album: {title or "Unknown"} — {artist_name or "Unknown"}', - log_type='info' + log_line=( + f'Album: {title or "Unknown"} — ' + f'{artist_name or "Unknown"}' + ), + log_type='info', ) - album_ids = { - 'spotify': spotify_album_id or '', - 'itunes': itunes_album_id or '', - 'deezer': deezer_album_id or '', - 'discogs': discogs_album_id or '', - 'hydrabase': hydrabase_album_id or '', - } - - # A complete canonical pair identifies one exact edition, - # independently of provider. Its tracklist must drive both the - # expected total and the missing-track calculation; neither a - # cached count nor another source may silently replace it. - canonical_items = None + album_ids = self._source_ids_from_row(row) resolved_source = primary_source - resolved_album_id = self._get_album_id_for_source(primary_source, album_ids) or '' + resolved_album_id = ( + self._get_album_id_for_source(primary_source, album_ids) + or '' + ) + related_album_ids = ( + work_item.get('related_album_ids') + or [album_id] + ) + raw_local_count = work_item.get('raw_local_count') + missing_tracks = None + # A complete canonical pair is authoritative. `_prepare_work_items` + # performs exactly one lookup per canonical group and stores the + # resulting list — including an empty list when the lookup failed. if canonical_source and canonical_album_id: - api_tracks = self._get_album_tracks( - str(canonical_source), - str(canonical_album_id), - ) - canonical_items = self._extract_track_items(api_tracks) + canonical_items = work_item.get('canonical_items', []) expected_total = len(canonical_items) resolved_source = str(canonical_source) resolved_album_id = str(canonical_album_id) + + if canonical_items: + actual_count = int( + work_item.get( + 'effective_actual_count', + actual_count, + ) + ) + owned_reference_indexes = set( + work_item.get( + 'owned_reference_indexes', + set(), + ) + ) + missing_tracks = self._build_missing_tracks( + canonical_items, + owned_reference_indexes, + resolved_source, + ) + if raw_local_count is None: + raw_local_count = actual_count else: - # Preserve the existing behavior for albums that have not - # resolved a canonical edition yet. + # Preserve the existing behavior for albums without a pinned + # canonical edition. expected_total = cached_api_count if not expected_total: expected_total = self._get_expected_total( @@ -226,16 +301,18 @@ class AlbumCompletenessJob(RepairJob): primary_source, album_ids, ) - # Only persist positive results. Zero/None would keep - # re-triggering the lookup on every scan. - if expected_total and expected_total > 0 and has_api_track_count: + if ( + expected_total + and expected_total > 0 + and has_api_track_count + ): self._save_api_track_count( context, album_id, expected_total, ) - # Skip singles/EPs based on expected track count (not local count) + # Skip singles/EPs based on expected track count (not local count). if expected_total and expected_total < min_tracks: result.skipped += 1 if context.update_progress and (i + 1) % 5 == 0: @@ -248,13 +325,15 @@ class AlbumCompletenessJob(RepairJob): context.update_progress(i + 1, total) continue - # Skip albums with zero local tracks — nothing to auto-fill from - if actual_count == 0: + effective_raw_local_count = ( + raw_local_count + if raw_local_count is not None + else int(row['actual_count'] or 0) + ) + if actual_count == 0 and effective_raw_local_count == 0: result.skipped += 1 continue - # Skip albums below minimum completion percentage - # (filters out "1 track from a playlist import" false positives) if min_completion_pct > 0 and expected_total > 0: completion = (actual_count / expected_total) * 100 if completion < min_completion_pct: @@ -263,26 +342,23 @@ class AlbumCompletenessJob(RepairJob): context.update_progress(i + 1, total) continue - # Album is incomplete — identify missing tracks from the same - # canonical edition used for the expected total when one is pinned. - missing_tracks = self._find_missing_tracks( - context, - primary_source, - album_id, - album_ids, - resolved_source=( - resolved_source - if canonical_items is not None - else None - ), - resolved_items=canonical_items, - ) + if missing_tracks is None: + missing_tracks = self._find_missing_tracks( + context, + primary_source, + album_id, + album_ids, + ) if context.report_progress: context.report_progress( - log_line=f'Incomplete: {title or "Unknown"} ({actual_count}/{expected_total})', - log_type='skip' + log_line=( + f'Incomplete: {title or "Unknown"} ' + f'({actual_count}/{expected_total})' + ), + log_type='skip', ) + if context.create_finding: try: inserted = context.create_finding( @@ -292,10 +368,13 @@ class AlbumCompletenessJob(RepairJob): entity_type='album', entity_id=str(album_id), file_path=None, - title=f'Incomplete: {title or "Unknown"} ({actual_count}/{expected_total})', + title=( + f'Incomplete: {title or "Unknown"} ' + f'({actual_count}/{expected_total})' + ), description=( - f'Album "{title}" by {artist_name or "Unknown"} has {actual_count} of ' - f'{expected_total} tracks' + f'Album "{title}" by {artist_name or "Unknown"} ' + f'has {actual_count} of {expected_total} tracks' ), details={ 'album_id': album_id, @@ -310,19 +389,31 @@ class AlbumCompletenessJob(RepairJob): 'deezer_album_id': deezer_album_id or '', 'discogs_album_id': discogs_album_id or '', 'hydrabase_album_id': hydrabase_album_id or '', + 'musicbrainz_album_id': ( + musicbrainz_album_id or '' + ), 'expected_tracks': expected_total, 'actual_tracks': actual_count, + 'raw_local_tracks': effective_raw_local_count, + 'related_album_ids': [ + str(value) + for value in related_album_ids + ], 'missing_tracks': missing_tracks, 'album_thumb_url': album_thumb or None, 'artist_thumb_url': artist_thumb or None, - } + }, ) if inserted: result.findings_created += 1 else: result.findings_skipped_dedup += 1 except Exception as e: - logger.debug("Error creating completeness finding for album %s: %s", album_id, e) + logger.debug( + "Error creating completeness finding for album %s: %s", + album_id, + e, + ) result.errors += 1 if context.update_progress and (i + 1) % 5 == 0: @@ -331,42 +422,831 @@ class AlbumCompletenessJob(RepairJob): if context.update_progress: context.update_progress(total, total) - logger.info("Completeness check: %d albums checked, %d incomplete found", - result.scanned, result.findings_created) + logger.info( + "Completeness check: %d logical albums checked, %d incomplete found", + result.scanned, + result.findings_created, + ) return result - def _save_api_track_count(self, context, album_id, count): - """Persist a metadata-API track count via the shared worker helper. + def _prepare_work_items(self, context, albums): + """Collapse safely validated fragments into logical canonical albums.""" + groups = self._build_candidate_groups(albums) + work_items = [] - Enrichment workers call `set_album_api_track_count` inside their own - `_update_album` transaction. Here we're in the repair job's fallback - path (the album wasn't enriched yet), so we own the connection + - commit ourselves. A cache-write failure must never break the scan, - so all errors are swallowed into the debug log. - """ + for group in groups: + anchor = group['anchor'] + members = group['members'] + canonical_source = anchor.get('canonical_source') or '' + canonical_album_id = anchor.get('canonical_album_id') or '' + + if canonical_source and canonical_album_id: + canonical_tracks = self._get_album_tracks( + str(canonical_source), + str(canonical_album_id), + ) + canonical_items = self._extract_track_items( + canonical_tracks + ) + + if canonical_items: + local_by_album = { + str(member['album_id']): self._load_local_tracks( + context, + [member['album_id']], + ) + for member in members + } + anchor_tracks = local_by_album.get( + str(anchor['album_id']), + [], + ) + + anchor_reference, _ = self._match_tracks( + canonical_items, + anchor_tracks, + str(canonical_source), + ) + # The anchor's persisted disc/track slots remain + # authoritative even when titles or durations drift. + anchor_reference.update( + self._reference_slots_for_local_tracks( + canonical_items, + anchor_tracks, + ) + ) + + included = [anchor] + excluded = [] + supplemental_reference = set() + + for member in members: + if member is anchor: + continue + + member_tracks = local_by_album.get( + str(member['album_id']), + [], + ) + matched_reference, _ = ( + self._match_fragment_tracks( + canonical_items, + member_tracks, + str(canonical_source), + ) + ) + if matched_reference: + included.append(member) + supplemental_reference.update( + matched_reference - anchor_reference + ) + else: + excluded.append(member) + + combined_local = [] + for member in included: + combined_local.extend( + local_by_album.get( + str(member['album_id']), + [], + ) + ) + + owned_reference_indexes = ( + anchor_reference | supplemental_reference + ) + work_items.append({ + 'row': anchor, + 'canonical_items': canonical_items, + 'related_album_ids': [ + member['album_id'] + for member in included + ], + 'raw_local_count': len(combined_local), + 'effective_actual_count': ( + int(anchor.get('actual_count') or 0) + + len(supplemental_reference) + ), + 'owned_reference_indexes': ( + owned_reference_indexes + ), + '_scan_order': min( + member['_scan_order'] + for member in included + ), + }) + + for member in excluded: + work_items.append( + self._independent_work_item( + member, + canonical_items=( + canonical_items + if self._same_canonical_pair( + member, + canonical_source, + canonical_album_id, + ) + else None + ), + ) + ) + continue + + # The canonical lookup was attempted and returned no usable + # tracklist. Canonical rows must not fall back to another + # provider; non-canonical siblings remain independent. + for member in members: + work_items.append( + self._independent_work_item( + member, + canonical_items=( + [] + if self._same_canonical_pair( + member, + canonical_source, + canonical_album_id, + ) + else None + ), + ) + ) + continue + + for member in members: + work_items.append( + self._independent_work_item(member) + ) + + work_items.sort(key=lambda item: item['_scan_order']) + return work_items + + def _independent_work_item(self, row, canonical_items=None): + item = { + 'row': row, + 'related_album_ids': [row['album_id']], + '_scan_order': row['_scan_order'], + } + if canonical_items is not None: + item['canonical_items'] = canonical_items + return item + + def _same_canonical_pair( + self, + row, + canonical_source, + canonical_album_id, + ): + return ( + str(row.get('canonical_source') or '') + == str(canonical_source) + and str(row.get('canonical_album_id') or '') + == str(canonical_album_id) + ) + + def _build_candidate_groups(self, albums): + """Group non-canonical rows around unambiguous canonical anchors.""" + canonical_groups = defaultdict(list) + + for album in albums: + source = album.get('canonical_source') or '' + canonical_id = album.get('canonical_album_id') or '' + if source and canonical_id: + key = ( + str(album.get('artist_id') or ''), + str(source), + str(canonical_id), + ) + canonical_groups[key].append(album) + + canonical_row_ids = { + str(row['album_id']) + for rows in canonical_groups.values() + for row in rows + } + + alias_to_groups = defaultdict(set) + for key, anchor_rows in canonical_groups.items(): + for row in anchor_rows: + artist_id = str(row.get('artist_id') or '') + for source, source_id in ( + self._source_ids_from_row(row).items() + ): + if source_id: + alias_to_groups[ + (artist_id, source, str(source_id)) + ].add(key) + + canonical_source = ( + row.get('canonical_source') or '' + ) + canonical_id = ( + row.get('canonical_album_id') or '' + ) + alias_to_groups[ + ( + artist_id, + str(canonical_source), + str(canonical_id), + ) + ].add(key) + + assigned_candidate_ids = set() + groups = [] + + for key, anchor_rows in canonical_groups.items(): + anchor = max( + anchor_rows, + key=lambda row: ( + int(row.get('actual_count') or 0), + 1 if row.get('album_title') else 0, + -int(row.get('_scan_order') or 0), + ), + ) + members = list(anchor_rows) + + for row in albums: + row_id = str(row['album_id']) + if ( + row_id in canonical_row_ids + or row_id in assigned_candidate_ids + ): + continue + + artist_id = str(row.get('artist_id') or '') + matches = set() + for source, source_id in ( + self._source_ids_from_row(row).items() + ): + if source_id: + matches.update( + alias_to_groups.get( + ( + artist_id, + source, + str(source_id), + ), + set(), + ) + ) + + if matches == {key}: + members.append(row) + assigned_candidate_ids.add(row_id) + + groups.append({ + 'anchor': anchor, + 'members': sorted( + members, + key=lambda row: row['_scan_order'], + ), + '_scan_order': min( + row['_scan_order'] + for row in members + ), + }) + + grouped_ids = canonical_row_ids | assigned_candidate_ids + for row in albums: + if str(row['album_id']) in grouped_ids: + continue + groups.append({ + 'anchor': row, + 'members': [row], + '_scan_order': row['_scan_order'], + }) + + groups.sort(key=lambda group: group['_scan_order']) + return groups + + def _source_ids_from_row(self, row): + """Return every stored release ID available on an album row.""" + return { + 'spotify': row.get('spotify_album_id') or '', + 'itunes': row.get('itunes_album_id') or '', + 'deezer': row.get('deezer_album_id') or '', + 'discogs': row.get('discogs_album_id') or '', + 'hydrabase': row.get('hydrabase_album_id') or '', + 'musicbrainz': row.get('musicbrainz_album_id') or '', + } + + def _load_local_tracks(self, context, album_ids): + """Load local tracks while tolerating older track-table schemas.""" + ids = [ + str(album_id) + for album_id in album_ids + if album_id is not None + ] + if not ids: + return [] + + placeholders = ','.join('?' for _ in ids) conn = None try: conn = context.db._get_connection() cursor = conn.cursor() - set_album_api_track_count(cursor, album_id, count) - conn.commit() + cursor.execute("PRAGMA table_info(tracks)") + columns = { + row[1] + for row in cursor.fetchall() + } + + select_cols = [ + ( + 'id' if 'id' in columns else 'NULL', + 'id', + ), + ('album_id', 'album_id'), + ( + 'title' if 'title' in columns else "''", + 'title', + ), + ( + 'track_number' + if 'track_number' in columns + else 'NULL', + 'track_number', + ), + ( + 'disc_number' + if 'disc_number' in columns + else '1', + 'disc_number', + ), + ( + 'duration' + if 'duration' in columns + else '0', + 'duration', + ), + ( + 'musicbrainz_recording_id' + if 'musicbrainz_recording_id' in columns + else "''", + 'musicbrainz_recording_id', + ), + ] + select_sql = ', '.join( + f'{expression} AS {alias}' + for expression, alias in select_cols + ) + cursor.execute( + f""" + SELECT {select_sql} + FROM tracks + WHERE album_id IN ({placeholders}) + """, + ids, + ) + return [ + { + 'id': row[0], + 'album_id': row[1], + 'title': row[2] or '', + 'track_number': row[3], + 'disc_number': ( + row[4] + if row[4] is not None + else 1 + ), + 'duration_ms': row[5] or 0, + 'musicbrainz_recording_id': row[6] or '', + } + for row in cursor.fetchall() + ] except Exception as e: - logger.debug("Failed to cache api_track_count for album %s: %s", album_id, e) + logger.debug( + "Failed loading local tracks for albums %s: %s", + ids, + e, + ) + return [] finally: if conn: conn.close() - def _get_expected_total(self, context, primary_source, album_ids): - """Try to get the expected track count from the active metadata provider first.""" + def _normalize_title(self, value): + text = unicodedata.normalize('NFKD', str(value or '')) + text = ''.join( + char + for char in text + if not unicodedata.combining(char) + ) + text = text.casefold() + text = re.sub(r'[\W_]+', ' ', text, flags=re.UNICODE) + return ' '.join(text.split()) + + def _as_int(self, value, default=None): + if value is None or value == '': + return default + if isinstance(value, bool): + return int(value) + if isinstance(value, (int, float)): + return int(value) + match = re.search(r'\d+', str(value)) + return int(match.group(0)) if match else default + + def _reference_duration_ms(self, item): + value = item.get('duration_ms') + if value not in (None, ''): + try: + return int(float(value)) + except (TypeError, ValueError): + return 0 + + value = item.get('duration') + if value not in (None, ''): + try: + return int(float(value) * 1000) + except (TypeError, ValueError): + return 0 + return 0 + + def _reference_slots_for_local_tracks( + self, + reference_items, + local_tracks, + ): + """Map unambiguous local disc/track slots to reference indexes.""" + slots = defaultdict(list) + for index, item in enumerate(reference_items): + number = self._as_int(item.get('track_number')) + disc = self._as_int( + item.get('disc_number'), + 1, + ) + if number is not None: + slots[(disc, number)].append(index) + + matched = set() + for track in local_tracks: + number = self._as_int(track.get('track_number')) + disc = self._as_int( + track.get('disc_number'), + 1, + ) + indexes = slots.get((disc, number), []) + if len(indexes) == 1: + matched.add(indexes[0]) + return matched + + def _track_match_score( + self, + reference, + local, + reference_source, + ): + """Return a score when a local track plausibly matches a reference.""" + reference_id = str(reference.get('id') or '') + local_mbid = str( + local.get('musicbrainz_recording_id') or '' + ) + if ( + reference_source == 'musicbrainz' + and reference_id + and reference_id == local_mbid + ): + return 1000 + + reference_number = self._as_int( + reference.get('track_number') + ) + local_number = self._as_int( + local.get('track_number') + ) + reference_disc = self._as_int( + reference.get('disc_number'), + 1, + ) + local_disc = self._as_int( + local.get('disc_number'), + 1, + ) + + same_number = ( + reference_number is not None + and local_number is not None + and reference_number == local_number + ) + same_slot = same_number and reference_disc == local_disc + + reference_title = self._normalize_title( + reference.get('name') + or reference.get('title') + ) + local_title = self._normalize_title( + local.get('title') + ) + title_ratio = ( + SequenceMatcher( + None, + reference_title, + local_title, + ).ratio() + if reference_title and local_title + else 0.0 + ) + exact_title = bool( + reference_title + and reference_title == local_title + ) + + reference_duration = self._reference_duration_ms( + reference + ) + local_duration = ( + self._as_int( + local.get('duration_ms'), + 0, + ) + or 0 + ) + has_durations = ( + reference_duration > 0 + and local_duration > 0 + ) + duration_close = ( + has_durations + and abs( + reference_duration - local_duration + ) <= 15000 + ) + + if same_slot and title_ratio >= 0.65: + return ( + 800 + + int(title_ratio * 100) + + (30 if duration_close else 0) + ) + if same_slot and duration_close: + return 740 + int(title_ratio * 100) + if exact_title: + return ( + 700 + + (50 if duration_close else 0) + + (20 if same_number else 0) + ) + if ( + title_ratio >= 0.92 + and (duration_close or not has_durations) + ): + return 650 + int(title_ratio * 100) + if ( + same_number + and title_ratio >= 0.80 + and duration_close + ): + return 620 + int(title_ratio * 100) + return None + + def _fragment_track_match_score( + self, + reference, + local, + reference_source, + ): + """Use stricter matching when accepting a sibling fragment.""" + reference_id = str(reference.get('id') or '') + local_mbid = str( + local.get('musicbrainz_recording_id') or '' + ) + if ( + reference_source == 'musicbrainz' + and reference_id + and reference_id == local_mbid + ): + return 1000 + + reference_title = self._normalize_title( + reference.get('name') + or reference.get('title') + ) + local_title = self._normalize_title( + local.get('title') + ) + if not reference_title or not local_title: + return None + + title_ratio = SequenceMatcher( + None, + reference_title, + local_title, + ).ratio() + exact_title = reference_title == local_title + + reference_number = self._as_int( + reference.get('track_number') + ) + local_number = self._as_int( + local.get('track_number') + ) + reference_disc = self._as_int( + reference.get('disc_number'), + 1, + ) + local_disc = self._as_int( + local.get('disc_number'), + 1, + ) + same_slot = ( + reference_number is not None + and local_number is not None + and reference_number == local_number + and reference_disc == local_disc + ) + + reference_duration = self._reference_duration_ms( + reference + ) + local_duration = ( + self._as_int( + local.get('duration_ms'), + 0, + ) + or 0 + ) + duration_close = ( + reference_duration > 0 + and local_duration > 0 + and abs( + reference_duration - local_duration + ) <= 15000 + ) + + if exact_title: + return ( + 900 + + (30 if same_slot else 0) + + (20 if duration_close else 0) + ) + if ( + title_ratio >= 0.95 + and (same_slot or duration_close) + ): + return 800 + int(title_ratio * 100) + if ( + title_ratio >= 0.85 + and same_slot + and duration_close + ): + return 700 + int(title_ratio * 100) + return None + + def _match_fragment_tracks( + self, + reference_items, + local_tracks, + reference_source, + ): + """One-to-one strict match used only for candidate siblings.""" + return self._greedy_match( + reference_items, + local_tracks, + reference_source, + self._fragment_track_match_score, + ) + + def _match_tracks( + self, + reference_items, + local_tracks, + reference_source, + ): + """One-to-one general match for tracks already on the anchor.""" + return self._greedy_match( + reference_items, + local_tracks, + reference_source, + self._track_match_score, + ) + + def _greedy_match( + self, + reference_items, + local_tracks, + reference_source, + scorer, + ): + candidates = [] + for reference_index, reference in enumerate( + reference_items + ): + for local_index, local in enumerate(local_tracks): + score = scorer( + reference, + local, + reference_source, + ) + if score is not None: + candidates.append( + ( + score, + reference_index, + local_index, + ) + ) + + candidates.sort(reverse=True) + matched_reference = set() + matched_local = set() + + for _, reference_index, local_index in candidates: + if ( + reference_index in matched_reference + or local_index in matched_local + ): + continue + matched_reference.add(reference_index) + matched_local.add(local_index) + + return matched_reference, matched_local + + def _build_missing_tracks( + self, + reference_items, + matched_reference, + reference_source, + ): + missing_tracks = [] + for index, item in enumerate(reference_items): + if index in matched_reference: + continue + + track_artists = [] + for artist in item.get('artists', []): + if isinstance(artist, dict): + track_artists.append( + artist.get('name', '') + ) + elif isinstance(artist, str): + track_artists.append(artist) + + source_track_id = item.get('id', '') + missing_tracks.append({ + 'track_number': item.get('track_number'), + 'name': ( + item.get('name') + or item.get('title') + or '' + ), + 'disc_number': item.get('disc_number', 1), + 'source': item.get( + '_source', + reference_source, + ), + 'source_track_id': source_track_id, + 'track_id': source_track_id, + 'spotify_track_id': source_track_id, + 'duration_ms': self._reference_duration_ms( + item + ), + 'artists': track_artists, + }) + return missing_tracks + + def _save_api_track_count(self, context, album_id, count): + """Persist a metadata-API track count via the shared worker helper.""" + conn = None + try: + conn = context.db._get_connection() + cursor = conn.cursor() + set_album_api_track_count( + cursor, + album_id, + count, + ) + conn.commit() + except Exception as e: + logger.debug( + "Failed to cache api_track_count for album %s: %s", + album_id, + e, + ) + finally: + if conn: + conn.close() + + def _get_expected_total( + self, + context, + primary_source, + album_ids, + ): + """Get the expected count from the active provider priority.""" for source in get_source_priority(primary_source): - album_id = self._get_album_id_for_source(source, album_ids) + album_id = self._get_album_id_for_source( + source, + album_ids, + ) if not album_id: continue - api_tracks = self._get_album_tracks(source, album_id) + api_tracks = self._get_album_tracks( + source, + album_id, + ) items = self._extract_track_items(api_tracks) if items: return len(items) - return 0 def _find_missing_tracks( @@ -378,22 +1258,20 @@ class AlbumCompletenessJob(RepairJob): resolved_source=None, resolved_items=None, ): - """Identify missing tracks from one resolved metadata edition. - - Without a supplied edition, preserve the existing source-priority lookup. - """ - # Get track numbers we already have + """Identify missing tracks from one resolved metadata edition.""" owned_numbers = set() conn = None try: conn = context.db._get_connection() cursor = conn.cursor() cursor.execute( - "SELECT track_number FROM tracks WHERE album_id = ? AND track_number IS NOT NULL", - (album_id,) + "SELECT track_number FROM tracks " + "WHERE album_id = ? " + "AND track_number IS NOT NULL", + (album_id,), ) - for tr in cursor.fetchall(): - owned_numbers.add(tr[0]) + for track in cursor.fetchall(): + owned_numbers.add(track[0]) except Exception: return [] finally: @@ -402,14 +1280,17 @@ class AlbumCompletenessJob(RepairJob): if resolved_items is None: api_tracks = None - for source in get_source_priority(primary_source): - source_album_id = self._get_album_id_for_source( - source, - album_ids, + for source in get_source_priority( + primary_source + ): + source_album_id = ( + self._get_album_id_for_source( + source, + album_ids, + ) ) if not source_album_id: continue - api_tracks = self._get_album_tracks( source, source_album_id, @@ -417,7 +1298,6 @@ class AlbumCompletenessJob(RepairJob): if self._extract_track_items(api_tracks): resolved_source = source break - items = self._extract_track_items(api_tracks) else: items = resolved_items @@ -426,64 +1306,77 @@ class AlbumCompletenessJob(RepairJob): return [] track_source = resolved_source or primary_source - - # All supported provider responses expose the same core fields once normalized. - # items[].track_number, items[].name, items[].disc_number, items[].id, items[].artists - missing_tracks = [] - for item in items: - tn = item.get('track_number') - if tn and tn not in owned_numbers: - track_artists = [] - for a in item.get('artists', []): - if isinstance(a, dict): - track_artists.append(a.get('name', '')) - elif isinstance(a, str): - track_artists.append(a) - missing_tracks.append({ - 'track_number': tn, - 'name': item.get('name', ''), - 'disc_number': item.get('disc_number', 1), - 'source': item.get('_source', track_source), - 'source_track_id': item.get('id', ''), - 'track_id': item.get('id', ''), - 'spotify_track_id': item.get('id', ''), - 'duration_ms': item.get('duration_ms', 0), - 'artists': track_artists, - }) - return missing_tracks + matched_reference = { + index + for index, item in enumerate(items) + if ( + item.get('track_number') + and item.get('track_number') in owned_numbers + ) + } + return self._build_missing_tracks( + items, + matched_reference, + track_source, + ) def _get_settings(self, context: JobContext) -> dict: if not context.config_manager: return self.default_settings.copy() - cfg = context.config_manager.get(f'repair.jobs.{self.job_id}.settings', {}) + cfg = context.config_manager.get( + f'repair.jobs.{self.job_id}.settings', + {}, + ) merged = self.default_settings.copy() merged.update(cfg) return merged def _get_primary_source(self) -> str: - """Return the active metadata source used for source prioritization.""" + """Return the active metadata source for prioritization.""" try: return get_primary_source() except Exception: return 'deezer' - def _get_album_id_for_source(self, source: str, album_ids: dict) -> str: + def _get_album_id_for_source( + self, + source: str, + album_ids: dict, + ) -> str: return album_ids.get(source, '') - def _get_album_tracks(self, source: str, album_id: str): + def _get_album_tracks( + self, + source: str, + album_id: str, + ): """Fetch album tracks from a specific source.""" try: - return get_album_tracks_for_source(source, album_id) + return get_album_tracks_for_source( + source, + album_id, + ) except Exception as e: - logger.debug("Error getting %s album tracks for %s: %s", source.capitalize(), album_id, e) + logger.debug( + "Error getting %s album tracks for %s: %s", + source.capitalize(), + album_id, + e, + ) return None def _extract_track_items(self, api_tracks): - """Normalize album track responses to a list of item dicts.""" + """Normalize provider responses to a list of track dicts.""" if not api_tracks: return [] if isinstance(api_tracks, dict): - items = api_tracks.get('items') or [] + items = ( + api_tracks.get('items') + or api_tracks.get('tracks') + or [] + ) + if isinstance(items, dict): + items = items.get('items') or [] return items if items else [] if isinstance(api_tracks, list): return api_tracks @@ -495,26 +1388,50 @@ class AlbumCompletenessJob(RepairJob): conn = context.db._get_connection() cursor = conn.cursor() - # Check which columns exist cursor.execute("PRAGMA table_info(albums)") - columns = {row[1] for row in cursor.fetchall()} + columns = { + row[1] + for row in cursor.fetchall() + } - where_parts = ["(spotify_album_id IS NOT NULL AND spotify_album_id != '')"] + where_parts = [ + "(spotify_album_id IS NOT NULL " + "AND spotify_album_id != '')", + ] if 'itunes_album_id' in columns: - where_parts.append("(itunes_album_id IS NOT NULL AND itunes_album_id != '')") + where_parts.append( + "(itunes_album_id IS NOT NULL " + "AND itunes_album_id != '')" + ) if 'deezer_id' in columns: - where_parts.append("(deezer_id IS NOT NULL AND deezer_id != '')") + where_parts.append( + "(deezer_id IS NOT NULL " + "AND deezer_id != '')" + ) if 'discogs_id' in columns: - where_parts.append("(discogs_id IS NOT NULL AND discogs_id != '')") + where_parts.append( + "(discogs_id IS NOT NULL " + "AND discogs_id != '')" + ) if 'soul_id' in columns: - where_parts.append("(soul_id IS NOT NULL AND soul_id != '')") + where_parts.append( + "(soul_id IS NOT NULL " + "AND soul_id != '')" + ) + if 'musicbrainz_release_id' in columns: + where_parts.append( + "(musicbrainz_release_id IS NOT NULL " + "AND musicbrainz_release_id != '')" + ) if ( 'canonical_source' in columns and 'canonical_album_id' in columns ): where_parts.append( - "(canonical_source IS NOT NULL AND canonical_source != '' " - "AND canonical_album_id IS NOT NULL AND canonical_album_id != '')" + "(canonical_source IS NOT NULL " + "AND canonical_source != '' " + "AND canonical_album_id IS NOT NULL " + "AND canonical_album_id != '')" ) cursor.execute(f""" diff --git a/tests/test_album_completeness_fragmented_rows.py b/tests/test_album_completeness_fragmented_rows.py new file mode 100644 index 00000000..bfec2a16 --- /dev/null +++ b/tests/test_album_completeness_fragmented_rows.py @@ -0,0 +1,496 @@ +import sqlite3 +import sys +import types +import uuid + + +class _DummyConfigManager: + def get(self, key, default=None): + return default + + def get_active_media_server(self): + return "plex" + + +if "spotipy" not in sys.modules: + spotipy = types.ModuleType("spotipy") + + class _DummySpotify: + def __init__(self, *args, **kwargs): + pass + + oauth2 = types.ModuleType("spotipy.oauth2") + + class _DummyOAuth: + def __init__(self, *args, **kwargs): + pass + + spotipy.Spotify = _DummySpotify + oauth2.SpotifyOAuth = _DummyOAuth + oauth2.SpotifyClientCredentials = _DummyOAuth + spotipy.oauth2 = oauth2 + sys.modules["spotipy"] = spotipy + sys.modules["spotipy.oauth2"] = oauth2 + +if "config.settings" not in sys.modules: + config_pkg = types.ModuleType("config") + settings_mod = types.ModuleType("config.settings") + + settings_mod.config_manager = _DummyConfigManager() + config_pkg.settings = settings_mod + sys.modules["config"] = config_pkg + sys.modules["config.settings"] = settings_mod + +from core.repair_jobs.album_completeness import AlbumCompletenessJob +import core.repair_jobs.album_completeness as album_completeness_module + + +class _SharedMemoryDB: + def __init__(self): + self.uri = ( + f"file:testdb_{uuid.uuid4().hex}" + "?mode=memory&cache=shared" + ) + self._keepalive = sqlite3.connect(self.uri, uri=True) + self._keepalive.executescript( + """ + CREATE TABLE artists ( + id TEXT PRIMARY KEY, + name TEXT, + thumb_url TEXT + ); + + CREATE TABLE albums ( + id TEXT PRIMARY KEY, + artist_id TEXT, + title TEXT, + thumb_url TEXT, + spotify_album_id TEXT, + itunes_album_id TEXT, + deezer_id TEXT, + discogs_id TEXT, + soul_id TEXT, + musicbrainz_release_id TEXT, + canonical_source TEXT, + canonical_album_id TEXT, + api_track_count INTEGER + ); + + CREATE TABLE tracks ( + id TEXT PRIMARY KEY, + album_id TEXT, + title TEXT, + track_number INTEGER, + disc_number INTEGER, + duration INTEGER, + musicbrainz_recording_id TEXT + ); + """ + ) + self._keepalive.commit() + + def _get_connection(self): + return sqlite3.connect(self.uri, uri=True) + + def insert_artist(self, artist_id, name): + self._keepalive.execute( + """ + INSERT INTO artists (id, name) + VALUES (?, ?) + """, + (artist_id, name), + ) + self._keepalive.commit() + + def insert_album( + self, + album_id, + artist_id, + title, + *, + spotify_id=None, + musicbrainz_id=None, + canonical_source=None, + canonical_album_id=None, + api_track_count=None, + ): + self._keepalive.execute( + """ + INSERT INTO albums ( + id, + artist_id, + title, + spotify_album_id, + musicbrainz_release_id, + canonical_source, + canonical_album_id, + api_track_count + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + album_id, + artist_id, + title, + spotify_id, + musicbrainz_id, + canonical_source, + canonical_album_id, + api_track_count, + ), + ) + self._keepalive.commit() + + def insert_track( + self, + album_id, + number, + title, + *, + disc=1, + duration_ms=180000, + mbid=None, + ): + track_id = f"{album_id}-{disc}-{number}-{uuid.uuid4().hex}" + self._keepalive.execute( + """ + INSERT INTO tracks ( + id, + album_id, + title, + track_number, + disc_number, + duration, + musicbrainz_recording_id + ) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + track_id, + album_id, + title, + number, + disc, + duration_ms, + mbid, + ), + ) + self._keepalive.commit() + + +def _context(db, findings): + return types.SimpleNamespace( + db=db, + transfer_folder='', + config_manager=_DummyConfigManager(), + spotify_client=None, + is_spotify_rate_limited=lambda: False, + stop_event=None, + create_finding=lambda **kwargs: ( + findings.append(kwargs) or True + ), + should_stop=None, + is_paused=None, + update_progress=None, + report_progress=None, + check_stop=lambda: False, + wait_if_paused=lambda: False, + ) + + +def _canonical_tracks(count, *, prefix="Canonical"): + return { + "items": [ + { + "id": f"track-{number}", + "name": f"{prefix} Track {number}", + "track_number": number, + "disc_number": 1, + "duration_ms": 180000 + number, + "artists": [], + } + for number in range(1, count + 1) + ], + } + + +def test_scan_groups_validated_fragmented_rows_into_one_finding( + monkeypatch, +): + db = _SharedMemoryDB() + db.insert_artist("artist-1", "Artist") + db.insert_album( + "anchor", + "artist-1", + "Album", + spotify_id="shared-release", + canonical_source="deezer", + canonical_album_id="canonical-release", + ) + db.insert_album( + "fragment", + "artist-1", + "ALBUM", + spotify_id="shared-release", + api_track_count=2, + ) + + db.insert_track("anchor", 1, "Canonical Track 1") + db.insert_track("anchor", 2, "Canonical Track 2") + db.insert_track("fragment", 3, "Canonical Track 3") + db.insert_track("fragment", 4, "Canonical Track 4") + + calls = [] + + def get_tracks(source, album_id): + calls.append((source, album_id)) + assert source == "deezer" + assert album_id == "canonical-release" + return _canonical_tracks(5) + + monkeypatch.setattr( + album_completeness_module, + "get_album_tracks_for_source", + get_tracks, + ) + monkeypatch.setattr( + album_completeness_module, + "get_primary_source", + lambda: "spotify", + ) + monkeypatch.setattr( + album_completeness_module, + "get_source_priority", + lambda primary: ["spotify", "deezer"], + ) + + findings = [] + result = AlbumCompletenessJob().scan( + _context(db, findings) + ) + + assert result.scanned == 1 + assert result.findings_created == 1 + assert calls == [("deezer", "canonical-release")] + + details = findings[0]["details"] + assert details["album_id"] == "anchor" + assert details["expected_tracks"] == 5 + assert details["actual_tracks"] == 4 + assert details["raw_local_tracks"] == 4 + assert details["related_album_ids"] == [ + "anchor", + "fragment", + ] + assert [ + track["track_number"] + for track in details["missing_tracks"] + ] == [5] + + +def test_shared_id_without_track_match_stays_independent( + monkeypatch, +): + db = _SharedMemoryDB() + db.insert_artist("artist-1", "Artist") + db.insert_album( + "anchor", + "artist-1", + "Album", + spotify_id="shared-release", + canonical_source="deezer", + canonical_album_id="canonical-release", + ) + db.insert_album( + "unrelated", + "artist-1", + "Different Album", + spotify_id="shared-release", + api_track_count=1, + ) + + db.insert_track("anchor", 1, "Canonical Track 1") + db.insert_track( + "unrelated", + 99, + "Completely Unrelated", + duration_ms=900000, + ) + + calls = [] + + def get_tracks(source, album_id): + calls.append((source, album_id)) + return _canonical_tracks(3) + + monkeypatch.setattr( + album_completeness_module, + "get_album_tracks_for_source", + get_tracks, + ) + monkeypatch.setattr( + album_completeness_module, + "get_primary_source", + lambda: "spotify", + ) + monkeypatch.setattr( + album_completeness_module, + "get_source_priority", + lambda primary: ["spotify", "deezer"], + ) + + findings = [] + result = AlbumCompletenessJob().scan( + _context(db, findings) + ) + + assert result.scanned == 2 + assert result.findings_created == 1 + assert calls == [("deezer", "canonical-release")] + assert findings[0]["entity_id"] == "anchor" + assert findings[0]["details"]["related_album_ids"] == [ + "anchor", + ] + + +def test_fragment_grouping_never_crosses_artist_boundary( + monkeypatch, +): + db = _SharedMemoryDB() + db.insert_artist("artist-1", "Artist One") + db.insert_artist("artist-2", "Artist Two") + db.insert_album( + "anchor", + "artist-1", + "Album", + spotify_id="shared-release", + canonical_source="deezer", + canonical_album_id="canonical-release", + ) + db.insert_album( + "other-artist", + "artist-2", + "Album", + spotify_id="shared-release", + api_track_count=1, + ) + + db.insert_track("anchor", 1, "Canonical Track 1") + db.insert_track( + "other-artist", + 2, + "Canonical Track 2", + ) + + calls = [] + + def get_tracks(source, album_id): + calls.append((source, album_id)) + return _canonical_tracks(3) + + monkeypatch.setattr( + album_completeness_module, + "get_album_tracks_for_source", + get_tracks, + ) + monkeypatch.setattr( + album_completeness_module, + "get_primary_source", + lambda: "spotify", + ) + monkeypatch.setattr( + album_completeness_module, + "get_source_priority", + lambda primary: ["spotify", "deezer"], + ) + + findings = [] + result = AlbumCompletenessJob().scan( + _context(db, findings) + ) + + assert result.scanned == 2 + assert result.findings_created == 1 + assert calls == [("deezer", "canonical-release")] + assert findings[0]["details"]["related_album_ids"] == [ + "anchor", + ] + + +def test_musicbrainz_recording_id_validates_fragment( + monkeypatch, +): + db = _SharedMemoryDB() + db.insert_artist("artist-1", "Artist") + db.insert_album( + "anchor", + "artist-1", + "Album", + spotify_id="shared-release", + musicbrainz_id="mb-release", + canonical_source="musicbrainz", + canonical_album_id="mb-release", + ) + db.insert_album( + "fragment", + "artist-1", + "Album Fragment", + spotify_id="shared-release", + api_track_count=1, + ) + + db.insert_track( + "anchor", + 1, + "Canonical Track 1", + mbid="track-1", + ) + db.insert_track( + "fragment", + 99, + "Wrong title and position", + duration_ms=999999, + mbid="track-2", + ) + + calls = [] + + def get_tracks(source, album_id): + calls.append((source, album_id)) + return _canonical_tracks(3) + + monkeypatch.setattr( + album_completeness_module, + "get_album_tracks_for_source", + get_tracks, + ) + monkeypatch.setattr( + album_completeness_module, + "get_primary_source", + lambda: "spotify", + ) + monkeypatch.setattr( + album_completeness_module, + "get_source_priority", + lambda primary: ["spotify", "musicbrainz"], + ) + + findings = [] + result = AlbumCompletenessJob().scan( + _context(db, findings) + ) + + assert result.scanned == 1 + assert result.findings_created == 1 + assert calls == [("musicbrainz", "mb-release")] + + details = findings[0]["details"] + assert details["actual_tracks"] == 2 + assert details["related_album_ids"] == [ + "anchor", + "fragment", + ] + assert [ + track["track_number"] + for track in details["missing_tracks"] + ] == [3]