diff --git a/core/matching_engine.py b/core/matching_engine.py index 9abb2d06..32113bcc 100644 --- a/core/matching_engine.py +++ b/core/matching_engine.py @@ -74,7 +74,22 @@ class MusicMatchingEngine: # Skip unidecode for CJK text — it converts Japanese kanji to Chinese pinyin, # producing gibberish like "tvanimedei" for "命の灯火". Preserve original characters # so Soulseek searches use the real title. Only apply unidecode to non-CJK text. - if any('\u2e80' <= c <= '\u9fff' or '\u3040' <= c <= '\u30ff' or '\uff00' <= c <= '\uffef' or '\uac00' <= c <= '\ud7af' for c in text): + # Issue #722 — flag CJK presence here so the alphanumeric strip + # below preserves CJK ranges instead of nuking them. Pre-fix the + # strip pattern ``[^a-z0-9\s$]`` deleted every CJK character, + # which left every Japanese title normalised to ``''``. Two empty + # strings produce 0.0 title similarity, the matcher fell back to + # duration+artist alone, and multiple iTunes tracks mapped to the + # same Tidal candidate, so the user got duplicate downloads under + # different track positions. + has_cjk = any( + '\u2e80' <= c <= '\u9fff' # CJK Unified Ideographs + radicals + or '\u3040' <= c <= '\u30ff' # Hiragana + Katakana + or '\uff00' <= c <= '\uffef' # Halfwidth / Fullwidth forms + or '\uac00' <= c <= '\ud7af' # Hangul syllables + for c in text + ) + if has_cjk: # CJK detected — just lowercase, don't transliterate text = text.lower() else: @@ -103,7 +118,17 @@ class MusicMatchingEngine: text = re.sub(r'[._/&-]', ' ', text) # Keep alphanumeric characters, spaces, AND the '$' sign. - text = re.sub(r'[^a-z0-9\s$]', '', text) + # When CJK was detected upstream, also preserve CJK Unified + # Ideographs / Hiragana / Katakana / Hangul / Halfwidth-Fullwidth + # ranges so Japanese / Chinese / Korean titles produce a + # comparable normalised form instead of an empty string. + if has_cjk: + text = re.sub( + r'[^a-z0-9\s$\u2e80-\u9fff\u3040-\u30ff\uff00-\uffef\uac00-\ud7af]', + '', text, + ) + else: + text = re.sub(r'[^a-z0-9\s$]', '', text) # Consolidate multiple spaces into one text = re.sub(r'\s+', ' ', text).strip() diff --git a/tests/test_matching_engine_cjk.py b/tests/test_matching_engine_cjk.py new file mode 100644 index 00000000..452b2074 --- /dev/null +++ b/tests/test_matching_engine_cjk.py @@ -0,0 +1,194 @@ +"""Tests for ``MusicMatchingEngine.normalize_string`` CJK preservation. + +Issue #722 (@Sokhii): downloading a Japanese OST through Apple Music +metadata + Tidal download produced duplicate tracks — same audio +landed under multiple track positions in the album. + +Root cause: ``normalize_string`` detected CJK presence and SKIPPED +unidecode (correct — kanji→pinyin would have been gibberish), but +then ran ``re.sub(r'[^a-z0-9\\s$]', '', text)`` which stripped EVERY +CJK character. Every Japanese title normalised to ``''``. The +``similarity_score`` guard at ``if not str1 or not str2: return 0.0`` +made every CJK-vs-CJK comparison return ``0.000``, so the matcher +fell back to duration+artist alone — multiple iTunes tracks mapped +to the same Tidal candidate, and the user got duplicate downloads +under different track positions. + +These tests pin the new behaviour: CJK characters survive +normalisation, identical CJK titles score 1.0, disjoint CJK titles +score low, mixed CJK+Latin titles work, and Latin-only titles are +completely unaffected. +""" + +from __future__ import annotations + +import pytest + +from core.matching_engine import MusicMatchingEngine + + +@pytest.fixture +def engine() -> MusicMatchingEngine: + return MusicMatchingEngine() + + +# --------------------------------------------------------------------------- +# Normalisation preserves CJK ranges. +# --------------------------------------------------------------------------- + + +def test_normalize_preserves_kanji_characters(engine: MusicMatchingEngine): + """Japanese kanji must survive normalisation, not get stripped.""" + assert engine.normalize_string('命の灯火') == '命の灯火' + + +def test_normalize_preserves_hiragana_characters(engine: MusicMatchingEngine): + """Hiragana also survives.""" + assert engine.normalize_string('あいうえお') == 'あいうえお' + + +def test_normalize_preserves_katakana_characters(engine: MusicMatchingEngine): + """Katakana — common in Japanese song titles for foreign loanwords — + survives. Pre-fix this was the most-visible failure since OST titles + are often Katakana.""" + assert engine.normalize_string('ハッピーデイズ') == 'ハッピーデイズ' + + +def test_normalize_preserves_hangul_characters(engine: MusicMatchingEngine): + """Korean Hangul survives (same root cause hits K-Pop OST tracks).""" + assert engine.normalize_string('안녕하세요') == '안녕하세요' + + +def test_normalize_preserves_simplified_chinese_characters(engine: MusicMatchingEngine): + """Chinese hanzi survives (same root cause hits Mandarin / Cantonese + releases). All three CJK ideograph users were broken together; the + fix covers all three.""" + assert engine.normalize_string('你好世界') == '你好世界' + + +def test_normalize_lowercases_cjk_branch_does_not_uppercase_ascii(engine: MusicMatchingEngine): + """Mixed CJK + Latin string — CJK branch was supposed to keep CJK and + only lowercase; verify the Latin half also gets lowercased and isn't + accidentally left as-is.""" + assert engine.normalize_string('Happy 命') == 'happy 命' + + +def test_normalize_strips_latin_punctuation_in_cjk_branch(engine: MusicMatchingEngine): + """The CJK branch must still strip Latin punctuation — only CJK + ranges are preserved, not random symbols. ``!`` should still go, + same as in the Latin branch.""" + assert engine.normalize_string('命の灯火!') == '命の灯火' + + +# --------------------------------------------------------------------------- +# Similarity scoring on CJK titles. +# --------------------------------------------------------------------------- + + +def test_identical_cjk_titles_score_perfect_match(engine: MusicMatchingEngine): + """Same Japanese title twice → 1.0. Pre-fix this was 0.0 because + both normalised to '' and the empty-string guard short-circuited.""" + a = engine.clean_title('命の灯火') + b = engine.clean_title('命の灯火') + assert engine.similarity_score(a, b) == 1.0 + + +def test_completely_disjoint_cjk_titles_score_low(engine: MusicMatchingEngine): + """Two unrelated Japanese titles share no characters → similarity + near 0. The point is that they're DIFFERENT — pre-fix they both + normalised to '' so were treated the same as "identical".""" + a = engine.clean_title('命の灯火') + b = engine.clean_title('無職転生') + score = engine.similarity_score(a, b) + assert score < 0.3 + + +def test_partially_overlapping_cjk_titles_score_partial(engine: MusicMatchingEngine): + """Sequential matching gives a midrange score for partial overlap — + proves the comparator is actually looking at the characters, not + just returning 0 or 1.""" + a = engine.clean_title('命の灯火') + b = engine.clean_title('命の音') + score = engine.similarity_score(a, b) + assert 0.3 < score < 1.0 + + +def test_cjk_title_does_not_falsely_match_unrelated_latin_title(engine: MusicMatchingEngine): + """Pre-fix bug: a CJK title normalised to '' would short-circuit + similarity scoring against ANY Latin title (also returning 0 + because of the empty guard). That's still 0 in both directions + so the symptom isn't directly observable here — but pin that a + real CJK string vs a real Latin string returns a meaningful low + score, not a coincidental match.""" + a = engine.clean_title('命の灯火') + b = engine.clean_title('Happy Days') + score = engine.similarity_score(a, b) + assert score < 0.2 + + +# --------------------------------------------------------------------------- +# Regression: Latin-only titles are untouched. +# --------------------------------------------------------------------------- + + +def test_latin_normalisation_unchanged_for_simple_title(engine: MusicMatchingEngine): + """No CJK in input → unidecode + lowercase path, exactly as before.""" + assert engine.normalize_string('Happy Days') == 'happy days' + + +def test_latin_normalisation_unchanged_for_unidecode_target(engine: MusicMatchingEngine): + """Cyrillic / accented Latin still goes through unidecode.""" + assert engine.normalize_string('Björk') == 'bjork' + + +def test_latin_normalisation_unchanged_for_dollar_sign(engine: MusicMatchingEngine): + """The ``$`` preservation rule still applies in the Latin branch + (A$AP Rocky etc.) — pinned so the CJK refactor doesn't accidentally + drop it.""" + norm = engine.normalize_string('A$AP Rocky') + assert '$' in norm + + +def test_latin_similarity_unchanged_for_baseline_comparison(engine: MusicMatchingEngine): + """Sanity: the existing Latin-Latin scoring behaviour didn't shift. + Identical strings still score 1.0; different strings score below + 1.0. Pin a specific pair from the regression report so a future + normaliser tweak doesn't quietly change Latin-side semantics.""" + a = engine.clean_title('Happy Days') + b = engine.clean_title('Happy Days') + assert engine.similarity_score(a, b) == 1.0 + + c = engine.clean_title('Happy Days') + d = engine.clean_title('My Past Self') + assert engine.similarity_score(c, d) < 0.5 + + +# --------------------------------------------------------------------------- +# Real-world scenarios from issue #722. +# --------------------------------------------------------------------------- + + +def test_mushoku_tensei_ost_track_titles_distinguishable(): + """End-to-end: the exact scenario from #722. Two different iTunes + tracks from Mushoku Tensei Original Soundtrack II — pre-fix both + normalised to '' so the matcher couldn't tell them apart and + routed both to the same Tidal candidate. Post-fix they're + distinguishable. + + Track titles taken from the iTunes album response for id 1753240110; + when the storefront returns Japanese titles instead of the English + romanisations (depends on user's region + storefront config), this + is the comparator the matcher will use.""" + engine = MusicMatchingEngine() + # Two distinct OST tracks rendered in Japanese. + track_a = engine.clean_title('幸せの日々') # 'Happy Days' + track_b = engine.clean_title('家探し') # 'Home Search' + score = engine.similarity_score(track_a, track_b) + # The match must be well below the 0.7+ threshold the candidate + # scorer uses to accept a match — otherwise both iTunes tracks + # would still pick the same Tidal candidate and duplicate. + assert score < 0.5 + + +if __name__ == '__main__': + pytest.main([__file__, '-v']) diff --git a/webui/static/helper.js b/webui/static/helper.js index 043c9b50..d03f4014 100644 --- a/webui/static/helper.js +++ b/webui/static/helper.js @@ -3413,6 +3413,10 @@ function closeHelperSearch() { // projects that span multiple commits before shipping. Strip the flag at // release time and add a real `date:` line at the top of the version block. const WHATS_NEW = { + '2.6.5': [ + { unreleased: true }, + { title: 'Fix: duplicate tracks in albums with Japanese / CJK titles (#722)', desc: 'Japanese OST downloads via Apple Music + Tidal produced duplicate library entries — the same audio file landed under multiple track positions in the album. Root cause: ``MusicMatchingEngine.normalize_string`` correctly skipped unidecode for CJK text (kanji→pinyin would have been gibberish) but then ran ``re.sub(r"[^a-z0-9\\s$]", "", text)`` which stripped EVERY CJK character. Every Japanese title normalised to ``""`` and ``similarity_score`` short-circuited to 0.000 on the empty-string guard. The matcher fell back to duration+artist alone, so multiple iTunes tracks (different songs with same artist + similar duration) mapped to the same Tidal candidate. User got the same audio downloaded N times under different track positions. Fix: when CJK is detected, the alphanumeric-strip step preserves CJK Unified Ideographs, Hiragana, Katakana, Hangul, and Halfwidth/Fullwidth ranges, so CJK titles produce a comparable normalised form. Two different Japanese tracks now score appropriately low; two identical Japanese tracks now score 1.0. Latin-only normalisation is completely unchanged. 16 new unit tests pin every CJK family + every regression-prone Latin baseline. Closes #722.', page: 'downloads' }, + ], '2.6.4': [ { date: 'May 28, 2026 — 2.6.4 release' }, { title: 'Fix: Usenet album bundle stuck on "downloading release" when SAB History flips before storage lands (#721)', desc: 'follow-up to the 2.6.3 queue→history handoff fix. The poll now correctly handles the second-stage gap: SAB flips a job\'s ``status`` to ``Completed`` in History a few seconds before its post-processing pipeline writes the final ``storage`` field. Pre-fix the bundle poll returned ``None`` on the first ``Completed`` read with no save_path, the plugin marked the batch failed, and the UI froze on the last 61% progress emit. ``poll_album_download`` now tolerates up to ``transient_miss_threshold`` consecutive "completed but no save_path" reads, so SAB gets a window to finish writing the path. When it lands, the poll resolves normally. If the path never lands past the threshold, the poll fails loudly with an explicit error pointing at the missing save_path field instead of letting the UI sit on "downloading 61%" until the 6-hour deadline. Also widened the SAB adapter\'s ``_parse_history_slot`` save_path fallback chain to try ``storage`` → ``path`` → ``download_path`` → ``dirname`` → ``incomplete_path`` (last resort), so SAB version / fork variations resolve cleanly. Whitespace-only values are skipped. 9 new unit tests pin every case: late-save_path arrival, sticky save_path from earlier downloading emits, threshold-exhausted failure, plus 6 SAB-adapter field-fallback variants. Closes #721.', page: 'downloads' },