Fix Japanese/CJK text mangled in Soulseek search queries
normalize_string() was running unidecode on all text, converting Japanese kanji to Chinese pinyin gibberish (命の灯火 → "tvanimedei"). Now detects CJK characters (kanji, hiragana, katakana, hangul, fullwidth forms) and skips unidecode for text containing them — just lowercases instead. Non-CJK text (Latin accents, Cyrillic) still goes through unidecode normally.
This commit is contained in:
parent
1646c3d9e1
commit
d944d4a7d2
3 changed files with 20 additions and 2 deletions
|
|
@ -67,8 +67,16 @@ class MusicMatchingEngine:
|
||||||
# Apply the character replacements before other normalization steps
|
# Apply the character replacements before other normalization steps
|
||||||
for original, replacement in char_map.items():
|
for original, replacement in char_map.items():
|
||||||
text = text.replace(original, replacement)
|
text = text.replace(original, replacement)
|
||||||
text = unidecode(text)
|
|
||||||
text = text.lower()
|
# Skip unidecode for CJK text — it converts Japanese kanji to Chinese pinyin,
|
||||||
|
# producing gibberish like "tvanimedei" for "命の灯火". Preserve original characters
|
||||||
|
# so Soulseek searches use the real title. Only apply unidecode to non-CJK text.
|
||||||
|
if any('\u2e80' <= c <= '\u9fff' or '\u3040' <= c <= '\u30ff' or '\uff00' <= c <= '\uffef' or '\uac00' <= c <= '\ud7af' for c in text):
|
||||||
|
# CJK detected — just lowercase, don't transliterate
|
||||||
|
text = text.lower()
|
||||||
|
else:
|
||||||
|
text = unidecode(text)
|
||||||
|
text = text.lower()
|
||||||
|
|
||||||
# Expand specific abbreviations for better matching
|
# Expand specific abbreviations for better matching
|
||||||
abbreviation_map = {
|
abbreviation_map = {
|
||||||
|
|
|
||||||
|
|
@ -19246,6 +19246,15 @@ def get_version_info():
|
||||||
"title": "What's New in SoulSync",
|
"title": "What's New in SoulSync",
|
||||||
"subtitle": f"Version {SOULSYNC_VERSION} — Latest Changes",
|
"subtitle": f"Version {SOULSYNC_VERSION} — Latest Changes",
|
||||||
"sections": [
|
"sections": [
|
||||||
|
{
|
||||||
|
"title": "🔧 Fix Japanese Song Searches Producing Gibberish",
|
||||||
|
"description": "CJK text no longer mangled by unidecode in Soulseek search queries",
|
||||||
|
"features": [
|
||||||
|
"• Japanese kanji, hiragana, katakana, and Korean hangul preserved in search queries",
|
||||||
|
"• unidecode was converting Japanese to Chinese pinyin (e.g. 命の灯火 → 'tvanimedei')",
|
||||||
|
"• Soulseek users typically share files with original CJK characters in filenames"
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"title": "🔧 Fix Partial Name Matching False Positives (#225)",
|
"title": "🔧 Fix Partial Name Matching False Positives (#225)",
|
||||||
"description": "Track ownership check no longer falsely matches prefix/suffix variations",
|
"description": "Track ownership check no longer falsely matches prefix/suffix variations",
|
||||||
|
|
|
||||||
|
|
@ -3403,6 +3403,7 @@ function closeHelperSearch() {
|
||||||
const WHATS_NEW = {
|
const WHATS_NEW = {
|
||||||
'2.1': [
|
'2.1': [
|
||||||
// Newest features first
|
// Newest features first
|
||||||
|
{ title: 'Fix Japanese/CJK Soulseek Searches', desc: 'Japanese kanji no longer mangled into Chinese pinyin — searches now use original characters' },
|
||||||
{ title: 'Fix Partial Title Matching', desc: '"Believe" no longer falsely matches "Believe In Me" — length ratio penalty prevents prefix false positives' },
|
{ title: 'Fix Partial Title Matching', desc: '"Believe" no longer falsely matches "Believe In Me" — length ratio penalty prevents prefix false positives' },
|
||||||
{ title: 'Fix Pipeline Blocking on Discovery Fail', desc: 'Playlist sync no longer drops tracks that failed metadata discovery — continues with original name/artist for download' },
|
{ title: 'Fix Pipeline Blocking on Discovery Fail', desc: 'Playlist sync no longer drops tracks that failed metadata discovery — continues with original name/artist for download' },
|
||||||
{ title: 'Playlist Explorer', desc: 'New page: expand playlists into visual discovery trees of albums and discographies — select and add to wishlist', page: 'playlist-explorer' },
|
{ title: 'Playlist Explorer', desc: 'New page: expand playlists into visual discovery trees of albums and discographies — select and add to wishlist', page: 'playlist-explorer' },
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue