danvics/soulsync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Split runtime builders into owning modules

Browse source 
- Move the import pipeline runtime factory into core.imports.pipeline
- Move the metadata runtime factory into core.metadata.enrichment
- Keep the web server wiring thin and drop the shared glue module
- Add contract tests that keep the two runtime bundles separate
This commit is contained in:
Antti Kettunen 2026-04-25 16:43:17 +03:00
parent bcab54095e
commit 9b2b6d856f
No known key found for this signature in database
GPG key ID: C6B2A3D250359BD7
5 changed files with 206 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
core/imports/pipeline.py
View file

@ -6,6 +6,8 @@ import json
import os
import threading
import time
from types import SimpleNamespace
from typing import Any
from config.settings import config_manager
from core.imports.file_ops import (
@ -68,12 +70,35 @@ from utils.logging_config import get_logger
logger = get_logger("imports.pipeline")
pp_logger = get_logger("post_processing")
__all__ = [
"build_import_pipeline_runtime",
"post_process_matched_download",
"post_process_matched_download_with_verification",
]
def post_process_matched_download(context_key, context, file_path, runtime):
def build_import_pipeline_runtime(
*,
automation_engine: Any | None = None,
on_download_completed: Any | None = None,
web_scan_manager: Any | None = None,
repair_worker: Any | None = None,
) -> SimpleNamespace:
"""Build the runtime object consumed by core.imports.pipeline."""
return SimpleNamespace(
automation_engine=automation_engine,
on_download_completed=on_download_completed,
web_scan_manager=web_scan_manager,
repair_worker=repair_worker,
)
def post_process_matched_download(context_key, context, file_path, runtime, metadata_runtime=None):
on_download_completed = getattr(runtime, "on_download_completed", None)
automation_engine = getattr(runtime, "automation_engine", None)
web_scan_manager = getattr(runtime, "web_scan_manager", None)
repair_worker = getattr(runtime, "repair_worker", None)
metadata_runtime = metadata_runtime or runtime
def _notify_download_completed(batch_id, task_id, success=True):
if on_download_completed:
@ -364,7 +389,7 @@ def post_process_matched_download(context_key, context, file_path, runtime):
f"[Metadata Input] Playlist mode - artist: '{artist_context.get('name', 'MISSING')}' "
f"(id: {artist_context.get('id', 'MISSING')})"
)
enhance_file_metadata(file_path, context, artist_context, None, runtime=runtime)
enhance_file_metadata(file_path, context, artist_context, None, runtime=metadata_runtime)
except Exception as meta_err:
import traceback
pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}")
@ -529,7 +554,7 @@ def post_process_matched_download(context_key, context, file_path, runtime):
)
else:
logger.info("[Metadata Input] album_info: None (single track)")
enhance_file_metadata(file_path, context, artist_context, album_info, runtime=runtime)
enhance_file_metadata(file_path, context, artist_context, album_info, runtime=metadata_runtime)
except Exception as meta_err:
import traceback
pp_logger.info(f"[inner] Metadata enhancement FAILED for {context_key}: {meta_err}\n{traceback.format_exc()}")
@ -762,7 +787,7 @@ def post_process_matched_download(context_key, context, file_path, runtime):
post_process_locks.pop(context_key, None)
def post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id, runtime):
def post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id, runtime, metadata_runtime=None):
on_download_completed = getattr(runtime, "on_download_completed", None)
def _notify_download_completed(batch_id, task_id, success=True):
@ -773,7 +798,7 @@ def post_process_matched_download_with_verification(context_key, context, file_p
try:
original_task_id = context.pop('task_id', None)
original_batch_id = context.pop('batch_id', None)
post_process_matched_download(context_key, context, file_path, runtime)
post_process_matched_download(context_key, context, file_path, runtime, metadata_runtime=metadata_runtime)
if original_task_id:
context['task_id'] = original_task_id
if original_batch_id:

29
core/metadata/enrichment.py
View file

@ -3,6 +3,8 @@
from __future__ import annotations
import os
from types import SimpleNamespace
from typing import Any
from core.metadata.artwork import embed_album_art_metadata
from core.metadata.common import (
@ -19,6 +21,7 @@ from utils.logging_config import get_logger as _create_logger
__all__ = [
"build_metadata_enrichment_runtime",
"enhance_file_metadata",
"extract_source_metadata",
"embed_source_ids",
@ -28,6 +31,32 @@ __all__ = [
logger = _create_logger("metadata.enrichment")
def build_metadata_enrichment_runtime(
*,
mb_worker: Any | None = None,
deezer_worker: Any | None = None,
audiodb_worker: Any | None = None,
tidal_client: Any | None = None,
qobuz_enrichment_worker: Any | None = None,
lastfm_worker: Any | None = None,
genius_worker: Any | None = None,
spotify_enrichment_worker: Any | None = None,
itunes_enrichment_worker: Any | None = None,
) -> SimpleNamespace:
"""Build the runtime object consumed by core.metadata.enrichment/source."""
return SimpleNamespace(
mb_worker=mb_worker,
deezer_worker=deezer_worker,
audiodb_worker=audiodb_worker,
tidal_client=tidal_client,
qobuz_enrichment_worker=qobuz_enrichment_worker,
lastfm_worker=lastfm_worker,
genius_worker=genius_worker,
spotify_enrichment_worker=spotify_enrichment_worker,
itunes_enrichment_worker=itunes_enrichment_worker,
)
def enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool:
cfg = get_config_manager()
if cfg.get("metadata_enhancement.enabled", True) is False:

81
tests/imports/test_import_pipeline.py
View file

@ -126,3 +126,84 @@ def test_verification_wrapper_handles_simple_download(tmp_path, monkeypatch):
runtime_state.processed_download_ids.update(original_processed_ids)
runtime_state.post_process_locks.clear()
runtime_state.post_process_locks.update(original_post_locks)
def test_post_process_matched_download_forwards_separate_metadata_runtime(tmp_path, monkeypatch):
source_path = tmp_path / "source.flac"
source_path.write_bytes(b"audio")
target_path = tmp_path / "Album Folder" / "track.flac"
runtime = types.SimpleNamespace(
automation_engine=None,
on_download_completed=None,
web_scan_manager=None,
repair_worker=None,
)
metadata_runtime = types.SimpleNamespace(marker="metadata-runtime")
seen = {}
monkeypatch.setattr(import_pipeline, "config_manager", types.SimpleNamespace(
get=lambda key, default=None: {
"post_processing.replaygain_enabled": False,
"lossy_copy.enabled": False,
"lossy_copy.delete_original": False,
"import.replace_lower_quality": False,
"soulseek.download_path": str(tmp_path / "downloads"),
}.get(key, default)
))
monkeypatch.setattr(import_pipeline, "normalize_import_context", lambda context: context)
monkeypatch.setattr(import_pipeline, "get_import_track_info", lambda context: {"_playlist_folder_mode": True, "_playlist_name": "Playlist"})
monkeypatch.setattr(import_pipeline, "get_import_original_search", lambda context: {"title": "Track", "album": "Album"})
monkeypatch.setattr(import_pipeline, "get_import_context_artist", lambda context: {"name": "Artist"})
monkeypatch.setattr(import_pipeline, "get_import_has_clean_metadata", lambda context: True)
monkeypatch.setattr(
import_pipeline,
"build_import_album_info",
lambda context, force_album=False: {
"is_album": True,
"album_name": "Album",
"track_number": 1,
"disc_number": 1,
"clean_track_name": "Track",
"source": "spotify",
},
)
monkeypatch.setattr(import_pipeline, "resolve_album_group", lambda artist_context, album_info, original_album: album_info["album_name"])
monkeypatch.setattr(import_pipeline, "get_import_clean_title", lambda *args, **kwargs: "Track")
monkeypatch.setattr(import_pipeline, "get_audio_quality_string", lambda file_path: "")
monkeypatch.setattr(import_pipeline, "check_flac_bit_depth", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "build_final_path_for_track", lambda *args, **kwargs: (str(target_path), None))
def _capture_enhance(file_path, context, artist, album_info, runtime=None):
seen["runtime"] = runtime
return True
monkeypatch.setattr(import_pipeline, "enhance_file_metadata", _capture_enhance)
monkeypatch.setattr(import_pipeline, "safe_move_file", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "download_cover_art", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "generate_lrc_file", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "downsample_hires_flac", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "create_lossy_copy", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "cleanup_empty_directories", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "emit_track_downloaded", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "record_library_history_download", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "record_download_provenance", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "record_soulsync_library_entry", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "check_and_remove_from_wishlist", lambda *args, **kwargs: None)
monkeypatch.setattr(import_pipeline, "record_retag_download", lambda *args, **kwargs: None)
context = {
"track_info": {"_playlist_folder_mode": True, "_playlist_name": "Playlist"},
"original_search_result": {"title": "Track", "album": "Album"},
"is_album_download": False,
}
import_pipeline.post_process_matched_download(
"ctx-1",
context,
str(source_path),
runtime,
metadata_runtime=metadata_runtime,
)
assert seen["runtime"] is metadata_runtime

56
tests/metadata/test_runtime_bundle.py Normal file
View file

@ -0,0 +1,56 @@
import types
from core.imports.pipeline import build_import_pipeline_runtime
from core.metadata.enrichment import build_metadata_enrichment_runtime
def test_build_import_pipeline_runtime_exposes_expected_contract():
import_fields = {
"automation_engine": object(),
"on_download_completed": object(),
"web_scan_manager": object(),
"repair_worker": object(),
}
runtime = build_import_pipeline_runtime(**import_fields)
assert isinstance(runtime, types.SimpleNamespace)
for name, value in import_fields.items():
assert hasattr(runtime, name)
assert getattr(runtime, name) is value
for name in (
"mb_worker",
"deezer_worker",
"audiodb_worker",
"tidal_client",
"qobuz_enrichment_worker",
"lastfm_worker",
"genius_worker",
"spotify_enrichment_worker",
"itunes_enrichment_worker",
):
assert not hasattr(runtime, name)
def test_build_metadata_enrichment_runtime_exposes_expected_contract():
metadata_fields = {
"mb_worker": object(),
"deezer_worker": object(),
"audiodb_worker": object(),
"tidal_client": object(),
"qobuz_enrichment_worker": object(),
"lastfm_worker": object(),
"genius_worker": object(),
"spotify_enrichment_worker": object(),
"itunes_enrichment_worker": object(),
}
runtime = build_metadata_enrichment_runtime(**metadata_fields)
assert isinstance(runtime, types.SimpleNamespace)
for name, value in metadata_fields.items():
assert hasattr(runtime, name)
assert getattr(runtime, name) is value
for name in ("automation_engine", "on_download_completed", "web_scan_manager", "repair_worker"):
assert not hasattr(runtime, name)

68
web_server.py
View file

@ -132,7 +132,9 @@ from core.imports.staging import (
start_import_suggestions_cache,
)
from core.imports.paths import build_final_path_for_track as _build_final_path_for_track
from core.imports.pipeline import build_import_pipeline_runtime as _build_import_pipeline_runtime
from core.metadata.common import get_file_lock
from core.metadata.enrichment import build_metadata_enrichment_runtime as _build_metadata_enrichment_runtime
from core.metadata.source import (
mb_release_cache,
mb_release_cache_lock,
@ -19678,13 +19680,13 @@ import urllib.request
def _wipe_source_tags(file_path: str) -> bool:
return metadata_enrichment.wipe_source_tags(file_path)
def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool:
def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, metadata_runtime=None) -> bool:
return metadata_enrichment.enhance_file_metadata(
file_path,
context,
artist,
album_info,
runtime=runtime or _build_import_pipeline_runtime(),
runtime=metadata_runtime or _build_metadata_enrichment_runtime(),
)
@ -19778,24 +19780,7 @@ def _post_process_matched_download_with_verification(context_key, context, file_
task_id,
batch_id,
_build_import_pipeline_runtime(),
)
def _build_import_pipeline_runtime():
"""Collect live controller dependencies for the shared import pipeline."""
return types.SimpleNamespace(
automation_engine=automation_engine,
on_download_completed=_on_download_completed,
web_scan_manager=web_scan_manager,
repair_worker=repair_worker,
mb_worker=mb_worker,
deezer_worker=deezer_worker,
audiodb_worker=audiodb_worker,
tidal_client=tidal_client,
qobuz_enrichment_worker=qobuz_enrichment_worker,
lastfm_worker=lastfm_worker,
genius_worker=genius_worker,
spotify_enrichment_worker=spotify_enrichment_worker,
itunes_enrichment_worker=itunes_enrichment_worker,
_build_metadata_enrichment_runtime(),
)
@ -19903,45 +19888,12 @@ def _post_process_matched_download(context_key, context, file_path):
just move files to /Transfer without metadata enhancement.
"""
from core.imports.pipeline import post_process_matched_download
return post_process_matched_download(context_key, context, file_path, _build_import_pipeline_runtime())
def _build_import_pipeline_runtime():
"""Collect the live controller dependencies needed by core.imports.pipeline."""
return types.SimpleNamespace(
automation_engine=automation_engine,
on_download_completed=_on_download_completed,
web_scan_manager=web_scan_manager,
repair_worker=repair_worker,
mb_worker=mb_worker,
deezer_worker=deezer_worker,
audiodb_worker=audiodb_worker,
tidal_client=tidal_client,
qobuz_enrichment_worker=qobuz_enrichment_worker,
lastfm_worker=lastfm_worker,
genius_worker=genius_worker,
spotify_enrichment_worker=spotify_enrichment_worker,
itunes_enrichment_worker=itunes_enrichment_worker,
)
def _wipe_source_tags(file_path: str) -> bool:
return metadata_enrichment.wipe_source_tags(file_path)
def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, runtime=None) -> bool:
return metadata_enrichment.enhance_file_metadata(
return post_process_matched_download(
context_key,
context,
file_path,
context,
artist,
album_info,
runtime=runtime or _build_import_pipeline_runtime(),
)
def _download_cover_art(album_info: dict, target_dir: str, context: dict = None):
return metadata_enrichment.download_cover_art(
album_info,
target_dir,
context,
_build_import_pipeline_runtime(),
metadata_runtime=_build_metadata_enrichment_runtime(),
)
# Track stale transfer keys (completed in slskd but no context — e.g., from before app restart)