From e10df4caf28c819a7a93be5b936d6174e94b6103 Mon Sep 17 00:00:00 2001 From: Antti Kettunen Date: Sat, 25 Apr 2026 12:36:24 +0300 Subject: [PATCH] Rehome import helpers into core/imports - Move import flow modules into a dedicated package - Update app and test imports to the new namespace - Group the import-focused tests under tests/imports --- api/downloads.py | 2 +- api/system.py | 4 +-- core/imports/__init__.py | 1 + core/{import_album.py => imports/album.py} | 6 ++-- .../album_naming.py} | 4 +-- .../{import_context.py => imports/context.py} | 0 .../file_ops.py} | 10 ++----- .../filename.py} | 0 core/{import_guards.py => imports/guards.py} | 6 ++-- core/{import_paths.py => imports/paths.py} | 8 +++--- .../pipeline.py} | 18 ++++++------ .../resolution.py} | 2 +- .../runtime_state.py} | 2 +- .../side_effects.py} | 4 +-- .../{import_staging.py => imports/staging.py} | 6 ++-- core/metadata_artwork.py | 2 +- core/metadata_lyrics.py | 2 +- core/metadata_service.py | 2 +- core/metadata_source.py | 2 +- core/soulseek_client.py | 2 +- tests/{ => imports}/test_import_album.py | 2 +- tests/{ => imports}/test_import_context.py | 2 +- tests/{ => imports}/test_import_file_ops.py | 6 ++-- tests/{ => imports}/test_import_filename.py | 2 +- tests/{ => imports}/test_import_guards.py | 2 +- tests/{ => imports}/test_import_paths.py | 4 +-- tests/{ => imports}/test_import_pipeline.py | 6 ++-- .../{ => imports}/test_import_side_effects.py | 2 +- tests/{ => imports}/test_import_staging.py | 2 +- tests/test_downloads_pagination.py | 2 +- web_server.py | 28 +++++++++---------- 31 files changed, 68 insertions(+), 73 deletions(-) create mode 100644 core/imports/__init__.py rename core/{import_album.py => imports/album.py} (99%) rename core/{import_album_naming.py => imports/album_naming.py} (98%) rename core/{import_context.py => imports/context.py} (100%) rename core/{import_file_ops.py => imports/file_ops.py} (97%) rename core/{import_filename.py => imports/filename.py} (100%) rename core/{import_guards.py => imports/guards.py} (97%) rename core/{import_paths.py => imports/paths.py} (99%) rename core/{import_pipeline.py => imports/pipeline.py} (99%) rename core/{import_resolution.py => imports/resolution.py} (99%) rename core/{import_runtime_state.py => imports/runtime_state.py} (99%) rename core/{import_side_effects.py => imports/side_effects.py} (99%) rename core/{import_staging.py => imports/staging.py} (99%) rename tests/{ => imports}/test_import_album.py (99%) rename tests/{ => imports}/test_import_context.py (99%) rename tests/{ => imports}/test_import_file_ops.py (94%) rename tests/{ => imports}/test_import_filename.py (94%) rename tests/{ => imports}/test_import_guards.py (97%) rename tests/{ => imports}/test_import_paths.py (98%) rename tests/{ => imports}/test_import_pipeline.py (97%) rename tests/{ => imports}/test_import_side_effects.py (98%) rename tests/{ => imports}/test_import_staging.py (99%) diff --git a/api/downloads.py b/api/downloads.py index fd36e91e..88fd690c 100644 --- a/api/downloads.py +++ b/api/downloads.py @@ -5,7 +5,7 @@ Download management endpoints — list, cancel active downloads. from flask import request, current_app from .auth import require_api_key from .helpers import api_success, api_error -from core.import_runtime_state import download_tasks, tasks_lock +from core.imports.runtime_state import download_tasks, tasks_lock def _serialize_download(task_id, task): diff --git a/api/system.py b/api/system.py index b6a72141..9bdf107b 100644 --- a/api/system.py +++ b/api/system.py @@ -55,7 +55,7 @@ def register_routes(bp): def system_activity(): """Recent activity feed.""" try: - from core.import_runtime_state import activity_feed + from core.imports.runtime_state import activity_feed items = list(activity_feed) if activity_feed else [] return api_success({"activities": items}) except Exception as e: @@ -74,7 +74,7 @@ def register_routes(bp): # Active download count download_count = 0 try: - from core.import_runtime_state import download_tasks, tasks_lock + from core.imports.runtime_state import download_tasks, tasks_lock with tasks_lock: download_count = sum( 1 for t in download_tasks.values() diff --git a/core/imports/__init__.py b/core/imports/__init__.py new file mode 100644 index 00000000..d5d5418f --- /dev/null +++ b/core/imports/__init__.py @@ -0,0 +1 @@ +"""Import flow helpers package.""" diff --git a/core/import_album.py b/core/imports/album.py similarity index 99% rename from core/import_album.py rename to core/imports/album.py index 1f439f1b..db6c7d01 100644 --- a/core/import_album.py +++ b/core/imports/album.py @@ -4,12 +4,12 @@ from __future__ import annotations from typing import Any, Dict, Iterable, List, Optional, Set -from core.import_context import normalize_import_context -from core.import_staging import collect_staging_files +from core.imports.context import normalize_import_context +from core.imports.staging import collect_staging_files from utils.logging_config import get_logger -logger = get_logger("import_album") +logger = get_logger("imports.album") def get_client_for_source(source: str): from core.metadata_service import get_client_for_source as _get_client_for_source diff --git a/core/import_album_naming.py b/core/imports/album_naming.py similarity index 98% rename from core/import_album_naming.py rename to core/imports/album_naming.py index 338f326d..6d3f379a 100644 --- a/core/import_album_naming.py +++ b/core/imports/album_naming.py @@ -6,11 +6,11 @@ import re import threading from typing import Any, Dict -from core.import_context import extract_artist_name +from core.imports.context import extract_artist_name from utils.logging_config import get_logger -logger = get_logger("import_album_naming") +logger = get_logger("imports.album_naming") _album_cache_lock = threading.Lock() _album_editions: dict[str, str] = {} diff --git a/core/import_context.py b/core/imports/context.py similarity index 100% rename from core/import_context.py rename to core/imports/context.py diff --git a/core/import_file_ops.py b/core/imports/file_ops.py similarity index 97% rename from core/import_file_ops.py rename to core/imports/file_ops.py index 21cf9318..73185aac 100644 --- a/core/import_file_ops.py +++ b/core/imports/file_ops.py @@ -1,4 +1,4 @@ -"""Shared file and path helpers for import processing.""" +"""File operation helpers for the import flow.""" from __future__ import annotations @@ -8,14 +8,8 @@ import shutil import subprocess import time from pathlib import Path -from typing import Any, Dict, Optional -# Backward-compatible re-exports; canonical homes are core.import_filename -# and core.import_staging. -from core.import_filename import extract_track_number_from_filename -from core.import_staging import read_staging_file_metadata - -logger = logging.getLogger("import_file_ops") +logger = logging.getLogger("imports.file_ops") def _get_config_manager(): diff --git a/core/import_filename.py b/core/imports/filename.py similarity index 100% rename from core/import_filename.py rename to core/imports/filename.py diff --git a/core/import_guards.py b/core/imports/guards.py similarity index 97% rename from core/import_guards.py rename to core/imports/guards.py index de2a3185..4d97c9fb 100644 --- a/core/import_guards.py +++ b/core/imports/guards.py @@ -8,7 +8,7 @@ from datetime import datetime from pathlib import Path from typing import Any, Dict, Optional -from core.import_context import ( +from core.imports.context import ( get_import_clean_artist, get_import_clean_title, get_import_context_artist, @@ -16,12 +16,12 @@ from core.import_context import ( get_import_track_info, normalize_import_context, ) -from core.import_file_ops import safe_move_file +from core.imports.file_ops import safe_move_file from database.music_database import MusicDatabase from utils.logging_config import get_logger -logger = get_logger("import_guards") +logger = get_logger("imports.guards") def _get_config_manager(): diff --git a/core/import_paths.py b/core/imports/paths.py similarity index 99% rename from core/import_paths.py rename to core/imports/paths.py index 39e280c8..5ac357d1 100644 --- a/core/import_paths.py +++ b/core/imports/paths.py @@ -9,10 +9,10 @@ import re from pathlib import Path from typing import Any -# Album grouping lives in core.import_album_naming; this module keeps the +# Album grouping lives in core.imports.album_naming; this module keeps the # imported helper because the path builder still needs it. -from core.import_album_naming import resolve_album_group -from core.import_context import ( +from core.imports.album_naming import resolve_album_group +from core.imports.context import ( extract_artist_name, get_import_clean_title, get_import_context_album, @@ -22,7 +22,7 @@ from core.import_context import ( normalize_import_context, ) -logger = logging.getLogger("import_paths") +logger = logging.getLogger("imports.paths") def _get_config_manager(): diff --git a/core/import_pipeline.py b/core/imports/pipeline.py similarity index 99% rename from core/import_pipeline.py rename to core/imports/pipeline.py index 62bc1f74..99eee1a5 100644 --- a/core/import_pipeline.py +++ b/core/imports/pipeline.py @@ -8,7 +8,7 @@ import threading import time from config.settings import config_manager -from core.import_file_ops import ( +from core.imports.file_ops import ( cleanup_empty_directories, create_lossy_copy, downsample_hires_flac, @@ -16,7 +16,7 @@ from core.import_file_ops import ( get_quality_tier_from_extension, safe_move_file, ) -from core.import_context import ( +from core.imports.context import ( build_import_album_info, extract_artist_name, get_import_clean_artist, @@ -28,9 +28,9 @@ from core.import_context import ( get_import_track_info, normalize_import_context, ) -from core.import_filename import extract_track_number_from_filename -from core.import_guards import check_flac_bit_depth, move_to_quarantine -from core.import_side_effects import ( +from core.imports.filename import extract_track_number_from_filename +from core.imports.guards import check_flac_bit_depth, move_to_quarantine +from core.imports.side_effects import ( check_and_remove_from_wishlist, emit_track_downloaded, record_download_provenance, @@ -38,7 +38,7 @@ from core.import_side_effects import ( record_retag_download, record_soulsync_library_entry, ) -from core.import_runtime_state import ( +from core.imports.runtime_state import ( add_activity_item, detect_album_info_web, download_batches, @@ -54,18 +54,18 @@ from core.import_runtime_state import ( from core.metadata_artwork import download_cover_art from core.metadata_common import wipe_source_tags from core.metadata_enrichment import enhance_file_metadata -from core.import_paths import ( +from core.imports.paths import ( build_final_path_for_track, build_simple_download_destination, docker_resolve_path, ) -from core.import_album_naming import resolve_album_group +from core.imports.album_naming import resolve_album_group from core.metadata_lyrics import generate_lrc_file from database.music_database import get_database from utils.logging_config import get_logger -logger = get_logger("import_pipeline") +logger = get_logger("imports.pipeline") pp_logger = get_logger("post_processing") diff --git a/core/import_resolution.py b/core/imports/resolution.py similarity index 99% rename from core/import_resolution.py rename to core/imports/resolution.py index 8f9c712c..40e4149c 100644 --- a/core/import_resolution.py +++ b/core/imports/resolution.py @@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional from utils.logging_config import get_logger -logger = get_logger("import_resolution") +logger = get_logger("imports.resolution") def _get_metadata_service(): diff --git a/core/import_runtime_state.py b/core/imports/runtime_state.py similarity index 99% rename from core/import_runtime_state.py rename to core/imports/runtime_state.py index 9abf2baa..4e24d06f 100644 --- a/core/import_runtime_state.py +++ b/core/imports/runtime_state.py @@ -6,7 +6,7 @@ import threading import time from typing import Any, Dict, Optional -from core.import_context import ( +from core.imports.context import ( build_import_album_info, extract_artist_name, get_import_clean_artist, diff --git a/core/import_side_effects.py b/core/imports/side_effects.py similarity index 99% rename from core/import_side_effects.py rename to core/imports/side_effects.py index 6c456e3d..dbc5bbf7 100644 --- a/core/import_side_effects.py +++ b/core/imports/side_effects.py @@ -7,7 +7,7 @@ import json import os from typing import Any, Dict, List, Optional -from core.import_context import ( +from core.imports.context import ( extract_artist_name, get_import_clean_album, get_import_clean_artist, @@ -26,7 +26,7 @@ from database.music_database import get_database from utils.logging_config import get_logger -logger = get_logger("import_side_effects") +logger = get_logger("imports.side_effects") def _get_config_manager(): diff --git a/core/import_staging.py b/core/imports/staging.py similarity index 99% rename from core/import_staging.py rename to core/imports/staging.py index 8f8fc6f0..44edb0bf 100644 --- a/core/import_staging.py +++ b/core/imports/staging.py @@ -6,11 +6,11 @@ import os import threading from typing import Any, Dict, Iterable, List, Optional, Tuple -from core.import_paths import docker_resolve_path -from core.import_filename import extract_track_number_from_filename +from core.imports.paths import docker_resolve_path +from core.imports.filename import extract_track_number_from_filename from utils.logging_config import get_logger -logger = get_logger("import_staging") +logger = get_logger("imports.staging") AUDIO_EXTENSIONS = {".mp3", ".flac", ".ogg", ".opus", ".m4a", ".aac", ".wav", ".wma", ".aiff", ".aif", ".ape"} diff --git a/core/metadata_artwork.py b/core/metadata_artwork.py index b84f5274..8d43b0af 100644 --- a/core/metadata_artwork.py +++ b/core/metadata_artwork.py @@ -6,7 +6,7 @@ import os import re import urllib.request -from core.import_context import get_import_context_album +from core.imports.context import get_import_context_album from core.metadata_common import ( get_config_manager, get_image_dimensions, diff --git a/core/metadata_lyrics.py b/core/metadata_lyrics.py index ead86fc9..00719c5c 100644 --- a/core/metadata_lyrics.py +++ b/core/metadata_lyrics.py @@ -2,7 +2,7 @@ from __future__ import annotations -from core.import_context import ( +from core.imports.context import ( get_import_clean_album, get_import_clean_title, get_import_context_album, diff --git a/core/metadata_service.py b/core/metadata_service.py index 68cf0b92..66487d60 100644 --- a/core/metadata_service.py +++ b/core/metadata_service.py @@ -849,7 +849,7 @@ def get_single_track_import_context( source_override: Optional[str] = None, ) -> Dict[str, Any]: """Compatibility wrapper for the single-track import resolver.""" - from core.import_resolution import get_single_track_import_context as _get_single_track_import_context + from core.imports.resolution import get_single_track_import_context as _get_single_track_import_context return _get_single_track_import_context( title, diff --git a/core/metadata_source.py b/core/metadata_source.py index a7c9d86d..7f4411e9 100644 --- a/core/metadata_source.py +++ b/core/metadata_source.py @@ -7,7 +7,7 @@ import threading import time from typing import Any, Dict -from core.import_context import ( +from core.imports.context import ( extract_artist_name, get_import_clean_artist, get_import_clean_title, diff --git a/core/soulseek_client.py b/core/soulseek_client.py index e859fd4e..3110c305 100644 --- a/core/soulseek_client.py +++ b/core/soulseek_client.py @@ -8,7 +8,7 @@ import time from pathlib import Path from utils.logging_config import get_logger from config.settings import config_manager -from core.import_filename import parse_filename_metadata +from core.imports.filename import parse_filename_metadata logger = get_logger("soulseek_client") diff --git a/tests/test_import_album.py b/tests/imports/test_import_album.py similarity index 99% rename from tests/test_import_album.py rename to tests/imports/test_import_album.py index 45706da3..5b11ca8e 100644 --- a/tests/test_import_album.py +++ b/tests/imports/test_import_album.py @@ -1,4 +1,4 @@ -import core.import_album as import_album +import core.imports.album as import_album class _FakeEngine: diff --git a/tests/test_import_context.py b/tests/imports/test_import_context.py similarity index 99% rename from tests/test_import_context.py rename to tests/imports/test_import_context.py index f91acf19..7e2f242a 100644 --- a/tests/test_import_context.py +++ b/tests/imports/test_import_context.py @@ -1,6 +1,6 @@ import pytest -from core.import_context import ( +from core.imports.context import ( build_import_album_info, get_import_clean_album, get_import_clean_artist, diff --git a/tests/test_import_file_ops.py b/tests/imports/test_import_file_ops.py similarity index 94% rename from tests/test_import_file_ops.py rename to tests/imports/test_import_file_ops.py index 056c95bf..a07c5c4a 100644 --- a/tests/test_import_file_ops.py +++ b/tests/imports/test_import_file_ops.py @@ -1,12 +1,12 @@ import sys import types -from core.import_file_ops import ( +from core.imports.file_ops import ( cleanup_empty_directories, safe_move_file, ) -from core.import_filename import extract_track_number_from_filename -from core.import_staging import read_staging_file_metadata +from core.imports.filename import extract_track_number_from_filename +from core.imports.staging import read_staging_file_metadata def test_extract_track_number_from_filename_handles_common_patterns(): diff --git a/tests/test_import_filename.py b/tests/imports/test_import_filename.py similarity index 94% rename from tests/test_import_filename.py rename to tests/imports/test_import_filename.py index a5b32ff2..59c29dcf 100644 --- a/tests/test_import_filename.py +++ b/tests/imports/test_import_filename.py @@ -1,6 +1,6 @@ import pytest -from core.import_filename import parse_filename_metadata +from core.imports.filename import parse_filename_metadata @pytest.mark.parametrize( diff --git a/tests/test_import_guards.py b/tests/imports/test_import_guards.py similarity index 97% rename from tests/test_import_guards.py rename to tests/imports/test_import_guards.py index a7b95372..ee71a1dd 100644 --- a/tests/test_import_guards.py +++ b/tests/imports/test_import_guards.py @@ -1,6 +1,6 @@ from types import SimpleNamespace -from core import import_guards as guards +from core.imports import guards class _FakeDB: diff --git a/tests/test_import_paths.py b/tests/imports/test_import_paths.py similarity index 98% rename from tests/test_import_paths.py rename to tests/imports/test_import_paths.py index a3e766f9..656279f9 100644 --- a/tests/test_import_paths.py +++ b/tests/imports/test_import_paths.py @@ -1,5 +1,5 @@ -import core.import_album_naming as album_naming -import core.import_paths as import_paths +import core.imports.album_naming as album_naming +import core.imports.paths as import_paths class _Config: diff --git a/tests/test_import_pipeline.py b/tests/imports/test_import_pipeline.py similarity index 97% rename from tests/test_import_pipeline.py rename to tests/imports/test_import_pipeline.py index 768a8788..7c50c942 100644 --- a/tests/test_import_pipeline.py +++ b/tests/imports/test_import_pipeline.py @@ -2,9 +2,9 @@ import logging import sys import types -import core.import_pipeline as import_pipeline -import core.import_paths as import_paths -import core.import_runtime_state as runtime_state +import core.imports.pipeline as import_pipeline +import core.imports.paths as import_paths +import core.imports.runtime_state as runtime_state class _Config: diff --git a/tests/test_import_side_effects.py b/tests/imports/test_import_side_effects.py similarity index 98% rename from tests/test_import_side_effects.py rename to tests/imports/test_import_side_effects.py index 510a9ab0..697d845b 100644 --- a/tests/test_import_side_effects.py +++ b/tests/imports/test_import_side_effects.py @@ -1,7 +1,7 @@ import sqlite3 from types import SimpleNamespace -from core import import_side_effects as side_effects +from core.imports import side_effects class _FakeDB: diff --git a/tests/test_import_staging.py b/tests/imports/test_import_staging.py similarity index 99% rename from tests/test_import_staging.py rename to tests/imports/test_import_staging.py index 1cd5e15b..56571ede 100644 --- a/tests/test_import_staging.py +++ b/tests/imports/test_import_staging.py @@ -1,6 +1,6 @@ from types import SimpleNamespace -import core.import_staging as import_staging +import core.imports.staging as import_staging class FakeClient: diff --git a/tests/test_downloads_pagination.py b/tests/test_downloads_pagination.py index f85dd96d..24340b66 100644 --- a/tests/test_downloads_pagination.py +++ b/tests/test_downloads_pagination.py @@ -44,7 +44,7 @@ _install_flask_limiter_stub() from flask import Flask, Blueprint # noqa: E402 from api import downloads as downloads_mod # noqa: E402 -import core.import_runtime_state as runtime_state # noqa: E402 +import core.imports.runtime_state as runtime_state # noqa: E402 def _make_task(status="downloading", when=None): diff --git a/web_server.py b/web_server.py index e6cd49c2..6d7da994 100644 --- a/web_server.py +++ b/web_server.py @@ -97,7 +97,7 @@ from core.database_update_worker import DatabaseUpdateWorker from core.web_scan_manager import WebScanManager from core.lyrics_client import lyrics_client from core.metadata_cache import get_metadata_cache -from core.import_context import ( +from core.imports.context import ( build_import_album_info, get_import_clean_album, get_import_clean_artist, @@ -114,14 +114,14 @@ from core.import_context import ( get_source_tag_names, normalize_import_context, ) -from core.import_album import ( +from core.imports.album import ( build_album_import_context, build_album_import_match_payload, resolve_album_artist_context, ) -from core.import_album_naming import resolve_album_group as _resolve_album_group -from core.import_filename import extract_track_number_from_filename, parse_filename_metadata -from core.import_staging import ( +from core.imports.album_naming import resolve_album_group as _resolve_album_group +from core.imports.filename import extract_track_number_from_filename, parse_filename_metadata +from core.imports.staging import ( get_import_suggestions_cache, get_primary_source, get_staging_path, @@ -131,9 +131,9 @@ from core.import_staging import ( search_import_tracks, start_import_suggestions_cache, ) -from core.import_paths import build_final_path_for_track as _build_final_path_for_track +from core.imports.paths import build_final_path_for_track as _build_final_path_for_track from core.metadata_common import get_file_lock -from core.import_runtime_state import ( +from core.imports.runtime_state import ( activity_feed, activity_feed_lock, add_activity_item, @@ -747,7 +747,7 @@ retag_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="RetagWork # Download Missing Tracks Modal State Management # Thread-safe state tracking for modal download functionality. -# Shared task/batch state now lives in core.import_runtime_state. +# Shared task/batch state now lives in core.imports.runtime_state. missing_download_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="MissingTrackWorker") # Automatic Wishlist / Watchlist Processing Flags @@ -845,7 +845,7 @@ def _regenerate_batch_m3u(batch, tracks): except ImportError: _unidecode = lambda x: x - from core.import_paths import sanitize_filename as _sanitize_filename + from core.imports.paths import sanitize_filename as _sanitize_filename def _norm(text): return _unidecode(text).lower().strip() if text else '' @@ -965,7 +965,7 @@ def _sanitize_filename(filename: str) -> str: def _compute_m3u_folder(transfer_dir, context_type, playlist_name, artist_name='', album_name='', year=''): """Compute the target folder for an M3U file using the template system.""" - from core.import_paths import get_file_path_from_template + from core.imports.paths import get_file_path_from_template if context_type == 'album' and artist_name and album_name: template_context = { @@ -3967,7 +3967,7 @@ def _update_automation_progress(automation_id, **kwargs): pass # --- Global Matched Downloads Context Management --- -# Shared with core.import_runtime_state so the refactored pipeline and web +# Shared with core.imports.runtime_state so the refactored pipeline and web # server operate on the same context registry. _orphaned_download_keys = set() # Context keys of downloads abandoned during retry @@ -19773,7 +19773,7 @@ def _post_process_matched_download_with_verification(context_key, context, file_ NEW VERIFICATION WORKFLOW: Enhanced post-processing with file verification. Only sets task status to 'completed' after successful file verification and move operation. """ - from core.import_pipeline import post_process_matched_download_with_verification + from core.imports.pipeline import post_process_matched_download_with_verification return post_process_matched_download_with_verification( context_key, context, @@ -19896,11 +19896,11 @@ def _post_process_matched_download(context_key, context, file_path): Also handles simple downloads (from search page "Download" button) which just move files to /Transfer without metadata enhancement. """ - from core.import_pipeline import post_process_matched_download + from core.imports.pipeline import post_process_matched_download return post_process_matched_download(context_key, context, file_path, _build_import_pipeline_runtime()) def _build_import_pipeline_runtime(): - """Collect the live controller dependencies needed by core.import_pipeline.""" + """Collect the live controller dependencies needed by core.imports.pipeline.""" return types.SimpleNamespace( automation_engine=automation_engine, on_download_completed=_on_download_completed,