Rehome import helpers into core/imports
- Move import flow modules into a dedicated package - Update app and test imports to the new namespace - Group the import-focused tests under tests/imports
This commit is contained in:
parent
b9269b4f16
commit
e10df4caf2
31 changed files with 68 additions and 73 deletions
|
|
@ -5,7 +5,7 @@ Download management endpoints — list, cancel active downloads.
|
|||
from flask import request, current_app
|
||||
from .auth import require_api_key
|
||||
from .helpers import api_success, api_error
|
||||
from core.import_runtime_state import download_tasks, tasks_lock
|
||||
from core.imports.runtime_state import download_tasks, tasks_lock
|
||||
|
||||
|
||||
def _serialize_download(task_id, task):
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ def register_routes(bp):
|
|||
def system_activity():
|
||||
"""Recent activity feed."""
|
||||
try:
|
||||
from core.import_runtime_state import activity_feed
|
||||
from core.imports.runtime_state import activity_feed
|
||||
items = list(activity_feed) if activity_feed else []
|
||||
return api_success({"activities": items})
|
||||
except Exception as e:
|
||||
|
|
@ -74,7 +74,7 @@ def register_routes(bp):
|
|||
# Active download count
|
||||
download_count = 0
|
||||
try:
|
||||
from core.import_runtime_state import download_tasks, tasks_lock
|
||||
from core.imports.runtime_state import download_tasks, tasks_lock
|
||||
with tasks_lock:
|
||||
download_count = sum(
|
||||
1 for t in download_tasks.values()
|
||||
|
|
|
|||
1
core/imports/__init__.py
Normal file
1
core/imports/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
"""Import flow helpers package."""
|
||||
|
|
@ -4,12 +4,12 @@ from __future__ import annotations
|
|||
|
||||
from typing import Any, Dict, Iterable, List, Optional, Set
|
||||
|
||||
from core.import_context import normalize_import_context
|
||||
from core.import_staging import collect_staging_files
|
||||
from core.imports.context import normalize_import_context
|
||||
from core.imports.staging import collect_staging_files
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
|
||||
logger = get_logger("import_album")
|
||||
logger = get_logger("imports.album")
|
||||
|
||||
def get_client_for_source(source: str):
|
||||
from core.metadata_service import get_client_for_source as _get_client_for_source
|
||||
|
|
@ -6,11 +6,11 @@ import re
|
|||
import threading
|
||||
from typing import Any, Dict
|
||||
|
||||
from core.import_context import extract_artist_name
|
||||
from core.imports.context import extract_artist_name
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
|
||||
logger = get_logger("import_album_naming")
|
||||
logger = get_logger("imports.album_naming")
|
||||
|
||||
_album_cache_lock = threading.Lock()
|
||||
_album_editions: dict[str, str] = {}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
"""Shared file and path helpers for import processing."""
|
||||
"""File operation helpers for the import flow."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
|
@ -8,14 +8,8 @@ import shutil
|
|||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
# Backward-compatible re-exports; canonical homes are core.import_filename
|
||||
# and core.import_staging.
|
||||
from core.import_filename import extract_track_number_from_filename
|
||||
from core.import_staging import read_staging_file_metadata
|
||||
|
||||
logger = logging.getLogger("import_file_ops")
|
||||
logger = logging.getLogger("imports.file_ops")
|
||||
|
||||
|
||||
def _get_config_manager():
|
||||
|
|
@ -8,7 +8,7 @@ from datetime import datetime
|
|||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from core.import_context import (
|
||||
from core.imports.context import (
|
||||
get_import_clean_artist,
|
||||
get_import_clean_title,
|
||||
get_import_context_artist,
|
||||
|
|
@ -16,12 +16,12 @@ from core.import_context import (
|
|||
get_import_track_info,
|
||||
normalize_import_context,
|
||||
)
|
||||
from core.import_file_ops import safe_move_file
|
||||
from core.imports.file_ops import safe_move_file
|
||||
from database.music_database import MusicDatabase
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
|
||||
logger = get_logger("import_guards")
|
||||
logger = get_logger("imports.guards")
|
||||
|
||||
|
||||
def _get_config_manager():
|
||||
|
|
@ -9,10 +9,10 @@ import re
|
|||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Album grouping lives in core.import_album_naming; this module keeps the
|
||||
# Album grouping lives in core.imports.album_naming; this module keeps the
|
||||
# imported helper because the path builder still needs it.
|
||||
from core.import_album_naming import resolve_album_group
|
||||
from core.import_context import (
|
||||
from core.imports.album_naming import resolve_album_group
|
||||
from core.imports.context import (
|
||||
extract_artist_name,
|
||||
get_import_clean_title,
|
||||
get_import_context_album,
|
||||
|
|
@ -22,7 +22,7 @@ from core.import_context import (
|
|||
normalize_import_context,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("import_paths")
|
||||
logger = logging.getLogger("imports.paths")
|
||||
|
||||
|
||||
def _get_config_manager():
|
||||
|
|
@ -8,7 +8,7 @@ import threading
|
|||
import time
|
||||
|
||||
from config.settings import config_manager
|
||||
from core.import_file_ops import (
|
||||
from core.imports.file_ops import (
|
||||
cleanup_empty_directories,
|
||||
create_lossy_copy,
|
||||
downsample_hires_flac,
|
||||
|
|
@ -16,7 +16,7 @@ from core.import_file_ops import (
|
|||
get_quality_tier_from_extension,
|
||||
safe_move_file,
|
||||
)
|
||||
from core.import_context import (
|
||||
from core.imports.context import (
|
||||
build_import_album_info,
|
||||
extract_artist_name,
|
||||
get_import_clean_artist,
|
||||
|
|
@ -28,9 +28,9 @@ from core.import_context import (
|
|||
get_import_track_info,
|
||||
normalize_import_context,
|
||||
)
|
||||
from core.import_filename import extract_track_number_from_filename
|
||||
from core.import_guards import check_flac_bit_depth, move_to_quarantine
|
||||
from core.import_side_effects import (
|
||||
from core.imports.filename import extract_track_number_from_filename
|
||||
from core.imports.guards import check_flac_bit_depth, move_to_quarantine
|
||||
from core.imports.side_effects import (
|
||||
check_and_remove_from_wishlist,
|
||||
emit_track_downloaded,
|
||||
record_download_provenance,
|
||||
|
|
@ -38,7 +38,7 @@ from core.import_side_effects import (
|
|||
record_retag_download,
|
||||
record_soulsync_library_entry,
|
||||
)
|
||||
from core.import_runtime_state import (
|
||||
from core.imports.runtime_state import (
|
||||
add_activity_item,
|
||||
detect_album_info_web,
|
||||
download_batches,
|
||||
|
|
@ -54,18 +54,18 @@ from core.import_runtime_state import (
|
|||
from core.metadata_artwork import download_cover_art
|
||||
from core.metadata_common import wipe_source_tags
|
||||
from core.metadata_enrichment import enhance_file_metadata
|
||||
from core.import_paths import (
|
||||
from core.imports.paths import (
|
||||
build_final_path_for_track,
|
||||
build_simple_download_destination,
|
||||
docker_resolve_path,
|
||||
)
|
||||
from core.import_album_naming import resolve_album_group
|
||||
from core.imports.album_naming import resolve_album_group
|
||||
from core.metadata_lyrics import generate_lrc_file
|
||||
from database.music_database import get_database
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
|
||||
logger = get_logger("import_pipeline")
|
||||
logger = get_logger("imports.pipeline")
|
||||
pp_logger = get_logger("post_processing")
|
||||
|
||||
|
||||
|
|
@ -7,7 +7,7 @@ from typing import Any, Dict, List, Optional
|
|||
from utils.logging_config import get_logger
|
||||
|
||||
|
||||
logger = get_logger("import_resolution")
|
||||
logger = get_logger("imports.resolution")
|
||||
|
||||
|
||||
def _get_metadata_service():
|
||||
|
|
@ -6,7 +6,7 @@ import threading
|
|||
import time
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from core.import_context import (
|
||||
from core.imports.context import (
|
||||
build_import_album_info,
|
||||
extract_artist_name,
|
||||
get_import_clean_artist,
|
||||
|
|
@ -7,7 +7,7 @@ import json
|
|||
import os
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from core.import_context import (
|
||||
from core.imports.context import (
|
||||
extract_artist_name,
|
||||
get_import_clean_album,
|
||||
get_import_clean_artist,
|
||||
|
|
@ -26,7 +26,7 @@ from database.music_database import get_database
|
|||
from utils.logging_config import get_logger
|
||||
|
||||
|
||||
logger = get_logger("import_side_effects")
|
||||
logger = get_logger("imports.side_effects")
|
||||
|
||||
|
||||
def _get_config_manager():
|
||||
|
|
@ -6,11 +6,11 @@ import os
|
|||
import threading
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
from core.import_paths import docker_resolve_path
|
||||
from core.import_filename import extract_track_number_from_filename
|
||||
from core.imports.paths import docker_resolve_path
|
||||
from core.imports.filename import extract_track_number_from_filename
|
||||
from utils.logging_config import get_logger
|
||||
|
||||
logger = get_logger("import_staging")
|
||||
logger = get_logger("imports.staging")
|
||||
|
||||
AUDIO_EXTENSIONS = {".mp3", ".flac", ".ogg", ".opus", ".m4a", ".aac", ".wav", ".wma", ".aiff", ".aif", ".ape"}
|
||||
|
||||
|
|
@ -6,7 +6,7 @@ import os
|
|||
import re
|
||||
import urllib.request
|
||||
|
||||
from core.import_context import get_import_context_album
|
||||
from core.imports.context import get_import_context_album
|
||||
from core.metadata_common import (
|
||||
get_config_manager,
|
||||
get_image_dimensions,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from core.import_context import (
|
||||
from core.imports.context import (
|
||||
get_import_clean_album,
|
||||
get_import_clean_title,
|
||||
get_import_context_album,
|
||||
|
|
|
|||
|
|
@ -849,7 +849,7 @@ def get_single_track_import_context(
|
|||
source_override: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Compatibility wrapper for the single-track import resolver."""
|
||||
from core.import_resolution import get_single_track_import_context as _get_single_track_import_context
|
||||
from core.imports.resolution import get_single_track_import_context as _get_single_track_import_context
|
||||
|
||||
return _get_single_track_import_context(
|
||||
title,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import threading
|
|||
import time
|
||||
from typing import Any, Dict
|
||||
|
||||
from core.import_context import (
|
||||
from core.imports.context import (
|
||||
extract_artist_name,
|
||||
get_import_clean_artist,
|
||||
get_import_clean_title,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import time
|
|||
from pathlib import Path
|
||||
from utils.logging_config import get_logger
|
||||
from config.settings import config_manager
|
||||
from core.import_filename import parse_filename_metadata
|
||||
from core.imports.filename import parse_filename_metadata
|
||||
|
||||
logger = get_logger("soulseek_client")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import core.import_album as import_album
|
||||
import core.imports.album as import_album
|
||||
|
||||
|
||||
class _FakeEngine:
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import pytest
|
||||
|
||||
from core.import_context import (
|
||||
from core.imports.context import (
|
||||
build_import_album_info,
|
||||
get_import_clean_album,
|
||||
get_import_clean_artist,
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
import sys
|
||||
import types
|
||||
|
||||
from core.import_file_ops import (
|
||||
from core.imports.file_ops import (
|
||||
cleanup_empty_directories,
|
||||
safe_move_file,
|
||||
)
|
||||
from core.import_filename import extract_track_number_from_filename
|
||||
from core.import_staging import read_staging_file_metadata
|
||||
from core.imports.filename import extract_track_number_from_filename
|
||||
from core.imports.staging import read_staging_file_metadata
|
||||
|
||||
|
||||
def test_extract_track_number_from_filename_handles_common_patterns():
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import pytest
|
||||
|
||||
from core.import_filename import parse_filename_metadata
|
||||
from core.imports.filename import parse_filename_metadata
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
from types import SimpleNamespace
|
||||
|
||||
from core import import_guards as guards
|
||||
from core.imports import guards
|
||||
|
||||
|
||||
class _FakeDB:
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import core.import_album_naming as album_naming
|
||||
import core.import_paths as import_paths
|
||||
import core.imports.album_naming as album_naming
|
||||
import core.imports.paths as import_paths
|
||||
|
||||
|
||||
class _Config:
|
||||
|
|
@ -2,9 +2,9 @@ import logging
|
|||
import sys
|
||||
import types
|
||||
|
||||
import core.import_pipeline as import_pipeline
|
||||
import core.import_paths as import_paths
|
||||
import core.import_runtime_state as runtime_state
|
||||
import core.imports.pipeline as import_pipeline
|
||||
import core.imports.paths as import_paths
|
||||
import core.imports.runtime_state as runtime_state
|
||||
|
||||
|
||||
class _Config:
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
import sqlite3
|
||||
from types import SimpleNamespace
|
||||
|
||||
from core import import_side_effects as side_effects
|
||||
from core.imports import side_effects
|
||||
|
||||
|
||||
class _FakeDB:
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
from types import SimpleNamespace
|
||||
|
||||
import core.import_staging as import_staging
|
||||
import core.imports.staging as import_staging
|
||||
|
||||
|
||||
class FakeClient:
|
||||
|
|
@ -44,7 +44,7 @@ _install_flask_limiter_stub()
|
|||
from flask import Flask, Blueprint # noqa: E402
|
||||
|
||||
from api import downloads as downloads_mod # noqa: E402
|
||||
import core.import_runtime_state as runtime_state # noqa: E402
|
||||
import core.imports.runtime_state as runtime_state # noqa: E402
|
||||
|
||||
|
||||
def _make_task(status="downloading", when=None):
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ from core.database_update_worker import DatabaseUpdateWorker
|
|||
from core.web_scan_manager import WebScanManager
|
||||
from core.lyrics_client import lyrics_client
|
||||
from core.metadata_cache import get_metadata_cache
|
||||
from core.import_context import (
|
||||
from core.imports.context import (
|
||||
build_import_album_info,
|
||||
get_import_clean_album,
|
||||
get_import_clean_artist,
|
||||
|
|
@ -114,14 +114,14 @@ from core.import_context import (
|
|||
get_source_tag_names,
|
||||
normalize_import_context,
|
||||
)
|
||||
from core.import_album import (
|
||||
from core.imports.album import (
|
||||
build_album_import_context,
|
||||
build_album_import_match_payload,
|
||||
resolve_album_artist_context,
|
||||
)
|
||||
from core.import_album_naming import resolve_album_group as _resolve_album_group
|
||||
from core.import_filename import extract_track_number_from_filename, parse_filename_metadata
|
||||
from core.import_staging import (
|
||||
from core.imports.album_naming import resolve_album_group as _resolve_album_group
|
||||
from core.imports.filename import extract_track_number_from_filename, parse_filename_metadata
|
||||
from core.imports.staging import (
|
||||
get_import_suggestions_cache,
|
||||
get_primary_source,
|
||||
get_staging_path,
|
||||
|
|
@ -131,9 +131,9 @@ from core.import_staging import (
|
|||
search_import_tracks,
|
||||
start_import_suggestions_cache,
|
||||
)
|
||||
from core.import_paths import build_final_path_for_track as _build_final_path_for_track
|
||||
from core.imports.paths import build_final_path_for_track as _build_final_path_for_track
|
||||
from core.metadata_common import get_file_lock
|
||||
from core.import_runtime_state import (
|
||||
from core.imports.runtime_state import (
|
||||
activity_feed,
|
||||
activity_feed_lock,
|
||||
add_activity_item,
|
||||
|
|
@ -747,7 +747,7 @@ retag_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="RetagWork
|
|||
|
||||
# Download Missing Tracks Modal State Management
|
||||
# Thread-safe state tracking for modal download functionality.
|
||||
# Shared task/batch state now lives in core.import_runtime_state.
|
||||
# Shared task/batch state now lives in core.imports.runtime_state.
|
||||
missing_download_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="MissingTrackWorker")
|
||||
|
||||
# Automatic Wishlist / Watchlist Processing Flags
|
||||
|
|
@ -845,7 +845,7 @@ def _regenerate_batch_m3u(batch, tracks):
|
|||
except ImportError:
|
||||
_unidecode = lambda x: x
|
||||
|
||||
from core.import_paths import sanitize_filename as _sanitize_filename
|
||||
from core.imports.paths import sanitize_filename as _sanitize_filename
|
||||
|
||||
def _norm(text):
|
||||
return _unidecode(text).lower().strip() if text else ''
|
||||
|
|
@ -965,7 +965,7 @@ def _sanitize_filename(filename: str) -> str:
|
|||
|
||||
def _compute_m3u_folder(transfer_dir, context_type, playlist_name, artist_name='', album_name='', year=''):
|
||||
"""Compute the target folder for an M3U file using the template system."""
|
||||
from core.import_paths import get_file_path_from_template
|
||||
from core.imports.paths import get_file_path_from_template
|
||||
|
||||
if context_type == 'album' and artist_name and album_name:
|
||||
template_context = {
|
||||
|
|
@ -3967,7 +3967,7 @@ def _update_automation_progress(automation_id, **kwargs):
|
|||
pass
|
||||
|
||||
# --- Global Matched Downloads Context Management ---
|
||||
# Shared with core.import_runtime_state so the refactored pipeline and web
|
||||
# Shared with core.imports.runtime_state so the refactored pipeline and web
|
||||
# server operate on the same context registry.
|
||||
_orphaned_download_keys = set() # Context keys of downloads abandoned during retry
|
||||
|
||||
|
|
@ -19773,7 +19773,7 @@ def _post_process_matched_download_with_verification(context_key, context, file_
|
|||
NEW VERIFICATION WORKFLOW: Enhanced post-processing with file verification.
|
||||
Only sets task status to 'completed' after successful file verification and move operation.
|
||||
"""
|
||||
from core.import_pipeline import post_process_matched_download_with_verification
|
||||
from core.imports.pipeline import post_process_matched_download_with_verification
|
||||
return post_process_matched_download_with_verification(
|
||||
context_key,
|
||||
context,
|
||||
|
|
@ -19896,11 +19896,11 @@ def _post_process_matched_download(context_key, context, file_path):
|
|||
Also handles simple downloads (from search page "Download" button) which
|
||||
just move files to /Transfer without metadata enhancement.
|
||||
"""
|
||||
from core.import_pipeline import post_process_matched_download
|
||||
from core.imports.pipeline import post_process_matched_download
|
||||
return post_process_matched_download(context_key, context, file_path, _build_import_pipeline_runtime())
|
||||
|
||||
def _build_import_pipeline_runtime():
|
||||
"""Collect the live controller dependencies needed by core.import_pipeline."""
|
||||
"""Collect the live controller dependencies needed by core.imports.pipeline."""
|
||||
return types.SimpleNamespace(
|
||||
automation_engine=automation_engine,
|
||||
on_download_completed=_on_download_completed,
|
||||
|
|
|
|||
Loading…
Reference in a new issue