import base64 import copy import glob import ipaddress import json import logging import os import re import socket import time import traceback import uuid from datetime import UTC, datetime, timedelta from functools import lru_cache, wraps from http.cookiejar import MozillaCookieJar from pathlib import Path from typing import Any from Crypto.Cipher import AES from app.library.log import get_logger LOG = get_logger() ALLOWED_SUBS_EXTENSIONS: set[str] = {".srt", ".vtt", ".ass"} "Allowed subtitle file extensions." FILES_TYPE: list = [ {"rx": re.compile(r"\.(avi|ts|mkv|mp4|mp3|mpv|ogm|m4v|webm|m4b)$", re.IGNORECASE), "type": "video"}, {"rx": re.compile(r"\.(mp3|flac|aac|opus|wav|m4a)$", re.IGNORECASE), "type": "audio"}, {"rx": re.compile(r"\.(srt|ass|ssa|smi|sub|idx)$", re.IGNORECASE), "type": "subtitle"}, {"rx": re.compile(r"\.(jpg|jpeg|png|gif|bmp|webp)$", re.IGNORECASE), "type": "image"}, {"rx": re.compile(r"\.(txt|nfo|md|json|yml|yaml|plexmatch)$", re.IGNORECASE), "type": "text"}, {"rx": re.compile(r"\.(nfo|json|jpg|torrent|\.info\.json)$", re.IGNORECASE), "type": "metadata"}, ] "File type patterns for sidecar files." TAG_REGEX: re.Pattern[str] = re.compile(r"%{([^:}]+)(?::([^}]*))?}c") "Regex to find tags in templates." class FileLogFormatter(logging.Formatter): def formatTime(self, record, datefmt=None): # noqa: N802 _ = datefmt return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds") LOG_RECORD_ATTRS: set[str] = set(logging.makeLogRecord({}).__dict__) | {"asctime", "message"} SKIP_LOG_FIELD = object() class JsonLogFormatter(logging.Formatter): def format(self, record: logging.LogRecord) -> str: source = { "path": record.pathname, "file": record.filename, "module": record.module, "function": record.funcName, "line": record.lineno, } data: dict[str, Any] = { "id": str(uuid.uuid4()), "datetime": self.formatTime(record), "level": record.levelname.lower(), "levelno": record.levelno, "logger": record.name, "message": record.getMessage(), "source": source, "process": {"id": record.process, "name": record.processName}, "thread": {"id": record.thread, "name": record.threadName}, "fields": self._extra(record), } if record.exc_info: exception = self._exception(record.exc_info) if exception is not None: data["exception"] = exception data["source"].update(self._exception_source(exception)) return json.dumps(data, ensure_ascii=False, default=str) def formatTime(self, record, datefmt=None): # noqa: N802 _ = datefmt return datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds") @staticmethod def _extra(record: logging.LogRecord) -> dict[str, Any]: extra: dict[str, Any] = {} for key, value in record.__dict__.items(): if key in LOG_RECORD_ATTRS or key.startswith("_"): continue value = JsonLogFormatter._field(value) if value is not SKIP_LOG_FIELD: extra[key] = value return extra @staticmethod def _field(value: Any) -> Any: if isinstance(value, str | int | float | bool) or value is None: return value if isinstance(value, Path): return str(value) if isinstance(value, dict): data: dict[str, Any] = {} for key, item in value.items(): if not isinstance(key, str) or key.startswith("_"): continue item = JsonLogFormatter._field(item) if item is not SKIP_LOG_FIELD: data[key] = item return data if isinstance(value, list): data: list[Any] = [] for item in value: item = JsonLogFormatter._field(item) if item is not SKIP_LOG_FIELD: data.append(item) return data return SKIP_LOG_FIELD @staticmethod def _exception( exc_info: tuple[type[BaseException], BaseException, Any] | tuple[None, None, None], ) -> dict[str, Any] | None: exc = exc_info[1] if exc is None: return None stack = JsonLogFormatter._exception_stack(exc_info) data: dict[str, Any] = {"type": JsonLogFormatter._exception_type(exc)} message = str(exc).strip() if message: data["message"] = message if stack: origin = stack[-1] data["file"] = origin["path"] data["line"] = origin["line"] data["stack"] = stack return data @staticmethod def _exception_type(exc: BaseException) -> str: module = exc.__class__.__module__ name = exc.__class__.__qualname__ return name if module == "builtins" else f"{module}.{name}" @staticmethod def _exception_stack( exc_info: tuple[type[BaseException], BaseException, Any] | tuple[None, None, None], ) -> list[dict[str, Any]]: exc = exc_info[1] tb = exc_info[2] if len(exc_info) > 2 else None if tb is None and exc is not None: tb = exc.__traceback__ if tb is None: return [] return [JsonLogFormatter._frame(frame) for frame in traceback.extract_tb(tb)] @staticmethod def _frame(frame: traceback.FrameSummary) -> dict[str, Any]: path = frame.filename file_path = Path(path) return { "path": path, "file": file_path.name, "module": file_path.stem, "function": frame.name, "line": frame.lineno, } @staticmethod def _exception_source(exception: dict[str, Any]) -> dict[str, Any]: source: dict[str, Any] = {} path = exception.get("file") if isinstance(path, str) and path: file_path = Path(path) source["path"] = path source["file"] = file_path.name source["module"] = file_path.stem line = exception.get("line") if isinstance(line, int): source["line"] = line stack = exception.get("stack") if isinstance(stack, list) and stack: frame = stack[-1] if isinstance(frame, dict): function = frame.get("function") if isinstance(function, str) and function: source["function"] = function module = frame.get("module") if isinstance(module, str) and module: source["module"] = module return source def timed_lru_cache(ttl_seconds: float, max_size: int = 128): """ Decorator that applies an LRU cache with a time-to-live (TTL) to a function. Supports both synchronous and asynchronous functions. Args: ttl_seconds (int): Time-to-live in seconds. max_size (int): Maximum size of the cache. Returns: Decorated function with caching capabilities. """ import inspect def decorator(func): is_async = inspect.iscoroutinefunction(func) if is_async: # For async functions, we need to cache the actual result, not the coroutine cache = {} cache_expiry = {} @wraps(func) async def async_wrapper(*args, **kwargs): key = str(args) + str(sorted(kwargs.items())) current_time = time.monotonic() # Check if we have a cached result that hasn't expired if key in cache and key in cache_expiry and current_time < cache_expiry[key]: cached_result = cache[key] try: return copy.deepcopy(cached_result) except Exception: return cached_result # Call the function and cache the result result = await func(*args, **kwargs) try: cached_value = copy.deepcopy(result) except Exception: cached_value = result cache[key] = cached_value cache_expiry[key] = current_time + ttl_seconds # Limit cache size if len(cache) > max_size: # Remove oldest entries oldest_keys = sorted(cache_expiry.keys(), key=lambda k: cache_expiry[k])[: len(cache) - max_size] for old_key in oldest_keys: cache.pop(old_key, None) cache_expiry.pop(old_key, None) try: return copy.deepcopy(cached_value) except Exception: return cached_value # Expose cache management methods def cache_clear(): cache.clear() cache_expiry.clear() def cache_info(): # Use functools._CacheInfo for compatibility with standard lru_cache from functools import _CacheInfo return _CacheInfo(hits=0, misses=len(cache), maxsize=max_size, currsize=len(cache)) async_wrapper_cache: Any = async_wrapper async_wrapper_cache.cache_clear = cache_clear async_wrapper_cache.cache_info = cache_info return async_wrapper # For sync functions, use the original implementation @lru_cache(maxsize=max_size) def cached(*args, **kwargs): result = func(*args, **kwargs) try: return copy.deepcopy(result) except Exception: return result cached_expiry = time.monotonic() + ttl_seconds @wraps(func) def sync_wrapper(*args, **kwargs): nonlocal cached_expiry if time.monotonic() >= cached_expiry: cached.cache_clear() cached_expiry = time.monotonic() + ttl_seconds result = cached(*args, **kwargs) try: return copy.deepcopy(result) except Exception: return result # expose cache_clear, cache_info sync_wrapper_cache: Any = sync_wrapper sync_wrapper_cache.cache_clear = cached.cache_clear sync_wrapper_cache.cache_info = cached.cache_info return sync_wrapper return decorator def calc_download_path(base_path: str | Path, folder: str | None = None, create_path: bool = True) -> str: """ Calculates download path. Args: base_path (str): Base download path. folder (str): Folder to add to the base path. create_path (bool): Create the path if it does not exist. Returns: Download path with base folder factored in. """ if not isinstance(base_path, Path): base_path = Path(base_path) if not folder: return str(base_path) folder = folder.removeprefix("/") realBasePath: Path = base_path.resolve() download_path: Path = Path(realBasePath).joinpath(folder).resolve(strict=False) if not str(download_path).startswith(str(realBasePath)): msg: str = f'Folder "{folder}" must resolve inside the base download folder "{realBasePath}".' raise Exception(msg) try: download_path.relative_to(realBasePath) except ValueError as e: msg = f'Folder "{folder}" must resolve inside the base download folder "{realBasePath}".' raise Exception(msg) from e if not download_path.is_dir() and create_path: download_path.mkdir(parents=True, exist_ok=True) return str(download_path) def _is_safe_key(key: Any) -> bool: """ Check if a dictionary key is safe for merging. Blocks: - All dunder attributes (__*__) - Non-string keys (for consistency) - Keys that could be dangerous variations """ # Only allow string keys if not isinstance(key, str): return False # Block empty keys and whitespace-only keys if not key or not key.strip(): return False # Block only truly dangerous dunder patterns key_stripped: str = key.strip() # Block dunder attributes (starts AND ends with __) return not (key_stripped.startswith("__") and key_stripped.endswith("__")) def merge_dict( source: dict, destination: dict, max_depth: int = 50, max_list_size: int = 10000, _depth: int = 0, _seen: set | None = None, ) -> dict: """ Merge data from source into destination. Args: source (dict): Source data destination (dict): Destination data max_depth (int): Maximum recursion depth allowed (default: 50) max_list_size (int): Maximum list size allowed (default: 10000) _depth (int): Internal recursion depth counter _seen (set): Internal circular reference tracker Returns: dict: The merged dictionary Raises: TypeError: If source or destination are not dictionaries RecursionError: If recursion depth exceeds safety limit ValueError: If circular reference is detected """ if not isinstance(source, dict) or not isinstance(destination, dict): msg = "Both source and destination must be dictionaries." raise TypeError(msg) # Prevent deep recursion DoS if _depth > max_depth: msg: str = f"Recursion depth limit exceeded ({max_depth})" raise RecursionError(msg) # Initialize circular reference tracking if _seen is None: _seen = set() # Check for circular references source_id = id(source) dest_id = id(destination) if source_id in _seen or dest_id in _seen: msg = "Circular reference detected" raise ValueError(msg) # Track current objects current_seen: set[Any | int] = _seen | {source_id, dest_id} # Create a clean copy of destination with only safe keys destination_copy: dict = {} for k, v in destination.items(): if _is_safe_key(k): # Prevent memory DoS from large lists if isinstance(v, list) and len(v) > max_list_size: destination_copy[k] = v[:max_list_size] # Truncate large lists else: destination_copy[k] = copy.deepcopy(v) for key, value in source.items(): # Skip unsafe keys if not _is_safe_key(key): continue destination_value: Any | None = destination_copy.get(key) # Recursively merge dictionaries with safety checks if isinstance(value, dict) and isinstance(destination_value, dict): destination_copy[key] = merge_dict( value, destination_value, max_depth, max_list_size, _depth + 1, current_seen ) # Safely extend lists with size limits elif isinstance(value, list) and isinstance(destination_value, list): # Prevent memory DoS combined_size = len(value) + len(destination_value) if combined_size > max_list_size: # Truncate to stay within limits available_space: int = max_list_size - len(destination_value) truncated_value: list = value[: max(0, available_space)] destination_copy[key] = copy.deepcopy(destination_value) + copy.deepcopy(truncated_value) else: destination_copy[key] = copy.deepcopy(destination_value) + copy.deepcopy(value) # For non-dict values or when destination doesn't have the key elif isinstance(value, dict): destination_copy[key] = merge_dict(value, {}, max_depth, max_list_size, _depth + 1, current_seen) elif isinstance(value, list) and len(value) > max_list_size: # Truncate large lists destination_copy[key] = copy.deepcopy(value[:max_list_size]) else: destination_copy[key] = copy.deepcopy(value) return destination_copy def check_id(file: Path) -> bool | Path: """ Check if we are able to get an id from the file name. if so check if any video file with the same id exists. Args: file (Path): File to check. Returns: bool|str: False if no file found, else the file path. """ match: re.Match[str] | None = re.search( r"(?<=\[)(?:youtube-)?(?P[a-zA-Z0-9\-_]{11})(?=\])", file.stem, re.IGNORECASE ) if not match: return False id: str | None = match.groupdict().get("id") if id is None: return False try: if not file.parent.exists(): return False for f in file.parent.iterdir(): if id not in f.stem: continue if f.suffix != file.suffix: continue return f.absolute() except OSError as e: LOG.exception( "Failed to check for a matching file for '%s'.", file, extra={"file_path": str(file), "operation": "check_id", "exception_type": type(e).__name__}, ) return False return False @timed_lru_cache(ttl_seconds=300, max_size=256) def is_private_address(hostname: str) -> bool: ip: str = socket.gethostbyname(hostname) ip_obj: ipaddress.IPv4Address | ipaddress.IPv6Address = ipaddress.ip_address(ip) return ip_obj.is_private or ip_obj.is_loopback or ip_obj.is_reserved or ip_obj.is_link_local def validate_url(url: str, allow_internal: bool = False) -> bool: """ Validate if the url is valid and allowed. Args: url (str): URL to validate. allow_internal (bool): If True, allow internal URLs or private networks. Returns: bool: True if the URL is valid and allowed. Raises: ValueError: If the URL is invalid or not allowed. """ if not url: msg = "URL is required." raise ValueError(msg) try: from yarl import URL parsed_url = URL(url) except ValueError as exc: msg = "Invalid URL." raise ValueError(msg) from exc # Check allowed schemes if parsed_url.scheme not in ["http", "https"]: msg = "Invalid scheme usage. Only HTTP or HTTPS allowed." raise ValueError(msg) hostname: str | None = parsed_url.host if not hostname: msg = "Invalid hostname." raise ValueError(msg) if allow_internal is False: try: if is_private_address(hostname): msg = "Access to internal urls or private networks is not allowed." raise ValueError(msg) except socket.gaierror as e: LOG.exception( "Failed to resolve hostname '%s'.", hostname, extra={"hostname": hostname, "exception_type": type(e).__name__}, ) msg = "Invalid hostname." raise ValueError(msg) from e return True def validate_uuid(uuid_str: str, version: int = 4) -> bool: """ Validate if the UUID is valid. Args: uuid_str (str): UUID to validate. version (int): The UUID version Returns: bool: True if the UUID is valid. """ try: uuid.UUID(uuid_str, version=version) return True except ValueError: return False @timed_lru_cache(ttl_seconds=60, max_size=256) def get_file_sidecar(file: Path | None = None) -> dict[str, list[dict[str, Any]]]: """ Get sidecar files for the given file. Args: file (Path|None): The video file. Returns: dict: A dictionary with sidecar files categorized by type. """ files: dict = {} if not file: return files for i, f in enumerate(file.parent.glob(f"{glob.escape(file.stem)}.*")): if f == file or f.is_file() is False or f.stem.startswith("."): continue if f.stat().st_size < 1: continue content_type = "Unknown" for pattern in FILES_TYPE: if pattern["rx"].search(f.name): content_type: str = pattern["type"] break if content_type == "subtitle": if f.suffix not in ALLOWED_SUBS_EXTENSIONS: continue lg: re.Match[str] | None = re.search(r"\.(?P\w{2,3})\.\w{3}$", f.name) lang: str | None = lg.groupdict().get("lang") if lg else "und" content: dict[str, Any] = {"file": f, "lang": lang, "name": f"{f.suffix[1:].upper()} ({i}) - {lang}"} else: content = {"file": f} if content_type not in files: files[content_type] = [] files[content_type].append(content) images: list[dict] = get_possible_images(str(file.parent)) if len(images) > 0: if "image" not in files: files["image"] = [] files["image"].extend(images) return files def rename_file(old_path: Path, new_name: str) -> tuple[Path, list[tuple[Path, Path]]]: """ Rename a file along with all its sidecar files. Args: old_path (Path): The original file path. new_name (str): The new name for the file (not full path, just the name). Returns: tuple[Path, list[tuple[Path, Path]]]: Tuple of (new_path, list of (old_sidecar_path, new_sidecar_path) tuples). Raises: OSError: If renaming fails. ValueError: If new_name would cause a collision with existing files. """ new_path: Path = old_path.parent / new_name if new_path.exists(): msg: str = f"Destination '{new_name}' already exists" raise ValueError(msg) sidecar_data: dict[str, dict] = get_file_sidecar(old_path) sidecar_files: list[Path] = [] for category in sidecar_data.values(): sidecar_files.extend([item["file"] for item in category if "file" in item and isinstance(item["file"], Path)]) renamed_sidecars: list[tuple[Path, Path]] = [] new_stem: str = new_path.stem rename_operations: list[tuple[Path, Path]] = [] for sidecar in sidecar_files: sidecar_suffix: str = sidecar.name[len(old_path.stem) :] # Everything after the original stem new_sidecar_path: Path = old_path.parent / f"{new_stem}{sidecar_suffix}" if new_sidecar_path.exists(): msg = f"Sidecar destination '{new_sidecar_path.name}' already exists" raise ValueError(msg) rename_operations.append((sidecar, new_sidecar_path)) try: renamed_main: Path = old_path.rename(new_path) for old_sidecar, new_sidecar in rename_operations: try: renamed_sidecar: Path = old_sidecar.rename(new_sidecar) renamed_sidecars.append((old_sidecar, renamed_sidecar)) except OSError as e: LOG.exception( "Failed to rename sidecar '%s' to '%s'.", old_sidecar, new_sidecar, extra={ "old_path": str(old_sidecar), "new_path": str(new_sidecar), "operation": "rename_sidecar", "exception_type": type(e).__name__, }, ) try: renamed_main.rename(old_path) except OSError as rollback_error: LOG.exception( "Failed to roll back main file rename from '%s' to '%s'.", renamed_main, old_path, extra={ "old_path": str(renamed_main), "new_path": str(old_path), "operation": "rename_rollback", "exception_type": type(rollback_error).__name__, }, ) raise return renamed_main, renamed_sidecars except OSError as e: LOG.exception( "Failed to rename '%s' to '%s'.", old_path, new_path, extra={ "old_path": str(old_path), "new_path": str(new_path), "operation": "rename", "exception_type": type(e).__name__, }, ) raise def move_file(old_path: Path, target_dir: Path) -> tuple[Path, list[tuple[Path, Path]]]: """ Move a file along with all its sidecar files to a target directory. Args: old_path (Path): The original file path. target_dir (Path): The target directory to move the file to. Returns: tuple[Path, list[tuple[Path, Path]]]: Tuple of (new_path, list of (old_sidecar_path, new_sidecar_path) tuples). Raises: OSError: If moving fails. ValueError: If target directory doesn't exist or if destination would cause collision. """ if not target_dir.exists(): msg: str = f"Target directory '{target_dir}' does not exist" raise ValueError(msg) if not target_dir.is_dir(): msg = f"Target '{target_dir}' is not a directory" raise ValueError(msg) new_path: Path = target_dir / old_path.name if new_path.exists(): msg = f"Destination '{new_path}' already exists" raise ValueError(msg) sidecar_data: dict[str, dict] = get_file_sidecar(old_path) sidecar_files: list[Path] = [] for category in sidecar_data.values(): sidecar_files.extend([item["file"] for item in category if "file" in item and isinstance(item["file"], Path)]) renamed_sidecars: list[tuple[Path, Path]] = [] move_operations: list[tuple[Path, Path]] = [] for sidecar in sidecar_files: new_sidecar_path: Path = target_dir / sidecar.name if new_sidecar_path.exists(): msg = f"Sidecar destination '{new_sidecar_path}' already exists" raise ValueError(msg) move_operations.append((sidecar, new_sidecar_path)) try: moved_main: Path = old_path.rename(new_path) for old_sidecar, new_sidecar in move_operations: try: moved_sidecar: Path = old_sidecar.rename(new_sidecar) renamed_sidecars.append((old_sidecar, moved_sidecar)) except OSError as e: LOG.exception( "Failed to move sidecar '%s' to '%s'.", old_sidecar, new_sidecar, extra={ "old_path": str(old_sidecar), "new_path": str(new_sidecar), "operation": "move_sidecar", "exception_type": type(e).__name__, }, ) try: moved_main.rename(old_path) # Rollback any sidecars that were already moved for rolled_back_old, rolled_back_new in renamed_sidecars: try: rolled_back_new.rename(rolled_back_old) except OSError as rollback_error: LOG.exception( "Failed to roll back sidecar move from '%s' to '%s'.", rolled_back_new, rolled_back_old, extra={ "old_path": str(rolled_back_new), "new_path": str(rolled_back_old), "operation": "move_sidecar_rollback", "exception_type": type(rollback_error).__name__, }, ) except OSError as rollback_error: LOG.exception( "Failed to roll back main file move from '%s' to '%s'.", moved_main, old_path, extra={ "old_path": str(moved_main), "new_path": str(old_path), "operation": "move_rollback", "exception_type": type(rollback_error).__name__, }, ) raise return moved_main, renamed_sidecars except OSError as e: LOG.exception( "Failed to move '%s' to '%s'.", old_path, new_path, extra={ "old_path": str(old_path), "new_path": str(new_path), "operation": "move", "exception_type": type(e).__name__, }, ) raise def get_possible_images(dir: str) -> list[dict]: images: list = [] path_loc = Path(dir, "test.jpg") for filename in ["poster", "thumbnail", "artwork", "cover", "fanart"]: for ext in [".jpg", ".jpeg", ".png", ".webp"]: f = path_loc.with_stem(filename).with_suffix(ext) if f.exists(): images.append({"file": f}) return images def get_mime_type(metadata: dict, file_path: Path) -> str: """ Determine the correct MIME type for a video file based on ffprobe metadata. Args: metadata (dict): Parsed JSON output from ffprobe. file_path (str): The path to the video file for fallback detection. Returns: str: MIME type compatible with HTML5