From fa8ba1af34592c54a291958916b4c68a7e38904a Mon Sep 17 00:00:00 2001 From: arabcoders Date: Thu, 28 May 2026 21:26:23 +0300 Subject: [PATCH] feat: make log level runtime managable. --- API.md | 27 ++++++ app/library/HttpAPI.py | 27 ++++++ app/library/config.py | 43 +++++---- app/library/log.py | 6 +- app/library/log_control.py | 33 +++++++ app/main.py | 5 +- app/routes/api/logs.py | 40 ++++++++ app/tests/test_images_routes.py | 7 +- ui/app/composables/useYtpConfig.ts | 2 + ui/app/pages/logs.vue | 145 +++++++++++++++++++++++------ ui/app/types/config.d.ts | 4 + 11 files changed, 290 insertions(+), 49 deletions(-) create mode 100644 app/library/log_control.py diff --git a/API.md b/API.md index 546a48ec..5c1c9642 100644 --- a/API.md +++ b/API.md @@ -88,6 +88,8 @@ This document describes the available endpoints and their usage. All endpoints r - [DELETE /api/conditions/{id}](#delete-apiconditionsid) - [GET /api/logs](#get-apilogs) - [GET /api/logs/stream](#get-apilogsstream) + - [GET /api/logs/level](#get-apilogslevel) + - [POST /api/logs/level/{level}](#post-apilogslevellevel) - [GET /api/notifications/](#get-apinotifications) - [GET /api/notifications/events/](#get-apinotificationsevents) - [POST /api/notifications/](#post-apinotifications) @@ -2421,6 +2423,31 @@ Binary image data with appropriate headers --- +### GET /api/logs/level +**Purpose**: Read the active runtime log level. + +**Response**: +```json +{ + "conf": "info", + "active": "info", + "levels": ["debug", "info", "warning", "error"] +} +``` + +--- + +### POST /api/logs/level/{level} +**Purpose**: Change the active runtime log level. + +**Path Parameter**: +- `level`: One of `debug`, `info`, `warning`, `error`. + +**Response**: +- `204 No Content` on success. + +--- + ### GET /api/notifications/ **Purpose**: Retrieve notification targets with pagination. diff --git a/app/library/HttpAPI.py b/app/library/HttpAPI.py index 503fcf70..609ed7e4 100644 --- a/app/library/HttpAPI.py +++ b/app/library/HttpAPI.py @@ -8,6 +8,7 @@ from pathlib import Path import anyio from aiohttp import web from aiohttp.web import Request, RequestHandler, Response +from aiohttp.web_log import AccessLogger from app.library.log import get_logger from app.library.Services import Services @@ -22,6 +23,32 @@ from .Utils import decrypt_data, encrypt_data, get_file, load_modules LOG = get_logger("http") +class HttpAccessLogger(AccessLogger): + def log(self, request: Request, response: Response, time: float) -> None: + try: + fmt_info = self._format_line(request, response, time) + + values: list[object] = [] + extra: dict[str, object] = {"elapsed_ms": round(time * 1000.0, 2)} + for key, value in fmt_info: + values.append(value) + + if isinstance(key, str): + extra[key] = value + else: + parent, child = key + group = extra.get(parent, {}) + if not isinstance(group, dict): + group = {} + group[child] = value + extra[parent] = group + + message = self._log_format % tuple(values) + self.logger.info(message, extra=extra) + except Exception: + self.logger.exception("Error in logging") + + class HttpAPI: def __init__(self, root_path: Path): self.encoder: Encoder = Encoder() diff --git a/app/library/config.py b/app/library/config.py index e9c38243..e1224d39 100644 --- a/app/library/config.py +++ b/app/library/config.py @@ -18,6 +18,15 @@ from .Singleton import Singleton from .Utils import JsonLogFormatter from .version import APP_BRANCH, APP_BUILD_DATE, APP_COMMIT_SHA, APP_VERSION +APP_THIRD_PARTY_LOG_LEVELS: tuple[tuple[str, int], ...] = ( + ("httpx", logging.WARNING), + ("urllib3.connectionpool", logging.WARNING), + ("apprise", logging.WARNING), + ("httpcore", logging.INFO), + ("aiosqlite", logging.INFO), + ("asyncio", logging.INFO), +) + if TYPE_CHECKING: from subprocess import CompletedProcess @@ -312,6 +321,7 @@ class Config(metaclass=Singleton): _frontend_vars: tuple = ( "download_path", "keep_archive", + "log_level", "output_template", "started", "remove_files", @@ -359,12 +369,13 @@ class Config(metaclass=Singleton): def __init__(self, is_native: bool = False): baseDefaultPath: str = str(Path(__file__).parent.parent.parent.absolute()) + LOG = get_logger() self.config_path = os.environ.get("YTP_CONFIG_PATH", None) or str(Path(baseDefaultPath) / "var" / "config") envFile: str = Path(self.config_path) / ".env" if envFile.exists(): - logging.info(f"Loading environment variables from '{envFile}'.") + LOG.info("Loading environment variables from '%s'.", envFile) load_dotenv(envFile) self.is_native = is_native @@ -394,7 +405,7 @@ class Config(metaclass=Singleton): for key in re.findall(r"\{.*?\}", v): localKey: str = key[1:-1] if localKey not in self.__dict__: - logging.error(f"Config variable '{k}' had non-existing config reference '{key}'.") + LOG.error("Config variable '%s' had non-existing config reference '%s'.", k, key) sys.exit(1) v: str = v.replace(key, str(getattr(self, localKey))) @@ -432,8 +443,6 @@ class Config(metaclass=Singleton): encoding="utf-8", ) - LOG = get_logger() - if self.debug: try: import debugpy @@ -499,15 +508,7 @@ class Config(metaclass=Singleton): self.started = time.time() - _log_levels = ( - ("httpx", logging.WARNING), - ("urllib3.connectionpool", logging.WARNING), - ("apprise", logging.WARNING), - ("httpcore", logging.INFO), - ("aiosqlite", logging.INFO), - ("asyncio", logging.INFO), - ) - for _tool, _level in _log_levels: + for _tool, _level in APP_THIRD_PARTY_LOG_LEVELS: logging.getLogger(_tool).setLevel(_level) if self.app_env not in ("production", "development"): @@ -582,6 +583,9 @@ class Config(metaclass=Singleton): data: dict[str, Any] = {k: getattr(self, k) for k in self._frontend_vars} data["ytdlp_version"] = Config._ytdlp_version() + from app.library.log_control import get_runtime_log_level + + data["runtime_log_level"] = get_runtime_log_level() return data def get_replacers(self) -> dict[str, str]: @@ -609,6 +613,7 @@ class Config(metaclass=Singleton): Updates the version of the application using git tags. This is used to set the version to the latest git tag. """ + LOG = get_logger() git_path: str = Path(__file__).parent / ".." / ".." / ".git" if not git_path.exists(): return @@ -626,12 +631,12 @@ class Config(metaclass=Singleton): ) if 0 != branch_result.returncode: - logging.error(f"Git rev-parse failed: {branch_result.stderr.strip()}") + LOG.error("Git rev-parse failed: %s", branch_result.stderr.strip()) return branch_name: str = branch_result.stdout.strip() if not branch_name: - logging.warning("Git branch name is empty.") + LOG.warning("Git branch name is empty.") return commit_result: CompletedProcess[str] = subprocess.run( @@ -644,12 +649,12 @@ class Config(metaclass=Singleton): ) if 0 != commit_result.returncode: - logging.error(f"Git log failed: {commit_result.stderr.strip()}") + LOG.error("Git log failed: %s", commit_result.stderr.strip()) return commit_info: str = commit_result.stdout.strip() if not commit_info: - logging.warning("Git commit info is empty.") + LOG.warning("Git commit info is empty.") return commit_date, commit_sha = commit_info.split("_", 1) @@ -665,6 +670,6 @@ class Config(metaclass=Singleton): "commit": self.app_commit_sha, "build_date": self.app_build_date, } - logging.info(f"Application version info set to '{version_data}'") + LOG.info("Application version info set to '%s'", version_data) except Exception as e: - logging.error(f"Error while getting git version: {e!s}") + LOG.error("Error while getting git version: %s", e) diff --git a/app/library/log.py b/app/library/log.py index e8a081c2..229985b8 100644 --- a/app/library/log.py +++ b/app/library/log.py @@ -9,5 +9,9 @@ HTTP_LOGGER_NAME = "http_api" LoggerKind = Literal["app", "http"] +def get_logger_name(kind: LoggerKind = "app") -> str: + return HTTP_LOGGER_NAME if kind == "http" else APP_LOGGER_NAME + + def get_logger(kind: LoggerKind = "app") -> logging.Logger: - return logging.getLogger(HTTP_LOGGER_NAME if kind == "http" else APP_LOGGER_NAME) + return logging.getLogger(get_logger_name(kind)) diff --git a/app/library/log_control.py b/app/library/log_control.py new file mode 100644 index 00000000..bc51d049 --- /dev/null +++ b/app/library/log_control.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import logging + +from app.library.log import get_logger, get_logger_name + +SUPPORTED_LOG_LEVELS: tuple[str, ...] = ("debug", "info", "warning", "error") + + +def normalize_log_level(level: str) -> str: + value: str = level.strip().lower() + if value not in SUPPORTED_LOG_LEVELS: + msg: str = f"Unsupported log level '{level}'." + raise ValueError(msg) + + return value + + +def get_runtime_log_level() -> str: + return logging.getLevelName(logging.getLogger(get_logger_name()).getEffectiveLevel()).lower() + + +def set_runtime_log_level(level: str) -> str: + normalized: str = normalize_log_level(level) + numeric_level: int | None = getattr(logging, normalized.upper(), None) + if not isinstance(numeric_level, int): + msg: str = f"Unsupported log level '{level}'." + raise ValueError(msg) + + for _logger in (get_logger(),): + _logger.setLevel(numeric_level) + + return normalized diff --git a/app/main.py b/app/main.py index 75b61c03..f7ec5eca 100644 --- a/app/main.py +++ b/app/main.py @@ -30,7 +30,7 @@ from app.library.cache import Cache from app.library.config import Config from app.library.downloads import DownloadQueue from app.library.Events import EventBus, Events -from app.library.HttpAPI import HttpAPI +from app.library.HttpAPI import HttpAccessLogger, HttpAPI from app.library.HttpSocket import HttpSocket from app.library.httpx_client import close_shared_clients from app.library.log import get_logger @@ -190,12 +190,14 @@ class Main: cb() HTTP_LOGGER = None + HTTP_LOGGER_CLASS = None if self._config.access_log: from app.library.HttpAPI import LOG as HTTP_LOGGER HTTP_LOGGER.addFilter( lambda record: f"GET {str(self._app.router['ping'].url_for()).rstrip('/')}" not in record.getMessage() ) + HTTP_LOGGER_CLASS = HttpAccessLogger web.run_app( self._app, @@ -203,6 +205,7 @@ class Main: port=port, loop=asyncio.get_event_loop(), access_log=HTTP_LOGGER, + access_log_class=HTTP_LOGGER_CLASS, print=started, handle_signals=cb is None, ) diff --git a/app/routes/api/logs.py b/app/routes/api/logs.py index 79078ffd..1c363e3b 100644 --- a/app/routes/api/logs.py +++ b/app/routes/api/logs.py @@ -9,6 +9,12 @@ from aiohttp.web import Request, Response from app.library.config import Config from app.library.encoder import Encoder from app.library.log import get_logger +from app.library.log_control import ( + SUPPORTED_LOG_LEVELS, + get_runtime_log_level, + normalize_log_level, + set_runtime_log_level, +) from app.library.router import route LOG = get_logger() @@ -176,6 +182,40 @@ async def logs(request: Request, config: Config, encoder: Encoder) -> Response: ) +@route("GET", "api/logs/level", "logs.level") +async def get_logs_level(config: Config, encoder: Encoder) -> Response: + configured = normalize_log_level(config.log_level) + active = get_runtime_log_level() + return web.json_response( + data={ + "conf": configured, + "active": active, + "levels": list(SUPPORTED_LOG_LEVELS), + }, + status=web.HTTPOk.status_code, + dumps=encoder.encode, + ) + + +@route("POST", "api/logs/level/{level}", "logs.level.set") +async def set_logs_level(request: Request) -> Response: + if not (level := request.match_info.get("level")): + return web.json_response( + {"error": "Log level is required."}, + status=web.HTTPBadRequest.status_code, + ) + + try: + set_runtime_log_level(level) + except ValueError as e: + return web.json_response( + {"error": f"{e!s} Available levels: {', '.join(SUPPORTED_LOG_LEVELS)}."}, + status=web.HTTPBadRequest.status_code, + ) + + return web.Response(status=web.HTTPNoContent.status_code) + + @route("GET", "api/logs/stream", "logs.stream") async def stream_logs(request: Request, config: Config, encoder: Encoder) -> Response | web.StreamResponse: if not config.file_logging: diff --git a/app/tests/test_images_routes.py b/app/tests/test_images_routes.py index 2d9bb9e4..9557a0ee 100644 --- a/app/tests/test_images_routes.py +++ b/app/tests/test_images_routes.py @@ -65,7 +65,12 @@ async def test_bg_log_redact(caplog: pytest.LogCaptureFixture, monkeypatch: pyte response = await images.get_background(req, config, DummyCache()) assert response.status == web.HTTPInternalServerError.status_code - record = next(record for record in caplog.records if record.name == images.LOG.name) + record = next( + record + for record in caplog.records + if record.name == images.LOG.name + and record.getMessage().startswith("Failed to request random background image") + ) assert "apitoken=secret" not in record.getMessage() assert "user:pass@" not in record.getMessage() assert record.url == "https://redacted:redacted@example.com/bg.jpg?redacted#redacted" diff --git a/ui/app/composables/useYtpConfig.ts b/ui/app/composables/useYtpConfig.ts index b66f71dd..522475e2 100644 --- a/ui/app/composables/useYtpConfig.ts +++ b/ui/app/composables/useYtpConfig.ts @@ -33,6 +33,8 @@ const state = reactive({ app_env: 'production', simple_mode: false, default_pagination: 50, + log_level: '', + runtime_log_level: '', check_for_updates: true, new_version: '', yt_new_version: '', diff --git a/ui/app/pages/logs.vue b/ui/app/pages/logs.vue index 1113c86a..db9b0fa4 100644 --- a/ui/app/pages/logs.vue +++ b/ui/app/pages/logs.vue @@ -80,6 +80,19 @@ + + Apply + + [{{ entry.log.logger }}]

- {{ emptyStateTitle }} + {{ hasActiveFilter ? 'No logs match these filters' : 'No log lines available' }}

- {{ emptyStateDescription }} + {{ hasActiveFilter ? 'No logs match these filters' : 'No log lines available' }}

@@ -425,6 +440,10 @@ const LOG_LEVEL_ICON: Record = { warning: 'i-lucide-triangle-alert', error: 'i-lucide-circle-x', }; +const formatLogLevel = (level: LogLevel): string => level.charAt(0).toUpperCase() + level.slice(1); +const getStoredRuntimeLogLevel = (): LogLevel | null => { + return config.app.runtime_log_level ? getLogLevel(config.app.runtime_log_level) : null; +}; let scrollTimeout: NodeJS.Timeout | null = null; @@ -441,6 +460,8 @@ const rawJsonOpen = useStorage('logs_raw_json_open', false); const sourceOpen = useStorage('logs_source_open', true); const selectedLevels = useStorage('logs_level_filter', [...LOG_LEVELS]); const sseController = ref(null); +const runtimeLogLevel = ref(null); +const runtimeLogLevelLoading = ref(false); const logs = ref>([]); const selectedLog = ref(null); @@ -512,15 +533,39 @@ const levelCounts = computed>(() => { return counts; }); + const levelFilterItems = computed(() => [ ...LOG_LEVELS.map((level) => ({ - label: `${level.charAt(0).toUpperCase()}${level.slice(1)} (${levelCounts.value[level]})`, + label: `${formatLogLevel(level)} (${levelCounts.value[level]})`, value: level, })), ]); + +const runtimeLogLevelCandidate = computed(() => { + for (const level of LOG_LEVELS) { + const thresholdLevels = LOG_LEVELS.slice(LOG_LEVELS.indexOf(level)); + if (thresholdLevels.length !== selectedLevelSet.value.size) { + continue; + } + + if (thresholdLevels.every((item) => selectedLevelSet.value.has(item))) { + return level; + } + } + + return null; +}); + +const canApplyRuntimeLogLevel = computed( + () => + runtimeLogLevelCandidate.value !== null && + runtimeLogLevel.value !== null && + runtimeLogLevelCandidate.value !== runtimeLogLevel.value, +); + const levelFilterLabel = computed(() => { if (selectedLevelSet.value.size === LOG_LEVELS.length) { - return `All levels (${logs.value.length})`; + return `All levels`; } if (selectedLevelSet.value.size === 0) { @@ -529,29 +574,7 @@ const levelFilterLabel = computed(() => { return LOG_LEVELS.filter((level) => selectedLevelSet.value.has(level)).join(', '); }); -const activeFilterLabel = computed(() => { - const parts: string[] = []; - if (hasTextFilter.value) { - parts.push(`query "${searchTerm.value}"`); - } - if (hasLevelFilter.value) { - const levels = LOG_LEVELS.filter((level) => selectedLevelSet.value.has(level)); - parts.push(levels.length ? `levels ${levels.join(', ')}` : 'no selected levels'); - } - - return parts.join(' and '); -}); -const emptyStateTitle = computed(() => - hasActiveFilter.value ? 'No logs match these filters' : 'No log lines available', -); -const emptyStateDescription = computed(() => { - if (!hasActiveFilter.value) { - return 'No log lines are available yet.'; - } - - return `No loaded log lines match ${activeFilterLabel.value}. Adjust filters or load older lines.`; -}); watch(toggleFilter, () => { if (!toggleFilter.value) { query.value = ''; @@ -718,6 +741,72 @@ const scrollToBottom = async (fast = false): Promise => { await scrollLogContainerToBottom(fast ? 'auto' : 'smooth'); }; +const loadRuntimeLogLevel = async (): Promise => { + try { + const response = await request('/api/logs/level'); + if (!response.ok) { + runtimeLogLevel.value = getStoredRuntimeLogLevel(); + return; + } + + const data = await response.json(); + if (typeof data.conf === 'string' && data.conf) { + config.app.log_level = getLogLevel(data.conf); + } + + if (!config.app.log_level && getStoredRuntimeLogLevel() === null) { + runtimeLogLevel.value = null; + return; + } + + if (typeof data.active === 'string' && data.active) { + const level = getLogLevel(data.active); + runtimeLogLevel.value = level; + config.app.runtime_log_level = level; + return; + } + + runtimeLogLevel.value = getStoredRuntimeLogLevel(); + } catch (error) { + runtimeLogLevel.value = getStoredRuntimeLogLevel(); + console.error('Failed to load runtime log level:', error); + } +}; + +const applyRuntimeLogLevel = async (): Promise => { + const normalized = runtimeLogLevelCandidate.value; + if (!normalized || runtimeLogLevel.value === null || normalized === runtimeLogLevel.value) { + return; + } + + const previous = runtimeLogLevel.value; + + try { + runtimeLogLevelLoading.value = true; + const response = await request(`/api/logs/level/${normalized}`, { + method: 'POST', + }); + + if (!response.ok) { + const data = await response.json(); + const message = await parse_api_error(data); + runtimeLogLevel.value = previous; + toast.error(`Failed to change log level. ${message}`); + return; + } + + runtimeLogLevel.value = normalized; + config.app.runtime_log_level = normalized; + toast.success('log level updated until next restart.'); + } catch (error: any) { + runtimeLogLevel.value = previous; + const message = error?.message || 'Unknown error'; + toast.error(`Failed to change log level. ${message}`); + } finally { + runtimeLogLevelLoading.value = false; + } +}; + const handleStreamMessage = (event: EventSourceMessage): void => { if (event.event !== 'log_lines' || !event.data) { return; @@ -967,6 +1056,8 @@ onMounted(async () => { return; } + runtimeLogLevel.value = getStoredRuntimeLogLevel(); + await loadRuntimeLogLevel(); await fetchLogs(); await startLogStream(); }); diff --git a/ui/app/types/config.d.ts b/ui/app/types/config.d.ts index 1197e408..58b14913 100644 --- a/ui/app/types/config.d.ts +++ b/ui/app/types/config.d.ts @@ -43,6 +43,10 @@ type AppConfig = { app_env: 'production' | 'development'; /** Default number of items per page for pagination */ default_pagination: number; + /** Configured default log level */ + log_level: 'debug' | 'info' | 'warning' | 'error' | ''; + /** Active runtime log level */ + runtime_log_level: 'debug' | 'info' | 'warning' | 'error' | ''; /** Indicates if the app should check for updates */ check_for_updates: boolean; /** New version available, empty string if none */