From 2e9684de5dd6cd967a385c787574b956cebfb0c5 Mon Sep 17 00:00:00 2001 From: arabcoders Date: Sun, 24 May 2026 21:19:25 +0300 Subject: [PATCH] refactor: standardize JSONL structure. --- API.md | 45 +++++++++++++---- app/library/Utils.py | 107 +++++++++++++++++++++++++++++++++------- app/routes/api/logs.py | 2 +- app/tests/test_utils.py | 44 ++++++++++++++++- ui/app/pages/logs.vue | 41 +++++++-------- ui/app/types/logs.ts | 22 +++++++-- 6 files changed, 208 insertions(+), 53 deletions(-) diff --git a/API.md b/API.md index 57d30de0..b360a8ff 100644 --- a/API.md +++ b/API.md @@ -2315,11 +2315,25 @@ Binary image data with appropriate headers { "id": "", "datetime": "2026-05-18T12:00:00.000+00:00", - "level": "info", - "levelno": 20, + "level": "error", + "levelno": 40, "logger": "downloads.queue", - "message": "Download started", - "exception_message": null, + "message": "Download failed", + "exception": { + "type": "ValueError", + "message": "bad", + "file": "/app/library/downloads/queue_manager.py", + "line": 123, + "stack": [ + { + "path": "/app/library/downloads/queue_manager.py", + "file": "queue_manager.py", + "module": "queue_manager", + "function": "start", + "line": 123 + } + ] + }, "source": { "path": "/app/library/downloads/queue_manager.py", "file": "queue_manager.py", @@ -2337,7 +2351,6 @@ Binary image data with appropriate headers } ``` - Returns `404 Not Found` if file logging is not enabled. -- `fields` contains scalar `logging.extra` values only. Exceptions are returned in `exception` when present. --- @@ -2353,11 +2366,25 @@ Binary image data with appropriate headers { "id": "", "datetime": "2026-05-18T12:00:00.000+00:00", - "level": "info", - "levelno": 20, + "level": "error", + "levelno": 40, "logger": "downloads.queue", - "message": "Download started", - "exception_message": null, + "message": "Download failed", + "exception": { + "type": "ValueError", + "message": "bad", + "file": "/app/library/downloads/queue_manager.py", + "line": 123, + "stack": [ + { + "path": "/app/library/downloads/queue_manager.py", + "file": "queue_manager.py", + "module": "queue_manager", + "function": "start", + "line": 123 + } + ] + }, "source": { "path": "/app/library/downloads/queue_manager.py", "file": "queue_manager.py", diff --git a/app/library/Utils.py b/app/library/Utils.py index bcc79005..4c383524 100644 --- a/app/library/Utils.py +++ b/app/library/Utils.py @@ -8,6 +8,7 @@ import os import re import socket import time +import traceback import uuid from datetime import UTC, datetime, timedelta from functools import lru_cache, wraps @@ -46,6 +47,13 @@ LOG_RECORD_ATTRS: set[str] = set(logging.makeLogRecord({}).__dict__) | {"asctime class JsonLogFormatter(logging.Formatter): def format(self, record: logging.LogRecord) -> str: + source = { + "path": record.pathname, + "file": record.filename, + "module": record.module, + "function": record.funcName, + "line": record.lineno, + } data: dict[str, Any] = { "id": str(uuid.uuid4()), "datetime": self.formatTime(record), @@ -53,24 +61,17 @@ class JsonLogFormatter(logging.Formatter): "levelno": record.levelno, "logger": record.name, "message": record.getMessage(), - "source": { - "path": record.pathname, - "file": record.filename, - "module": record.module, - "function": record.funcName, - "line": record.lineno, - }, + "source": source, "process": {"id": record.process, "name": record.processName}, "thread": {"id": record.thread, "name": record.threadName}, "fields": self._extra(record), } if record.exc_info: - data["exception"] = self.formatException(record.exc_info) - data["exception_message"] = self._exception_message(record.exc_info) - - if record.stack_info: - data["stack"] = self.formatStack(record.stack_info) + exception = self._exception(record.exc_info) + if exception is not None: + data["exception"] = exception + data["source"].update(self._exception_source(exception)) return json.dumps(data, ensure_ascii=False, default=str) @@ -90,16 +91,88 @@ class JsonLogFormatter(logging.Formatter): return extra @staticmethod - def _exception_message( + def _exception( exc_info: tuple[type[BaseException], BaseException, Any] | tuple[None, None, None], - ) -> str | None: + ) -> dict[str, Any] | None: exc = exc_info[1] if exc is None: return None - name = exc.__class__.__name__ - msg = str(exc).strip() - return f"{name}: {msg}" if msg else name + stack = JsonLogFormatter._exception_stack(exc_info) + data: dict[str, Any] = {"type": JsonLogFormatter._exception_type(exc)} + + message = str(exc).strip() + if message: + data["message"] = message + + if stack: + origin = stack[-1] + data["file"] = origin["path"] + data["line"] = origin["line"] + data["stack"] = stack + + return data + + @staticmethod + def _exception_type(exc: BaseException) -> str: + module = exc.__class__.__module__ + name = exc.__class__.__qualname__ + return name if module == "builtins" else f"{module}.{name}" + + @staticmethod + def _exception_stack( + exc_info: tuple[type[BaseException], BaseException, Any] | tuple[None, None, None], + ) -> list[dict[str, Any]]: + exc = exc_info[1] + tb = exc_info[2] if len(exc_info) > 2 else None + if tb is None and exc is not None: + tb = exc.__traceback__ + + if tb is None: + return [] + + return [JsonLogFormatter._frame(frame) for frame in traceback.extract_tb(tb)] + + @staticmethod + def _frame(frame: traceback.FrameSummary) -> dict[str, Any]: + path = frame.filename + file_path = Path(path) + return { + "path": path, + "file": file_path.name, + "module": file_path.stem, + "function": frame.name, + "line": frame.lineno, + } + + @staticmethod + def _exception_source(exception: dict[str, Any]) -> dict[str, Any]: + source: dict[str, Any] = {} + + path = exception.get("file") + if isinstance(path, str) and path: + file_path = Path(path) + source["path"] = path + source["file"] = file_path.name + source["module"] = file_path.stem + + line = exception.get("line") + if isinstance(line, int): + source["line"] = line + + stack = exception.get("stack") + if isinstance(stack, list) and stack: + frame = stack[-1] + if isinstance(frame, dict): + function = frame.get("function") + if isinstance(function, str) and function: + source["function"] = function + + module = frame.get("module") + if isinstance(module, str) and module: + source["module"] = module + + return source def timed_lru_cache(ttl_seconds: int, max_size: int = 128): diff --git a/app/routes/api/logs.py b/app/routes/api/logs.py index 2a9efe54..07c7aec4 100644 --- a/app/routes/api/logs.py +++ b/app/routes/api/logs.py @@ -38,7 +38,7 @@ def _parse_jsonl_line(line: bytes | str) -> dict | None: "message": str(payload["message"]).strip(), } - for key in ("levelno", "source", "process", "thread", "fields", "exception", "exception_message", "stack"): + for key in ("levelno", "source", "process", "thread", "fields", "exception"): if key not in payload: continue diff --git a/app/tests/test_utils.py b/app/tests/test_utils.py index bf618da1..96493605 100644 --- a/app/tests/test_utils.py +++ b/app/tests/test_utils.py @@ -310,8 +310,32 @@ class TestJsonLogFormatter: data = json.loads(formatter.format(record)) assert data["message"] == "failed" - assert data["exception_message"] == "ValueError: bad" - assert "ValueError: bad" in data["exception"] + assert data["exception"] == { + "type": "ValueError", + "message": "bad", + "file": __file__, + "line": data["exception"]["line"], + "stack": data["exception"]["stack"], + } + assert data["exception"]["line"] > 0 + assert data["exception"]["stack"][-1] == { + "path": __file__, + "file": Path(__file__).name, + "module": Path(__file__).stem, + "function": "test_exception", + "line": data["exception"]["line"], + } + assert data["source"] == data["exception"]["stack"][-1] + assert "exception_message" not in data + + def test_no_raw_stack(self): + formatter = JsonLogFormatter() + record = logging.LogRecord("test", logging.ERROR, __file__, 1, "failed", (), None) + record.stack_info = 'Stack (most recent call last):\n File "x", line 1, in y' + + data = json.loads(formatter.format(record)) + + assert "stack" not in data class TestCalcDownloadPath: @@ -1581,6 +1605,21 @@ class TestReadLogfile: "level": "error", "logger": "test", "message": "line 3", + "exception": { + "type": "ValueError", + "message": "bad", + "file": "/tmp/test.py", + "line": 3, + "stack": [ + { + "path": "/tmp/test.py", + "file": "test.py", + "module": "test", + "function": "run", + "line": 3, + } + ], + }, }, ] self.log_file.write_text("\n".join(json.dumps(line) for line in lines) + "\n") @@ -1593,6 +1632,7 @@ class TestReadLogfile: assert all("line" not in line for line in result["logs"]) assert [line["message"] for line in result["logs"]] == ["line 2", "line 3"] assert result["logs"][0]["level"] == "warning" + assert result["logs"][1]["exception"]["type"] == "ValueError" assert result["next_offset"] == 2 assert result["end_is_reached"] is False diff --git a/ui/app/pages/logs.vue b/ui/app/pages/logs.vue index 2b205cc3..1113c86a 100644 --- a/ui/app/pages/logs.vue +++ b/ui/app/pages/logs.vue @@ -177,8 +177,8 @@ > {{ entry.log.message }} - - : {{ entry.log.exception_message }} + + : {{ exceptionSummary(entry.log) }}

@@ -266,11 +266,11 @@

@@ -292,20 +292,7 @@ 
{{ selectedLog.exception }}
- - -
-

- - Stack -

- 
{{ selectedLog.stack }}
{{ exceptionText(selectedLog) }}
@@ -816,13 +803,27 @@ const logTimeTitle = (value?: string): string => const logRaw = (log: log_line): string => JSON.stringify(log, null, 2); +const exceptionSummary = (log: log_line): string => { + const type = log.exception?.type?.trim() ?? ''; + const message = log.exception?.message?.trim() ?? ''; + + if (type && message) { + return `${type}: ${message}`; + } + + return type || message; +}; + +const exceptionText = (log: log_line): string => + log.exception ? JSON.stringify(log.exception, null, 2) : ''; + const searchableLog = (log: log_line): string => [ log.message, log.level, log.logger, - log.exception, - log.stack, + exceptionSummary(log), + log.exception ? JSON.stringify(log.exception) : '', log.source ? JSON.stringify(log.source) : '', log.process ? JSON.stringify(log.process) : '', log.thread ? JSON.stringify(log.thread) : '', diff --git a/ui/app/types/logs.ts b/ui/app/types/logs.ts index 48cccbda..3fddb9ac 100644 --- a/ui/app/types/logs.ts +++ b/ui/app/types/logs.ts @@ -16,6 +16,22 @@ type log_thread = { name?: string; }; +type log_exception_frame = { + path?: string; + file?: string; + module?: string; + function?: string; + line?: number; +}; + +type log_exception = { + type: string; + message?: string; + file?: string; + line?: number; + stack?: Array; +}; + type log_line = { id: string; message: string; @@ -27,9 +43,7 @@ type log_line = { process?: log_process; thread?: log_thread; fields?: Record; - exception?: string | null; - exception_message?: string | null; - stack?: string | null; + exception?: log_exception | null; }; -export type { log_line, log_process, log_source, log_thread }; +export type { log_exception, log_exception_frame, log_line, log_process, log_source, log_thread };