refactor: standardize JSONL structure.

This commit is contained in:
arabcoders 2026-05-24 21:19:25 +03:00
parent ca7227ffc5
commit 2e9684de5d
6 changed files with 208 additions and 53 deletions

45
API.md
View file

@ -2315,11 +2315,25 @@ Binary image data with appropriate headers
{ {
"id": "<uuid>", "id": "<uuid>",
"datetime": "2026-05-18T12:00:00.000+00:00", "datetime": "2026-05-18T12:00:00.000+00:00",
"level": "info", "level": "error",
"levelno": 20, "levelno": 40,
"logger": "downloads.queue", "logger": "downloads.queue",
"message": "Download started", "message": "Download failed",
"exception_message": null, "exception": {
"type": "ValueError",
"message": "bad",
"file": "/app/library/downloads/queue_manager.py",
"line": 123,
"stack": [
{
"path": "/app/library/downloads/queue_manager.py",
"file": "queue_manager.py",
"module": "queue_manager",
"function": "start",
"line": 123
}
]
},
"source": { "source": {
"path": "/app/library/downloads/queue_manager.py", "path": "/app/library/downloads/queue_manager.py",
"file": "queue_manager.py", "file": "queue_manager.py",
@ -2337,7 +2351,6 @@ Binary image data with appropriate headers
} }
``` ```
- Returns `404 Not Found` if file logging is not enabled. - Returns `404 Not Found` if file logging is not enabled.
- `fields` contains scalar `logging.extra` values only. Exceptions are returned in `exception` when present.
--- ---
@ -2353,11 +2366,25 @@ Binary image data with appropriate headers
{ {
"id": "<uuid>", "id": "<uuid>",
"datetime": "2026-05-18T12:00:00.000+00:00", "datetime": "2026-05-18T12:00:00.000+00:00",
"level": "info", "level": "error",
"levelno": 20, "levelno": 40,
"logger": "downloads.queue", "logger": "downloads.queue",
"message": "Download started", "message": "Download failed",
"exception_message": null, "exception": {
"type": "ValueError",
"message": "bad",
"file": "/app/library/downloads/queue_manager.py",
"line": 123,
"stack": [
{
"path": "/app/library/downloads/queue_manager.py",
"file": "queue_manager.py",
"module": "queue_manager",
"function": "start",
"line": 123
}
]
},
"source": { "source": {
"path": "/app/library/downloads/queue_manager.py", "path": "/app/library/downloads/queue_manager.py",
"file": "queue_manager.py", "file": "queue_manager.py",

View file

@ -8,6 +8,7 @@ import os
import re import re
import socket import socket
import time import time
import traceback
import uuid import uuid
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from functools import lru_cache, wraps from functools import lru_cache, wraps
@ -46,6 +47,13 @@ LOG_RECORD_ATTRS: set[str] = set(logging.makeLogRecord({}).__dict__) | {"asctime
class JsonLogFormatter(logging.Formatter): class JsonLogFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str: def format(self, record: logging.LogRecord) -> str:
source = {
"path": record.pathname,
"file": record.filename,
"module": record.module,
"function": record.funcName,
"line": record.lineno,
}
data: dict[str, Any] = { data: dict[str, Any] = {
"id": str(uuid.uuid4()), "id": str(uuid.uuid4()),
"datetime": self.formatTime(record), "datetime": self.formatTime(record),
@ -53,24 +61,17 @@ class JsonLogFormatter(logging.Formatter):
"levelno": record.levelno, "levelno": record.levelno,
"logger": record.name, "logger": record.name,
"message": record.getMessage(), "message": record.getMessage(),
"source": { "source": source,
"path": record.pathname,
"file": record.filename,
"module": record.module,
"function": record.funcName,
"line": record.lineno,
},
"process": {"id": record.process, "name": record.processName}, "process": {"id": record.process, "name": record.processName},
"thread": {"id": record.thread, "name": record.threadName}, "thread": {"id": record.thread, "name": record.threadName},
"fields": self._extra(record), "fields": self._extra(record),
} }
if record.exc_info: if record.exc_info:
data["exception"] = self.formatException(record.exc_info) exception = self._exception(record.exc_info)
data["exception_message"] = self._exception_message(record.exc_info) if exception is not None:
data["exception"] = exception
if record.stack_info: data["source"].update(self._exception_source(exception))
data["stack"] = self.formatStack(record.stack_info)
return json.dumps(data, ensure_ascii=False, default=str) return json.dumps(data, ensure_ascii=False, default=str)
@ -90,16 +91,88 @@ class JsonLogFormatter(logging.Formatter):
return extra return extra
@staticmethod @staticmethod
def _exception_message( def _exception(
exc_info: tuple[type[BaseException], BaseException, Any] | tuple[None, None, None], exc_info: tuple[type[BaseException], BaseException, Any] | tuple[None, None, None],
) -> str | None: ) -> dict[str, Any] | None:
exc = exc_info[1] exc = exc_info[1]
if exc is None: if exc is None:
return None return None
name = exc.__class__.__name__ stack = JsonLogFormatter._exception_stack(exc_info)
msg = str(exc).strip() data: dict[str, Any] = {"type": JsonLogFormatter._exception_type(exc)}
return f"{name}: {msg}" if msg else name
message = str(exc).strip()
if message:
data["message"] = message
if stack:
origin = stack[-1]
data["file"] = origin["path"]
data["line"] = origin["line"]
data["stack"] = stack
return data
@staticmethod
def _exception_type(exc: BaseException) -> str:
module = exc.__class__.__module__
name = exc.__class__.__qualname__
return name if module == "builtins" else f"{module}.{name}"
@staticmethod
def _exception_stack(
exc_info: tuple[type[BaseException], BaseException, Any] | tuple[None, None, None],
) -> list[dict[str, Any]]:
exc = exc_info[1]
tb = exc_info[2] if len(exc_info) > 2 else None
if tb is None and exc is not None:
tb = exc.__traceback__
if tb is None:
return []
return [JsonLogFormatter._frame(frame) for frame in traceback.extract_tb(tb)]
@staticmethod
def _frame(frame: traceback.FrameSummary) -> dict[str, Any]:
path = frame.filename
file_path = Path(path)
return {
"path": path,
"file": file_path.name,
"module": file_path.stem,
"function": frame.name,
"line": frame.lineno,
}
@staticmethod
def _exception_source(exception: dict[str, Any]) -> dict[str, Any]:
source: dict[str, Any] = {}
path = exception.get("file")
if isinstance(path, str) and path:
file_path = Path(path)
source["path"] = path
source["file"] = file_path.name
source["module"] = file_path.stem
line = exception.get("line")
if isinstance(line, int):
source["line"] = line
stack = exception.get("stack")
if isinstance(stack, list) and stack:
frame = stack[-1]
if isinstance(frame, dict):
function = frame.get("function")
if isinstance(function, str) and function:
source["function"] = function
module = frame.get("module")
if isinstance(module, str) and module:
source["module"] = module
return source
def timed_lru_cache(ttl_seconds: int, max_size: int = 128): def timed_lru_cache(ttl_seconds: int, max_size: int = 128):

View file

@ -38,7 +38,7 @@ def _parse_jsonl_line(line: bytes | str) -> dict | None:
"message": str(payload["message"]).strip(), "message": str(payload["message"]).strip(),
} }
for key in ("levelno", "source", "process", "thread", "fields", "exception", "exception_message", "stack"): for key in ("levelno", "source", "process", "thread", "fields", "exception"):
if key not in payload: if key not in payload:
continue continue

View file

@ -310,8 +310,32 @@ class TestJsonLogFormatter:
data = json.loads(formatter.format(record)) data = json.loads(formatter.format(record))
assert data["message"] == "failed" assert data["message"] == "failed"
assert data["exception_message"] == "ValueError: bad" assert data["exception"] == {
assert "ValueError: bad" in data["exception"] "type": "ValueError",
"message": "bad",
"file": __file__,
"line": data["exception"]["line"],
"stack": data["exception"]["stack"],
}
assert data["exception"]["line"] > 0
assert data["exception"]["stack"][-1] == {
"path": __file__,
"file": Path(__file__).name,
"module": Path(__file__).stem,
"function": "test_exception",
"line": data["exception"]["line"],
}
assert data["source"] == data["exception"]["stack"][-1]
assert "exception_message" not in data
def test_no_raw_stack(self):
formatter = JsonLogFormatter()
record = logging.LogRecord("test", logging.ERROR, __file__, 1, "failed", (), None)
record.stack_info = 'Stack (most recent call last):\n File "x", line 1, in y'
data = json.loads(formatter.format(record))
assert "stack" not in data
class TestCalcDownloadPath: class TestCalcDownloadPath:
@ -1581,6 +1605,21 @@ class TestReadLogfile:
"level": "error", "level": "error",
"logger": "test", "logger": "test",
"message": "line 3", "message": "line 3",
"exception": {
"type": "ValueError",
"message": "bad",
"file": "/tmp/test.py",
"line": 3,
"stack": [
{
"path": "/tmp/test.py",
"file": "test.py",
"module": "test",
"function": "run",
"line": 3,
}
],
},
}, },
] ]
self.log_file.write_text("\n".join(json.dumps(line) for line in lines) + "\n") self.log_file.write_text("\n".join(json.dumps(line) for line in lines) + "\n")
@ -1593,6 +1632,7 @@ class TestReadLogfile:
assert all("line" not in line for line in result["logs"]) assert all("line" not in line for line in result["logs"])
assert [line["message"] for line in result["logs"]] == ["line 2", "line 3"] assert [line["message"] for line in result["logs"]] == ["line 2", "line 3"]
assert result["logs"][0]["level"] == "warning" assert result["logs"][0]["level"] == "warning"
assert result["logs"][1]["exception"]["type"] == "ValueError"
assert result["next_offset"] == 2 assert result["next_offset"] == 2
assert result["end_is_reached"] is False assert result["end_is_reached"] is False

View file

@ -177,8 +177,8 @@
> >
</span> </span>
<span class="ml-2">{{ entry.log.message }}</span> <span class="ml-2">{{ entry.log.message }}</span>
<span v-if="entry.log.exception_message" class="ml-1 text-error/90"> <span v-if="exceptionSummary(entry.log)" class="ml-1 text-error/90">
: {{ entry.log.exception_message }} : {{ exceptionSummary(entry.log) }}
</span> </span>
</p> </p>
</div> </div>
@ -266,11 +266,11 @@
</p> </p>
<UAlert <UAlert
v-if="selectedLog.exception_message" v-if="exceptionSummary(selectedLog)"
color="error" color="error"
variant="soft" variant="soft"
icon="i-lucide-badge-alert" icon="i-lucide-badge-alert"
:title="selectedLog.exception_message" :title="exceptionSummary(selectedLog)"
class="w-full" class="w-full"
/> />
</div> </div>
@ -292,20 +292,7 @@
<pre <pre
v-if="exceptionOpen" v-if="exceptionOpen"
class="max-h-72 overflow-auto rounded-sm border border-error/30 bg-error/5 p-3 text-xs whitespace-pre-wrap text-error" class="max-h-72 overflow-auto rounded-sm border border-error/30 bg-error/5 p-3 text-xs whitespace-pre-wrap text-error"
>{{ selectedLog.exception }}</pre >{{ exceptionText(selectedLog) }}</pre
>
</section>
<section v-if="selectedLog.stack" class="space-y-2">
<h3
class="inline-flex items-center gap-2 text-xs font-semibold uppercase tracking-[0.18em] text-toned"
>
<UIcon name="i-lucide-layers" class="size-4 text-toned" />
Stack
</h3>
<pre
class="max-h-72 overflow-auto rounded-sm border border-default bg-elevated/50 p-3 text-xs whitespace-pre-wrap text-default"
>{{ selectedLog.stack }}</pre
> >
</section> </section>
@ -816,13 +803,27 @@ const logTimeTitle = (value?: string): string =>
const logRaw = (log: log_line): string => JSON.stringify(log, null, 2); const logRaw = (log: log_line): string => JSON.stringify(log, null, 2);
const exceptionSummary = (log: log_line): string => {
const type = log.exception?.type?.trim() ?? '';
const message = log.exception?.message?.trim() ?? '';
if (type && message) {
return `${type}: ${message}`;
}
return type || message;
};
const exceptionText = (log: log_line): string =>
log.exception ? JSON.stringify(log.exception, null, 2) : '';
const searchableLog = (log: log_line): string => const searchableLog = (log: log_line): string =>
[ [
log.message, log.message,
log.level, log.level,
log.logger, log.logger,
log.exception, exceptionSummary(log),
log.stack, log.exception ? JSON.stringify(log.exception) : '',
log.source ? JSON.stringify(log.source) : '', log.source ? JSON.stringify(log.source) : '',
log.process ? JSON.stringify(log.process) : '', log.process ? JSON.stringify(log.process) : '',
log.thread ? JSON.stringify(log.thread) : '', log.thread ? JSON.stringify(log.thread) : '',

View file

@ -16,6 +16,22 @@ type log_thread = {
name?: string; name?: string;
}; };
type log_exception_frame = {
path?: string;
file?: string;
module?: string;
function?: string;
line?: number;
};
type log_exception = {
type: string;
message?: string;
file?: string;
line?: number;
stack?: Array<log_exception_frame>;
};
type log_line = { type log_line = {
id: string; id: string;
message: string; message: string;
@ -27,9 +43,7 @@ type log_line = {
process?: log_process; process?: log_process;
thread?: log_thread; thread?: log_thread;
fields?: Record<string, unknown>; fields?: Record<string, unknown>;
exception?: string | null; exception?: log_exception | null;
exception_message?: string | null;
stack?: string | null;
}; };
export type { log_line, log_process, log_source, log_thread }; export type { log_exception, log_exception_frame, log_line, log_process, log_source, log_thread };