Add logs viewing and enable file logging by default

This commit is contained in:
arabcoders 2025-04-04 20:18:03 +03:00
parent f17a27d35c
commit 48e388b0fe
12 changed files with 588 additions and 27 deletions

View file

@ -18,7 +18,6 @@
"arrowless",
"attl",
"autonumber",
"changeslog",
"consoletitle",
"cookiesfrombrowser",
"copyts",
@ -49,6 +48,7 @@
"preferredquality",
"printtraffic",
"quicktime",
"timespec",
"tmpfilename",
"upgrader",
"urandom",

View file

@ -111,6 +111,9 @@ class Events:
PRESETS_UPDATE = "presets_update"
SCHEDULE_ADD = "schedule_add"
SUBSCRIBED = "subscribed"
UNSUBSCRIBED = "unsubscribed"
def get_all() -> list:
"""
Get all the events.

View file

@ -46,6 +46,7 @@ from .Utils import (
get_file_sidecar,
get_files,
get_mime_type,
read_logfile,
validate_url,
validate_uuid,
)
@ -73,7 +74,8 @@ class HttpAPI(Common):
"/presets",
"/tasks",
"/notifications",
"/changeslog",
"/changelog",
"/logs",
"/browser",
"/browser/{path:.*}",
]
@ -728,6 +730,42 @@ class HttpAPI(Common):
dumps=self.encoder.encode,
)
@route("GET", "api/logs")
async def logs(self, request: Request) -> Response:
"""
Get recent logs
Args:
request (Request): The request object.
Returns:
Response: The response object.
"""
if not self.config.file_logging:
return web.json_response(
data={"error": "File logging is not enabled."}, status=web.HTTPNotFound.status_code
)
offset = int(request.query.get("offset", 0))
limit = int(request.query.get("limit", 50))
if limit < 1 or limit > 150:
limit = 50
return web.json_response(
data={
"logs": await read_logfile(
file=os.path.join(self.config.config_path, "logs", "app.log"),
offset=offset,
limit=limit,
),
"offset": offset,
"limit": limit,
},
status=web.HTTPOk.status_code,
dumps=self.encoder.encode,
)
@route("GET", "api/presets")
async def presets(self, request: Request) -> Response:
"""

View file

@ -19,7 +19,7 @@ from .encoder import Encoder
from .Events import Event, EventBus, Events, error
from .ItemDTO import Item
from .Presets import Presets
from .Utils import is_downloaded
from .Utils import is_downloaded, tail_log
LOG = logging.getLogger("socket_api")
@ -33,6 +33,12 @@ class HttpSocket(Common):
sio: socketio.AsyncServer
queue: DownloadQueue
subscribers: dict[str, list[str]] = {}
"""Event subscriber list."""
log_task = None
"""Task to tail the log file."""
def __init__(
self,
queue: DownloadQueue | None = None,
@ -288,3 +294,92 @@ class HttpSocket(Common):
async def resume(self, *_, **__):
self.queue.resume()
await self._notify.emit(Events.PAUSED, data={"paused": False, "at": time.time()})
@ws_event
async def subscribe(self, sid: str, event: str):
"""
Subscribe to a specific event.
Args:
sid (str): The session ID of the client.
event (str): The event to subscribe to.
"""
if not isinstance(event, str):
await self._notify.emit(Events.ERROR, data=error("Invalid event."), to=sid)
return
if event not in self.subscribers:
self.subscribers[event] = []
if sid not in self.subscribers[event]:
self.subscribers[event].append(sid)
LOG.debug(f"Client '{sid}' subscribed to event '{event}'.")
await self.sio.emit(Events.SUBSCRIBED, data={"event": event}, to=sid)
async def emit_logs(data: dict):
await self.subscribe_emit(event=event, data=data)
if "log_lines" == event and self.log_task is None:
LOG.debug("Starting log tailing task.")
self.log_task = asyncio.create_task(
tail_log(
file=os.path.join(self.config.config_path, "logs", "app.log"),
emitter=emit_logs,
),
name="tail_log",
)
@ws_event
async def unsubscribe(self, sid: str, event: str):
"""
Unsubscribe from a specific event.
Args:
sid (str): The session ID of the client.
event (str): The event to unsubscribe from.
"""
if event not in self.subscribers:
return
if sid not in self.subscribers[event]:
return
self.subscribers[event].remove(sid)
await self.sio.emit(Events.UNSUBSCRIBED, data={"event": event}, to=sid)
LOG.debug(f"Client '{sid}' unsubscribed from event '{event}'.")
if "log_lines" != event or not self.log_task or "log_lines" not in self.subscribers:
return
if len(self.subscribers["log_lines"]) < 1:
try:
LOG.debug("Stopping log tailing task.")
self.log_task.cancel()
self.log_task = None
except asyncio.CancelledError:
pass
@ws_event
async def disconnect(self, sid: str):
"""
Handle client disconnection.
Args:
sid (str): The session ID of the client.
"""
LOG.debug(f"Client '{sid}' disconnected.")
for event in self.subscribers:
if sid in self.subscribers[event]:
await self.unsubscribe(sid=sid, event=event)
async def subscribe_emit(self, event: str, data: dict):
if event not in self.subscribers or len(self.subscribers[event]) < 1:
return
for sid in self.subscribers[event]:
await self.sio.emit(event=event, data=data, to=sid)

View file

@ -70,11 +70,18 @@ FILES_TYPE: list = [
{"rx": re.compile(r"\.(nfo|json|jpg|torrent|\.info\.json)$", re.IGNORECASE), "type": "metadata"},
]
DATETIME_PATTERN = re.compile(r"^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[+-]\d{2}:\d{2}))\s?")
class StreamingError(Exception):
"""Raised when an error occurs during streaming."""
class FileLogFormatter(logging.Formatter):
def formatTime(self, record, datefmt=None): # noqa: ARG002, N802
return datetime.datetime.fromtimestamp(record.created).astimezone().isoformat(timespec="milliseconds")
def calc_download_path(base_path: str, folder: str | None = None, create_path: bool = True) -> str:
"""
Calculates download path and prevents folder traversal.
@ -944,3 +951,103 @@ def strip_newline(string: str) -> str:
res = re.sub(r"(\r\n|\r|\n)", " ", string)
return res.strip() if res else ""
async def read_logfile(file: str, offset: int = 0, limit: int = 50) -> list[str]:
"""
Read log file and return limited lines from the end.
Args:
file (str): The log file path.
offset (int): The number of lines to skip from the end.
limit (int): The number of lines to read.
Returns:
list[str]: A list of log lines.
"""
from anyio import open_file
if not os.path.exists(file):
return []
from hashlib import sha256
try:
async with await open_file(file, "rb") as f:
await f.seek(0, os.SEEK_END)
size = await f.tell()
block_size = 1024
block_end = size
buffer = b""
lines = []
while len(lines) < (offset + limit) and block_end > 0:
block_start = max(0, block_end - block_size)
await f.seek(block_start)
chunk = await f.read(block_end - block_start)
buffer = chunk + buffer
lines = buffer.splitlines()
block_end = block_start
selected_lines = lines[-(offset + limit) :] if offset else lines[-limit:]
if offset:
selected_lines = selected_lines[:-offset] or []
result = []
for line in selected_lines:
line_bytes = line if isinstance(line, bytes) else line.encode()
msg = line.decode(errors="replace")
dt_match = DATETIME_PATTERN.match(msg)
result.append(
{
"id": sha256(line_bytes).hexdigest(),
"line": msg[dt_match.end() :] if dt_match else msg,
"datetime": dt_match.group(1) if dt_match else None,
}
)
return result
except Exception:
return []
async def tail_log(file: str, emitter: callable, sleep_time: float = 0.5):
"""
Continuously read a log file and emit new lines.
Args:
file (str): The log file path.
emitter (callable): A callable to emit new lines.
sleep_time (float): The time to sleep between reads.
"""
from asyncio import sleep as asyncio_sleep
from hashlib import sha256
from anyio import open_file
if not os.path.exists(file):
return
async with await open_file(file, "rb") as f:
await f.seek(0, os.SEEK_END)
while True:
line = await f.readline()
if not line:
await asyncio_sleep(sleep_time)
continue
msg = line.decode(errors="replace")
dt_match = DATETIME_PATTERN.match(msg)
await emitter(
{
"id": sha256(line if isinstance(line, bytes) else line.encode()).hexdigest(),
"line": msg[dt_match.end() :] if dt_match else msg,
"datetime": dt_match.group(1) if dt_match else None,
}
)

View file

@ -4,14 +4,14 @@ import os
import re
import sys
import time
from logging.handlers import RotatingFileHandler
from logging.handlers import TimedRotatingFileHandler
from multiprocessing.managers import SyncManager
from pathlib import Path
import coloredlogs
from dotenv import load_dotenv
from .Utils import arg_converter
from .Utils import FileLogFormatter, arg_converter
from .version import APP_VERSION
@ -55,6 +55,9 @@ class Config:
log_level: str = "info"
"""The log level to use for the application."""
log_level_file: str = "info"
"""The log level to use for the file logging."""
max_workers: int = 1
"""The maximum number of workers to use for downloading."""
@ -130,7 +133,7 @@ class Config:
instance_title: str | None = None
"The title of the instance."
file_logging: bool = False
file_logging: bool = True
"Enable file logging."
sentry_dsn: str | None = None
@ -222,6 +225,7 @@ class Config:
"console_enabled",
"browser_enabled",
"ytdlp_cli",
"file_logging",
)
"The variables that are relevant to the frontend."
@ -377,11 +381,22 @@ class Config:
LOG.info("The frontend is running in basic mode.")
if self.file_logging:
handler = RotatingFileHandler(
os.path.join(self.config_path, "app.log"), maxBytes=1 * 1024 * 1024, backupCount=3
log_level_file = getattr(logging, self.log_level_file.upper(), None)
if not isinstance(log_level_file, int):
msg = f"Invalid file log level '{self.log_level_file}' specified."
raise TypeError(msg)
loggingPath = os.path.join(self.config_path, "logs")
if not os.path.exists(loggingPath):
os.makedirs(loggingPath, exist_ok=True)
handler = TimedRotatingFileHandler(
filename=os.path.join(self.config_path, "logs", "app.log"),
when="midnight",
backupCount=3,
)
handler.setLevel(logging.ERROR)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler.setLevel(log_level_file)
formatter = FileLogFormatter("%(asctime)s [%(levelname)s.%(name)s]: %(message)s")
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)

View file

@ -51,11 +51,26 @@ hr {
border-bottom: 1px dotted;
}
.has-tooltip {
cursor: help;
border-bottom: 1px dotted;
}
@media (prefers-color-scheme: light) {
.has-tooltip {
border-bottom-color: #000;
}
}
@media (prefers-color-scheme: dark) {
* {
unicode-bidi: plaintext;
}
.has-tooltip {
border-bottom-color: rgba(255, 255, 255, 0.3);
}
.container {
padding: 1em;
margin-top: 1em;
@ -266,6 +281,10 @@ hr {
white-space: pre;
}
.is-pre-wrap {
white-space: pre-wrap;
}
.has-text-bold {
font-weight: bold;
}

View file

@ -77,7 +77,6 @@
</div>
</div>
<div class="columns is-multiline is-mobile" v-if="showAdvanced && !config.app.basic_mode">
<div class="column is-12">
<div class="field">
<label class="label is-inline is-unselectable" for="output_format"
@ -98,7 +97,7 @@
</div>
</div>
<div class="column is-6-tablet is-4-mobile">
<div class="column is-6-tablet is-12-mobile">
<div class="field">
<label class="label is-inline is-unselectable" for="cli_options">
<span class="icon"><i class="fa-solid fa-terminal" /></span>
@ -121,7 +120,7 @@
</div>
</div>
<div class="column is-6-tablet is-4-mobile">
<div class="column is-6-tablet is-12-mobile">
<div class="field">
<label class="label is-inline is-unselectable" for="ytdlpCookies">
<span class="icon"><i class="fa-solid fa-cookie" /></span>

View file

@ -41,15 +41,6 @@
</NuxtLink>
</div>
<div class="navbar-item" v-if="!config.app.basic_mode && config.app.console_enabled">
<NuxtLink class="button is-dark has-tooltip-bottom" to="/console">
<span class="icon-text">
<span class="icon"><i class="fa-solid fa-terminal" /></span>
<span class="is-hidden-mobile">Terminal</span>
</span>
</NuxtLink>
</div>
<div class="navbar-item" v-if="!config.app.basic_mode">
<NuxtLink class="button is-dark has-tooltip-bottom" to="/tasks">
<span class="icon-text">
@ -59,7 +50,7 @@
</NuxtLink>
</div>
<div class="navbar-item" v-if="!config.app.basic_mode" v-tooltip.bottom="'Notifications'">
<div class="navbar-item" v-if="!config.app.basic_mode">
<NuxtLink class="button is-dark has-tooltip-bottom" to="/notifications">
<span class="icon-text">
<span class="icon"><i class="fa-solid fa-paper-plane" /></span>
@ -68,6 +59,24 @@
</NuxtLink>
</div>
<div class="navbar-item" v-if="!config.app.basic_mode && config.app.console_enabled">
<NuxtLink class="button is-dark has-tooltip-bottom" to="/console">
<span class="icon-text">
<span class="icon"><i class="fa-solid fa-terminal" /></span>
<span class="is-hidden-mobile">Terminal</span>
</span>
</NuxtLink>
</div>
<div class="navbar-item" v-if="!config.app.basic_mode && config.app.file_logging">
<NuxtLink class="button is-dark has-tooltip-bottom" to="/logs">
<span class="icon-text">
<span class="icon"><i class="fa-solid fa-file-lines" /></span>
<span class="is-hidden-mobile">Logs</span>
</span>
</NuxtLink>
</div>
<div class="navbar-item is-hidden-mobile">
<button class="button is-dark" @click="reloadPage">
<span class="icon"><i class="fas fa-refresh"></i></span>
@ -96,7 +105,7 @@
<span class="is-hidden-mobile">&nbsp;({{ config?.app?.version || 'unknown' }})</span>
- <NuxtLink target="_blank" href="https://github.com/yt-dlp/yt-dlp">yt-dlp</NuxtLink>
<span class="is-hidden-mobile">&nbsp;({{ config?.app?.ytdlp_version || 'unknown' }})</span>
- <NuxtLink :to="`/changeslog?version=${config?.app?.version || 'unknown'}`">CHANGELOG</NuxtLink>
- <NuxtLink :to="`/changelog?version=${config?.app?.version || 'unknown'}`">CHANGELOG</NuxtLink>
</div>
</div>
<div class="column is-4-mobile" v-if="config.app?.started">

View file

@ -2,12 +2,12 @@
<div>
<div class="mt-1 columns is-multiline">
<div class="column is-12 is-clearfix is-unselectable">
<span id="env_page_title" class="title is-4">
<span class="title is-4">
<span class="icon"><i class="fas fa-cogs" /></span>
Changes log
CHANGELOG
</span>
<div class="is-hidden-mobile">
<span class="subtitle">This page displays the application change logs.</span>
<span class="subtitle">This page display the latest changes and updates from the project.</span>
</div>
</div>
</div>

275
ui/pages/logs.vue Normal file
View file

@ -0,0 +1,275 @@
<style scoped>
#logView {
min-height: 72vh;
min-width: inherit;
max-width: 100%;
}
#logView>span:nth-child(even) {
color: #ffc9d4;
}
#logView>span:nth-child(odd) {
color: #e3c981;
}
.logbox {
background-color: #333;
box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2), 0 6px 20px 0 rgba(0, 0, 0, 0.19);
min-width: 100%;
max-height: 73vh;
overflow-y: scroll;
overflow-x: auto;
}
div.logbox pre {
background-color: rgb(31, 34, 41);
}
.logline {
word-break: break-all;
line-height: 2.3em;
padding: 1em;
color: #fff1b8;
}
</style>
<template>
<div>
<div class="mt-1 columns is-multiline">
<div class="column is-12 is-clearfix is-unselectable">
<span class="title is-4">
<span class="icon"><i class="fas fa-file-lines" :class="{ 'fa-spin': loading }" /></span>
Logs
</span>
<div class="is-pulled-right">
<div class="field is-grouped">
<div class="control">
<button v-if="!autoScroll" @click="scrollToBottom" class="button is-primary">
<span class="icon"><i class="fas fa-arrow-down"></i></span>
<span>Go to Bottom</span>
</button>
</div>
<div class="control has-icons-left" v-if="toggleFilter || query">
<input type="search" v-model.lazy="query" class="input" id="filter"
placeholder="Filter displayed content">
<span class="icon is-left"><i class="fas fa-filter" /></span>
</div>
<div class="control">
<button class="button is-danger is-light" @click="toggleFilter = !toggleFilter">
<span class="icon"><i class="fas fa-filter" /></span>
</button>
</div>
<p class="control">
<button class="button is-purple" @click="textWrap = !textWrap" v-tooltip.bottom="'Toggle text wrap'">
<span class="icon"><i class="fas fa-text-width"></i></span>
</button>
</p>
</div>
</div>
<div class="is-hidden-mobile">
<span class="subtitle">The logs are displayed in real-time. You can scroll up to view older logs.</span>
</div>
</div>
</div>
<div class="column is-12">
<div class="logbox is-grid" ref="logContainer" @scroll.passive="handleScroll">
<pre class="p-1 m-1"><code id="logView" class="p-0 logline is-block"
:class="{ 'is-pre-wrap': true === textWrap, 'is-pre': false === textWrap }"><span
v-for="log in filteredItems" :key="log.id" class="is-block"><template
v-if="log?.datetime">[<span class="has-tooltip" :title="log.datetime">{{ moment(log.datetime).format('HH:mm:ss') }}</span>]&nbsp;</template>{{
log.line }}</span><span class="is-block" v-if="filteredItems.length < 1"><span v-if="query"><span
class="has-text-danger">No logs found for the filter: </span><span class="has-text-info">{{ query
}}</span></span><span v-else><span class="has-text-danger">No logs available</span></span>
</span></code></pre>
<div ref="bottomMarker"></div>
</div>
</div>
</div>
</template>
<script setup>
import moment from 'moment'
import { request } from '~/utils/index'
import { ref, onMounted, nextTick } from 'vue'
let scrollTimeout = null
const toast = useToast()
const socket = useSocketStore()
const config = useConfigStore()
const logs = ref([])
const offset = ref(0)
const limit = 50
const maxLogLimit = 1000
const loading = ref(false)
const logContainer = ref(null)
const bottomMarker = ref(null)
const autoScroll = ref(true)
const textWrap = ref(true)
const query = ref(useRoute().query.filter ?? '')
const toggleFilter = ref(false)
watch(toggleFilter, () => {
if (!toggleFilter.value) {
query.value = ''
}
});
watch(() => config.app.basic_mode, async v => {
if (!v) {
return
}
await navigateTo('/')
})
watch(() => config.app.file_logging, async v => {
if (v) {
return
}
await navigateTo('/')
})
const filteredItems = computed(() => {
const raw = query.value.trim().toLowerCase()
const contextMatch = raw.match(/context:(\d+)/)
const context = contextMatch ? parseInt(contextMatch[1], 10) : 0
const searchTerm = raw.replace(/context:\d+/, '').trim()
if (!searchTerm) return logs.value
const result = []
const matchedIndexes = new Set()
logs.value.forEach((log, i) => {
if (log.line.toLowerCase().includes(searchTerm)) {
for (let j = Math.max(0, i - context); j <= Math.min(logs.value.length - 1, i + context); j++) {
matchedIndexes.add(j)
}
}
})
Array.from(matchedIndexes).sort((a, b) => a - b).forEach(index => {
result.push(logs.value[index])
})
return result
})
const fetchLogs = async () => {
if (logs.value.length >= maxLogLimit) {
return
}
loading.value = true
try {
const req = await request(`/api/logs?offset=${offset.value}&limit=${limit}`)
if (!req.ok) {
toast.error('Failed to fetch logs');
return
}
const response = await req.json()
if (response.error) {
toast.error(response.error)
return
}
const lines = response.logs ?? [];
if (lines.length) {
logs.value.unshift(...response.logs)
offset.value += lines.length;
if (logs.value.length > maxLogLimit) {
logs.value.splice(0, logs.value.length - maxLogLimit)
}
}
// Auto-scroll only if the user was already at the bottom
nextTick(() => {
if (autoScroll.value && bottomMarker.value) {
bottomMarker.value.scrollIntoView({ behavior: 'auto' })
}
})
} catch (err) {
console.error('Failed to fetch logs:', err)
} finally {
loading.value = false
}
}
const handleScroll = () => {
if (!logContainer.value || query.value) {
return
}
const container = logContainer.value
const nearBottom = container.scrollTop + container.clientHeight >= container.scrollHeight - 50
const nearTop = container.scrollTop < 50
autoScroll.value = nearBottom
if (nearTop && !loading.value && !scrollTimeout) {
scrollTimeout = setTimeout(async () => {
const previousHeight = container.scrollHeight
await fetchLogs()
nextTick(() => {
const newHeight = container.scrollHeight
container.scrollTop += newHeight - previousHeight
})
scrollTimeout = null
}, 300)
}
}
const scrollToBottom = () => {
autoScroll.value = true
nextTick(() => {
if (bottomMarker.value) {
bottomMarker.value.scrollIntoView({ behavior: 'smooth' })
}
})
}
onMounted(async () => {
await fetchLogs()
socket.emit('subscribe', 'log_lines')
window.socket = socket
socket.on('log_lines', data => {
if (logs.value.length >= maxLogLimit) {
logs.value.shift()
}
logs.value.push(data)
nextTick(() => {
if (autoScroll.value && bottomMarker.value) {
bottomMarker.value.scrollIntoView({ behavior: 'smooth' })
}
})
})
})
onUnmounted(() => {
socket.emit('unsubscribe', 'log_lines')
socket.off('log_lines')
})
onBeforeUnmount(() => {
socket.emit('unsubscribe', 'log_lines')
socket.off('log_lines')
if (scrollTimeout) clearTimeout(scrollTimeout)
})
useHead({ title: 'Logs' })
</script>

View file

@ -19,6 +19,7 @@ const CONFIG_KEYS = {
console_enabled: false,
browser_enabled: false,
ytdlp_cli: '',
file_logging: false,
},
presets: [
{