danvics/ytptube
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Increase log retrieval limit and enhance log pagination metadata

Browse source
This commit is contained in:
arabcoders 2025-04-10 02:32:38 +03:00
parent 17dd007ff0
commit bcdc4ee06e
3 changed files with 92 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
app/library/HttpAPI.py
View file

@ -749,19 +749,22 @@ class HttpAPI(Common):
)
offset = int(request.query.get("offset", 0))
limit = int(request.query.get("limit", 50))
limit = int(request.query.get("limit", 100))
if limit < 1 or limit > 150:
limit = 50
logs_data = await read_logfile(
file=os.path.join(self.config.config_path, "logs", "app.log"),
offset=offset,
limit=limit,
)
return web.json_response(
data={
"logs": await read_logfile(
file=os.path.join(self.config.config_path, "logs", "app.log"),
offset=offset,
limit=limit,
),
"logs": logs_data["logs"],
"offset": offset,
"limit": limit,
"next_offset": logs_data["next_offset"],
"end_is_reached": logs_data["end_is_reached"],
},
status=web.HTTPOk.status_code,
dumps=self.encoder.encode,

55
app/library/Utils.py
View file

@ -942,54 +942,64 @@ def strip_newline(string: str) -> str:
return res.strip() if res else ""
async def read_logfile(file: str, offset: int = 0, limit: int = 50) -> list[str]:
async def read_logfile(file: str, offset: int = 0, limit: int = 50) -> dict:
"""
Read log file and return limited lines from the end.
Read a log file and return a set of log lines along with pagination metadata.
Args:
file (str): The log file path.
offset (int): The number of lines to skip from the end.
limit (int): The number of lines to read.
offset (int): Number of lines to skip from the end (newer entries).
limit (int): Number of lines to return.
Returns:
list[str]: A list of log lines.
dict: A dictionary containing:
- logs: List of log entries.
- next_offset: Offset for the next page or None.
- end_is_reached: True if there are no older logs.
"""
from anyio import open_file
if not os.path.exists(file):
return []
from hashlib import sha256
if not pathlib.Path(file).exists():
return {"logs": [], "next_offset": None, "end_is_reached": True}
result = []
try:
async with await open_file(file, "rb") as f:
await f.seek(0, os.SEEK_END)
size = await f.tell()
file_size = await f.tell()
block_size = 1024
block_end = size
block_end = file_size
buffer = b""
lines = []
while len(lines) < (offset + limit) and block_end > 0:
required_count = offset + limit + 1
while len(lines) < required_count and block_end > 0:
block_start = max(0, block_end - block_size)
await f.seek(block_start)
chunk = await f.read(block_end - block_start)
buffer = chunk + buffer
buffer = chunk + buffer # prepend the chunk
lines = buffer.splitlines()
block_end = block_start
selected_lines = lines[-(offset + limit) :] if offset else lines[-limit:]
if offset:
selected_lines = selected_lines[:-offset] or []
if len(lines) > offset + limit:
next_offset = offset + limit
end_is_reached = False
else:
next_offset = None
end_is_reached = True
if offset:
selected_lines = lines[-(offset + limit) : -offset]
else:
selected_lines = lines[-limit:]
result = []
for line in selected_lines:
line_bytes = line if isinstance(line, bytes) else line.encode()
msg = line.decode(errors="replace")
dt_match = DATETIME_PATTERN.match(msg)
result.append(
{
"id": sha256(line_bytes).hexdigest(),
@ -998,10 +1008,9 @@ async def read_logfile(file: str, offset: int = 0, limit: int = 50) -> list[str]
}
)
return result
return {"logs": result, "next_offset": next_offset, "end_is_reached": end_is_reached}
except Exception:
return []
return {"logs": [], "next_offset": None, "end_is_reached": True}
async def tail_log(file: str, emitter: callable, sleep_time: float = 0.5):
@ -1019,7 +1028,7 @@ async def tail_log(file: str, emitter: callable, sleep_time: float = 0.5):
from anyio import open_file
if not os.path.exists(file):
if not pathlib(file).exists():
return
async with await open_file(file, "rb") as f:

88
ui/pages/logs.vue
View file

@ -14,7 +14,7 @@
}
.logbox {
background-color: #333;
background-color: #1f2229 !important;
box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2), 0 6px 20px 0 rgba(0, 0, 0, 0.19);
min-width: 100%;
max-height: 73vh;
@ -22,13 +22,13 @@
overflow-x: auto;
}
div.logbox pre {
background-color: rgb(31, 34, 41);
code {
background-color: unset;
}
.logline {
word-break: break-all;
line-height: 2.3em;
line-height: 1.75rem;
padding: 1em;
color: #fff1b8;
}
@ -39,13 +39,13 @@ div.logbox pre {
<div class="mt-1 columns is-multiline">
<div class="column is-12 is-clearfix is-unselectable">
<span class="title is-4">
<span class="icon"><i class="fas fa-file-lines" :class="{ 'fa-spin': loading }" /></span>
<span class="icon"><i class="fas fa-file-lines" /></span>
Logs
</span>
<div class="is-pulled-right">
<div class="field is-grouped">
<div class="control">
<button v-if="!autoScroll" @click="scrollToBottom" class="button is-primary">
<button v-if="!autoScroll" @click="scrollToBottom(false)" class="button is-primary">
<span class="icon"><i class="fas fa-arrow-down"></i></span>
<span>Go to Bottom</span>
</button>
@ -73,26 +73,42 @@ div.logbox pre {
</div>
<div class="is-hidden-mobile">
<span class="subtitle">The logs are displayed in real-time. You can scroll up to view older logs.</span>
<span class="subtitle">The logs are being streamed in real-time. You can scroll up to view older logs.</span>
</div>
</div>
</div>
<div class="column is-12">
<div class="logbox is-grid" ref="logContainer" @scroll.passive="handleScroll">
<pre class="p-1 m-1"><code id="logView" class="p-0 logline is-block"
:class="{ 'is-pre-wrap': true === textWrap, 'is-pre': false === textWrap }"><span
v-for="log in filteredItems" :key="log.id" class="is-block"><template
v-if="log?.datetime">[<span class="has-tooltip" :title="log.datetime">{{ moment(log.datetime).format('HH:mm:ss') }}</span>]&nbsp;</template>{{
log.line }}</span><span class="is-block" v-if="filteredItems.length < 1"><span v-if="query"><span
class="has-text-danger">No logs found for the filter: </span><span class="has-text-info">{{ query
}}</span></span><span v-else><span class="has-text-danger">No logs available</span></span>
</span></code></pre>
<div ref="bottomMarker"></div>
<div class="column is-12">
<div class="logbox is-grid" ref="logContainer" @scroll.passive="handleScroll">
<code id="logView" class="p-2 logline is-block"
:class="{ 'is-pre-wrap': true === textWrap, 'is-pre': false === textWrap }">
<span class="is-block m-0 notification is-info is-dark has-text-centered" v-if="reachedEnd && !query">
<span class="notification-title">
<span class="icon"><i class="fas fa-exclamation-triangle"/></span>
No more logs available for this file.
</span>
</span>
<span v-for="log in filteredItems" :key="log.id" class="is-block">
<template v-if="log?.datetime">[<span class="has-tooltip" :title="log.datetime">{{ moment(log.datetime).format('HH:mm:ss') }}</span>]</template>
{{ log.line }}
</span>
<span class="is-block" v-if="filteredItems.length < 1">
<span class="is-block m-0 notification is-warning is-dark has-text-centered" v-if="query">
<span class="notification-title is-danger">
<span class="icon"><i class="fas fa-filter"/></span>
No logs match this query: <u>{{ query }}</u>
</span>
</span>
<span v-else>
<span class="has-text-danger">No logs available</span></span>
</span>
</code>
<div ref="bottomMarker"></div>
</div>
</div>
</div>
</div>
</template>
<script setup>
import moment from 'moment'
import { request } from '~/utils/index'
@ -107,13 +123,12 @@ const config = useConfigStore()
const logs = ref([])
const offset = ref(0)
const limit = 50
const maxLogLimit = 1000
const loading = ref(false)
const logContainer = ref(null)
const bottomMarker = ref(null)
const autoScroll = ref(true)
const textWrap = ref(true)
const reachedEnd = ref(false)
const bg_enable = useStorage('random_bg', true)
const bg_opacity = useStorage('random_bg_opacity', 0.85)
@ -122,6 +137,7 @@ const toggleFilter = ref(false)
watch(toggleFilter, () => {
if (!toggleFilter.value) {
query.value = ''
scrollToBottom(true)
}
});
@ -166,14 +182,14 @@ const filteredItems = computed(() => {
})
const fetchLogs = async () => {
if (logs.value.length >= maxLogLimit) {
loading.value = true
if (reachedEnd.value || query.value) {
return
}
loading.value = true
try {
const req = await request(`/api/logs?offset=${offset.value}&limit=${limit}`)
const req = await request(`/api/logs?offset=${offset.value}`)
if (!req.ok) {
toast.error('Failed to fetch logs');
return
@ -189,13 +205,16 @@ const fetchLogs = async () => {
if (lines.length) {
logs.value.unshift(...response.logs)
offset.value += lines.length;
if (logs.value.length > maxLogLimit) {
logs.value.splice(0, logs.value.length - maxLogLimit)
}
}
// Auto-scroll only if the user was already at the bottom
if (response?.next_offset) {
offset.value = response.next_offset
}
if (response?.end_is_reached) {
reachedEnd.value = true
}
nextTick(() => {
if (autoScroll.value && bottomMarker.value) {
bottomMarker.value.scrollIntoView({ behavior: 'auto' })
@ -208,7 +227,6 @@ const fetchLogs = async () => {
}
}
const handleScroll = () => {
if (!logContainer.value || query.value) {
return
@ -233,11 +251,11 @@ const handleScroll = () => {
}
}
const scrollToBottom = () => {
const scrollToBottom = (fast=false) => {
autoScroll.value = true
nextTick(() => {
if (bottomMarker.value) {
bottomMarker.value.scrollIntoView({ behavior: 'smooth' })
bottomMarker.value.scrollIntoView({ behavior: fast ? 'auto': 'smooth' })
}
})
}
@ -246,10 +264,6 @@ onMounted(async () => {
await fetchLogs()
socket.emit('subscribe', 'log_lines')
socket.on('log_lines', data => {
if (logs.value.length >= maxLogLimit) {
logs.value.shift()
}
logs.value.push(data)
nextTick(() => {