Increase log retrieval limit and enhance log pagination metadata

app/library/HttpAPI.py

@@ -749,19 +749,22 @@ class HttpAPI(Common):
             )
 
         offset = int(request.query.get("offset", 0))
-        limit = int(request.query.get("limit", 50))
+        limit = int(request.query.get("limit", 100))
         if limit < 1 or limit > 150:
             limit = 50
 
+        logs_data = await read_logfile(
+            file=os.path.join(self.config.config_path, "logs", "app.log"),
+            offset=offset,
+            limit=limit,
+        )
         return web.json_response(
             data={
-                "logs": await read_logfile(
+                "logs": logs_data["logs"],
-                    file=os.path.join(self.config.config_path, "logs", "app.log"),
-                    offset=offset,
-                    limit=limit,
-                ),
                 "offset": offset,
                 "limit": limit,
+                "next_offset": logs_data["next_offset"],
+                "end_is_reached": logs_data["end_is_reached"],
             },
             status=web.HTTPOk.status_code,
             dumps=self.encoder.encode,

app/library/Utils.py

@@ -942,54 +942,64 @@ def strip_newline(string: str) -> str:
     return res.strip() if res else ""
 
 
-async def read_logfile(file: str, offset: int = 0, limit: int = 50) -> list[str]:
+async def read_logfile(file: str, offset: int = 0, limit: int = 50) -> dict:
     """
-    Read log file and return limited lines from the end.
+    Read a log file and return a set of log lines along with pagination metadata.
 
     Args:
         file (str): The log file path.
-        offset (int): The number of lines to skip from the end.
+        offset (int): Number of lines to skip from the end (newer entries).
-        limit (int): The number of lines to read.
+        limit (int): Number of lines to return.
 
     Returns:
-        list[str]: A list of log lines.
+        dict: A dictionary containing:
-
+            - logs: List of log entries.
+            - next_offset: Offset for the next page or None.
+            - end_is_reached: True if there are no older logs.
     """
     from anyio import open_file
-
-    if not os.path.exists(file):
-        return []
-
     from hashlib import sha256
 
+    if not pathlib.Path(file).exists():
+        return {"logs": [], "next_offset": None, "end_is_reached": True}
+
+    result = []
     try:
         async with await open_file(file, "rb") as f:
             await f.seek(0, os.SEEK_END)
-            size = await f.tell()
+            file_size = await f.tell()
 
             block_size = 1024
-            block_end = size
+            block_end = file_size
             buffer = b""
             lines = []
 
-            while len(lines) < (offset + limit) and block_end > 0:
+            required_count = offset + limit + 1
+
+            while len(lines) < required_count and block_end > 0:
                 block_start = max(0, block_end - block_size)
                 await f.seek(block_start)
                 chunk = await f.read(block_end - block_start)
-                buffer = chunk + buffer
+                buffer = chunk + buffer  # prepend the chunk
                 lines = buffer.splitlines()
                 block_end = block_start
 
-            selected_lines = lines[-(offset + limit) :] if offset else lines[-limit:]
+            if len(lines) > offset + limit:
-            if offset:
+                next_offset = offset + limit
-                selected_lines = selected_lines[:-offset] or []
+                end_is_reached = False
+            else:
+                next_offset = None
+                end_is_reached = True
+
+            if offset:
+                selected_lines = lines[-(offset + limit) : -offset]
+            else:
+                selected_lines = lines[-limit:]
 
-            result = []
             for line in selected_lines:
                 line_bytes = line if isinstance(line, bytes) else line.encode()
                 msg = line.decode(errors="replace")
                 dt_match = DATETIME_PATTERN.match(msg)
-
                 result.append(
                     {
                         "id": sha256(line_bytes).hexdigest(),
@@ -998,10 +1008,9 @@ async def read_logfile(file: str, offset: int = 0, limit: int = 50) -> list[str]
                     }
                 )
 
-            return result
+            return {"logs": result, "next_offset": next_offset, "end_is_reached": end_is_reached}
-
     except Exception:
-        return []
+        return {"logs": [], "next_offset": None, "end_is_reached": True}
 
 
 async def tail_log(file: str, emitter: callable, sleep_time: float = 0.5):
@@ -1019,7 +1028,7 @@ async def tail_log(file: str, emitter: callable, sleep_time: float = 0.5):
 
     from anyio import open_file
 
-    if not os.path.exists(file):
+    if not pathlib(file).exists():
         return
 
     async with await open_file(file, "rb") as f:

ui/pages/logs.vue

@@ -14,7 +14,7 @@
 }
 
 .logbox {
-  background-color: #333;
+  background-color: #1f2229 !important;
   box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2), 0 6px 20px 0 rgba(0, 0, 0, 0.19);
   min-width: 100%;
   max-height: 73vh;
@@ -22,13 +22,13 @@
   overflow-x: auto;
 }
 
-div.logbox pre {
+code {
-  background-color: rgb(31, 34, 41);
+  background-color: unset;
 }
 
 .logline {
   word-break: break-all;
-  line-height: 2.3em;
+  line-height: 1.75rem;
   padding: 1em;
   color: #fff1b8;
 }
@@ -39,13 +39,13 @@ div.logbox pre {
     <div class="mt-1 columns is-multiline">
       <div class="column is-12 is-clearfix is-unselectable">
         <span class="title is-4">
-          <span class="icon"><i class="fas fa-file-lines" :class="{ 'fa-spin': loading }" /></span>
+          <span class="icon"><i class="fas fa-file-lines" /></span>
           Logs
         </span>
         <div class="is-pulled-right">
           <div class="field is-grouped">
             <div class="control">
-              <button v-if="!autoScroll" @click="scrollToBottom" class="button is-primary">
+              <button v-if="!autoScroll" @click="scrollToBottom(false)" class="button is-primary">
                 <span class="icon"><i class="fas fa-arrow-down"></i></span>
                 <span>Go to Bottom</span>
               </button>
@@ -73,26 +73,42 @@ div.logbox pre {
         </div>
 
         <div class="is-hidden-mobile">
-          <span class="subtitle">The logs are displayed in real-time. You can scroll up to view older logs.</span>
+          <span class="subtitle">The logs are being streamed in real-time. You can scroll up to view older logs.</span>
         </div>
       </div>
-    </div>
 
-    <div class="column is-12">
+      <div class="column is-12">
-      <div class="logbox is-grid" ref="logContainer" @scroll.passive="handleScroll">
+        <div class="logbox is-grid" ref="logContainer" @scroll.passive="handleScroll">
-        <pre class="p-1 m-1"><code id="logView" class="p-0 logline is-block"
+          <code id="logView" class="p-2 logline is-block"
-                               :class="{ 'is-pre-wrap': true === textWrap, 'is-pre': false === textWrap }"><span
+            :class="{ 'is-pre-wrap': true === textWrap, 'is-pre': false === textWrap }">
-        v-for="log in filteredItems" :key="log.id" class="is-block"><template
+            <span class="is-block m-0 notification is-info is-dark has-text-centered" v-if="reachedEnd && !query">
-        v-if="log?.datetime">[<span class="has-tooltip" :title="log.datetime">{{ moment(log.datetime).format('HH:mm:ss') }}</span>]&nbsp;</template>{{
+              <span class="notification-title">
-          log.line }}</span><span class="is-block" v-if="filteredItems.length < 1"><span v-if="query"><span
+                <span class="icon"><i class="fas fa-exclamation-triangle"/></span>
-      class="has-text-danger">No logs found for the filter: </span><span class="has-text-info">{{ query
+                No more logs available for this file.
-      }}</span></span><span v-else><span class="has-text-danger">No logs available</span></span>
+              </span>
-</span></code></pre>
+            </span>
-        <div ref="bottomMarker"></div>
+            <span v-for="log in filteredItems" :key="log.id" class="is-block">
+              <template v-if="log?.datetime">[<span class="has-tooltip" :title="log.datetime">{{ moment(log.datetime).format('HH:mm:ss') }}</span>]</template>
+              {{ log.line }}
+            </span>
+            <span class="is-block" v-if="filteredItems.length < 1">
+                <span class="is-block m-0 notification is-warning is-dark has-text-centered" v-if="query">
+                  <span class="notification-title is-danger">
+                    <span class="icon"><i class="fas fa-filter"/></span>
+                    No logs match this query: <u>{{ query }}</u>
+                  </span>
+                </span>
+                <span v-else>
+                  <span class="has-text-danger">No logs available</span></span>
+              </span>
+          </code>
+          <div ref="bottomMarker"></div>
+        </div>
       </div>
     </div>
   </div>
 </template>
+
 <script setup>
 import moment from 'moment'
 import { request } from '~/utils/index'
@@ -107,13 +123,12 @@ const config = useConfigStore()
 
 const logs = ref([])
 const offset = ref(0)
-const limit = 50
-const maxLogLimit = 1000
 const loading = ref(false)
 const logContainer = ref(null)
 const bottomMarker = ref(null)
 const autoScroll = ref(true)
 const textWrap = ref(true)
+const reachedEnd = ref(false)
 const bg_enable = useStorage('random_bg', true)
 const bg_opacity = useStorage('random_bg_opacity', 0.85)
 
@@ -122,6 +137,7 @@ const toggleFilter = ref(false)
 watch(toggleFilter, () => {
   if (!toggleFilter.value) {
     query.value = ''
+    scrollToBottom(true)
   }
 });
 
@@ -166,14 +182,14 @@ const filteredItems = computed(() => {
 })
 
 const fetchLogs = async () => {
-  if (logs.value.length >= maxLogLimit) {
+  loading.value = true
+
+  if (reachedEnd.value || query.value) {
     return
   }
 
-  loading.value = true
-
   try {
-    const req = await request(`/api/logs?offset=${offset.value}&limit=${limit}`)
+    const req = await request(`/api/logs?offset=${offset.value}`)
     if (!req.ok) {
       toast.error('Failed to fetch logs');
       return
@@ -189,13 +205,16 @@ const fetchLogs = async () => {
 
     if (lines.length) {
       logs.value.unshift(...response.logs)
-      offset.value += lines.length;
-      if (logs.value.length > maxLogLimit) {
-        logs.value.splice(0, logs.value.length - maxLogLimit)
-      }
     }
 
-    // Auto-scroll only if the user was already at the bottom
+    if (response?.next_offset) {
+      offset.value = response.next_offset
+    }
+
+    if (response?.end_is_reached) {
+      reachedEnd.value = true
+    }
+
     nextTick(() => {
       if (autoScroll.value && bottomMarker.value) {
         bottomMarker.value.scrollIntoView({ behavior: 'auto' })
@@ -208,7 +227,6 @@ const fetchLogs = async () => {
   }
 }
 
-
 const handleScroll = () => {
   if (!logContainer.value || query.value) {
     return
@@ -233,11 +251,11 @@ const handleScroll = () => {
   }
 }
 
-const scrollToBottom = () => {
+const scrollToBottom = (fast=false) => {
   autoScroll.value = true
   nextTick(() => {
     if (bottomMarker.value) {
-      bottomMarker.value.scrollIntoView({ behavior: 'smooth' })
+      bottomMarker.value.scrollIntoView({ behavior: fast ? 'auto': 'smooth' })
     }
   })
 }
@@ -246,10 +264,6 @@ onMounted(async () => {
   await fetchLogs()
   socket.emit('subscribe', 'log_lines')
   socket.on('log_lines', data => {
-    if (logs.value.length >= maxLogLimit) {
-      logs.value.shift()
-    }
-
     logs.value.push(data)
 
     nextTick(() => {