Decouple queue from config endpoint; load via paginated API with correct ordering (#2)

* Initial plan

* Refactor: remove queue from /api/system/configuration, use existing paginated /api/history endpoint for queue data

Co-authored-by: jbatesy <51190172+jbatesy@users.noreply.github.com>

* Remove accidentally committed package-lock.json and add to gitignore

Co-authored-by: jbatesy <51190172+jbatesy@users.noreply.github.com>

* Simplify loadMoreQueue to use loadNextPage for reduced duplication

Co-authored-by: jbatesy <51190172+jbatesy@users.noreply.github.com>

* Order in-progress downloads oldest-to-newest (order of processing)

Co-authored-by: jbatesy <51190172+jbatesy@users.noreply.github.com>

* Fix queue ordering: remove client-side timestamp sort, rely on backend created_at ASC

Co-authored-by: jbatesy <51190172+jbatesy@users.noreply.github.com>

* Revert "Fix queue ordering: remove client-side timestamp sort, rely on backend created_at ASC"

This reverts commit 9b53112e0050a45e4db95c85064577d22ed47a50. Changes not needed

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: jbatesy <51190172+jbatesy@users.noreply.github.com>
Co-authored-by: Jesse Bate <noreply@idump.me>

.gitignore

@@ -40,3 +40,4 @@ version.txt
 test_impl.py
 ./eslint.config.js
 .pytest_cache
+ui/package-lock.json

app/routes/api/system.py

@@ -32,7 +32,7 @@ LOG: logging.Logger = logging.getLogger(__name__)
 @route("GET", "api/system/configuration", "system.configuration")
 async def system_config(queue: DownloadQueue, config: Config, encoder: Encoder) -> Response:
     """
-    Pause non-active downloads.
+    Get the system configuration.
 
     Args:
         queue (DownloadQueue): The download queue instance.
@@ -55,7 +55,6 @@ async def system_config(queue: DownloadQueue, config: Config, encoder: Encoder)
                 depth_limit=config.download_path_depth - 1,
             ),
             "history_count": await queue.done.get_total_count(),
-            "queue": (await queue.get("queue"))["queue"],
         },
         status=web.HTTPOk.status_code,
         dumps=encoder.encode,

app/tests/test_system_routes.py

@@ -1,10 +1,52 @@
+import json
+
 import pytest
-from unittest.mock import AsyncMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch
 
 from app.library.config import Config
 from app.library.encoder import Encoder
 from app.library.UpdateChecker import UpdateChecker
-from app.routes.api.system import check_updates
+from app.routes.api.system import check_updates, system_config
+
+
+class TestSystemConfigEndpoint:
+    """Tests for the system configuration endpoint."""
+
+    def setup_method(self):
+        """Reset singletons before each test."""
+        Config._reset_singleton()
+
+    @pytest.mark.asyncio
+    async def test_system_config_does_not_return_queue(self):
+        """Test that the configuration endpoint does not include queue data."""
+        config = Config.get_instance()
+        encoder = Encoder()
+
+        mock_queue = MagicMock()
+        mock_queue.is_paused.return_value = False
+        mock_done = AsyncMock()
+        mock_done.get_total_count = AsyncMock(return_value=0)
+        mock_queue.done = mock_done
+
+        with (
+            patch("app.routes.api.system.Presets") as mock_presets_cls,
+            patch("app.routes.api.system.DLFields") as mock_dl_fields_cls,
+            patch("app.routes.api.system.list_folders", return_value=[]),
+        ):
+            mock_presets_cls.get_instance.return_value.get_all.return_value = []
+            mock_dl_fields_cls.get_instance.return_value.get_all_serialized = AsyncMock(return_value=[])
+
+            response = await system_config(mock_queue, config, encoder)
+
+            assert 200 == response.status
+            body = json.loads(response.body.decode("utf-8"))
+            assert "queue" not in body, "Configuration response should not include queue data"
+            assert "app" in body, "Configuration response should include app data"
+            assert "paused" in body, "Configuration response should include paused status"
+            assert "history_count" in body, "Configuration response should include history_count"
+            assert "presets" in body, "Configuration response should include presets"
+            assert "dl_fields" in body, "Configuration response should include dl_fields"
+            assert "folders" in body, "Configuration response should include folders"
 
 
 class TestCheckUpdatesEndpoint:

ui/app/components/Queue.vue

@@ -470,6 +470,21 @@
     </div>
   </div>
 
+  <div
+    v-if="queuePagination.isLoaded && queuePagination.page < queuePagination.total_pages"
+    ref="loadMoreTrigger"
+    class="columns is-centered mt-4"
+  >
+    <div class="column is-narrow">
+      <div v-if="queuePagination.isLoading" class="has-text-centered">
+        <span class="icon is-large has-text-info">
+          <i class="fas fa-spinner fa-pulse fa-2x"></i>
+        </span>
+        <p class="is-size-7 has-text-grey mt-2">Loading more items...</p>
+      </div>
+    </div>
+  </div>
+
   <div class="modal is-active" v-if="embed_url">
     <div class="modal-background" @click="embed_url = ''"></div>
     <div class="modal-content is-unbounded-model">
@@ -481,7 +496,7 @@
 
 <script setup lang="ts">
 import moment from 'moment';
-import { useStorage } from '@vueuse/core';
+import { useStorage, useIntersectionObserver } from '@vueuse/core';
 import type { StoreItem } from '~/types/store';
 import { useConfirm } from '~/composables/useConfirm';
 import { deepIncludes } from '~/utils';
@@ -514,12 +529,15 @@ const selectedElms = ref<string[]>([]);
 const masterSelectAll = ref(false);
 const embed_url = ref('');
 const isRefreshing = ref(false);
+const loadMoreTrigger = ref<HTMLElement | null>(null);
 const autoRefreshInterval = ref<NodeJS.Timeout | null>(null);
 const autoRefreshEnabled = useStorage<boolean>('queue_auto_refresh', true);
 const autoRefreshDelay = useStorage<number>('queue_auto_refresh_delay', 10000);
 
 const showThumbnails = computed(() => !!props.thumbnails && !hideThumbnail.value);
 
+const queuePagination = computed(() => stateStore.getQueuePagination());
+
 const refreshQueue = async () => {
   isRefreshing.value = true;
   try {
@@ -531,6 +549,14 @@ const refreshQueue = async () => {
   }
 };
 
+const loadMoreQueue = async () => {
+  try {
+    await stateStore.loadNextPage('queue', true);
+  } catch {
+    toast.error('Failed to load more queue items');
+  }
+};
+
 const startAutoRefresh = () => {
   if (autoRefreshInterval.value) {
     clearInterval(autoRefreshInterval.value);
@@ -573,7 +599,14 @@ watch(autoRefreshEnabled, (enabled) => {
   }
 });
 
-onMounted(() => {
+onMounted(async () => {
+  if (!queuePagination.value.isLoaded) {
+    try {
+      await stateStore.loadPaginated('queue', 1, config.app.default_pagination, 'ASC', false);
+    } catch {
+      toast.error('Failed to load queue');
+    }
+  }
   if (!socket.isConnected && autoRefreshEnabled.value) {
     startAutoRefresh();
   }
@@ -581,6 +614,20 @@ onMounted(() => {
 
 onBeforeUnmount(() => stopAutoRefresh());
 
+useIntersectionObserver(
+  loadMoreTrigger,
+  ([entry]) => {
+    if (
+      entry?.isIntersecting &&
+      !queuePagination.value.isLoading &&
+      queuePagination.value.page < queuePagination.value.total_pages
+    ) {
+      loadMoreQueue();
+    }
+  },
+  { threshold: 0.5 },
+);
+
 watch(masterSelectAll, (value) => {
   if (value) {
     selectedElms.value = Object.values(stateStore.queue).map((element: StoreItem) => element._id);
@@ -590,13 +637,14 @@ watch(masterSelectAll, (value) => {
 });
 
 const filteredItems = computed<StoreItem[]>(() => {
+  const items = Object.values(stateStore.queue)
+    .slice()
+    .sort((a, b) => (a.timestamp ?? 0) - (b.timestamp ?? 0));
   const q = props.query?.toLowerCase();
   if (!q) {
-    return Object.values(stateStore.queue);
+    return items;
   }
-  return Object.values(stateStore.queue).filter((i: StoreItem) =>
+  return items.filter((i: StoreItem) => deepIncludes(i, q, new WeakSet()));
-    deepIncludes(i, q, new WeakSet()),
-  );
 });
 
 const hasSelected = computed(() => 0 < selectedElms.value.length);

ui/app/components/Simple.vue

@@ -407,6 +407,8 @@ const addInProgress = ref<boolean>(false);
 const showExtras = ref<boolean>(false);
 const isRefreshing = ref<boolean>(false);
 
+const queuePaginationInfo = computed(() => stateStore.getQueuePagination());
+
 const refreshQueue = async (): Promise<void> => {
   if (isRefreshing.value) {
     return;
@@ -426,7 +428,7 @@ const paginationInfo = computed(() => stateStore.getPagination());
 const queueItems = computed<StoreItem[]>(() =>
   Object.values(queue.value ?? {})
     .slice()
-    .sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0)),
+    .sort((a, b) => (a.timestamp ?? 0) - (b.timestamp ?? 0)),
 );
 const historyEntries = computed<StoreItem[]>(() => {
   const items = Object.values(history.value ?? {});
@@ -857,7 +859,7 @@ const connectionStatusColor = computed(() => {
   }
 });
 
-// Load history via API on mount
+// Load queue and history via API on mount
 onMounted(async () => {
   const route = useRoute();
 
@@ -870,6 +872,14 @@ onMounted(async () => {
     window.history.replaceState({}, '', url.toString());
   }
 
+  if (!queuePaginationInfo.value.isLoaded) {
+    try {
+      await stateStore.loadPaginated('queue', 1, DEFAULT_PAGE_SIZE, 'ASC');
+    } catch (error) {
+      console.error('Failed to load queue on mount:', error);
+    }
+  }
+
   if (!paginationInfo.value.isLoaded) {
     try {
       await stateStore.loadPaginated('history', 1, DEFAULT_PAGE_SIZE, 'DESC');

ui/app/stores/ConfigStore.ts

@@ -85,11 +85,6 @@ export const useConfigStore = defineStore('config', () => {
         delete data.history_count;
       }
 
-      if (data.queue) {
-        stateStore.addAll('queue', data.queue);
-        delete data.queue;
-      }
-
       setAll(data);
       state.is_loaded = true;
       last_reload = now;

ui/app/stores/StateStore.ts

@@ -6,10 +6,7 @@ import { request } from '~/utils';
 type StateType = 'queue' | 'history';
 type KeyType = string;
 
-interface State {
+interface PaginationState {
-  queue: Record<KeyType, StoreItem>;
-  history: Record<KeyType, StoreItem>;
-  pagination: {
   page: number;
   per_page: number;
   total: number;
@@ -18,14 +15,16 @@ interface State {
   has_prev: boolean;
   isLoaded: boolean;
   isLoading: boolean;
-  };
 }
 
-export const useStateStore = defineStore('state', () => {
+interface State {
-  const state = reactive<State>({
+  queue: Record<KeyType, StoreItem>;
-    queue: {},
+  history: Record<KeyType, StoreItem>;
-    history: {},
+  pagination: PaginationState;
-    pagination: {
+  queue_pagination: PaginationState;
+}
+
+const defaultPagination = (): PaginationState => ({
   page: 1,
   per_page: 50,
   total: 0,
@@ -34,7 +33,14 @@ export const useStateStore = defineStore('state', () => {
   has_prev: false,
   isLoaded: false,
   isLoading: false,
-    },
+});
+
+export const useStateStore = defineStore('state', () => {
+  const state = reactive<State>({
+    queue: {},
+    history: {},
+    pagination: defaultPagination(),
+    queue_pagination: defaultPagination(),
   });
 
   const add = (type: StateType, key: KeyType, value: StoreItem): void => {
@@ -73,17 +79,18 @@ export const useStateStore = defineStore('state', () => {
     return !!state[type][key];
   };
 
+  const paginationFor = (type: StateType): PaginationState => {
+    return type === 'queue' ? state.queue_pagination : state.pagination;
+  };
+
   const clearAll = (type: StateType): void => {
     state[type] = {};
-    if ('queue' === type) {
+    const pg = paginationFor(type);
-      return;
+    pg.total = 0;
-    }
+    pg.page = 1;
-
+    pg.total_pages = 0;
-    state.pagination.total = 0;
+    pg.has_next = false;
-    state.pagination.page = 1;
+    pg.has_prev = false;
-    state.pagination.total_pages = 0;
-    state.pagination.has_next = false;
-    state.pagination.has_prev = false;
   };
 
   const addAll = (type: StateType, data: Record<KeyType, StoreItem>): void => {
@@ -99,8 +106,9 @@ export const useStateStore = defineStore('state', () => {
   };
 
   const count = (type: StateType): number => {
-    if ('history' === type && state.pagination.total > 0) {
+    const pg = paginationFor(type);
-      return state.pagination.total;
+    if (pg.total > 0) {
+      return pg.total;
     }
     return Object.keys(state[type]).length;
   };
@@ -113,15 +121,14 @@ export const useStateStore = defineStore('state', () => {
     append: boolean = false,
     status?: string,
   ): Promise<void> => {
-    if ('history' !== type) {
+    const pg = paginationFor(type);
-      throw new Error('Pagination is only supported for history type');
+    pg.isLoading = true;
-    }
 
-    state.pagination.isLoading = true;
+    const apiType = type === 'queue' ? 'queue' : 'done';
 
     try {
       const params: Record<string, string> = {
-        type: 'done',
+        type: apiType,
         page: page.toString(),
         per_page: per_page.toString(),
         order,
@@ -137,7 +144,7 @@ export const useStateStore = defineStore('state', () => {
       const data = await response.json();
 
       if (data.pagination) {
-        state.pagination = { ...data.pagination, isLoaded: true, isLoading: false };
+        Object.assign(pg, { ...data.pagination, isLoaded: true, isLoading: false });
         const items: Record<KeyType, StoreItem> = {};
         for (const item of data.items || []) {
           items[item._id] = item;
@@ -147,47 +154,47 @@ export const useStateStore = defineStore('state', () => {
       }
     } catch (error) {
       console.error(`Failed to load ${type} page ${page}:`, error);
-      state.pagination.isLoading = false;
+      pg.isLoading = false;
     }
   };
 
+  const orderFor = (type: StateType): 'ASC' | 'DESC' => {
+    return type === 'queue' ? 'ASC' : 'DESC';
+  };
+
   const loadNextPage = async (type: StateType, append: boolean = false): Promise<void> => {
-    if ('history' !== type) {
+    const pg = paginationFor(type);
-      throw new Error('Pagination is only supported for history type');
-    }
 
-    if (!state.pagination.has_next || state.pagination.isLoading) {
+    if (!pg.has_next || pg.isLoading) {
       return;
     }
 
-    await loadPaginated(type, state.pagination.page + 1, state.pagination.per_page, 'DESC', append);
+    await loadPaginated(type, pg.page + 1, pg.per_page, orderFor(type), append);
   };
 
   const loadPreviousPage = async (type: StateType): Promise<void> => {
-    if ('history' !== type) {
+    const pg = paginationFor(type);
-      throw new Error('Pagination is only supported for history type');
-    }
 
-    if (!state.pagination.has_prev || state.pagination.isLoading) {
+    if (!pg.has_prev || pg.isLoading) {
       return;
     }
 
-    await loadPaginated(type, state.pagination.page - 1, state.pagination.per_page);
+    await loadPaginated(type, pg.page - 1, pg.per_page, orderFor(type));
   };
 
   const reloadCurrentPage = async (type: StateType): Promise<void> => {
-    if ('history' !== type) {
+    const pg = paginationFor(type);
-      throw new Error('Pagination is only supported for history type');
+    if (!pg.isLoaded) {
-    }
-    if (!state.pagination.isLoaded) {
       return;
     }
 
-    await loadPaginated(type, state.pagination.page, state.pagination.per_page);
+    await loadPaginated(type, pg.page, pg.per_page, orderFor(type));
   };
 
   const getPagination = () => state.pagination;
 
+  const getQueuePagination = () => state.queue_pagination;
+
   const setHistoryCount = (count: number) => {
     state.pagination.total = count;
     if (count > 0 && !state.pagination.isLoaded) {
@@ -196,25 +203,12 @@ export const useStateStore = defineStore('state', () => {
   };
 
   /**
-   * Load queue data from REST API.
+   * Load queue data from REST API using the paginated history endpoint.
-   * Uses the /live endpoint to get real-time in-memory data with live progress.
    *
    * @returns Promise that resolves when queue is loaded
    */
   const loadQueue = async (): Promise<void> => {
-    try {
+    await loadPaginated('queue', 1, state.queue_pagination.per_page, 'ASC');
-      const response = await request('/api/history/live');
-      const data = (await response.json()) as {
-        queue: Record<KeyType, StoreItem>;
-        history_count: number;
-      };
-
-      state.queue = data.queue || {};
-      setHistoryCount(data.history_count);
-    } catch (error) {
-      console.error('Failed to load queue:', error);
-      throw error;
-    }
   };
 
   /**
@@ -519,6 +513,7 @@ export const useStateStore = defineStore('state', () => {
     loadPreviousPage,
     reloadCurrentPage,
     getPagination,
+    getQueuePagination,
     setHistoryCount,
     loadQueue,
     addDownload,