Decouple queue from config endpoint; load via paginated API with correct ordering (#2)

* Initial plan * Refactor: remove queue from /api/system/configuration, use existing paginated /api/history endpoint for queue data Co-authored-by: jbatesy <51190172+jbatesy@users.noreply.github.com> * Remove accidentally committed package-lock.json and add to gitignore Co-authored-by: jbatesy <51190172+jbatesy@users.noreply.github.com> * Simplify loadMoreQueue to use loadNextPage for reduced duplication Co-authored-by: jbatesy <51190172+jbatesy@users.noreply.github.com> * Order in-progress downloads oldest-to-newest (order of processing) Co-authored-by: jbatesy <51190172+jbatesy@users.noreply.github.com> * Fix queue ordering: remove client-side timestamp sort, rely on backend created_at ASC Co-authored-by: jbatesy <51190172+jbatesy@users.noreply.github.com> * Revert "Fix queue ordering: remove client-side timestamp sort, rely on backend created_at ASC" This reverts commit 9b53112e0050a45e4db95c85064577d22ed47a50. Changes not needed --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: jbatesy <51190172+jbatesy@users.noreply.github.com> Co-authored-by: Jesse Bate <noreply@idump.me>
2026-03-09 15:05:22 +10:30 · 2026-03-09 15:05:22 +10:30 · b9c5b0b7ad
commit b9c5b0b7ad
parent ee2bddcb81
7 changed files with 175 additions and 85 deletions
--- a/.gitignore
+++ b/.gitignore
@ -40,3 +40,4 @@ version.txt
 test_impl.py
 ./eslint.config.js
 .pytest_cache
+ui/package-lock.json
--- a/app/routes/api/system.py
+++ b/app/routes/api/system.py
@ -32,7 +32,7 @@ LOG: logging.Logger = logging.getLogger(__name__)
@route("GET", "api/system/configuration", "system.configuration")
 async def system_config(queue: DownloadQueue, config: Config, encoder: Encoder) -> Response:
    """
-    Pause non-active downloads.
+    Get the system configuration.

    Args:
        queue (DownloadQueue): The download queue instance.
@ -55,7 +55,6 @@ async def system_config(queue: DownloadQueue, config: Config, encoder: Encoder)
                depth_limit=config.download_path_depth - 1,
            ),
            "history_count": await queue.done.get_total_count(),
-            "queue": (await queue.get("queue"))["queue"],
        },
        status=web.HTTPOk.status_code,
        dumps=encoder.encode,
--- a/app/tests/test_system_routes.py
+++ b/app/tests/test_system_routes.py
@ -1,10 +1,52 @@
+import json
+
 import pytest
-from unittest.mock import AsyncMock, patch
+from unittest.mock import AsyncMock, MagicMock, patch

 from app.library.config import Config
 from app.library.encoder import Encoder
 from app.library.UpdateChecker import UpdateChecker
-from app.routes.api.system import check_updates
+from app.routes.api.system import check_updates, system_config
+
+
+class TestSystemConfigEndpoint:
+    """Tests for the system configuration endpoint."""
+
+    def setup_method(self):
+        """Reset singletons before each test."""
+        Config._reset_singleton()
+
+    @pytest.mark.asyncio
+    async def test_system_config_does_not_return_queue(self):
+        """Test that the configuration endpoint does not include queue data."""
+        config = Config.get_instance()
+        encoder = Encoder()
+
+        mock_queue = MagicMock()
+        mock_queue.is_paused.return_value = False
+        mock_done = AsyncMock()
+        mock_done.get_total_count = AsyncMock(return_value=0)
+        mock_queue.done = mock_done
+
+        with (
+            patch("app.routes.api.system.Presets") as mock_presets_cls,
+            patch("app.routes.api.system.DLFields") as mock_dl_fields_cls,
+            patch("app.routes.api.system.list_folders", return_value=[]),
+        ):
+            mock_presets_cls.get_instance.return_value.get_all.return_value = []
+            mock_dl_fields_cls.get_instance.return_value.get_all_serialized = AsyncMock(return_value=[])
+
+            response = await system_config(mock_queue, config, encoder)
+
+            assert 200 == response.status
+            body = json.loads(response.body.decode("utf-8"))
+            assert "queue" not in body, "Configuration response should not include queue data"
+            assert "app" in body, "Configuration response should include app data"
+            assert "paused" in body, "Configuration response should include paused status"
+            assert "history_count" in body, "Configuration response should include history_count"
+            assert "presets" in body, "Configuration response should include presets"
+            assert "dl_fields" in body, "Configuration response should include dl_fields"
+            assert "folders" in body, "Configuration response should include folders"


 class TestCheckUpdatesEndpoint:
--- a/ui/app/components/Queue.vue
+++ b/ui/app/components/Queue.vue
@ -470,6 +470,21 @@
    </div>
  </div>

+  <div
+    v-if="queuePagination.isLoaded && queuePagination.page < queuePagination.total_pages"
+    ref="loadMoreTrigger"
+    class="columns is-centered mt-4"
+  >
+    <div class="column is-narrow">
+      <div v-if="queuePagination.isLoading" class="has-text-centered">
+        <span class="icon is-large has-text-info">
+          <i class="fas fa-spinner fa-pulse fa-2x"></i>
+        </span>
+        <p class="is-size-7 has-text-grey mt-2">Loading more items...</p>
+      </div>
+    </div>
+  </div>
+
  <div class="modal is-active" v-if="embed_url">
    <div class="modal-background" @click="embed_url = ''"></div>
    <div class="modal-content is-unbounded-model">
@ -481,7 +496,7 @@

 <script setup lang="ts">
 import moment from 'moment';
-import { useStorage } from '@vueuse/core';
+import { useStorage, useIntersectionObserver } from '@vueuse/core';
 import type { StoreItem } from '~/types/store';
 import { useConfirm } from '~/composables/useConfirm';
 import { deepIncludes } from '~/utils';
@ -514,12 +529,15 @@ const selectedElms = ref<string[]>([]);
 const masterSelectAll = ref(false);
 const embed_url = ref('');
 const isRefreshing = ref(false);
+const loadMoreTrigger = ref<HTMLElement | null>(null);
 const autoRefreshInterval = ref<NodeJS.Timeout | null>(null);
 const autoRefreshEnabled = useStorage<boolean>('queue_auto_refresh', true);
 const autoRefreshDelay = useStorage<number>('queue_auto_refresh_delay', 10000);

 const showThumbnails = computed(() => !!props.thumbnails && !hideThumbnail.value);

+const queuePagination = computed(() => stateStore.getQueuePagination());
+
 const refreshQueue = async () => {
  isRefreshing.value = true;
  try {
@ -531,6 +549,14 @@ const refreshQueue = async () => {
  }
 };

+const loadMoreQueue = async () => {
+  try {
+    await stateStore.loadNextPage('queue', true);
+  } catch {
+    toast.error('Failed to load more queue items');
+  }
+};
+
 const startAutoRefresh = () => {
  if (autoRefreshInterval.value) {
    clearInterval(autoRefreshInterval.value);
@ -573,7 +599,14 @@ watch(autoRefreshEnabled, (enabled) => {
  }
 });

-onMounted(() => {
+onMounted(async () => {
+  if (!queuePagination.value.isLoaded) {
+    try {
+      await stateStore.loadPaginated('queue', 1, config.app.default_pagination, 'ASC', false);
+    } catch {
+      toast.error('Failed to load queue');
+    }
+  }
  if (!socket.isConnected && autoRefreshEnabled.value) {
    startAutoRefresh();
  }
@ -581,6 +614,20 @@ onMounted(() => {

 onBeforeUnmount(() => stopAutoRefresh());

+useIntersectionObserver(
+  loadMoreTrigger,
+  ([entry]) => {
+    if (
+      entry?.isIntersecting &&
+      !queuePagination.value.isLoading &&
+      queuePagination.value.page < queuePagination.value.total_pages
+    ) {
+      loadMoreQueue();
+    }
+  },
+  { threshold: 0.5 },
+);
+
 watch(masterSelectAll, (value) => {
  if (value) {
    selectedElms.value = Object.values(stateStore.queue).map((element: StoreItem) => element._id);
@ -590,13 +637,14 @@ watch(masterSelectAll, (value) => {
 });

 const filteredItems = computed<StoreItem[]>(() => {
+  const items = Object.values(stateStore.queue)
+    .slice()
+    .sort((a, b) => (a.timestamp ?? 0) - (b.timestamp ?? 0));
  const q = props.query?.toLowerCase();
  if (!q) {
-    return Object.values(stateStore.queue);
+    return items;
  }
-  return Object.values(stateStore.queue).filter((i: StoreItem) =>
-    deepIncludes(i, q, new WeakSet()),
-  );
+  return items.filter((i: StoreItem) => deepIncludes(i, q, new WeakSet()));
 });

 const hasSelected = computed(() => 0 < selectedElms.value.length);
--- a/ui/app/components/Simple.vue
+++ b/ui/app/components/Simple.vue
@ -407,6 +407,8 @@ const addInProgress = ref<boolean>(false);
 const showExtras = ref<boolean>(false);
 const isRefreshing = ref<boolean>(false);

+const queuePaginationInfo = computed(() => stateStore.getQueuePagination());
+
 const refreshQueue = async (): Promise<void> => {
  if (isRefreshing.value) {
    return;
@ -426,7 +428,7 @@ const paginationInfo = computed(() => stateStore.getPagination());
 const queueItems = computed<StoreItem[]>(() =>
  Object.values(queue.value ?? {})
    .slice()
-    .sort((a, b) => (b.timestamp ?? 0) - (a.timestamp ?? 0)),
+    .sort((a, b) => (a.timestamp ?? 0) - (b.timestamp ?? 0)),
 );
 const historyEntries = computed<StoreItem[]>(() => {
  const items = Object.values(history.value ?? {});
@ -857,7 +859,7 @@ const connectionStatusColor = computed(() => {
  }
 });

-// Load history via API on mount
+// Load queue and history via API on mount
 onMounted(async () => {
  const route = useRoute();

@ -870,6 +872,14 @@ onMounted(async () => {
    window.history.replaceState({}, '', url.toString());
  }

+  if (!queuePaginationInfo.value.isLoaded) {
+    try {
+      await stateStore.loadPaginated('queue', 1, DEFAULT_PAGE_SIZE, 'ASC');
+    } catch (error) {
+      console.error('Failed to load queue on mount:', error);
+    }
+  }
+
  if (!paginationInfo.value.isLoaded) {
    try {
      await stateStore.loadPaginated('history', 1, DEFAULT_PAGE_SIZE, 'DESC');
--- a/ui/app/stores/ConfigStore.ts
+++ b/ui/app/stores/ConfigStore.ts
@ -85,11 +85,6 @@ export const useConfigStore = defineStore('config', () => {
        delete data.history_count;
      }

-      if (data.queue) {
-        stateStore.addAll('queue', data.queue);
-        delete data.queue;
-      }
-
      setAll(data);
      state.is_loaded = true;
      last_reload = now;
--- a/ui/app/stores/StateStore.ts
+++ b/ui/app/stores/StateStore.ts
@ -6,35 +6,41 @@ import { request } from '~/utils';
 type StateType = 'queue' | 'history';
 type KeyType = string;

+interface PaginationState {
+  page: number;
+  per_page: number;
+  total: number;
+  total_pages: number;
+  has_next: boolean;
+  has_prev: boolean;
+  isLoaded: boolean;
+  isLoading: boolean;
+}
+
 interface State {
  queue: Record<KeyType, StoreItem>;
  history: Record<KeyType, StoreItem>;
-  pagination: {
-    page: number;
-    per_page: number;
-    total: number;
-    total_pages: number;
-    has_next: boolean;
-    has_prev: boolean;
-    isLoaded: boolean;
-    isLoading: boolean;
-  };
+  pagination: PaginationState;
+  queue_pagination: PaginationState;
 }

+const defaultPagination = (): PaginationState => ({
+  page: 1,
+  per_page: 50,
+  total: 0,
+  total_pages: 0,
+  has_next: false,
+  has_prev: false,
+  isLoaded: false,
+  isLoading: false,
+});
+
 export const useStateStore = defineStore('state', () => {
  const state = reactive<State>({
    queue: {},
    history: {},
-    pagination: {
-      page: 1,
-      per_page: 50,
-      total: 0,
-      total_pages: 0,
-      has_next: false,
-      has_prev: false,
-      isLoaded: false,
-      isLoading: false,
-    },
+    pagination: defaultPagination(),
+    queue_pagination: defaultPagination(),
  });

  const add = (type: StateType, key: KeyType, value: StoreItem): void => {
@ -73,17 +79,18 @@ export const useStateStore = defineStore('state', () => {
    return !!state[type][key];
  };

+  const paginationFor = (type: StateType): PaginationState => {
+    return type === 'queue' ? state.queue_pagination : state.pagination;
+  };
+
  const clearAll = (type: StateType): void => {
    state[type] = {};
-    if ('queue' === type) {
-      return;
-    }
-
-    state.pagination.total = 0;
-    state.pagination.page = 1;
-    state.pagination.total_pages = 0;
-    state.pagination.has_next = false;
-    state.pagination.has_prev = false;
+    const pg = paginationFor(type);
+    pg.total = 0;
+    pg.page = 1;
+    pg.total_pages = 0;
+    pg.has_next = false;
+    pg.has_prev = false;
  };

  const addAll = (type: StateType, data: Record<KeyType, StoreItem>): void => {
@ -99,8 +106,9 @@ export const useStateStore = defineStore('state', () => {
  };

  const count = (type: StateType): number => {
-    if ('history' === type && state.pagination.total > 0) {
-      return state.pagination.total;
+    const pg = paginationFor(type);
+    if (pg.total > 0) {
+      return pg.total;
    }
    return Object.keys(state[type]).length;
  };
@ -113,15 +121,14 @@ export const useStateStore = defineStore('state', () => {
    append: boolean = false,
    status?: string,
  ): Promise<void> => {
-    if ('history' !== type) {
-      throw new Error('Pagination is only supported for history type');
-    }
+    const pg = paginationFor(type);
+    pg.isLoading = true;

-    state.pagination.isLoading = true;
+    const apiType = type === 'queue' ? 'queue' : 'done';

    try {
      const params: Record<string, string> = {
-        type: 'done',
+        type: apiType,
        page: page.toString(),
        per_page: per_page.toString(),
        order,
@ -137,7 +144,7 @@ export const useStateStore = defineStore('state', () => {
      const data = await response.json();

      if (data.pagination) {
-        state.pagination = { ...data.pagination, isLoaded: true, isLoading: false };
+        Object.assign(pg, { ...data.pagination, isLoaded: true, isLoading: false });
        const items: Record<KeyType, StoreItem> = {};
        for (const item of data.items || []) {
          items[item._id] = item;
@ -147,47 +154,47 @@ export const useStateStore = defineStore('state', () => {
      }
    } catch (error) {
      console.error(`Failed to load ${type} page ${page}:`, error);
-      state.pagination.isLoading = false;
+      pg.isLoading = false;
    }
  };

+  const orderFor = (type: StateType): 'ASC' | 'DESC' => {
+    return type === 'queue' ? 'ASC' : 'DESC';
+  };
+
  const loadNextPage = async (type: StateType, append: boolean = false): Promise<void> => {
-    if ('history' !== type) {
-      throw new Error('Pagination is only supported for history type');
-    }
+    const pg = paginationFor(type);

-    if (!state.pagination.has_next || state.pagination.isLoading) {
+    if (!pg.has_next || pg.isLoading) {
      return;
    }

-    await loadPaginated(type, state.pagination.page + 1, state.pagination.per_page, 'DESC', append);
+    await loadPaginated(type, pg.page + 1, pg.per_page, orderFor(type), append);
  };

  const loadPreviousPage = async (type: StateType): Promise<void> => {
-    if ('history' !== type) {
-      throw new Error('Pagination is only supported for history type');
-    }
+    const pg = paginationFor(type);

-    if (!state.pagination.has_prev || state.pagination.isLoading) {
+    if (!pg.has_prev || pg.isLoading) {
      return;
    }

-    await loadPaginated(type, state.pagination.page - 1, state.pagination.per_page);
+    await loadPaginated(type, pg.page - 1, pg.per_page, orderFor(type));
  };

  const reloadCurrentPage = async (type: StateType): Promise<void> => {
-    if ('history' !== type) {
-      throw new Error('Pagination is only supported for history type');
-    }
-    if (!state.pagination.isLoaded) {
+    const pg = paginationFor(type);
+    if (!pg.isLoaded) {
      return;
    }

-    await loadPaginated(type, state.pagination.page, state.pagination.per_page);
+    await loadPaginated(type, pg.page, pg.per_page, orderFor(type));
  };

  const getPagination = () => state.pagination;

+  const getQueuePagination = () => state.queue_pagination;
+
  const setHistoryCount = (count: number) => {
    state.pagination.total = count;
    if (count > 0 && !state.pagination.isLoaded) {
@ -196,25 +203,12 @@ export const useStateStore = defineStore('state', () => {
  };

  /**
-   * Load queue data from REST API.
-   * Uses the /live endpoint to get real-time in-memory data with live progress.
+   * Load queue data from REST API using the paginated history endpoint.
   *
   * @returns Promise that resolves when queue is loaded
   */
  const loadQueue = async (): Promise<void> => {
-    try {
-      const response = await request('/api/history/live');
-      const data = (await response.json()) as {
-        queue: Record<KeyType, StoreItem>;
-        history_count: number;
-      };
-
-      state.queue = data.queue || {};
-      setHistoryCount(data.history_count);
-    } catch (error) {
-      console.error('Failed to load queue:', error);
-      throw error;
-    }
+    await loadPaginated('queue', 1, state.queue_pagination.per_page, 'ASC');
  };

  /**
@ -519,6 +513,7 @@ export const useStateStore = defineStore('state', () => {
    loadPreviousPage,
    reloadCurrentPage,
    getPagination,
+    getQueuePagination,
    setHistoryCount,
    loadQueue,
    addDownload,