diff --git a/src/components/reader/SegmentsSidebar.tsx b/src/components/reader/SegmentsSidebar.tsx
index 5c9b2f1..8752f4a 100644
--- a/src/components/reader/SegmentsSidebar.tsx
+++ b/src/components/reader/SegmentsSidebar.tsx
@@ -185,6 +185,7 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId, epubBookRef }:
     isPlaying,
     playFromSegment,
     activeReaderType,
+    clearSegmentCaches,
   } = useTTS();
   const {
     providerRef,
@@ -241,6 +242,9 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId, epubBookRef }:
 
   const listRef = useRef<HTMLDivElement | null>(null);
   const didAutoScrollOnOpenRef = useRef(false);
+  const userScrollUntilMsRef = useRef(0);
+  const programmaticScrollUntilMsRef = useRef(0);
+  const lastSegmentRefreshKeyRef = useRef('');
   const segmentsQueryKey = useMemo(
     () => [SEGMENTS_MANIFEST_QUERY_KEY, documentId] as const,
     [documentId],
@@ -249,8 +253,6 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId, epubBookRef }:
   const segmentsQuery = useInfiniteQuery({
     queryKey: segmentsQueryKey,
     enabled: isOpen && !!documentId,
-    refetchInterval: isOpen ? 2500 : false,
-    refetchIntervalInBackground: false,
     initialPageParam: null as string | null,
     queryFn: async ({ pageParam, signal }) => {
       if (!documentId) {
@@ -321,6 +323,7 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId, epubBookRef }:
       return payload;
     },
     onSuccess: async (payload) => {
+      clearSegmentCaches();
       if (payload?.warning) {
         toast.error(`Segments cleared, but audio cleanup was partial: ${payload.warning}`);
       } else if (payload) {
@@ -354,20 +357,55 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId, epubBookRef }:
     await clearSegments();
   }, [documentId, isClearingSegments, clearSegments]);
 
+  useEffect(() => {
+    if (!isOpen || !isPlaying) return;
+    const locationKey = activeReaderType === 'epub'
+      ? String(currDocPage)
+      : String(currDocPageNumber);
+    const refreshKey = `${activeReaderType}|${locationKey}|${currentSentenceIndex}`;
+    if (lastSegmentRefreshKeyRef.current === refreshKey) return;
+    lastSegmentRefreshKeyRef.current = refreshKey;
+    void refetchManifest();
+  }, [
+    isOpen,
+    isPlaying,
+    activeReaderType,
+    currDocPage,
+    currDocPageNumber,
+    currentSentenceIndex,
+    refetchManifest,
+  ]);
+
   useEffect(() => {
     if (!isOpen) return;
     const node = listRef.current;
     if (!node) return;
 
+    const markUserScrollActive = () => {
+      userScrollUntilMsRef.current = Date.now() + 1200;
+    };
+
     const onScroll = () => {
+      if (Date.now() > programmaticScrollUntilMsRef.current) {
+        markUserScrollActive();
+      }
       if (!hasMoreManifestPages || isLoadingMoreManifest) return;
       const distance = node.scrollHeight - node.scrollTop - node.clientHeight;
       if (distance > 280) return;
       void fetchNextPage();
     };
 
+    const onWheel = () => markUserScrollActive();
+    const onTouchMove = () => markUserScrollActive();
+
     node.addEventListener('scroll', onScroll);
-    return () => node.removeEventListener('scroll', onScroll);
+    node.addEventListener('wheel', onWheel, { passive: true });
+    node.addEventListener('touchmove', onTouchMove, { passive: true });
+    return () => {
+      node.removeEventListener('scroll', onScroll);
+      node.removeEventListener('wheel', onWheel);
+      node.removeEventListener('touchmove', onTouchMove);
+    };
   }, [isOpen, hasMoreManifestPages, isLoadingMoreManifest, fetchNextPage]);
 
   const handleSelectVariant = useCallback(async (settings: TTSSegmentSettings | null) => {
@@ -553,10 +591,12 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId, epubBookRef }:
 
     const container = listRef.current;
     if (!container) return;
+    if (Date.now() < userScrollUntilMsRef.current) return;
     const activeRow = container.querySelector<HTMLElement>('[data-active-segment="true"]');
     if (!activeRow) return;
 
     requestAnimationFrame(() => {
+      programmaticScrollUntilMsRef.current = Date.now() + 300;
       activeRow.scrollIntoView({ block: 'center', behavior: 'auto' });
       didAutoScrollOnOpenRef.current = true;
     });
@@ -568,6 +608,7 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId, epubBookRef }:
 
     const root = listRef.current;
     if (!root) return;
+    if (Date.now() < userScrollUntilMsRef.current) return;
     const activeRow = root.querySelector<HTMLElement>('[data-active-segment="true"]');
     if (!activeRow) return;
 
@@ -576,6 +617,7 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId, epubBookRef }:
     if (isElementFullyVisibleWithinContainer(activeRow, scrollContainer)) return;
 
     requestAnimationFrame(() => {
+      programmaticScrollUntilMsRef.current = Date.now() + 300;
       activeRow.scrollIntoView({ block: 'center', behavior: 'auto' });
     });
   }, [
diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx
index d8b83eb..1972448 100644
--- a/src/contexts/TTSContext.tsx
+++ b/src/contexts/TTSContext.tsx
@@ -146,6 +146,7 @@ interface TTSContextType extends TTSPlaybackState {
   setSpeedAndRestart: (speed: number) => void;
   setAudioPlayerSpeedAndRestart: (speed: number) => void;
   setVoiceAndRestart: (voice: string) => void;
+  clearSegmentCaches: () => void;
   skipToLocation: (location: TTSLocation, shouldPause?: boolean) => void;
   registerLocationChangeHandler: (handler: ((location: TTSLocation) => void) | null) => void;  // EPUB-only: Handles chapter navigation
   registerEpubLocationWalker: (walker: EpubRenderedLocationWalker | null) => void;
@@ -2896,6 +2897,16 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
     setCurrentWordIndex(null);
   }, [abortAudio, clearWarmAudioCache, invalidatePlaybackRun, clearPendingEpubJump, bumpEpubPreloadGeneration]);
 
+  const clearSegmentCaches = useCallback(() => {
+    // Keep the current viewport/sentence list intact, but force all audio/manifest
+    // state to be re-resolved after a server-side clear.
+    abortAudio(true);
+    segmentManifestCacheRef.current.clear();
+    sentenceAlignmentCacheRef.current.clear();
+    setCurrentSentenceAlignment(undefined);
+    setCurrentWordIndex(null);
+  }, [abortAudio]);
+
   /**
    * Stops the current audio playback and starts playing from a specified index
    * 
@@ -3107,6 +3118,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
     setSpeedAndRestart,
     setAudioPlayerSpeedAndRestart,
     setVoiceAndRestart,
+    clearSegmentCaches,
     skipToLocation,
     registerLocationChangeHandler,
     registerEpubLocationWalker,
@@ -3137,6 +3149,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
     setSpeedAndRestart,
     setAudioPlayerSpeedAndRestart,
     setVoiceAndRestart,
+    clearSegmentCaches,
     skipToLocation,
     registerLocationChangeHandler,
     registerEpubLocationWalker,