From e9190b6dab2095aab08e7c946ad7db17dfa58a83 Mon Sep 17 00:00:00 2001
From: arabcoders <admin@arabcoders.org>
Date: Sat, 23 Aug 2025 21:41:37 +0300
Subject: [PATCH] added better fallback to hls.

---
 .vscode/settings.json             |   1 +
 ui/app/components/VideoPlayer.vue | 358 ++++++++++++++++++++++++------
 ui/app/types/video.d.ts           |  85 +++++++
 3 files changed, 379 insertions(+), 65 deletions(-)
 create mode 100644 ui/app/types/video.d.ts

diff --git a/.vscode/settings.json b/.vscode/settings.json
index b375b46a..a07cee3c 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -95,6 +95,7 @@
     "rejecttitle",
     "remux",
     "rtime",
+    "rvfc",
     "SIGUSR",
     "smhd",
     "socketio",
diff --git a/ui/app/components/VideoPlayer.vue b/ui/app/components/VideoPlayer.vue
index 701960f6..85778a85 100644
--- a/ui/app/components/VideoPlayer.vue
+++ b/ui/app/components/VideoPlayer.vue
@@ -8,6 +8,7 @@
   max-width: 80vw;
 }
 </style>
+
 <template>
   <div v-if="infoLoaded">
     <video class="player" ref="video" :poster="uri(thumbnail)" :title="title" playsinline controls
@@ -28,32 +29,10 @@
 <script setup lang="ts">
 import { useStorage } from '@vueuse/core'
 import Hls from 'hls.js'
-import type { StoreItem } from '~/types/store'
 import { disableOpacity, enableOpacity } from '~/utils'
 
-type video_track_element = {
-  file: string,
-  kind: string,
-  label: string,
-  lang: string,
-}
-
-type video_source_element = {
-  src: string,
-  type: string,
-  onerror: (e: Event) => void,
-}
-
-type file_info = {
-  title: string,
-  mimetype: string,
-  sidecar?: {
-    image?: Array<{ file: string }>,
-    text?: Array<{ file: string }>,
-    subtitle?: Array<{ name: string, lang: string, file: string }>,
-  }
-  error?: string,
-}
+import type { StoreItem } from '~/types/store'
+import type { file_info, video_source_element, video_track_element } from '~/types/video'
 
 const config = useConfigStore()
 const toast = useNotification()
@@ -67,7 +46,7 @@ const props = defineProps({
 
 const emitter = defineEmits(['closeModel'])
 
-const video = useTemplateRef<HTMLMediaElement>('video')
+const video = useTemplateRef<HTMLVideoElement>('video')
 const tracks = ref<Array<video_track_element>>([])
 const sources = ref<Array<video_source_element>>([])
 
@@ -75,43 +54,153 @@ const thumbnail = ref('/images/placeholder.png')
 const artist = ref('')
 const title = ref('')
 const isAudio = ref(false)
-const isApple = /(iPhone|iPod|iPad).*AppleWebKit/i.test(navigator.userAgent)
+const hasVideoStream = ref(false)
 const volume = useStorage('player_volume', 1)
 const notFirefox = !navigator.userAgent.toLowerCase().includes('firefox')
 const infoLoaded = ref(false)
 const destroyed = ref(false)
+const isApple = /(iPhone|iPod|iPad).*AppleWebKit/i.test(navigator.userAgent)
+const havePoster = ref(false)
 
+let unbindMediaSessionListeners: null | (() => void) = null
 let hls: Hls | null = null
+let detachDecodeGuard: null | (() => void) = null
 
 const handle_event = (e: KeyboardEvent) => {
-  if (e.key !== 'Escape') {
+  if ('Escape' !== e.key) {
     return
   }
   emitter('closeModel')
 }
 
+const bindMediaSessionListeners = (el: HTMLVideoElement) => {
+  const onLoadedMetadata = (e: Event) => updateMediaSessionPosition(e.currentTarget)
+  const onTimeUpdate = (e: Event) => updateMediaSessionPosition(e.currentTarget)
+  const onRateChange = (e: Event) => updateMediaSessionPosition(e.currentTarget)
+  const onSeeked = (e: Event) => updateMediaSessionPosition(e.currentTarget)
+  const onPause = async (e: Event) => {
+    const target = (e.currentTarget as HTMLVideoElement) ?? null
+    if (!target || destroyed.value) {
+      return
+    }
+    const dataUrl = await captureFrame(target)
+    if (dataUrl) {
+      thumbnail.value = dataUrl
+      havePoster.value = true
+      applyMediaSessionMetadata()
+    }
+  }
+
+  el.addEventListener('loadedmetadata', onLoadedMetadata)
+  el.addEventListener('timeupdate', onTimeUpdate)
+  el.addEventListener('ratechange', onRateChange)
+  el.addEventListener('seeked', onSeeked)
+  el.addEventListener('pause', onPause)
+
+  return () => {
+    el.removeEventListener('loadedmetadata', onLoadedMetadata)
+    el.removeEventListener('timeupdate', onTimeUpdate)
+    el.removeEventListener('ratechange', onRateChange)
+    el.removeEventListener('seeked', onSeeked)
+    el.removeEventListener('pause', onPause)
+  }
+}
+
+const updateMediaSessionPosition = (target: EventTarget | null) => {
+  if (false === ('mediaSession' in navigator)) {
+    return
+  }
+  const el = (target as HTMLVideoElement) ?? null
+  if (!el || destroyed.value) {
+    return
+  }
+  const d = el.duration
+  if (false === Number.isFinite(d) || 0 >= d) {
+    return
+  }
+  try {
+    navigator.mediaSession.setPositionState({
+      duration: d,
+      playbackRate: el.playbackRate,
+      position: el.currentTime,
+    })
+  } catch { }
+}
+
 const volume_change_handler = () => {
-  if (!video.value) {
+  const el = video.value
+  if (!el) {
+    return
+  }
+  volume.value = el.volume
+  updateMediaSessionPosition(el)
+}
+
+/**
+ * Unified frame capture helper (merges previous captureCurrentFrame/captureFirstFramePoster logic).
+ * Returns a JPEG data URL or '' on failure.
+ */
+const captureFrame = async (el: HTMLVideoElement): Promise<string> => {
+  if (!el || destroyed.value) {
+    return ''
+  }
+  if (0 === el.videoWidth || 0 === el.videoHeight) {
+    return ''
+  }
+
+  const w = el.videoWidth
+  const h = el.videoHeight
+
+  try {
+    if ('OffscreenCanvas' in window) {
+      const c = new (window as any).OffscreenCanvas(w, h)
+      const ctx = c.getContext('2d')
+      if (!ctx) { return '' }
+      ctx.drawImage(el, 0, 0, w, h)
+      const blob = await c.convertToBlob({ type: 'image/jpeg', quality: 0.86 })
+      return await new Promise<string>(r => {
+        const fr = new FileReader()
+        fr.onload = () => r(String(fr.result))
+        fr.readAsDataURL(blob)
+      })
+    } else {
+      const c = document.createElement('canvas')
+      c.width = w
+      c.height = h
+      const ctx = c.getContext('2d')
+      if (!ctx) { return '' }
+      ctx.drawImage(el, 0, 0, w, h)
+      return c.toDataURL('image/jpeg', 0.86)
+    }
+  } catch {
+    return ''
+  }
+}
+
+const captureFirstFramePoster = async (el: HTMLVideoElement): Promise<void> => {
+  if (!el || destroyed.value) {
+    return
+  }
+  if (havePoster.value) {
+    return
+  }
+  if (0 === el.videoWidth || 0 === el.videoHeight) {
     return
   }
 
-  volume.value = video.value.volume
-
-  if (false === ("mediaSession" in navigator)) {
+  const dataUrl = await captureFrame(el)
+  if (!dataUrl) {
     return
   }
 
-  navigator.mediaSession.setPositionState({
-    duration: video.value.duration,
-    playbackRate: video.value.playbackRate,
-    position: video.value.currentTime,
-  })
+  thumbnail.value = dataUrl
+  havePoster.value = true
+  applyMediaSessionMetadata()
 }
 
 onMounted(async () => {
   disableOpacity()
   const req = await request(makeDownload(config, props.item, 'api/file/info'))
-
   const response: file_info = await req.json()
 
   if (!req.ok) {
@@ -124,24 +213,37 @@ onMounted(async () => {
 
   if (props.item.extras?.thumbnail) {
     thumbnail.value = '/api/thumbnail?url=' + encodePath(props.item.extras.thumbnail)
+    havePoster.value = true
   } else {
     if (response.sidecar?.image?.[0]?.file) {
-      thumbnail.value = makeDownload(config, { "filename": response.sidecar.image[0].file })
+      thumbnail.value = makeDownload(config, { 'filename': response.sidecar.image[0].file })
+      havePoster.value = true
+    }
+  }
+
+  hasVideoStream.value = Array.isArray(response.ffprobe?.video)
+    && response.ffprobe.video.some(s => 'video' === s.codec_type)
+
+  if (!props.item.extras?.is_video && props.item.extras?.is_audio) {
+    isAudio.value = true
+  } else {
+    if (false === hasVideoStream.value) {
+      isAudio.value = true
     }
   }
 
-  // -- check if mimetype is video/mp4 and device is apple
-  // -- as always apple, apple like to be snowflakes.
   if (isApple) {
     const allowedCodec = response.mimetype && response.mimetype.includes('video/mp4')
+    const src = makeDownload(config, props.item, allowedCodec ? 'api/download' : 'm3u8')
     sources.value.push({
-      src: makeDownload(config, props.item, allowedCodec ? 'api/download' : 'm3u8'),
+      src,
       type: allowedCodec ? response.mimetype : 'application/x-mpegURL',
       onerror: (e: Event) => src_error(),
     })
   } else {
+    const src = makeDownload(config, props.item, 'api/download')
     sources.value.push({
-      src: makeDownload(config, props.item, 'api/download'),
+      src,
       type: response.mimetype,
       onerror: e => src_error(),
     })
@@ -157,20 +259,19 @@ onMounted(async () => {
 
   if (props.item?.title) {
     title.value = props.item.title
-  }
-  else {
+  } else {
     if (response?.title) {
       title.value = response.title
+    } else {
+      if (response.ffprobe?.metadata?.tags?.title) {
+        title.value = response.ffprobe.metadata.tags.title
+      }
     }
   }
 
-  if (!props.item.extras?.is_video && props.item.extras?.is_audio) {
-    isAudio.value = true
-  }
-
   response.sidecar?.subtitle?.forEach((cap, _) => {
     tracks.value.push({
-      kind: "captions",
+      kind: 'captions',
       label: cap.name,
       lang: cap.lang,
       file: `${makeDownload(config, { filename: cap.file }, 'api/player/subtitle')}.vtt`
@@ -185,9 +286,28 @@ onMounted(async () => {
   await nextTick()
 
   prepareVideoPlayer()
+
+  if (video.value) {
+    unbindMediaSessionListeners = bindMediaSessionListeners(video.value)
+  }
+
   document.addEventListener('keydown', handle_event)
 })
 
+const applyMediaSessionMetadata = () => {
+  if (false === ('mediaSession' in navigator)) {
+    return
+  }
+  const meta: MediaMetadataInit = { title: title.value }
+  if (artist.value) {
+    meta.artist = artist.value
+  }
+  if (thumbnail.value) {
+    meta.artwork = [{ src: thumbnail.value, sizes: '1920x1080', type: 'image/jpeg' }]
+  }
+  try { navigator.mediaSession.metadata = new MediaMetadata(meta) } catch { }
+}
+
 onUpdated(() => prepareVideoPlayer())
 
 onBeforeUnmount(() => {
@@ -198,6 +318,11 @@ onBeforeUnmount(() => {
 
   document.removeEventListener('keydown', handle_event)
 
+  if (unbindMediaSessionListeners) {
+    unbindMediaSessionListeners()
+    unbindMediaSessionListeners = null
+  }
+
   if (title.value) {
     window.document.title = 'YTPTube'
   }
@@ -209,8 +334,12 @@ onBeforeUnmount(() => {
 
   try {
     video.value.pause()
-    video.value.querySelectorAll("source").forEach(source => source.removeAttribute("src"))
+    video.value.querySelectorAll('source').forEach(source => source.removeAttribute('src'))
     video.value.removeEventListener('volumechange', volume_change_handler)
+    if (detachDecodeGuard) {
+      detachDecodeGuard()
+      detachDecodeGuard = null
+    }
     video.value.load()
   }
   catch (e) {
@@ -223,21 +352,8 @@ const prepareVideoPlayer = () => {
     return
   }
 
-  if (false === ("mediaSession" in navigator)) {
-    return
-  }
+  applyMediaSessionMetadata()
 
-  let mediaMetadata: MediaMetadataInit = { title: title.value }
-
-  if (thumbnail.value) {
-    mediaMetadata.artwork = [{ src: thumbnail.value, sizes: '1920x1080', type: 'image/jpeg' }]
-  }
-
-  if (artist.value) {
-    mediaMetadata.artist = artist.value
-  }
-
-  navigator.mediaSession.metadata = new MediaMetadata(mediaMetadata)
   if (title.value) {
     window.document.title = `YTPTube - Playing: ${title.value}`
   }
@@ -248,6 +364,102 @@ const prepareVideoPlayer = () => {
 
   video.value.volume = volume.value
   video.value.addEventListener('volumechange', volume_change_handler)
+
+  if (detachDecodeGuard) {
+    detachDecodeGuard()
+    detachDecodeGuard = null
+  }
+
+  if (hasVideoStream.value) {
+    if ('requestVideoFrameCallback' in video.value) {
+      ; (video.value as any).requestVideoFrameCallback(() => captureFirstFramePoster(video.value!))
+    } else {
+      const tryOnce = () => captureFirstFramePoster(video.value!)
+        ; (video.value as any).addEventListener('loadeddata', tryOnce, { once: true })
+    }
+
+    detachDecodeGuard = attachDecodeFailGuard(video.value, () => src_error())
+  }
+}
+
+function attachDecodeFailGuard(el: HTMLVideoElement, onFail: () => void): () => void {
+  let rvfcId: number | null = null
+  let timer: number | null = null
+  let armed = false
+  let done = false
+
+  const clearAll = () => {
+    if (null !== timer) {
+      clearTimeout(timer)
+      timer = null
+    }
+    if (null !== rvfcId && 'cancelVideoFrameCallback' in el) {
+      ; (el as any).cancelVideoFrameCallback(rvfcId)
+      rvfcId = null
+    }
+    el.removeEventListener('loadeddata', onLoadedData)
+    el.removeEventListener('error', onError)
+    el.removeEventListener('emptied', onError)
+  }
+
+  const ok = () => {
+    if (done) return
+    done = true
+    clearAll()
+  }
+
+  const fail = () => {
+    if (done) return
+    done = true
+    clearAll()
+    onFail()
+  }
+
+  const decodedFrames = (): number => {
+    try {
+      const q = 'getVideoPlaybackQuality' in el ? (el as any).getVideoPlaybackQuality() : null
+      if (q && 'totalVideoFrames' in q) {
+        return Number(q.totalVideoFrames || 0)
+      }
+      const anyEl = el as any
+      if ('webkitDecodedFrameCount' in anyEl) {
+        return Number(anyEl.webkitDecodedFrameCount || 0)
+      }
+    } catch { }
+    return 0
+  }
+
+  const armCheck = () => {
+    if (armed || done) return
+    armed = true
+
+    if ('requestVideoFrameCallback' in el) {
+      rvfcId = (el as any).requestVideoFrameCallback(() => ok())
+      timer = window.setTimeout(() => fail(), 1000)
+      return
+    }
+
+    timer = window.setTimeout(() => {
+      if (0 === (el as any).videoWidth || 0 === (el as any).videoHeight || 0 === decodedFrames()) {
+        fail()
+      } else {
+        ok()
+      }
+    }, 800)
+  }
+
+  const onLoadedData = () => armCheck()
+  const onError = () => fail()
+
+  el.addEventListener('loadeddata', onLoadedData, { once: true })
+  el.addEventListener('error', onError)
+  el.addEventListener('emptied', onError)
+
+  if (4 <= el.readyState || 2 <= el.readyState) {
+    queueMicrotask(armCheck)
+  }
+
+  return clearAll
 }
 
 const src_error = async () => {
@@ -258,7 +470,8 @@ const src_error = async () => {
   if (destroyed.value) {
     return
   }
-  console.warn('Direct play failed, trying HLS.')
+
+  console.warn('Source failed to load, attempting HLS fallback...')
   attach_hls(makeDownload(config, props.item, 'm3u8'))
 }
 
@@ -275,6 +488,21 @@ const attach_hls = (link: string) => {
     fragLoadingTimeOut: 200000,
   })
 
+  hls.on(Hls.Events.MANIFEST_PARSED, () => {
+    applyMediaSessionMetadata()
+  })
+
+  hls.on(Hls.Events.LEVEL_LOADED, () => {
+    if (video.value) {
+      if ('requestVideoFrameCallback' in video.value) {
+        ; (video.value as any).requestVideoFrameCallback(() => captureFirstFramePoster(video.value!))
+      } else {
+        const once = () => captureFirstFramePoster(video.value!)
+          ; (video.value as any).addEventListener('loadeddata', once, { once: true })
+      }
+    }
+  })
+
   hls.loadSource(link)
   hls.attachMedia(video.value)
 }
diff --git a/ui/app/types/video.d.ts b/ui/app/types/video.d.ts
new file mode 100644
index 00000000..c08f9205
--- /dev/null
+++ b/ui/app/types/video.d.ts
@@ -0,0 +1,85 @@
+
+type video_track_element = {
+  file: string,
+  kind: string,
+  label: string,
+  lang: string,
+}
+
+type video_source_element = {
+  src: string,
+  type: string,
+  onerror: (e: Event) => void,
+}
+
+type FFProbeTags = {
+  major_brand?: string,
+  minor_version?: string,
+  compatible_brands?: string,
+  encoder?: string,
+  title?: string,
+  artist?: string,
+  album_artist?: string,
+  album?: string,
+  date?: string,
+  track?: string,
+  genre?: string,
+}
+
+type FFProbeMetadata = {
+  filename: string,
+  nb_streams: number,
+  nb_programs: number,
+  format_name: string,
+  format_long_name: string,
+  start_time: string,
+  duration: string,
+  size: string,
+  bit_rate: string,
+  tags: FFProbeTags
+}
+
+type FFProbeStream = {
+  index: number,
+  codec_name: string,
+  codec_type: string,
+  width?: number,
+  height?: number,
+  tags?: {
+    language?: string,
+    title?: string,
+    handler_name?: string,
+  },
+  channels?: number,
+  sample_rate?: string,
+  bit_rate?: string,
+}
+
+type FFProbeResult = {
+  metadata: FFProbeMetadata,
+  video: Array<FFProbeStream>,
+  audio: Array<FFProbeStream>,
+  subtitle: Array<FFProbeStream>,
+  attachment: Array<FFProbeStream>,
+  is_video: boolean,
+  is_audio: boolean,
+}
+
+type file_info = {
+  title: string,
+  ffprobe: FFProbeResult,
+  mimetype: string,
+  sidecar?: {
+    image?: Array<{ file: string }>,
+    text?: Array<{ file: string }>,
+    subtitle?: Array<{ name: string, lang: string, file: string }>,
+  },
+  error?: string,
+}
+
+export type {
+  video_track_element,
+  video_source_element,
+  FFProbeResult,
+  file_info,
+};