Dashboard: YouTube date enricher as a 4th worker orb + Follow→Watchlist consistency
- The standalone enricher now reports orb telemetry: stats()/pause()/resume(); /enrichment/youtube/status (+pause/resume) special-cased in the route. Added the 4th header worker button (red ▶), WORKER_DEFS entry, and SERVICES wiring — it's not on the socket so video-enrichment.js polls it every 3s. It animates (idle → active orb + inbound pulses) while a followed channel's dates are being fetched, like the TMDB/TVDB/OMDb orbs. - Channel detail now uses the SAME standard watchlist button as shows/movies (library-artist-watchlist-btn, ✓/+ icon, 'In Watchlist'/'Watchlist') instead of a bespoke 'Follow' — consistency. Still wired to the channel follow action. 176 backend tests green; JS balanced; music untouched.
This commit is contained in:
parent
30dc587ebf
commit
b6d8f07516
9 changed files with 110 additions and 13 deletions
|
|
@ -30,6 +30,10 @@ def register_routes(bp):
|
||||||
from core.video.enrichment.engine import get_video_enrichment_engine
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
return get_video_enrichment_engine()
|
return get_video_enrichment_engine()
|
||||||
|
|
||||||
|
def _yt_enricher():
|
||||||
|
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||||
|
return get_youtube_date_enricher()
|
||||||
|
|
||||||
@bp.route("/enrichment/services", methods=["GET"])
|
@bp.route("/enrichment/services", methods=["GET"])
|
||||||
def video_enrichment_services():
|
def video_enrichment_services():
|
||||||
try:
|
try:
|
||||||
|
|
@ -108,6 +112,8 @@ def register_routes(bp):
|
||||||
|
|
||||||
@bp.route("/enrichment/<service>/status", methods=["GET"])
|
@bp.route("/enrichment/<service>/status", methods=["GET"])
|
||||||
def video_enrichment_status(service):
|
def video_enrichment_status(service):
|
||||||
|
if service == "youtube": # the standalone date enricher (not an engine worker)
|
||||||
|
return jsonify(_yt_enricher().stats())
|
||||||
w = engine().worker(service)
|
w = engine().worker(service)
|
||||||
if not w:
|
if not w:
|
||||||
return jsonify({"error": "unknown service"}), 404
|
return jsonify({"error": "unknown service"}), 404
|
||||||
|
|
@ -115,6 +121,9 @@ def register_routes(bp):
|
||||||
|
|
||||||
@bp.route("/enrichment/<service>/pause", methods=["POST"])
|
@bp.route("/enrichment/<service>/pause", methods=["POST"])
|
||||||
def video_enrichment_pause(service):
|
def video_enrichment_pause(service):
|
||||||
|
if service == "youtube":
|
||||||
|
_yt_enricher().pause()
|
||||||
|
return jsonify({"status": "paused"})
|
||||||
w = engine().worker(service)
|
w = engine().worker(service)
|
||||||
if not w:
|
if not w:
|
||||||
return jsonify({"error": "unknown service"}), 404
|
return jsonify({"error": "unknown service"}), 404
|
||||||
|
|
@ -123,6 +132,9 @@ def register_routes(bp):
|
||||||
|
|
||||||
@bp.route("/enrichment/<service>/resume", methods=["POST"])
|
@bp.route("/enrichment/<service>/resume", methods=["POST"])
|
||||||
def video_enrichment_resume(service):
|
def video_enrichment_resume(service):
|
||||||
|
if service == "youtube":
|
||||||
|
_yt_enricher().resume()
|
||||||
|
return jsonify({"status": "running"})
|
||||||
w = engine().worker(service)
|
w = engine().worker(service)
|
||||||
if not w:
|
if not w:
|
||||||
return jsonify({"error": "unknown service"}), 404
|
return jsonify({"error": "unknown service"}), 404
|
||||||
|
|
|
||||||
|
|
@ -84,7 +84,7 @@ def register_routes(bp):
|
||||||
if followed: # followed channels get their full upload-date catalog in the background
|
if followed: # followed channels get their full upload-date catalog in the background
|
||||||
try:
|
try:
|
||||||
from core.video.youtube_enrichment import get_youtube_date_enricher
|
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||||
get_youtube_date_enricher().enqueue(channel.get("youtube_id"))
|
get_youtube_date_enricher().enqueue(channel.get("youtube_id"), channel.get("title"))
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
return jsonify({"success": followed, "following": followed, "added_videos": added,
|
return jsonify({"success": followed, "following": followed, "added_videos": added,
|
||||||
|
|
@ -156,7 +156,7 @@ def register_routes(bp):
|
||||||
if following: # opening a followed channel → ensure its dates get enriched
|
if following: # opening a followed channel → ensure its dates get enriched
|
||||||
try:
|
try:
|
||||||
from core.video.youtube_enrichment import get_youtube_date_enricher
|
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||||
get_youtube_date_enricher().enqueue(cid)
|
get_youtube_date_enricher().enqueue(cid, channel.get("title"))
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
# backfill the real avatar onto wished rows (flat listing often omits it)
|
# backfill the real avatar onto wished rows (flat listing often omits it)
|
||||||
|
|
|
||||||
|
|
@ -31,21 +31,28 @@ class YoutubeDateEnricher:
|
||||||
self._db_factory = db_factory or self._default_db
|
self._db_factory = db_factory or self._default_db
|
||||||
self._q: "queue.Queue[str]" = queue.Queue()
|
self._q: "queue.Queue[str]" = queue.Queue()
|
||||||
self._inflight = set()
|
self._inflight = set()
|
||||||
|
self._titles = {}
|
||||||
self._thread = None
|
self._thread = None
|
||||||
self._lock = threading.Lock()
|
self._lock = threading.Lock()
|
||||||
|
self._current = None # channel being enriched right now (for the orb)
|
||||||
|
self._paused = False
|
||||||
|
self._channels_done = 0
|
||||||
|
self._dates_total = 0
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _default_db():
|
def _default_db():
|
||||||
from database.video_database import VideoDatabase
|
from database.video_database import VideoDatabase
|
||||||
return VideoDatabase()
|
return VideoDatabase()
|
||||||
|
|
||||||
def enqueue(self, channel_id):
|
def enqueue(self, channel_id, title=None):
|
||||||
"""Queue a followed channel for full date enrichment (deduped; starts the
|
"""Queue a followed channel for full date enrichment (deduped; starts the
|
||||||
worker thread on first use)."""
|
worker thread on first use)."""
|
||||||
cid = str(channel_id or "").strip()
|
cid = str(channel_id or "").strip()
|
||||||
if not cid:
|
if not cid:
|
||||||
return
|
return
|
||||||
with self._lock:
|
with self._lock:
|
||||||
|
if title:
|
||||||
|
self._titles[cid] = title
|
||||||
if cid in self._inflight:
|
if cid in self._inflight:
|
||||||
return
|
return
|
||||||
self._inflight.add(cid)
|
self._inflight.add(cid)
|
||||||
|
|
@ -54,8 +61,34 @@ class YoutubeDateEnricher:
|
||||||
self._thread = threading.Thread(target=self._run, name="yt-date-enricher", daemon=True)
|
self._thread = threading.Thread(target=self._run, name="yt-date-enricher", daemon=True)
|
||||||
self._thread.start()
|
self._thread.start()
|
||||||
|
|
||||||
|
def pause(self):
|
||||||
|
self._paused = True
|
||||||
|
|
||||||
|
def resume(self):
|
||||||
|
self._paused = False
|
||||||
|
|
||||||
|
def stats(self):
|
||||||
|
"""Dashboard-orb telemetry — same shape the enrichment workers report."""
|
||||||
|
queued = self._q.qsize()
|
||||||
|
running = bool(self._current) and not self._paused
|
||||||
|
cur = self._current
|
||||||
|
return {
|
||||||
|
"enabled": True,
|
||||||
|
"idle": False, # this worker idles, it never "completes"
|
||||||
|
"running": running,
|
||||||
|
"paused": self._paused,
|
||||||
|
"current_item": {"type": "channel", "name": cur} if cur else None,
|
||||||
|
"progress": {"channels": {"matched": self._channels_done,
|
||||||
|
"total": self._channels_done + queued + (1 if cur else 0)}},
|
||||||
|
"queued": queued,
|
||||||
|
"dates_cached": self._dates_total,
|
||||||
|
}
|
||||||
|
|
||||||
def _run(self):
|
def _run(self):
|
||||||
while True:
|
while True:
|
||||||
|
if self._paused:
|
||||||
|
time.sleep(0.5)
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
cid = self._q.get(timeout=45)
|
cid = self._q.get(timeout=45)
|
||||||
except queue.Empty:
|
except queue.Empty:
|
||||||
|
|
@ -65,6 +98,7 @@ class YoutubeDateEnricher:
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("YouTube date enrichment failed for %s", cid)
|
logger.exception("YouTube date enrichment failed for %s", cid)
|
||||||
finally:
|
finally:
|
||||||
|
self._current = None
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self._inflight.discard(cid)
|
self._inflight.discard(cid)
|
||||||
self._q.task_done()
|
self._q.task_done()
|
||||||
|
|
@ -76,6 +110,7 @@ class YoutubeDateEnricher:
|
||||||
cid = str(channel_id or "").strip()
|
cid = str(channel_id or "").strip()
|
||||||
if not cid or db.channel_dates_enriched_recently(cid):
|
if not cid or db.channel_dates_enriched_recently(cid):
|
||||||
return
|
return
|
||||||
|
self._current = self._titles.get(cid) or cid
|
||||||
|
|
||||||
dates = {}
|
dates = {}
|
||||||
try:
|
try:
|
||||||
|
|
@ -103,6 +138,8 @@ class YoutubeDateEnricher:
|
||||||
if filled:
|
if filled:
|
||||||
logger.info("YouTube dates: %d cached for %s (per-video fallback)", filled, cid)
|
logger.info("YouTube dates: %d cached for %s (per-video fallback)", filled, cid)
|
||||||
|
|
||||||
|
self._channels_done += 1
|
||||||
|
self._dates_total += len(dates) + filled
|
||||||
db.mark_channel_dates_enriched(cid, len(dates) + filled)
|
db.mark_channel_dates_enriched(cid, len(dates) + filled)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -716,3 +716,13 @@ def test_youtube_search_endpoint_hydrates_following(tmp_path, monkeypatch):
|
||||||
flags = {c["youtube_id"]: c["following"] for c in d["channels"]}
|
flags = {c["youtube_id"]: c["following"] for c in d["channels"]}
|
||||||
assert flags == {"UCaaa": True, "UCbbb": False}
|
assert flags == {"UCaaa": True, "UCbbb": False}
|
||||||
assert client.get("/api/video/youtube/search?q=").get_json()["channels"] == []
|
assert client.get("/api/video/youtube/search?q=").get_json()["channels"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_enrichment_youtube_status_route(tmp_path):
|
||||||
|
client, _ = _make_client(tmp_path)
|
||||||
|
d = client.get("/api/video/enrichment/youtube/status").get_json()
|
||||||
|
assert d["enabled"] is True and "running" in d and "progress" in d
|
||||||
|
assert client.post("/api/video/enrichment/youtube/pause").get_json()["status"] == "paused"
|
||||||
|
assert client.post("/api/video/enrichment/youtube/resume").get_json()["status"] == "running"
|
||||||
|
# unknown service still 404s
|
||||||
|
assert client.get("/api/video/enrichment/bogus/status").status_code == 404
|
||||||
|
|
|
||||||
|
|
@ -49,3 +49,12 @@ def test_enrich_falls_back_to_per_video_when_proxy_empty(db, monkeypatch):
|
||||||
def test_enricher_imports_nothing_from_music():
|
def test_enricher_imports_nothing_from_music():
|
||||||
src = Path("core/video/youtube_enrichment.py").read_text(encoding="utf-8")
|
src = Path("core/video/youtube_enrichment.py").read_text(encoding="utf-8")
|
||||||
assert "database.music_database" not in src and "from database import" not in src
|
assert "database.music_database" not in src and "from database import" not in src
|
||||||
|
|
||||||
|
|
||||||
|
def test_enricher_stats_shape_and_pause():
|
||||||
|
e = YoutubeDateEnricher(db_factory=lambda: None)
|
||||||
|
s = e.stats()
|
||||||
|
assert s["enabled"] is True and s["running"] is False and s["paused"] is False
|
||||||
|
assert s["current_item"] is None and "progress" in s
|
||||||
|
e.pause(); assert e.stats()["paused"] is True
|
||||||
|
e.resume(); assert e.stats()["paused"] is False
|
||||||
|
|
|
||||||
|
|
@ -488,6 +488,22 @@
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="video-enrich-container" style="--ve-accent: 255, 59, 59;">
|
||||||
|
<button class="video-enrich-button" type="button" data-video-enrich="youtube" title="YouTube Dates">
|
||||||
|
<span class="video-enrich-glyph" aria-hidden="true">▶</span>
|
||||||
|
<span class="video-enrich-spinner" aria-hidden="true"></span>
|
||||||
|
</button>
|
||||||
|
<div class="video-enrich-tooltip" data-video-enrich-tooltip="youtube">
|
||||||
|
<div class="tooltip-content">
|
||||||
|
<div class="tooltip-header">YouTube Dates</div>
|
||||||
|
<div class="tooltip-body">
|
||||||
|
<div class="tooltip-status">Status: <span data-video-enrich-status>Idle</span></div>
|
||||||
|
<div class="tooltip-current" data-video-enrich-current>No channels queued</div>
|
||||||
|
<div class="tooltip-progress" data-video-enrich-progress>Progress: 0 / 0</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
<!-- Reuses music's .em-manage-btn* classes verbatim so it's
|
<!-- Reuses music's .em-manage-btn* classes verbatim so it's
|
||||||
pixel-identical; wired by data-attribute (no inline
|
pixel-identical; wired by data-attribute (no inline
|
||||||
handler) via video-enrichment.js, never music's. -->
|
handler) via video-enrichment.js, never music's. -->
|
||||||
|
|
|
||||||
|
|
@ -335,10 +335,13 @@
|
||||||
var a = q('[data-vd-actions]');
|
var a = q('[data-vd-actions]');
|
||||||
if (!a) return;
|
if (!a) return;
|
||||||
if (d.source === 'youtube') {
|
if (d.source === 'youtube') {
|
||||||
|
// Use the SAME watchlist button as shows/movies (consistency); it just
|
||||||
|
// follows the channel instead of a tmdb show.
|
||||||
var on = !!d.following;
|
var on = !!d.following;
|
||||||
a.innerHTML =
|
a.innerHTML =
|
||||||
'<button class="vd-yt-follow' + (on ? ' vd-yt-follow--on' : '') + '" type="button" data-vd-act="yt-follow">' +
|
'<button class="library-artist-watchlist-btn' + (on ? ' watching' : '') + '" type="button" data-vd-act="yt-follow">' +
|
||||||
(on ? '✓ Following' : '+ Follow') + '</button>' +
|
'<span class="watchlist-icon">' + (on ? '✓' : '+') + '</span>' +
|
||||||
|
'<span class="watchlist-text">' + (on ? 'In Watchlist' : 'Watchlist') + '</span></button>' +
|
||||||
'<a class="vd-yt-link" href="https://www.youtube.com/channel/' + esc(d.id) +
|
'<a class="vd-yt-link" href="https://www.youtube.com/channel/' + esc(d.id) +
|
||||||
'" target="_blank" rel="noopener">Open on YouTube ↗</a>';
|
'" target="_blank" rel="noopener">Open on YouTube ↗</a>';
|
||||||
return;
|
return;
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,9 @@
|
||||||
(function () {
|
(function () {
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
var SERVICES = ['tmdb', 'tvdb', 'omdb'];
|
var SERVICES = ['tmdb', 'tvdb', 'omdb']; // pushed over the shared socket
|
||||||
|
var POLLED = ['youtube']; // standalone daemon → light polling
|
||||||
|
var ALL = SERVICES.concat(POLLED);
|
||||||
|
|
||||||
function onVideoSide() {
|
function onVideoSide() {
|
||||||
return document.body.getAttribute('data-side') === 'video';
|
return document.body.getAttribute('data-side') === 'video';
|
||||||
|
|
@ -72,7 +74,13 @@
|
||||||
|
|
||||||
// One-time fetch so the buttons aren't blank until the first socket push.
|
// One-time fetch so the buttons aren't blank until the first socket push.
|
||||||
function primeOnce() {
|
function primeOnce() {
|
||||||
if (onVideoSide() && !document.hidden) SERVICES.forEach(pollOne);
|
if (onVideoSide() && !document.hidden) ALL.forEach(pollOne);
|
||||||
|
}
|
||||||
|
// The YouTube date enricher isn't on the socket — poll it so its orb reflects
|
||||||
|
// activity (only while the video side is foregrounded).
|
||||||
|
function pollPolledLoop() {
|
||||||
|
if (onVideoSide() && !document.hidden) POLLED.forEach(pollOne);
|
||||||
|
setTimeout(pollPolledLoop, 3000);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Listen on the shared socket (set up by core.js) for the server-pushed
|
// Listen on the shared socket (set up by core.js) for the server-pushed
|
||||||
|
|
@ -104,7 +112,7 @@
|
||||||
}
|
}
|
||||||
|
|
||||||
function init() {
|
function init() {
|
||||||
SERVICES.forEach(function (svc) {
|
ALL.forEach(function (svc) {
|
||||||
var b = btn(svc);
|
var b = btn(svc);
|
||||||
if (b) b.addEventListener('click', function () { toggle(svc); });
|
if (b) b.addEventListener('click', function () { toggle(svc); });
|
||||||
});
|
});
|
||||||
|
|
@ -115,7 +123,8 @@
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
primeOnce(); // instant initial state
|
primeOnce(); // instant initial state
|
||||||
bindSocket(); // then the socket pushes updates — no polling
|
bindSocket(); // then the socket pushes updates for tmdb/tvdb/omdb
|
||||||
|
pollPolledLoop(); // ...and poll the youtube enricher
|
||||||
// Re-prime when the user returns to the tab (socket kept pushing, but a
|
// Re-prime when the user returns to the tab (socket kept pushing, but a
|
||||||
// quick fetch snaps the buttons current immediately).
|
// quick fetch snaps the buttons current immediately).
|
||||||
document.addEventListener('visibilitychange', function () {
|
document.addEventListener('visibilitychange', function () {
|
||||||
|
|
|
||||||
|
|
@ -20,10 +20,11 @@
|
||||||
// VIDEO side: the two enrichment buttons + the Manage Workers hub. `sel`
|
// VIDEO side: the two enrichment buttons + the Manage Workers hub. `sel`
|
||||||
// matches the BUTTON; the orb's positioned element is its container.
|
// matches the BUTTON; the orb's positioned element is its container.
|
||||||
const WORKER_DEFS = [
|
const WORKER_DEFS = [
|
||||||
{ sel: '[data-video-enrich="tmdb"]', color: [56, 189, 248], id: 'tmdb' },
|
{ sel: '[data-video-enrich="tmdb"]', color: [56, 189, 248], id: 'tmdb' },
|
||||||
{ sel: '[data-video-enrich="tvdb"]', color: [168, 85, 247], id: 'tvdb' },
|
{ sel: '[data-video-enrich="tvdb"]', color: [168, 85, 247], id: 'tvdb' },
|
||||||
{ sel: '[data-video-enrich="omdb"]', color: [245, 197, 24], id: 'omdb' },
|
{ sel: '[data-video-enrich="omdb"]', color: [245, 197, 24], id: 'omdb' },
|
||||||
{ sel: '[data-video-manage-workers]', color: [168, 85, 247], hub: true },
|
{ sel: '[data-video-enrich="youtube"]', color: [255, 59, 59], id: 'youtube' },
|
||||||
|
{ sel: '[data-video-manage-workers]', color: [168, 85, 247], hub: true },
|
||||||
];
|
];
|
||||||
|
|
||||||
const ERROR_COLOR = [255, 80, 80]; // pulses fired on real worker errors
|
const ERROR_COLOR = [255, 80, 80]; // pulses fired on real worker errors
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue