video enrichment: add DeArrow (crowd-sourced YouTube titles) — full parity
Fourth service. Rides the YouTube-video enrich path (like RYD/SponsorBlock), keyless,
on by default → toggle in the YouTube-Extras frame.
- DeArrowWorker: fetches the branding API and stores the first non-original crowd
title for a cached YouTube video. apply_youtube_dearrow + youtube_video_dearrow_title
DB methods; dearrow_status added to the youtube_enrich next/breakdown whitelists.
- Schema: dearrow_title/status/attempted on youtube_video_stats (video_schema.sql +
_COLUMN_MIGRATIONS for existing DBs).
- config GET/POST dearrow_enabled; manager orb (blue 🏹, video-kind) + status poll;
YT video-detail panel shows a 'DeArrow' alt-title (payload via youtube.py + CSS).
- 4 new tests + fixed-set/config assertions.
30 backfill tests green. (My change is ruff-clean; the 12 pre-existing S110s in
api/video/youtube.py are unrelated tech debt on this branch, left untouched.)
This commit is contained in:
BoulderBadgeDad
parent
6f8a2a3f7a
commit
c33810f39f
13 changed files with 160 additions and 20 deletions
|
|
@ -56,6 +56,7 @@ def register_routes(bp):
|
|||
"trakt_api_key": db.get_setting("trakt_api_key") or "",
|
||||
"ryd_enabled": (db.get_setting("ryd_enabled") or "1") == "1",
|
||||
"sponsorblock_enabled": (db.get_setting("sponsorblock_enabled") or "1") == "1",
|
||||
"dearrow_enabled": (db.get_setting("dearrow_enabled") or "1") == "1",
|
||||
"tvmaze_enabled": (db.get_setting("tvmaze_enabled") or "1") == "1",
|
||||
"anilist_enabled": (db.get_setting("anilist_enabled") or "0") == "1",
|
||||
"billboard_autoplay": (db.get_setting("billboard_autoplay") or "1") == "1",
|
||||
|
|
@ -93,7 +94,8 @@ def register_routes(bp):
|
|||
put_key("opensubtitles_api_key")
|
||||
put_key("trakt_api_key")
|
||||
# No-key worker on/off toggles (read live by the worker — no rebuild needed).
|
||||
for flag in ("ryd_enabled", "sponsorblock_enabled", "tvmaze_enabled", "anilist_enabled"):
|
||||
for flag in ("ryd_enabled", "sponsorblock_enabled", "dearrow_enabled",
|
||||
"tvmaze_enabled", "anilist_enabled"):
|
||||
if flag in body:
|
||||
db.set_setting(flag, "1" if body.get(flag) else "0")
|
||||
if "billboard_autoplay" in body:
|
||||
|
|
|
|||
|
|
@ -322,6 +322,12 @@ def register_routes(bp):
|
|||
if not v or not v.get("youtube_id"):
|
||||
return jsonify({"success": False, "error": "Video not found"}), 404
|
||||
db = get_video_db()
|
||||
# DeArrow crowd-sourced better title (if enriched) — surfaced as an
|
||||
# alternative on the detail page.
|
||||
try:
|
||||
v["dearrow_title"] = db.youtube_video_dearrow_title(v["youtube_id"])
|
||||
except Exception:
|
||||
v["dearrow_title"] = None
|
||||
if v.get("description"):
|
||||
try:
|
||||
db.set_wishlist_video_overview(v["youtube_id"], v["description"])
|
||||
|
|
|
|||
|
|
@ -698,9 +698,55 @@ class AniListWorker(VideoBackfillWorker):
|
|||
return self.db.backfill_breakdown("anilist")
|
||||
|
||||
|
||||
# ── DeArrow (no key) — crowd-sourced better titles for YouTube videos ─────────
|
||||
class DeArrowWorker(VideoBackfillWorker):
|
||||
URL = "https://sponsor.ajay.app/api/branding"
|
||||
|
||||
def __init__(self, db):
|
||||
super().__init__(db, "dearrow", "DeArrow", interval=0.6)
|
||||
|
||||
def _enabled(self):
|
||||
return str(self.db.get_setting("dearrow_enabled") or "1") != "0"
|
||||
|
||||
def test(self):
|
||||
try:
|
||||
j = _http_get_json(self.URL, {"videoID": "dQw4w9WgXcQ"})
|
||||
return (j is not None, "DeArrow reachable" if j is not None else "No response")
|
||||
except Exception as e:
|
||||
return (False, str(e))
|
||||
|
||||
def next_item(self):
|
||||
return self.db.youtube_enrich_next("dearrow_status")
|
||||
|
||||
def fetch(self, item):
|
||||
j = _http_get_json(self.URL, {"videoID": item["youtube_id"]})
|
||||
if not isinstance(j, dict):
|
||||
return None
|
||||
# DeArrow returns crowd titles in preference order; take the first that
|
||||
# isn't the YouTube original.
|
||||
for t in (j.get("titles") or []):
|
||||
if isinstance(t, dict) and not t.get("original"):
|
||||
title = (t.get("title") or "").strip()
|
||||
if title:
|
||||
return {"title": title}
|
||||
return None
|
||||
|
||||
def record_ok(self, item, data):
|
||||
self.db.apply_youtube_dearrow(item["youtube_id"], data.get("title"), "ok")
|
||||
|
||||
def record_empty(self, item):
|
||||
self.db.apply_youtube_dearrow(item["youtube_id"], None, "not_found")
|
||||
|
||||
def record_error(self, item):
|
||||
self.db.apply_youtube_dearrow(item["youtube_id"], None, "error")
|
||||
|
||||
def breakdown(self):
|
||||
return self.db.youtube_enrich_breakdown("dearrow_status")
|
||||
|
||||
|
||||
def build_backfill_workers(db) -> dict:
|
||||
"""All backfill workers, keyed by service id, for the engine registry."""
|
||||
return {w.service: w for w in (
|
||||
RydWorker(db), SponsorBlockWorker(db), FanartWorker(db), OpenSubtitlesWorker(db),
|
||||
TraktWorker(db), TVmazeWorker(db), AniListWorker(db),
|
||||
TraktWorker(db), TVmazeWorker(db), AniListWorker(db), DeArrowWorker(db),
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -176,6 +176,10 @@ _COLUMN_MIGRATIONS = [
|
|||
# AniList anime average score backfill (TV only, 0-100)
|
||||
("shows", "anilist_score", "INTEGER"),
|
||||
("shows", "anilist_status", "TEXT"), ("shows", "anilist_attempted", "TEXT"),
|
||||
# DeArrow crowd-sourced better titles for cached YouTube videos
|
||||
("youtube_video_stats", "dearrow_title", "TEXT"),
|
||||
("youtube_video_stats", "dearrow_status", "TEXT"),
|
||||
("youtube_video_stats", "dearrow_attempted", "TEXT"),
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -509,7 +513,7 @@ class VideoDatabase:
|
|||
"""Next cached YouTube video missing a per-video enrichment. status_col is
|
||||
'ryd_status' or 'sb_status'. Distinct by youtube_id (a video shared across
|
||||
playlists is enriched once). Returns {kind:'video', id, name, youtube_id}."""
|
||||
if status_col not in ("ryd_status", "sb_status"):
|
||||
if status_col not in ("ryd_status", "sb_status", "dearrow_status"):
|
||||
return None
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
|
|
@ -568,8 +572,38 @@ class VideoDatabase:
|
|||
finally:
|
||||
conn.close()
|
||||
|
||||
def apply_youtube_dearrow(self, youtube_id, title, status: str) -> None:
|
||||
"""Record DeArrow's crowd-sourced better title (+ status) for a video."""
|
||||
yid = str(youtube_id or "").strip()
|
||||
if not yid:
|
||||
return
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO youtube_video_stats (youtube_id, dearrow_title, dearrow_status, dearrow_attempted) "
|
||||
"VALUES (?,?,?,CURRENT_TIMESTAMP) ON CONFLICT(youtube_id) DO UPDATE SET "
|
||||
"dearrow_title=COALESCE(excluded.dearrow_title, dearrow_title), "
|
||||
"dearrow_status=excluded.dearrow_status, dearrow_attempted=CURRENT_TIMESTAMP",
|
||||
(yid, title, status))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def youtube_video_dearrow_title(self, youtube_id) -> str | None:
|
||||
"""The DeArrow crowd title for a video, if one was recorded (detail UI)."""
|
||||
yid = str(youtube_id or "").strip()
|
||||
if not yid:
|
||||
return None
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT dearrow_title FROM youtube_video_stats WHERE youtube_id=?", (yid,)).fetchone()
|
||||
return row["dearrow_title"] if row and row["dearrow_title"] else None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def youtube_enrich_breakdown(self, status_col: str) -> dict:
|
||||
if status_col not in ("ryd_status", "sb_status"):
|
||||
if status_col not in ("ryd_status", "sb_status", "dearrow_status"):
|
||||
return {}
|
||||
conn = self._get_connection()
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -317,13 +317,16 @@ CREATE TABLE IF NOT EXISTS youtube_channel_meta (
|
|||
-- sb_* SponsorBlock -> crowd-sourced segments (in youtube_video_segments)
|
||||
-- status columns: NULL = pending, 'ok' | 'not_found' | 'error'.
|
||||
CREATE TABLE IF NOT EXISTS youtube_video_stats (
|
||||
youtube_id TEXT PRIMARY KEY,
|
||||
like_count INTEGER,
|
||||
dislike_count INTEGER,
|
||||
ryd_status TEXT,
|
||||
ryd_attempted TEXT,
|
||||
sb_status TEXT,
|
||||
sb_attempted TEXT
|
||||
youtube_id TEXT PRIMARY KEY,
|
||||
like_count INTEGER,
|
||||
dislike_count INTEGER,
|
||||
ryd_status TEXT,
|
||||
ryd_attempted TEXT,
|
||||
sb_status TEXT,
|
||||
sb_attempted TEXT,
|
||||
dearrow_title TEXT, -- DeArrow crowd-sourced better title
|
||||
dearrow_status TEXT,
|
||||
dearrow_attempted TEXT
|
||||
);
|
||||
|
||||
-- SponsorBlock crowd segments (sponsor/intro/outro/selfpromo/…) for a video.
|
||||
|
|
|
|||
|
|
@ -335,8 +335,8 @@ def test_enrichment_config_save_load(tmp_path, monkeypatch):
|
|||
assert client.get("/api/video/enrichment/config").get_json() == {
|
||||
"tmdb_api_key": "", "tvdb_api_key": "", "omdb_api_key": "",
|
||||
"fanart_api_key": "", "opensubtitles_api_key": "", "trakt_api_key": "",
|
||||
"ryd_enabled": True, "sponsorblock_enabled": True, "tvmaze_enabled": True,
|
||||
"anilist_enabled": False,
|
||||
"ryd_enabled": True, "sponsorblock_enabled": True, "dearrow_enabled": True,
|
||||
"tvmaze_enabled": True, "anilist_enabled": False,
|
||||
"billboard_autoplay": True, "watch_region": "US"}
|
||||
client.post("/api/video/enrichment/config",
|
||||
json={"tmdb_api_key": "abc", "tvdb_api_key": "xyz", "omdb_api_key": "om",
|
||||
|
|
@ -346,8 +346,8 @@ def test_enrichment_config_save_load(tmp_path, monkeypatch):
|
|||
assert client.get("/api/video/enrichment/config").get_json() == {
|
||||
"tmdb_api_key": "abc", "tvdb_api_key": "xyz", "omdb_api_key": "om",
|
||||
"fanart_api_key": "fa", "opensubtitles_api_key": "os", "trakt_api_key": "",
|
||||
"ryd_enabled": False, "sponsorblock_enabled": True, "tvmaze_enabled": True,
|
||||
"anilist_enabled": False,
|
||||
"ryd_enabled": False, "sponsorblock_enabled": True, "dearrow_enabled": True,
|
||||
"tvmaze_enabled": True, "anilist_enabled": False,
|
||||
"billboard_autoplay": False, "watch_region": "GB"}
|
||||
assert db.get_setting("tmdb_api_key") == "abc" and db.get_setting("omdb_api_key") == "om"
|
||||
assert client.get("/api/video/prefs").get_json() == {
|
||||
|
|
|
|||
|
|
@ -13,7 +13,8 @@ import pytest
|
|||
from database.video_database import VideoDatabase
|
||||
from core.video.enrichment.backfill import (
|
||||
RydWorker, SponsorBlockWorker, FanartWorker, OpenSubtitlesWorker, TraktWorker, TVmazeWorker,
|
||||
AniListWorker, VideoBackfillWorker, _RateLimited, _Unauthorized, build_backfill_workers,
|
||||
AniListWorker, DeArrowWorker, VideoBackfillWorker, _RateLimited, _Unauthorized,
|
||||
build_backfill_workers,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -179,7 +180,38 @@ def test_get_stats_shape_matches_matcher_worker(db):
|
|||
|
||||
def test_build_backfill_workers_set(db):
|
||||
assert set(build_backfill_workers(db)) == {
|
||||
"ryd", "sponsorblock", "fanart", "opensubtitles", "trakt", "tvmaze", "anilist"}
|
||||
"ryd", "sponsorblock", "fanart", "opensubtitles", "trakt", "tvmaze", "anilist", "dearrow"}
|
||||
|
||||
|
||||
# ── DeArrow (no-key, YouTube crowd titles) ────────────────────────────────────
|
||||
def test_dearrow_queue_and_apply(db):
|
||||
db.cache_channel_videos("UC1", [{"youtube_id": "v", "title": "clickbait TITLE!!!"}])
|
||||
nxt = db.youtube_enrich_next("dearrow_status")
|
||||
assert nxt and nxt["youtube_id"] == "v" and nxt["kind"] == "video"
|
||||
db.apply_youtube_dearrow("v", "A calm, accurate title", "ok")
|
||||
assert db.youtube_video_dearrow_title("v") == "A calm, accurate title"
|
||||
assert db.youtube_enrich_breakdown("dearrow_status")["video"]["matched"] == 1
|
||||
|
||||
|
||||
def test_dearrow_worker_records_title(db):
|
||||
db.cache_channel_videos("UC1", [{"youtube_id": "v", "title": "X"}])
|
||||
w = DeArrowWorker(db)
|
||||
assert w.enabled is True
|
||||
w.fetch = lambda item: {"title": "Better Crowd Title"}
|
||||
assert w.process_one() is True
|
||||
assert db.youtube_video_dearrow_title("v") == "Better Crowd Title"
|
||||
|
||||
|
||||
def test_dearrow_fetch_parses_branding(db, monkeypatch):
|
||||
import core.video.enrichment.backfill as bf
|
||||
branding = {"titles": [
|
||||
{"title": "Original", "original": True},
|
||||
{"title": "Crowd Title", "original": False},
|
||||
]}
|
||||
monkeypatch.setattr(bf, "_http_get_json",
|
||||
lambda url, params=None, headers=None, timeout=12: branding)
|
||||
out = DeArrowWorker(db).fetch({"youtube_id": "v"})
|
||||
assert out == {"title": "Crowd Title"}
|
||||
|
||||
|
||||
# ── Trakt (community rating backfill, keyed on imdb id) ────────────────────────
|
||||
|
|
|
|||
|
|
@ -5619,6 +5619,10 @@
|
|||
<input type="checkbox" id="video-sponsorblock-enabled" checked>
|
||||
<span>SponsorBlock (crowd-sourced segments)</span>
|
||||
</label>
|
||||
<label class="vid-pref-row">
|
||||
<input type="checkbox" id="video-dearrow-enabled" checked>
|
||||
<span>DeArrow (crowd-sourced better titles)</span>
|
||||
</label>
|
||||
<div class="callback-info">
|
||||
<div class="callback-help">Free community APIs — no key needed. Enrich cached YouTube videos in the background.</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1103,7 +1103,11 @@
|
|||
var yc = window.VideoYoutube, stats = [];
|
||||
var lk = yc && yc.compactCount(v.like_count); if (lk) stats.push(lk + ' likes');
|
||||
var vw = yc && yc.compactCount(v.view_count); if (vw) stats.push(vw + ' views');
|
||||
panel.innerHTML = '<div class="vd-ep-extra-body">' +
|
||||
var dearrow = v.dearrow_title
|
||||
? '<div class="vd-dearrow"><span class="vd-dearrow-tag">DeArrow</span>' +
|
||||
esc(v.dearrow_title) + '</div>'
|
||||
: '';
|
||||
panel.innerHTML = '<div class="vd-ep-extra-body">' + dearrow +
|
||||
(stats.length ? '<div class="vd-ep-extra-gh">' + esc(stats.join(' · ')) + '</div>' : '') +
|
||||
'<p class="vd-ep-extra-ov">' + esc(v.description || 'No description.') + '</p></div>';
|
||||
})
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@
|
|||
{ id: 'opensubtitles', name: 'OpenSubtitles', color: '#22a079', rgb: '34, 160, 121', kinds: ['movie', 'show'], glyph: '💬' },
|
||||
{ id: 'ryd', name: 'YouTube Votes', color: '#ef4444', rgb: '239, 68, 68', kinds: ['video'], glyph: '👍' },
|
||||
{ id: 'sponsorblock', name: 'SponsorBlock', color: '#00b4a0', rgb: '0, 180, 160', kinds: ['video'], glyph: '⏭' },
|
||||
{ id: 'dearrow', name: 'DeArrow', color: '#1f77ce', rgb: '31, 119, 206', kinds: ['video'], glyph: '🏹' },
|
||||
{ id: 'trakt', name: 'Trakt', color: '#ed1c24', rgb: '237, 28, 36', kinds: ['movie', 'show'], glyph: '★' },
|
||||
{ id: 'tvmaze', name: 'TVmaze', color: '#3dd6c0', rgb: '61, 214, 192', kinds: ['show'], glyph: '📺' },
|
||||
{ id: 'anilist', name: 'AniList', color: '#02a9ff', rgb: '2, 169, 255', kinds: ['show'], glyph: '🎌' },
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
// 'enrichment:<svc>') — including the standalone YouTube date enricher — so the
|
||||
// browser never polls /api/video/enrichment/<svc>/status.
|
||||
var SERVICES = ['tmdb', 'tvdb', 'omdb', 'youtube',
|
||||
'fanart', 'opensubtitles', 'ryd', 'sponsorblock', 'trakt', 'tvmaze', 'anilist'];
|
||||
'fanart', 'opensubtitles', 'ryd', 'sponsorblock', 'dearrow', 'trakt', 'tvmaze', 'anilist'];
|
||||
|
||||
function onVideoSide() {
|
||||
return document.body.getAttribute('data-side') === 'video';
|
||||
|
|
|
|||
|
|
@ -222,6 +222,8 @@
|
|||
if (ryd && d.ryd_enabled != null) ryd.checked = !!d.ryd_enabled;
|
||||
var sb = document.getElementById('video-sponsorblock-enabled');
|
||||
if (sb && d.sponsorblock_enabled != null) sb.checked = !!d.sponsorblock_enabled;
|
||||
var dea = document.getElementById('video-dearrow-enabled');
|
||||
if (dea && d.dearrow_enabled != null) dea.checked = !!d.dearrow_enabled;
|
||||
var tvm = document.getElementById('video-tvmaze-enabled');
|
||||
if (tvm && d.tvmaze_enabled != null) tvm.checked = !!d.tvmaze_enabled;
|
||||
var anl = document.getElementById('video-anilist-enabled');
|
||||
|
|
@ -256,6 +258,7 @@
|
|||
var trakt = document.getElementById('trakt-api-key');
|
||||
var ryd = document.getElementById('video-ryd-enabled');
|
||||
var sb = document.getElementById('video-sponsorblock-enabled');
|
||||
var dea = document.getElementById('video-dearrow-enabled');
|
||||
var tvm = document.getElementById('video-tvmaze-enabled');
|
||||
var anl = document.getElementById('video-anilist-enabled');
|
||||
return fetch(CONFIG_URL, {
|
||||
|
|
@ -269,6 +272,7 @@
|
|||
trakt_api_key: trakt ? trakt.value : '',
|
||||
ryd_enabled: ryd ? ryd.checked : true,
|
||||
sponsorblock_enabled: sb ? sb.checked : true,
|
||||
dearrow_enabled: dea ? dea.checked : true,
|
||||
tvmaze_enabled: tvm ? tvm.checked : true,
|
||||
anilist_enabled: anl ? anl.checked : false,
|
||||
})
|
||||
|
|
@ -314,7 +318,7 @@
|
|||
// Enrichment keys save on blur/change (turns the workers on).
|
||||
['tmdb-api-key', 'tvdb-api-key', 'omdb-api-key',
|
||||
'fanart-api-key', 'opensubtitles-api-key', 'trakt-api-key',
|
||||
'video-ryd-enabled', 'video-sponsorblock-enabled',
|
||||
'video-ryd-enabled', 'video-sponsorblock-enabled', 'video-dearrow-enabled',
|
||||
'video-tvmaze-enabled', 'video-anilist-enabled'].forEach(function (id) {
|
||||
var el = document.getElementById(id);
|
||||
if (el) el.addEventListener('change', function () { saveKeys(); });
|
||||
|
|
|
|||
|
|
@ -790,6 +790,10 @@ a.vd-prov:hover img, a.vd-prov:hover .vd-prov-ph { border-color: rgba(var(--vd-a
|
|||
.vd-rt--trakt .vd-rt-tag { background: #ed1c24; color: #fff; }
|
||||
.vd-rt--tvmaze .vd-rt-tag { background: #3dd6c0; color: #00211d; }
|
||||
.vd-rt--anilist .vd-rt-tag { background: #02a9ff; color: #fff; }
|
||||
/* DeArrow crowd title (YouTube video expand panel) */
|
||||
.vd-dearrow { display: flex; align-items: center; gap: 8px; margin-bottom: 8px; font-weight: 600; color: #cfe0ff; }
|
||||
.vd-dearrow-tag { font-size: 10px; font-weight: 900; letter-spacing: 0.04em; padding: 2px 6px; border-radius: 4px;
|
||||
background: #1f77ce; color: #fff; }
|
||||
|
||||
/* ── OMDb worker uses a ★ glyph (no clean brand logo) ─────────────────────── */
|
||||
.video-enrich-glyph {
|
||||
|
|
|
|||
Loading…
Reference in a new issue