video enrichment: add Wikidata (official website link) — full parity, completes the batch
Fifth/final service. Keyless, movies + shows, on by default → toggle in the
'Community Data (No Key)' frame.
- WikidataWorker: two-step lookup — find the entity by IMDb id (haswbstatement P345)
then read its official website (P856); stores wikidata_url. Registered in
build_backfill_workers.
- DB: wikidata_url/status/attempted on movies + shows + _BACKFILL/_BACKFILL_COLS;
both detail payloads return wikidata_url.
- config GET/POST wikidata_enabled; manager orb (green 🔗) + status poll; detail page
'Official Site' link badge alongside IMDb/TMDB/TVDB.
- 3 new tests (incl. the two-step fetch) + fixed-set/config assertions.
34 backfill tests green, ruff clean. All five new services (Trakt, TVmaze, AniList,
DeArrow, Wikidata) now have the full worker/DB/connections/orb/manager/detail parity.
This commit is contained in:
BoulderBadgeDad
parent
c33810f39f
commit
b3f704fbfd
10 changed files with 128 additions and 7 deletions
|
|
@ -59,6 +59,7 @@ def register_routes(bp):
|
|||
"dearrow_enabled": (db.get_setting("dearrow_enabled") or "1") == "1",
|
||||
"tvmaze_enabled": (db.get_setting("tvmaze_enabled") or "1") == "1",
|
||||
"anilist_enabled": (db.get_setting("anilist_enabled") or "0") == "1",
|
||||
"wikidata_enabled": (db.get_setting("wikidata_enabled") or "1") == "1",
|
||||
"billboard_autoplay": (db.get_setting("billboard_autoplay") or "1") == "1",
|
||||
"watch_region": (db.get_setting("watch_region") or "US").upper(),
|
||||
})
|
||||
|
|
@ -95,7 +96,7 @@ def register_routes(bp):
|
|||
put_key("trakt_api_key")
|
||||
# No-key worker on/off toggles (read live by the worker — no rebuild needed).
|
||||
for flag in ("ryd_enabled", "sponsorblock_enabled", "dearrow_enabled",
|
||||
"tvmaze_enabled", "anilist_enabled"):
|
||||
"tvmaze_enabled", "anilist_enabled", "wikidata_enabled"):
|
||||
if flag in body:
|
||||
db.set_setting(flag, "1" if body.get(flag) else "0")
|
||||
if "billboard_autoplay" in body:
|
||||
|
|
|
|||
|
|
@ -744,9 +744,66 @@ class DeArrowWorker(VideoBackfillWorker):
|
|||
return self.db.youtube_enrich_breakdown("dearrow_status")
|
||||
|
||||
|
||||
# ── Wikidata (no key) — the title's official website ──────────────────────────
|
||||
class WikidataWorker(VideoBackfillWorker):
|
||||
API = "https://www.wikidata.org/w/api.php"
|
||||
|
||||
def __init__(self, db):
|
||||
super().__init__(db, "wikidata", "Wikidata", interval=1.0)
|
||||
|
||||
def _enabled(self):
|
||||
return str(self.db.get_setting("wikidata_enabled") or "1") != "0"
|
||||
|
||||
def _entity_for_imdb(self, imdb):
|
||||
s = _http_get_json(self.API, {"action": "query", "list": "search",
|
||||
"srsearch": "haswbstatement:P345=" + imdb,
|
||||
"srlimit": 1, "format": "json"})
|
||||
hits = (((s or {}).get("query") or {}).get("search") or [])
|
||||
return hits[0].get("title") if hits else None
|
||||
|
||||
def test(self):
|
||||
try:
|
||||
ok = self._entity_for_imdb("tt0137523") is not None # Fight Club
|
||||
return (ok, "Wikidata reachable" if ok else "No response")
|
||||
except Exception as e:
|
||||
return (False, str(e))
|
||||
|
||||
def next_item(self):
|
||||
return self.db.backfill_next("wikidata")
|
||||
|
||||
def fetch(self, item):
|
||||
imdb = str(item.get("imdb_id") or "").strip()
|
||||
if not imdb.lower().startswith("tt"):
|
||||
return None
|
||||
qid = self._entity_for_imdb(imdb)
|
||||
if not qid:
|
||||
return None
|
||||
e = _http_get_json(self.API, {"action": "wbgetentities", "ids": qid,
|
||||
"props": "claims", "format": "json"})
|
||||
claims = (((e or {}).get("entities") or {}).get(qid) or {}).get("claims") or {}
|
||||
for c in (claims.get("P856") or []): # P856 = official website
|
||||
url = ((c.get("mainsnak") or {}).get("datavalue") or {}).get("value")
|
||||
if isinstance(url, str) and url.startswith("http"):
|
||||
return {"wikidata_url": url}
|
||||
return None
|
||||
|
||||
def record_ok(self, item, data):
|
||||
self.db.backfill_mark("wikidata", item["kind"], item["id"], "ok", columns=data)
|
||||
|
||||
def record_empty(self, item):
|
||||
self.db.backfill_mark("wikidata", item["kind"], item["id"], "not_found")
|
||||
|
||||
def record_error(self, item):
|
||||
self.db.backfill_mark("wikidata", item["kind"], item["id"], "error")
|
||||
|
||||
def breakdown(self):
|
||||
return self.db.backfill_breakdown("wikidata")
|
||||
|
||||
|
||||
def build_backfill_workers(db) -> dict:
|
||||
"""All backfill workers, keyed by service id, for the engine registry."""
|
||||
return {w.service: w for w in (
|
||||
RydWorker(db), SponsorBlockWorker(db), FanartWorker(db), OpenSubtitlesWorker(db),
|
||||
TraktWorker(db), TVmazeWorker(db), AniListWorker(db), DeArrowWorker(db),
|
||||
WikidataWorker(db),
|
||||
)}
|
||||
|
|
|
|||
|
|
@ -99,6 +99,10 @@ _BACKFILL = {
|
|||
"show": ("shows", "anilist_status", "anilist_attempted",
|
||||
"title IS NOT NULL AND title <> ''"),
|
||||
},
|
||||
"wikidata": { # official website lookup by imdb id (movies + shows)
|
||||
"movie": ("movies", "wikidata_status", "wikidata_attempted", "imdb_id IS NOT NULL"),
|
||||
"show": ("shows", "wikidata_status", "wikidata_attempted", "imdb_id IS NOT NULL"),
|
||||
},
|
||||
}
|
||||
# Columns each backfill service may gap-fill (whitelist; never clobbers server data).
|
||||
# A worker visits each item once (status IS NULL), so these NULL columns are written
|
||||
|
|
@ -109,6 +113,7 @@ _BACKFILL_COLS = {
|
|||
"trakt": {"trakt_rating", "trakt_votes"},
|
||||
"tvmaze": {"tvmaze_rating"},
|
||||
"anilist": {"anilist_score"},
|
||||
"wikidata": {"wikidata_url"},
|
||||
}
|
||||
|
||||
# Columns ensured on existing DBs (ALTER TABLE ADD COLUMN; idempotent).
|
||||
|
|
@ -176,6 +181,11 @@ _COLUMN_MIGRATIONS = [
|
|||
# AniList anime average score backfill (TV only, 0-100)
|
||||
("shows", "anilist_score", "INTEGER"),
|
||||
("shows", "anilist_status", "TEXT"), ("shows", "anilist_attempted", "TEXT"),
|
||||
# Wikidata official-website backfill (movies + shows)
|
||||
("movies", "wikidata_url", "TEXT"),
|
||||
("movies", "wikidata_status", "TEXT"), ("movies", "wikidata_attempted", "TEXT"),
|
||||
("shows", "wikidata_url", "TEXT"),
|
||||
("shows", "wikidata_status", "TEXT"), ("shows", "wikidata_attempted", "TEXT"),
|
||||
# DeArrow crowd-sourced better titles for cached YouTube videos
|
||||
("youtube_video_stats", "dearrow_title", "TEXT"),
|
||||
("youtube_video_stats", "dearrow_status", "TEXT"),
|
||||
|
|
@ -1625,6 +1635,7 @@ class VideoDatabase:
|
|||
"trakt_rating": show["trakt_rating"], "trakt_votes": show["trakt_votes"],
|
||||
"tvmaze_rating": show["tvmaze_rating"],
|
||||
"anilist_score": show["anilist_score"],
|
||||
"wikidata_url": show["wikidata_url"],
|
||||
"genres": genres, "cast": credits["cast"], "crew": credits["crew"],
|
||||
"tmdb_id": show["tmdb_id"], "tvdb_id": show["tvdb_id"], "imdb_id": show["imdb_id"],
|
||||
"has_poster": bool(show["poster_url"]), "has_backdrop": bool(show["backdrop_url"]),
|
||||
|
|
@ -2674,6 +2685,7 @@ class VideoDatabase:
|
|||
"rating": m["rating"], "rating_critic": m["rating_critic"], "genres": genres,
|
||||
"imdb_rating": m["imdb_rating"], "rt_rating": m["rt_rating"], "metacritic": m["metacritic"],
|
||||
"trakt_rating": m["trakt_rating"], "trakt_votes": m["trakt_votes"],
|
||||
"wikidata_url": m["wikidata_url"],
|
||||
"cast": credits["cast"], "crew": credits["crew"],
|
||||
"tmdb_id": m["tmdb_id"], "imdb_id": m["imdb_id"],
|
||||
"has_poster": bool(m["poster_url"]), "has_backdrop": bool(m["backdrop_url"]),
|
||||
|
|
|
|||
|
|
@ -336,7 +336,7 @@ def test_enrichment_config_save_load(tmp_path, monkeypatch):
|
|||
"tmdb_api_key": "", "tvdb_api_key": "", "omdb_api_key": "",
|
||||
"fanart_api_key": "", "opensubtitles_api_key": "", "trakt_api_key": "",
|
||||
"ryd_enabled": True, "sponsorblock_enabled": True, "dearrow_enabled": True,
|
||||
"tvmaze_enabled": True, "anilist_enabled": False,
|
||||
"tvmaze_enabled": True, "anilist_enabled": False, "wikidata_enabled": True,
|
||||
"billboard_autoplay": True, "watch_region": "US"}
|
||||
client.post("/api/video/enrichment/config",
|
||||
json={"tmdb_api_key": "abc", "tvdb_api_key": "xyz", "omdb_api_key": "om",
|
||||
|
|
@ -347,7 +347,7 @@ def test_enrichment_config_save_load(tmp_path, monkeypatch):
|
|||
"tmdb_api_key": "abc", "tvdb_api_key": "xyz", "omdb_api_key": "om",
|
||||
"fanart_api_key": "fa", "opensubtitles_api_key": "os", "trakt_api_key": "",
|
||||
"ryd_enabled": False, "sponsorblock_enabled": True, "dearrow_enabled": True,
|
||||
"tvmaze_enabled": True, "anilist_enabled": False,
|
||||
"tvmaze_enabled": True, "anilist_enabled": False, "wikidata_enabled": True,
|
||||
"billboard_autoplay": False, "watch_region": "GB"}
|
||||
assert db.get_setting("tmdb_api_key") == "abc" and db.get_setting("omdb_api_key") == "om"
|
||||
assert client.get("/api/video/prefs").get_json() == {
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import pytest
|
|||
from database.video_database import VideoDatabase
|
||||
from core.video.enrichment.backfill import (
|
||||
RydWorker, SponsorBlockWorker, FanartWorker, OpenSubtitlesWorker, TraktWorker, TVmazeWorker,
|
||||
AniListWorker, DeArrowWorker, VideoBackfillWorker, _RateLimited, _Unauthorized,
|
||||
AniListWorker, DeArrowWorker, WikidataWorker, VideoBackfillWorker, _RateLimited, _Unauthorized,
|
||||
build_backfill_workers,
|
||||
)
|
||||
|
||||
|
|
@ -180,7 +180,46 @@ def test_get_stats_shape_matches_matcher_worker(db):
|
|||
|
||||
def test_build_backfill_workers_set(db):
|
||||
assert set(build_backfill_workers(db)) == {
|
||||
"ryd", "sponsorblock", "fanart", "opensubtitles", "trakt", "tvmaze", "anilist", "dearrow"}
|
||||
"ryd", "sponsorblock", "fanart", "opensubtitles",
|
||||
"trakt", "tvmaze", "anilist", "dearrow", "wikidata"}
|
||||
|
||||
|
||||
# ── Wikidata (no-key, official-website lookup by imdb id) ─────────────────────
|
||||
def test_wikidata_queue_keyed_on_imdb(db):
|
||||
mid = db.upsert_movie("plex", {"server_id": "m1", "title": "Fight Club", "year": 1999})
|
||||
assert db.backfill_next("wikidata") is None # no imdb id → not queued
|
||||
with db.connect() as c:
|
||||
c.execute("UPDATE movies SET imdb_id='tt0137523' WHERE id=?", (mid,))
|
||||
c.commit()
|
||||
assert db.backfill_next("wikidata")["kind"] == "movie"
|
||||
|
||||
|
||||
def test_wikidata_worker_records_url(db):
|
||||
mid = db.upsert_movie("plex", {"server_id": "m1", "title": "M"})
|
||||
with db.connect() as c:
|
||||
c.execute("UPDATE movies SET imdb_id='tt1' WHERE id=?", (mid,))
|
||||
c.commit()
|
||||
w = WikidataWorker(db)
|
||||
assert w.enabled is True
|
||||
w.fetch = lambda item: {"wikidata_url": "https://example.com"}
|
||||
assert w.process_one() is True
|
||||
with db.connect() as c:
|
||||
r = c.execute("SELECT wikidata_url, wikidata_status FROM movies WHERE id=?", (mid,)).fetchone()
|
||||
assert r["wikidata_url"] == "https://example.com" and r["wikidata_status"] == "ok"
|
||||
|
||||
|
||||
def test_wikidata_fetch_two_step_lookup(db, monkeypatch):
|
||||
import core.video.enrichment.backfill as bf
|
||||
|
||||
def fake(url, params=None, headers=None, timeout=12):
|
||||
if (params or {}).get("action") == "query":
|
||||
return {"query": {"search": [{"title": "Q190050"}]}}
|
||||
return {"entities": {"Q190050": {"claims": {"P856": [
|
||||
{"mainsnak": {"datavalue": {"value": "https://officialsite.example"}}}]}}}}
|
||||
|
||||
monkeypatch.setattr(bf, "_http_get_json", fake)
|
||||
out = WikidataWorker(db).fetch({"kind": "movie", "imdb_id": "tt0137523"})
|
||||
assert out == {"wikidata_url": "https://officialsite.example"}
|
||||
|
||||
|
||||
# ── DeArrow (no-key, YouTube crowd titles) ────────────────────────────────────
|
||||
|
|
|
|||
|
|
@ -5599,6 +5599,10 @@
|
|||
<input type="checkbox" id="video-anilist-enabled">
|
||||
<span>AniList (anime score) — off by default; enable if you have anime</span>
|
||||
</label>
|
||||
<label class="vid-pref-row">
|
||||
<input type="checkbox" id="video-wikidata-enabled" checked>
|
||||
<span>Wikidata (official website link)</span>
|
||||
</label>
|
||||
<div class="callback-info">
|
||||
<div class="callback-help">Free community APIs — no key needed. Gap-fill extra data on your library in the background.</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -219,6 +219,7 @@
|
|||
if (d.tmdb_id) badges.push(badge(TMDB_LOGO, 'TMDB', 'TMDB',
|
||||
'https://www.themoviedb.org/' + (d.kind === 'movie' ? 'movie' : 'tv') + '/' + d.tmdb_id));
|
||||
if (d.tvdb_id) badges.push(badge(TVDB_LOGO, 'TVDB', 'TVDB', 'https://thetvdb.com/?id=' + d.tvdb_id + '&tab=series'));
|
||||
if (d.wikidata_url) badges.push(badge('', 'Official Site', 'Official Site', d.wikidata_url));
|
||||
l.innerHTML = badges.join('');
|
||||
}
|
||||
var g = q('[data-vd-genres]');
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@
|
|||
{ id: 'trakt', name: 'Trakt', color: '#ed1c24', rgb: '237, 28, 36', kinds: ['movie', 'show'], glyph: '★' },
|
||||
{ id: 'tvmaze', name: 'TVmaze', color: '#3dd6c0', rgb: '61, 214, 192', kinds: ['show'], glyph: '📺' },
|
||||
{ id: 'anilist', name: 'AniList', color: '#02a9ff', rgb: '2, 169, 255', kinds: ['show'], glyph: '🎌' },
|
||||
{ id: 'wikidata', name: 'Wikidata', color: '#339966', rgb: '51, 153, 102', kinds: ['movie', 'show'], glyph: '🔗' },
|
||||
];
|
||||
|
||||
function workerDef(id) {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,8 @@
|
|||
// 'enrichment:<svc>') — including the standalone YouTube date enricher — so the
|
||||
// browser never polls /api/video/enrichment/<svc>/status.
|
||||
var SERVICES = ['tmdb', 'tvdb', 'omdb', 'youtube',
|
||||
'fanart', 'opensubtitles', 'ryd', 'sponsorblock', 'dearrow', 'trakt', 'tvmaze', 'anilist'];
|
||||
'fanart', 'opensubtitles', 'ryd', 'sponsorblock', 'dearrow',
|
||||
'trakt', 'tvmaze', 'anilist', 'wikidata'];
|
||||
|
||||
function onVideoSide() {
|
||||
return document.body.getAttribute('data-side') === 'video';
|
||||
|
|
|
|||
|
|
@ -228,6 +228,8 @@
|
|||
if (tvm && d.tvmaze_enabled != null) tvm.checked = !!d.tvmaze_enabled;
|
||||
var anl = document.getElementById('video-anilist-enabled');
|
||||
if (anl && d.anilist_enabled != null) anl.checked = !!d.anilist_enabled;
|
||||
var wkd = document.getElementById('video-wikidata-enabled');
|
||||
if (wkd && d.wikidata_enabled != null) wkd.checked = !!d.wikidata_enabled;
|
||||
var ap = document.getElementById('video-billboard-autoplay');
|
||||
if (ap && d.billboard_autoplay != null) ap.checked = !!d.billboard_autoplay;
|
||||
var wr = document.getElementById('video-watch-region');
|
||||
|
|
@ -261,6 +263,7 @@
|
|||
var dea = document.getElementById('video-dearrow-enabled');
|
||||
var tvm = document.getElementById('video-tvmaze-enabled');
|
||||
var anl = document.getElementById('video-anilist-enabled');
|
||||
var wkd = document.getElementById('video-wikidata-enabled');
|
||||
return fetch(CONFIG_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'Accept': 'application/json' },
|
||||
|
|
@ -275,6 +278,7 @@
|
|||
dearrow_enabled: dea ? dea.checked : true,
|
||||
tvmaze_enabled: tvm ? tvm.checked : true,
|
||||
anilist_enabled: anl ? anl.checked : false,
|
||||
wikidata_enabled: wkd ? wkd.checked : true,
|
||||
})
|
||||
}).then(function () { if (!silent) toast('API keys saved', 'success'); })
|
||||
.catch(function () { /* ignore */ });
|
||||
|
|
@ -319,7 +323,8 @@
|
|||
['tmdb-api-key', 'tvdb-api-key', 'omdb-api-key',
|
||||
'fanart-api-key', 'opensubtitles-api-key', 'trakt-api-key',
|
||||
'video-ryd-enabled', 'video-sponsorblock-enabled', 'video-dearrow-enabled',
|
||||
'video-tvmaze-enabled', 'video-anilist-enabled'].forEach(function (id) {
|
||||
'video-tvmaze-enabled', 'video-anilist-enabled',
|
||||
'video-wikidata-enabled'].forEach(function (id) {
|
||||
var el = document.getElementById(id);
|
||||
if (el) el.addEventListener('change', function () { saveKeys(); });
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue