From b3f704fbfd94d836d96ae98845556a8786abad43 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Thu, 18 Jun 2026 23:13:22 -0700 Subject: [PATCH] =?UTF-8?q?video=20enrichment:=20add=20Wikidata=20(officia?= =?UTF-8?q?l=20website=20link)=20=E2=80=94=20full=20parity,=20completes=20?= =?UTF-8?q?the=20batch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifth/final service. Keyless, movies + shows, on by default → toggle in the 'Community Data (No Key)' frame. - WikidataWorker: two-step lookup — find the entity by IMDb id (haswbstatement P345) then read its official website (P856); stores wikidata_url. Registered in build_backfill_workers. - DB: wikidata_url/status/attempted on movies + shows + _BACKFILL/_BACKFILL_COLS; both detail payloads return wikidata_url. - config GET/POST wikidata_enabled; manager orb (green 🔗) + status poll; detail page 'Official Site' link badge alongside IMDb/TMDB/TVDB. - 3 new tests (incl. the two-step fetch) + fixed-set/config assertions. 34 backfill tests green, ruff clean. All five new services (Trakt, TVmaze, AniList, DeArrow, Wikidata) now have the full worker/DB/connections/orb/manager/detail parity. --- api/video/enrichment.py | 3 +- core/video/enrichment/backfill.py | 57 +++++++++++++++++++ database/video_database.py | 12 ++++ tests/test_video_api.py | 4 +- tests/test_video_backfill.py | 43 +++++++++++++- webui/index.html | 4 ++ webui/static/video/video-detail.js | 1 + .../static/video/video-enrichment-manager.js | 1 + webui/static/video/video-enrichment.js | 3 +- webui/static/video/video-settings.js | 7 ++- 10 files changed, 128 insertions(+), 7 deletions(-) diff --git a/api/video/enrichment.py b/api/video/enrichment.py index 34a153a0..64f323ac 100644 --- a/api/video/enrichment.py +++ b/api/video/enrichment.py @@ -59,6 +59,7 @@ def register_routes(bp): "dearrow_enabled": (db.get_setting("dearrow_enabled") or "1") == "1", "tvmaze_enabled": (db.get_setting("tvmaze_enabled") or "1") == "1", "anilist_enabled": (db.get_setting("anilist_enabled") or "0") == "1", + "wikidata_enabled": (db.get_setting("wikidata_enabled") or "1") == "1", "billboard_autoplay": (db.get_setting("billboard_autoplay") or "1") == "1", "watch_region": (db.get_setting("watch_region") or "US").upper(), }) @@ -95,7 +96,7 @@ def register_routes(bp): put_key("trakt_api_key") # No-key worker on/off toggles (read live by the worker — no rebuild needed). for flag in ("ryd_enabled", "sponsorblock_enabled", "dearrow_enabled", - "tvmaze_enabled", "anilist_enabled"): + "tvmaze_enabled", "anilist_enabled", "wikidata_enabled"): if flag in body: db.set_setting(flag, "1" if body.get(flag) else "0") if "billboard_autoplay" in body: diff --git a/core/video/enrichment/backfill.py b/core/video/enrichment/backfill.py index 262152ab..f0b4b6e8 100644 --- a/core/video/enrichment/backfill.py +++ b/core/video/enrichment/backfill.py @@ -744,9 +744,66 @@ class DeArrowWorker(VideoBackfillWorker): return self.db.youtube_enrich_breakdown("dearrow_status") +# ── Wikidata (no key) — the title's official website ────────────────────────── +class WikidataWorker(VideoBackfillWorker): + API = "https://www.wikidata.org/w/api.php" + + def __init__(self, db): + super().__init__(db, "wikidata", "Wikidata", interval=1.0) + + def _enabled(self): + return str(self.db.get_setting("wikidata_enabled") or "1") != "0" + + def _entity_for_imdb(self, imdb): + s = _http_get_json(self.API, {"action": "query", "list": "search", + "srsearch": "haswbstatement:P345=" + imdb, + "srlimit": 1, "format": "json"}) + hits = (((s or {}).get("query") or {}).get("search") or []) + return hits[0].get("title") if hits else None + + def test(self): + try: + ok = self._entity_for_imdb("tt0137523") is not None # Fight Club + return (ok, "Wikidata reachable" if ok else "No response") + except Exception as e: + return (False, str(e)) + + def next_item(self): + return self.db.backfill_next("wikidata") + + def fetch(self, item): + imdb = str(item.get("imdb_id") or "").strip() + if not imdb.lower().startswith("tt"): + return None + qid = self._entity_for_imdb(imdb) + if not qid: + return None + e = _http_get_json(self.API, {"action": "wbgetentities", "ids": qid, + "props": "claims", "format": "json"}) + claims = (((e or {}).get("entities") or {}).get(qid) or {}).get("claims") or {} + for c in (claims.get("P856") or []): # P856 = official website + url = ((c.get("mainsnak") or {}).get("datavalue") or {}).get("value") + if isinstance(url, str) and url.startswith("http"): + return {"wikidata_url": url} + return None + + def record_ok(self, item, data): + self.db.backfill_mark("wikidata", item["kind"], item["id"], "ok", columns=data) + + def record_empty(self, item): + self.db.backfill_mark("wikidata", item["kind"], item["id"], "not_found") + + def record_error(self, item): + self.db.backfill_mark("wikidata", item["kind"], item["id"], "error") + + def breakdown(self): + return self.db.backfill_breakdown("wikidata") + + def build_backfill_workers(db) -> dict: """All backfill workers, keyed by service id, for the engine registry.""" return {w.service: w for w in ( RydWorker(db), SponsorBlockWorker(db), FanartWorker(db), OpenSubtitlesWorker(db), TraktWorker(db), TVmazeWorker(db), AniListWorker(db), DeArrowWorker(db), + WikidataWorker(db), )} diff --git a/database/video_database.py b/database/video_database.py index b8fd0275..5ce0c0db 100644 --- a/database/video_database.py +++ b/database/video_database.py @@ -99,6 +99,10 @@ _BACKFILL = { "show": ("shows", "anilist_status", "anilist_attempted", "title IS NOT NULL AND title <> ''"), }, + "wikidata": { # official website lookup by imdb id (movies + shows) + "movie": ("movies", "wikidata_status", "wikidata_attempted", "imdb_id IS NOT NULL"), + "show": ("shows", "wikidata_status", "wikidata_attempted", "imdb_id IS NOT NULL"), + }, } # Columns each backfill service may gap-fill (whitelist; never clobbers server data). # A worker visits each item once (status IS NULL), so these NULL columns are written @@ -109,6 +113,7 @@ _BACKFILL_COLS = { "trakt": {"trakt_rating", "trakt_votes"}, "tvmaze": {"tvmaze_rating"}, "anilist": {"anilist_score"}, + "wikidata": {"wikidata_url"}, } # Columns ensured on existing DBs (ALTER TABLE ADD COLUMN; idempotent). @@ -176,6 +181,11 @@ _COLUMN_MIGRATIONS = [ # AniList anime average score backfill (TV only, 0-100) ("shows", "anilist_score", "INTEGER"), ("shows", "anilist_status", "TEXT"), ("shows", "anilist_attempted", "TEXT"), + # Wikidata official-website backfill (movies + shows) + ("movies", "wikidata_url", "TEXT"), + ("movies", "wikidata_status", "TEXT"), ("movies", "wikidata_attempted", "TEXT"), + ("shows", "wikidata_url", "TEXT"), + ("shows", "wikidata_status", "TEXT"), ("shows", "wikidata_attempted", "TEXT"), # DeArrow crowd-sourced better titles for cached YouTube videos ("youtube_video_stats", "dearrow_title", "TEXT"), ("youtube_video_stats", "dearrow_status", "TEXT"), @@ -1625,6 +1635,7 @@ class VideoDatabase: "trakt_rating": show["trakt_rating"], "trakt_votes": show["trakt_votes"], "tvmaze_rating": show["tvmaze_rating"], "anilist_score": show["anilist_score"], + "wikidata_url": show["wikidata_url"], "genres": genres, "cast": credits["cast"], "crew": credits["crew"], "tmdb_id": show["tmdb_id"], "tvdb_id": show["tvdb_id"], "imdb_id": show["imdb_id"], "has_poster": bool(show["poster_url"]), "has_backdrop": bool(show["backdrop_url"]), @@ -2674,6 +2685,7 @@ class VideoDatabase: "rating": m["rating"], "rating_critic": m["rating_critic"], "genres": genres, "imdb_rating": m["imdb_rating"], "rt_rating": m["rt_rating"], "metacritic": m["metacritic"], "trakt_rating": m["trakt_rating"], "trakt_votes": m["trakt_votes"], + "wikidata_url": m["wikidata_url"], "cast": credits["cast"], "crew": credits["crew"], "tmdb_id": m["tmdb_id"], "imdb_id": m["imdb_id"], "has_poster": bool(m["poster_url"]), "has_backdrop": bool(m["backdrop_url"]), diff --git a/tests/test_video_api.py b/tests/test_video_api.py index 21a90d49..56a66560 100644 --- a/tests/test_video_api.py +++ b/tests/test_video_api.py @@ -336,7 +336,7 @@ def test_enrichment_config_save_load(tmp_path, monkeypatch): "tmdb_api_key": "", "tvdb_api_key": "", "omdb_api_key": "", "fanart_api_key": "", "opensubtitles_api_key": "", "trakt_api_key": "", "ryd_enabled": True, "sponsorblock_enabled": True, "dearrow_enabled": True, - "tvmaze_enabled": True, "anilist_enabled": False, + "tvmaze_enabled": True, "anilist_enabled": False, "wikidata_enabled": True, "billboard_autoplay": True, "watch_region": "US"} client.post("/api/video/enrichment/config", json={"tmdb_api_key": "abc", "tvdb_api_key": "xyz", "omdb_api_key": "om", @@ -347,7 +347,7 @@ def test_enrichment_config_save_load(tmp_path, monkeypatch): "tmdb_api_key": "abc", "tvdb_api_key": "xyz", "omdb_api_key": "om", "fanart_api_key": "fa", "opensubtitles_api_key": "os", "trakt_api_key": "", "ryd_enabled": False, "sponsorblock_enabled": True, "dearrow_enabled": True, - "tvmaze_enabled": True, "anilist_enabled": False, + "tvmaze_enabled": True, "anilist_enabled": False, "wikidata_enabled": True, "billboard_autoplay": False, "watch_region": "GB"} assert db.get_setting("tmdb_api_key") == "abc" and db.get_setting("omdb_api_key") == "om" assert client.get("/api/video/prefs").get_json() == { diff --git a/tests/test_video_backfill.py b/tests/test_video_backfill.py index e72ce85d..8fdbfa52 100644 --- a/tests/test_video_backfill.py +++ b/tests/test_video_backfill.py @@ -13,7 +13,7 @@ import pytest from database.video_database import VideoDatabase from core.video.enrichment.backfill import ( RydWorker, SponsorBlockWorker, FanartWorker, OpenSubtitlesWorker, TraktWorker, TVmazeWorker, - AniListWorker, DeArrowWorker, VideoBackfillWorker, _RateLimited, _Unauthorized, + AniListWorker, DeArrowWorker, WikidataWorker, VideoBackfillWorker, _RateLimited, _Unauthorized, build_backfill_workers, ) @@ -180,7 +180,46 @@ def test_get_stats_shape_matches_matcher_worker(db): def test_build_backfill_workers_set(db): assert set(build_backfill_workers(db)) == { - "ryd", "sponsorblock", "fanart", "opensubtitles", "trakt", "tvmaze", "anilist", "dearrow"} + "ryd", "sponsorblock", "fanart", "opensubtitles", + "trakt", "tvmaze", "anilist", "dearrow", "wikidata"} + + +# ── Wikidata (no-key, official-website lookup by imdb id) ───────────────────── +def test_wikidata_queue_keyed_on_imdb(db): + mid = db.upsert_movie("plex", {"server_id": "m1", "title": "Fight Club", "year": 1999}) + assert db.backfill_next("wikidata") is None # no imdb id → not queued + with db.connect() as c: + c.execute("UPDATE movies SET imdb_id='tt0137523' WHERE id=?", (mid,)) + c.commit() + assert db.backfill_next("wikidata")["kind"] == "movie" + + +def test_wikidata_worker_records_url(db): + mid = db.upsert_movie("plex", {"server_id": "m1", "title": "M"}) + with db.connect() as c: + c.execute("UPDATE movies SET imdb_id='tt1' WHERE id=?", (mid,)) + c.commit() + w = WikidataWorker(db) + assert w.enabled is True + w.fetch = lambda item: {"wikidata_url": "https://example.com"} + assert w.process_one() is True + with db.connect() as c: + r = c.execute("SELECT wikidata_url, wikidata_status FROM movies WHERE id=?", (mid,)).fetchone() + assert r["wikidata_url"] == "https://example.com" and r["wikidata_status"] == "ok" + + +def test_wikidata_fetch_two_step_lookup(db, monkeypatch): + import core.video.enrichment.backfill as bf + + def fake(url, params=None, headers=None, timeout=12): + if (params or {}).get("action") == "query": + return {"query": {"search": [{"title": "Q190050"}]}} + return {"entities": {"Q190050": {"claims": {"P856": [ + {"mainsnak": {"datavalue": {"value": "https://officialsite.example"}}}]}}}} + + monkeypatch.setattr(bf, "_http_get_json", fake) + out = WikidataWorker(db).fetch({"kind": "movie", "imdb_id": "tt0137523"}) + assert out == {"wikidata_url": "https://officialsite.example"} # ── DeArrow (no-key, YouTube crowd titles) ──────────────────────────────────── diff --git a/webui/index.html b/webui/index.html index e009044f..fe895e3b 100644 --- a/webui/index.html +++ b/webui/index.html @@ -5599,6 +5599,10 @@ AniList (anime score) — off by default; enable if you have anime +
Free community APIs — no key needed. Gap-fill extra data on your library in the background.
diff --git a/webui/static/video/video-detail.js b/webui/static/video/video-detail.js index 8e63fa7f..19bf0b0d 100644 --- a/webui/static/video/video-detail.js +++ b/webui/static/video/video-detail.js @@ -219,6 +219,7 @@ if (d.tmdb_id) badges.push(badge(TMDB_LOGO, 'TMDB', 'TMDB', 'https://www.themoviedb.org/' + (d.kind === 'movie' ? 'movie' : 'tv') + '/' + d.tmdb_id)); if (d.tvdb_id) badges.push(badge(TVDB_LOGO, 'TVDB', 'TVDB', 'https://thetvdb.com/?id=' + d.tvdb_id + '&tab=series')); + if (d.wikidata_url) badges.push(badge('', 'Official Site', 'Official Site', d.wikidata_url)); l.innerHTML = badges.join(''); } var g = q('[data-vd-genres]'); diff --git a/webui/static/video/video-enrichment-manager.js b/webui/static/video/video-enrichment-manager.js index 9c0b8fc4..aa82a0c1 100644 --- a/webui/static/video/video-enrichment-manager.js +++ b/webui/static/video/video-enrichment-manager.js @@ -29,6 +29,7 @@ { id: 'trakt', name: 'Trakt', color: '#ed1c24', rgb: '237, 28, 36', kinds: ['movie', 'show'], glyph: '★' }, { id: 'tvmaze', name: 'TVmaze', color: '#3dd6c0', rgb: '61, 214, 192', kinds: ['show'], glyph: '📺' }, { id: 'anilist', name: 'AniList', color: '#02a9ff', rgb: '2, 169, 255', kinds: ['show'], glyph: '🎌' }, + { id: 'wikidata', name: 'Wikidata', color: '#339966', rgb: '51, 153, 102', kinds: ['movie', 'show'], glyph: '🔗' }, ]; function workerDef(id) { diff --git a/webui/static/video/video-enrichment.js b/webui/static/video/video-enrichment.js index faaf7a95..877b91ce 100644 --- a/webui/static/video/video-enrichment.js +++ b/webui/static/video/video-enrichment.js @@ -18,7 +18,8 @@ // 'enrichment:') — including the standalone YouTube date enricher — so the // browser never polls /api/video/enrichment//status. var SERVICES = ['tmdb', 'tvdb', 'omdb', 'youtube', - 'fanart', 'opensubtitles', 'ryd', 'sponsorblock', 'dearrow', 'trakt', 'tvmaze', 'anilist']; + 'fanart', 'opensubtitles', 'ryd', 'sponsorblock', 'dearrow', + 'trakt', 'tvmaze', 'anilist', 'wikidata']; function onVideoSide() { return document.body.getAttribute('data-side') === 'video'; diff --git a/webui/static/video/video-settings.js b/webui/static/video/video-settings.js index f8089d26..8a133d21 100644 --- a/webui/static/video/video-settings.js +++ b/webui/static/video/video-settings.js @@ -228,6 +228,8 @@ if (tvm && d.tvmaze_enabled != null) tvm.checked = !!d.tvmaze_enabled; var anl = document.getElementById('video-anilist-enabled'); if (anl && d.anilist_enabled != null) anl.checked = !!d.anilist_enabled; + var wkd = document.getElementById('video-wikidata-enabled'); + if (wkd && d.wikidata_enabled != null) wkd.checked = !!d.wikidata_enabled; var ap = document.getElementById('video-billboard-autoplay'); if (ap && d.billboard_autoplay != null) ap.checked = !!d.billboard_autoplay; var wr = document.getElementById('video-watch-region'); @@ -261,6 +263,7 @@ var dea = document.getElementById('video-dearrow-enabled'); var tvm = document.getElementById('video-tvmaze-enabled'); var anl = document.getElementById('video-anilist-enabled'); + var wkd = document.getElementById('video-wikidata-enabled'); return fetch(CONFIG_URL, { method: 'POST', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json' }, @@ -275,6 +278,7 @@ dearrow_enabled: dea ? dea.checked : true, tvmaze_enabled: tvm ? tvm.checked : true, anilist_enabled: anl ? anl.checked : false, + wikidata_enabled: wkd ? wkd.checked : true, }) }).then(function () { if (!silent) toast('API keys saved', 'success'); }) .catch(function () { /* ignore */ }); @@ -319,7 +323,8 @@ ['tmdb-api-key', 'tvdb-api-key', 'omdb-api-key', 'fanart-api-key', 'opensubtitles-api-key', 'trakt-api-key', 'video-ryd-enabled', 'video-sponsorblock-enabled', 'video-dearrow-enabled', - 'video-tvmaze-enabled', 'video-anilist-enabled'].forEach(function (id) { + 'video-tvmaze-enabled', 'video-anilist-enabled', + 'video-wikidata-enabled'].forEach(function (id) { var el = document.getElementById(id); if (el) el.addEventListener('change', function () { saveKeys(); }); });