From 21ed97ff26aabc6364f4a0661dc14eb3106d8c31 Mon Sep 17 00:00:00 2001 From: BoulderBadgeDad Date: Wed, 17 Jun 2026 13:51:10 -0700 Subject: [PATCH] YouTube enrichment: legacy channels auto-upgrade to InnerTube (backwards compat) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Channels enriched before InnerTube existed (partial coverage, e.g. CaseyNeistat 36, Good Mythical MORE 25) were locked behind the 24h coverage gate and would never reach the full ~240-date catalog on their own. Tag each enrichment row with the source ('innertube'|'fallback'); legacy rows (method NULL after the additive migration) bypass the gate ONCE so they re-enrich and upgrade, then settle under the normal window. Non-destructive: existing follows, wished videos (24), and cached dates (1176) are untouched — only coverage refreshes on next open/sweep. Migration is additive (method TEXT); regression test added. --- core/video/youtube_enrichment.py | 4 +++- database/video_database.py | 23 ++++++++++++++++------- database/video_schema.sql | 3 ++- tests/test_video_database.py | 10 ++++++++++ 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/core/video/youtube_enrichment.py b/core/video/youtube_enrichment.py index 6a701790..e4fffd58 100644 --- a/core/video/youtube_enrichment.py +++ b/core/video/youtube_enrichment.py @@ -205,7 +205,9 @@ class YoutubeDateEnricher: filled += 1 self._channels_done += 1 self._dates_total += len(dates) + filled - db.mark_channel_dates_enriched(cid, len(dates) + filled) + # Tag the source so legacy (pre-InnerTube) rows are recognisable and upgrade. + method = "innertube" if dates else "fallback" + db.mark_channel_dates_enriched(cid, len(dates) + filled, method=method) # Terse per-channel summary (like the worker's "Synced full episode list…"). logger.info("Dated %d videos for %s (%d bulk + %d per-video)", len(dates) + filled, self._current, len(dates), filled) diff --git a/database/video_database.py b/database/video_database.py index c648e01e..5ff55d5d 100644 --- a/database/video_database.py +++ b/database/video_database.py @@ -109,6 +109,9 @@ _COLUMN_MIGRATIONS = [ ("video_wishlist", "source", "TEXT NOT NULL DEFAULT 'tmdb'"), ("video_wishlist", "source_id", "TEXT"), ("video_wishlist", "parent_source_id", "TEXT"), # owning channel youtube id (video rows) + # which source produced a channel's dates — NULL on legacy (pre-InnerTube) rows + # so they re-enrich once and upgrade to the full InnerTube catalog. + ("youtube_channel_enrichment", "method", "TEXT"), ] @@ -2039,17 +2042,19 @@ class VideoDatabase: finally: conn.close() - def mark_channel_dates_enriched(self, channel_id, date_count=0) -> None: - """Record that the background enricher swept this channel (skip re-sweeps).""" + def mark_channel_dates_enriched(self, channel_id, date_count=0, method="innertube") -> None: + """Record that the background enricher swept this channel (skip re-sweeps). + ``method`` tags which source produced the dates; legacy rows have NULL and + get re-enriched once so they upgrade to the InnerTube catalog.""" if not channel_id: return conn = self._get_connection() try: conn.execute( - "INSERT INTO youtube_channel_enrichment (channel_id, enriched_at, date_count) " - "VALUES (?, CURRENT_TIMESTAMP, ?) ON CONFLICT(channel_id) DO UPDATE SET " - "enriched_at=CURRENT_TIMESTAMP, date_count=excluded.date_count", - (str(channel_id), int(date_count or 0))) + "INSERT INTO youtube_channel_enrichment (channel_id, enriched_at, date_count, method) " + "VALUES (?, CURRENT_TIMESTAMP, ?, ?) ON CONFLICT(channel_id) DO UPDATE SET " + "enriched_at=CURRENT_TIMESTAMP, date_count=excluded.date_count, method=excluded.method", + (str(channel_id), int(date_count or 0), method or None)) conn.commit() finally: conn.close() @@ -2064,10 +2069,14 @@ class VideoDatabase: conn = self._get_connection() try: row = conn.execute( - "SELECT enriched_at, date_count FROM youtube_channel_enrichment WHERE channel_id=?", + "SELECT enriched_at, date_count, method FROM youtube_channel_enrichment WHERE channel_id=?", (str(channel_id),)).fetchone() if not row or not row["enriched_at"]: return False + # Legacy rows (enriched before InnerTube, method NULL) → always re-enrich + # once so they upgrade to the full catalog, then settle under the window. + if not row["method"]: + return False try: when = datetime.strptime(row["enriched_at"], "%Y-%m-%d %H:%M:%S") except (ValueError, TypeError): diff --git a/database/video_schema.sql b/database/video_schema.sql index 649f267c..b3bb7d6d 100644 --- a/database/video_schema.sql +++ b/database/video_schema.sql @@ -276,7 +276,8 @@ CREATE TABLE IF NOT EXISTS youtube_video_dates ( CREATE TABLE IF NOT EXISTS youtube_channel_enrichment ( channel_id TEXT PRIMARY KEY, enriched_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, - date_count INTEGER NOT NULL DEFAULT 0 + date_count INTEGER NOT NULL DEFAULT 0, + method TEXT -- 'innertube' | 'fallback'; NULL = legacy → re-enrich once ); -- ── Owned media files (the Library = content that has a file) ──────────────── diff --git a/tests/test_video_database.py b/tests/test_video_database.py index e9af02f5..5ef60b96 100644 --- a/tests/test_video_database.py +++ b/tests/test_video_database.py @@ -1182,3 +1182,13 @@ def test_channel_enrichment_tracking(db): db.mark_channel_dates_enriched("UCx", date_count=50) assert db.channel_dates_enriched_recently("UCx", within_hours=24) is True assert db.channel_dates_enriched_recently("UCx", within_hours=0) is False # window respected + + +def test_legacy_enrichment_rows_upgrade(db): + # A pre-InnerTube row (method NULL) with otherwise-good coverage must NOT be + # locked for 24h — it re-enriches once so it upgrades to the full catalog. + db.mark_channel_dates_enriched("UCleg", date_count=50, method=None) + assert db.channel_dates_enriched_recently("UCleg") is False + # After the InnerTube re-run, the normal window applies again (no churn). + db.mark_channel_dates_enriched("UCleg", date_count=240, method="innertube") + assert db.channel_dates_enriched_recently("UCleg") is True