From a5846854e53a152651a9d0e721aef2035d5e79dd Mon Sep 17 00:00:00 2001
From: Jesse Bannon <jesse.bannon@datarobot.com>
Date: Fri, 10 Sep 2021 21:24:01 -0700
Subject: [PATCH] youtube working

---
 requirements.txt                |  2 +-
 subscriptions.yaml              | 50 ++++++++++++++++-----------------
 ytdl_subscribe/subscriptions.py | 32 +++++++++++++++------
 3 files changed, 48 insertions(+), 36 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 4d88d8dc..7c447aba 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,4 +6,4 @@ pathvalidate==2.4.1
 Pillow==8.3.2
 pyYAML==5.4.1
 sanitize-filename==1.2.0
-youtube_dl==2021.6.6
+yt-dlp==2021.9.2
diff --git a/subscriptions.yaml b/subscriptions.yaml
index 96b1cde4..67c33149 100644
--- a/subscriptions.yaml
+++ b/subscriptions.yaml
@@ -28,8 +28,7 @@ presets:
       noop: True
     ytdl_opts:
       format: 'best'
-      sleep_interval: 8
-      max_sleep_interval: 25
+      ignoreerrors: True
     post_process:
       file_name: "{sanitized_artist} - {sanitized_track}.{ext}"
       convert_thumbnail: "jpg"
@@ -43,31 +42,30 @@ presets:
         year: "{upload_year}"
 
 subscriptions:
-  ALISON:
-    preset: "soundcloud_with_id3_tags"
-    soundcloud:
-      username: alis_on
-    output_path: "/Users/jesse.bannon/workspace/ytdl_subscribe/alison"
-    overrides:
-      artist: "A.L.I.S.O.N."
-      genre: "Synthwave / Electronic"
-  delorra:
-    preset: "soundcloud_with_id3_tags"
-    soundcloud:
-      username: delorra
-    output_path: "/Users/jesse.bannon/workspace/ytdl_subscribe/delorra"
-    overrides:
-      artist: "DeLorra"
-      genre: "Synthwave / Electronic"
-
-#  tom_petty:
-#    url: "https://www.youtube.com/playlist?list=PLoopXDarluPBnuxs4PTC55Sc_2ShAXC0i"
-#    youtube:
-#      playlist_id: PLoopXDarluPBnuxs4PTC55Sc_2ShAXC0i
-#    preset: "music_videos"
-#    output_path: "tompetty_test"
+#  ALISON:
+#    preset: "soundcloud_with_id3_tags"
+#    soundcloud:
+#      username: alis_on
+#    output_path: "/Users/jesse.bannon/workspace/ytdl_subscribe/alison"
 #    overrides:
-#      artist: "Tom Petty and the Heartbreakers"
+#      artist: "A.L.I.S.O.N."
+#      genre: "Synthwave / Electronic"
+#  delorra:
+#    preset: "soundcloud_with_id3_tags"
+#    soundcloud:
+#      username: delorra
+#    output_path: "/Users/jesse.bannon/workspace/ytdl_subscribe/delorra"
+#    overrides:
+#      artist: "DeLorra"
+#      genre: "Synthwave / Electronic"
+
+  tom_petty:
+    youtube:
+      playlist_id: PLoopXDarluPBnuxs4PTC55Sc_2ShAXC0i
+    preset: "music_videos"
+    output_path: "tompetty_test"
+    overrides:
+      artist: "Tom Petty and the Heartbreakers"
 
 #  rammstein:
 #    url: "https://www.youtube.com/playlist?list=PLVTLbc6i-h_iuhdwUfuPDLFLXG2QQnz-x"
diff --git a/ytdl_subscribe/subscriptions.py b/ytdl_subscribe/subscriptions.py
index 06c0d961..82a30501 100644
--- a/ytdl_subscribe/subscriptions.py
+++ b/ytdl_subscribe/subscriptions.py
@@ -1,3 +1,4 @@
+import json
 import os
 from shutil import copyfile
 
@@ -8,7 +9,7 @@ import dicttoxml
 from PIL import Image
 from ytdl_subscribe import SubscriptionSource
 
-import youtube_dl as ytdl
+import yt_dlp as ytdl
 
 
 def _f(value, entry):
@@ -239,20 +240,33 @@ class YoutubeSubscription(Subscription):
         entry = super(YoutubeSubscription, self).parse_entry(entry)
         entry["upload_year"] = entry["upload_date"][:4]
 
-        entry["thumbnail_ext"] = "webp"
+        entry["thumbnail_ext"] = entry["thumbnail"].split(".")[-1]
 
         # Try to get the track, fall back on title
         entry["sanitized_track"] = sanitize(entry.get("track", entry["title"]))
         return entry
 
     def extract_info(self):
-        """
-        Extracts only the info of the source, does not download it
-        """
-        url = f"https://youtube.com/playlist?list={self.options['playlist_id']}"
-        with ytdl.YoutubeDL(self.ytdl_opts) as ytd:
-            info = ytd.extract_info(url)
+        playlist_id = self.options['playlist_id']
+        url = f"https://youtube.com/playlist?list={playlist_id}"
+        track_ytdl_opts = {
+            "download_archive": self.WORKING_DIRECTORY + "/ytdl-download-archive.txt",
+            "writeinfojson": True,
+        }
 
-        entries = [self.parse_entry(e) for e in info["entries"]]
+        # Do not get entries from the extract info, let it write to the info.json file and
+        # load that instead. This is because if the video is already downloaded, it will
+        # not fetch the metadata (maybe there is a way??)
+        with ytdl.YoutubeDL(dict(self.ytdl_opts, **track_ytdl_opts)) as ytd:
+            _ = ytd.extract_info(url)
+
+        # Load the entries from info.json, ignore the playlist entry
+        entries = []
+        for file_name in os.listdir(self.WORKING_DIRECTORY):
+            if file_name.endswith('.info.json') and not file_name.startswith(playlist_id):
+                with open(self.WORKING_DIRECTORY + '/' + file_name, 'r') as f:
+                    entries.append(json.load(f))
+
+        entries = [self.parse_entry(e) for e in entries]
         for e in entries:
             self.post_process_entry(e)