diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..e5c73bfe --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +dicttoxml==1.7.4 +music-tag==0.4.3 +pathvalidate==2.4.1 +Pillow==8.3.2 +pyYAML==5.4.1 +sanitize-filename==1.2.0 +youtube_dl==2021.6.6 diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..400a781a --- /dev/null +++ b/setup.py @@ -0,0 +1,5 @@ +from setuptools import setup, find_packages +setup( + name='ytdl_subscribe', + packages=find_packages(), +) \ No newline at end of file diff --git a/subscriptions.yaml b/subscriptions.yaml new file mode 100644 index 00000000..70f2ab43 --- /dev/null +++ b/subscriptions.yaml @@ -0,0 +1,61 @@ +config: + working_directory: '/Users/jesse.bannon/workspace/ytdl_subscribe/tmp' + +presets: + soundcloud: + source: "soundcloud" + ytdl_opts: + format: 'bestaudio/best' + extractaudio: True # only keep the audio + audioformat: "mp3" # convert to mp3 + noplaylist: True + post_process: + file_name: "[{upload_year}] {sanitized_title}/{sanitized_title}.{ext}" + thumbnail_name: "[{upload_year}] {sanitized_title}/folder.{thumbnail_ext}" + tagging: + artist: "{artist}" + title: "{title}" + album: "{title}" + year: "{upload_year}" + genre: "{genre}" + + music_videos: + source: "youtube" + ytdl_opts: + format: 'best' + sleep_interval: 8 + max_sleep_interval: 25 + post_process: + file_name: "{sanitized_artist} - {sanitized_track}.{ext}" + convert_thumbnail: "jpg" + thumbnail_name: "{sanitized_artist} - {sanitized_track}.jpg" + nfo_name: "{sanitized_artist} - {sanitized_track}.nfo" + nfo_root: "musicvideo" + nfo: + artist: "{artist}" + track: "{sanitized_track}" + album: "Music Videos" + year: "{upload_year}" + +subscriptions: +# delorra: +# url: "https://soundcloud.com/delorra/tracks" +# preset: "soundcloud" +# output_path: "/Users/jesse.bannon/workspace/ytdl_subscribe/delorra" +# overrides: +# artist: "DeLorra" +# genre: "Synthwave / Electronic" + +# tom_petty: +# url: "https://www.youtube.com/playlist?list=PLoopXDarluPBnuxs4PTC55Sc_2ShAXC0i" +# preset: "music_videos" +# output_path: "tompetty_test" +# overrides: +# artist: "Tom Petty and the Heartbreakers" + + rammstein: + url: "https://www.youtube.com/playlist?list=PLVTLbc6i-h_iuhdwUfuPDLFLXG2QQnz-x" + preset: "music_videos" + output_path: "rammstein test" + overrides: + artist: "Rammstein" diff --git a/ytdl_subscribe/__init__.py b/ytdl_subscribe/__init__.py new file mode 100644 index 00000000..dfbb933f --- /dev/null +++ b/ytdl_subscribe/__init__.py @@ -0,0 +1 @@ +from .enums import * \ No newline at end of file diff --git a/ytdl_subscribe/__pycache__/__init__.cpython-39.pyc b/ytdl_subscribe/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 00000000..808b36f8 Binary files /dev/null and b/ytdl_subscribe/__pycache__/__init__.cpython-39.pyc differ diff --git a/ytdl_subscribe/__pycache__/enums.cpython-39.pyc b/ytdl_subscribe/__pycache__/enums.cpython-39.pyc new file mode 100644 index 00000000..7e4d42c3 Binary files /dev/null and b/ytdl_subscribe/__pycache__/enums.cpython-39.pyc differ diff --git a/ytdl_subscribe/__pycache__/parse.cpython-39.pyc b/ytdl_subscribe/__pycache__/parse.cpython-39.pyc new file mode 100644 index 00000000..4923da61 Binary files /dev/null and b/ytdl_subscribe/__pycache__/parse.cpython-39.pyc differ diff --git a/ytdl_subscribe/__pycache__/subscriptions.cpython-39.pyc b/ytdl_subscribe/__pycache__/subscriptions.cpython-39.pyc new file mode 100644 index 00000000..dff31676 Binary files /dev/null and b/ytdl_subscribe/__pycache__/subscriptions.cpython-39.pyc differ diff --git a/ytdl_subscribe/enums.py b/ytdl_subscribe/enums.py new file mode 100644 index 00000000..49decf9b --- /dev/null +++ b/ytdl_subscribe/enums.py @@ -0,0 +1,10 @@ + + +class SubscriptionSource(object): + YOUTUBE = 'youtube' + SOUNDCLOUD = 'soundcloud' + +class YAMLSection(object): + CONFIG_KEY = 'config' + PRESET_KEY = 'presets' + SUBSCRIPTIONS_KEY = 'subscriptions' \ No newline at end of file diff --git a/ytdl_subscribe/main.py b/ytdl_subscribe/main.py new file mode 100644 index 00000000..8a4351b0 --- /dev/null +++ b/ytdl_subscribe/main.py @@ -0,0 +1,11 @@ + +from ytdl_subscribe.parse import parse_subscriptions + + + +if __name__ == "__main__": + # execute only if run as a script + subscriptions = parse_subscriptions('subscriptions.yaml') + for s in subscriptions: + s.extract_info() + print('hey') diff --git a/ytdl_subscribe/parse.py b/ytdl_subscribe/parse.py new file mode 100644 index 00000000..af244fd4 --- /dev/null +++ b/ytdl_subscribe/parse.py @@ -0,0 +1,33 @@ +import yaml +import time +from getpass import getpass +from ytdl_subscribe.subscriptions import Subscription + +from ytdl_subscribe.enums import YAMLSection + + + +def _set_config_variables(config): + Subscription.WORKING_DIRECTORY = config.get('working_directory', '') + Subscription.USERNAME = config.get('ytdl_username') + if Subscription.USERNAME: + print(f"Enter password for '{Subscription.USERNAME}': ") + time.sleep(0.2) + Subscription.PASSWORD = getpass() + +def parse_subscriptions(subscription_yaml_path): + with open(subscription_yaml_path, 'r') as f: + yaml_dict = yaml.safe_load(f) + + config = yaml_dict[YAMLSection.CONFIG_KEY] + _set_config_variables(config) + + presets = yaml_dict[YAMLSection.PRESET_KEY] + + subscriptions = [] + for name, subscription in yaml_dict[YAMLSection.SUBSCRIPTIONS_KEY].items(): + if subscription.get('preset') in presets: + preset = presets[subscription['preset']] + subscriptions.append(Subscription.from_dict(dict(preset, **subscription))) + + return subscriptions \ No newline at end of file diff --git a/ytdl_subscribe/subscriptions.py b/ytdl_subscribe/subscriptions.py new file mode 100644 index 00000000..1cf081e8 --- /dev/null +++ b/ytdl_subscribe/subscriptions.py @@ -0,0 +1,159 @@ +from datetime import datetime +import os +from shutil import copyfile +import urllib +import music_tag +from sanitize_filename import sanitize +import dicttoxml + +from PIL import Image +from ytdl_subscribe import SubscriptionSource + +import youtube_dl as ytdl + +def _f(value, entry): + return value.format(**entry) + + +def _ffile(file_value, path, entry, makedirs=False): + file_name = _f(file_value, entry) + output_file_path = f"{path}/{file_name}" + if makedirs: + os.makedirs(os.path.dirname(output_file_path), exist_ok=True) + + return output_file_path + + +class Subscription(object): + WORKING_DIRECTORY = '' + + source = None + + def __init__(self, url, ytdl_opts, post_process, overrides, output_path): + self.url = url + self.ytdl_opts = ytdl_opts or dict() + self.post_process = post_process + self.overrides = overrides + self.output_path = output_path + + # Always set outtmpl to the id and extension. Will be renamed using the subscription's output_path value + self.ytdl_opts['outtmpl'] = self.WORKING_DIRECTORY + '/%(id)s.%(ext)s' + + + def parse_entry(self, entry): + # Add overrides to the entry + entry = dict(entry, **self.overrides) + + # Add the file path to the entry, assert it exists + entry['file_path'] = f"{self.WORKING_DIRECTORY}/{entry['id']}.{entry['ext']}" + assert os.path.isfile(entry['file_path']) + + # Add sanitized values + entry['sanitized_artist'] = sanitize(entry['artist']) + entry['sanitized_title'] = sanitize(entry['title']) + + return entry + + def _post_process_tagging(self, entry): + t = music_tag.load_file(entry['file_path']) + for tag, tag_formatter in self.post_process['tagging'].items(): + t[tag] = _f(tag_formatter, entry) + t.save() + + def _post_process_nfo(self, entry): + nfo = {} + for tag, tag_formatter in self.post_process['nfo'].items(): + nfo[tag] = _f(tag_formatter, entry) + + xml = dicttoxml.dicttoxml( + obj=nfo, + root='nfo_root' in self.post_process, + custom_root=self.post_process.get('nfo_root'), + attr_type=False, + ) + nfo_file_path = _ffile(self.post_process['nfo_name'], self.output_path, entry, makedirs=True) + with open(nfo_file_path, 'wb') as f: + f.write(xml) + + def post_process_entry(self, entry): + if 'tagging' in self.post_process: + self._post_process_tagging(entry) + + # Move the file after all direct file modifications are complete + output_file_path = _ffile(self.post_process['file_name'], self.output_path, entry, makedirs=True) + copyfile(entry['file_path'], output_file_path) + + # Download the thumbnail if its present + if 'thumbnail_name' in self.post_process: + thumbnail_file_path = _ffile(self.post_process['thumbnail_name'], self.output_path, entry, makedirs=True) + urllib.request.urlretrieve(entry['thumbnail_url'], thumbnail_file_path) + if 'convert_thumbnail' in self.post_process: + # TODO: Clean with yaml definitions + if self.post_process['convert_thumbnail'] == 'jpg': + self.post_process['convert_thumbnail'] = 'jpeg' + im = Image.open(thumbnail_file_path).convert("RGB") + im.save(thumbnail_file_path, self.post_process['convert_thumbnail']) + + if 'nfo' in self.post_process: + self._post_process_nfo(entry) + + def extract_info(self): + """ + Extracts only the info of the source, does not download it + """ + with ytdl.YoutubeDL(self.ytdl_opts) as ytd: + info = ytd.extract_info(self.url) + + entries = [self.parse_entry(e) for e in info['entries']] + for e in entries: + self.post_process_entry(e) + + return + + @classmethod + def from_dict(cls, d): + source = d['source'] + if source == SubscriptionSource.SOUNDCLOUD: + dclass = SoundcloudSubscription + elif source == SubscriptionSource.YOUTUBE: + dclass = YoutubeSubscription + else: + raise ValueError('dne') + + return dclass( + url=d['url'], + ytdl_opts=d['ytdl_opts'], + post_process=d['post_process'], + overrides=d['overrides'], + output_path=d['output_path'] + ) + + +class SoundcloudSubscription(Subscription): + source = SubscriptionSource.SOUNDCLOUD + + def parse_entry(self, entry): + entry = super(SoundcloudSubscription, self).parse_entry(entry) + entry['upload_year'] = datetime.fromtimestamp(entry['timestamp']).year + + # Add thumbnail values + original_thumbnail = [t for t in entry['thumbnails'] if t['id'] == 'original'][0] + entry['thumbnail_url'] = original_thumbnail['url'] + entry['thumbnail_ext'] = original_thumbnail['url'].split('.')[-1] + + return entry + + +class YoutubeSubscription(Subscription): + source = SubscriptionSource.YOUTUBE + + def parse_entry(self, entry): + entry = super(YoutubeSubscription, self).parse_entry(entry) + entry['upload_year'] = entry['upload_date'][:4] + + entry['thumbnail_url'] = entry['thumbnail'] + entry['thumbnail_ext'] = 'webp' + + # Try to get the track, fall back on title + entry['sanitized_track'] = sanitize(entry.get('track', entry['title'])) + return entry \ No newline at end of file