From 981dc9efd9bb3f1b8c6cc9e8efbece878b017146 Mon Sep 17 00:00:00 2001 From: Jesse Bannon Date: Wed, 8 Sep 2021 21:37:54 -0700 Subject: [PATCH] initial commit --- requirements.txt | 7 + setup.py | 5 + subscriptions.yaml | 61 +++++++ ytdl_subscribe/__init__.py | 1 + .../__pycache__/__init__.cpython-39.pyc | Bin 0 -> 184 bytes .../__pycache__/enums.cpython-39.pyc | Bin 0 -> 611 bytes .../__pycache__/parse.cpython-39.pyc | Bin 0 -> 1193 bytes .../__pycache__/subscriptions.cpython-39.pyc | Bin 0 -> 5350 bytes ytdl_subscribe/enums.py | 10 ++ ytdl_subscribe/main.py | 11 ++ ytdl_subscribe/parse.py | 33 ++++ ytdl_subscribe/subscriptions.py | 159 ++++++++++++++++++ 12 files changed, 287 insertions(+) create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 subscriptions.yaml create mode 100644 ytdl_subscribe/__init__.py create mode 100644 ytdl_subscribe/__pycache__/__init__.cpython-39.pyc create mode 100644 ytdl_subscribe/__pycache__/enums.cpython-39.pyc create mode 100644 ytdl_subscribe/__pycache__/parse.cpython-39.pyc create mode 100644 ytdl_subscribe/__pycache__/subscriptions.cpython-39.pyc create mode 100644 ytdl_subscribe/enums.py create mode 100644 ytdl_subscribe/main.py create mode 100644 ytdl_subscribe/parse.py create mode 100644 ytdl_subscribe/subscriptions.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..e5c73bfe --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +dicttoxml==1.7.4 +music-tag==0.4.3 +pathvalidate==2.4.1 +Pillow==8.3.2 +pyYAML==5.4.1 +sanitize-filename==1.2.0 +youtube_dl==2021.6.6 diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..400a781a --- /dev/null +++ b/setup.py @@ -0,0 +1,5 @@ +from setuptools import setup, find_packages +setup( + name='ytdl_subscribe', + packages=find_packages(), +) \ No newline at end of file diff --git a/subscriptions.yaml b/subscriptions.yaml new file mode 100644 index 00000000..70f2ab43 --- /dev/null +++ b/subscriptions.yaml @@ -0,0 +1,61 @@ +config: + working_directory: '/Users/jesse.bannon/workspace/ytdl_subscribe/tmp' + +presets: + soundcloud: + source: "soundcloud" + ytdl_opts: + format: 'bestaudio/best' + extractaudio: True # only keep the audio + audioformat: "mp3" # convert to mp3 + noplaylist: True + post_process: + file_name: "[{upload_year}] {sanitized_title}/{sanitized_title}.{ext}" + thumbnail_name: "[{upload_year}] {sanitized_title}/folder.{thumbnail_ext}" + tagging: + artist: "{artist}" + title: "{title}" + album: "{title}" + year: "{upload_year}" + genre: "{genre}" + + music_videos: + source: "youtube" + ytdl_opts: + format: 'best' + sleep_interval: 8 + max_sleep_interval: 25 + post_process: + file_name: "{sanitized_artist} - {sanitized_track}.{ext}" + convert_thumbnail: "jpg" + thumbnail_name: "{sanitized_artist} - {sanitized_track}.jpg" + nfo_name: "{sanitized_artist} - {sanitized_track}.nfo" + nfo_root: "musicvideo" + nfo: + artist: "{artist}" + track: "{sanitized_track}" + album: "Music Videos" + year: "{upload_year}" + +subscriptions: +# delorra: +# url: "https://soundcloud.com/delorra/tracks" +# preset: "soundcloud" +# output_path: "/Users/jesse.bannon/workspace/ytdl_subscribe/delorra" +# overrides: +# artist: "DeLorra" +# genre: "Synthwave / Electronic" + +# tom_petty: +# url: "https://www.youtube.com/playlist?list=PLoopXDarluPBnuxs4PTC55Sc_2ShAXC0i" +# preset: "music_videos" +# output_path: "tompetty_test" +# overrides: +# artist: "Tom Petty and the Heartbreakers" + + rammstein: + url: "https://www.youtube.com/playlist?list=PLVTLbc6i-h_iuhdwUfuPDLFLXG2QQnz-x" + preset: "music_videos" + output_path: "rammstein test" + overrides: + artist: "Rammstein" diff --git a/ytdl_subscribe/__init__.py b/ytdl_subscribe/__init__.py new file mode 100644 index 00000000..dfbb933f --- /dev/null +++ b/ytdl_subscribe/__init__.py @@ -0,0 +1 @@ +from .enums import * \ No newline at end of file diff --git a/ytdl_subscribe/__pycache__/__init__.cpython-39.pyc b/ytdl_subscribe/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..808b36f8e623d81f7ce674aaa3477f360b60f99d GIT binary patch literal 184 zcmYe~<>g`kf<=F{6GedZV-N=!FabFZKwQiNBvKes7;_jx7*d#m88n$+G6ID(8E-La z`Drpnv8Lvg<`%DHC}IYRf{9=5`k}?CMaBAAsl~;qdP#|SdHH$z<@rU~#RZAUsrr>A vDLJ~urAfufMVUzuZajh;AD@|*SrQ+wS5SG2!zMRBr8Fni4rJqJAZ7pnqUA3r literal 0 HcmV?d00001 diff --git a/ytdl_subscribe/__pycache__/enums.cpython-39.pyc b/ytdl_subscribe/__pycache__/enums.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7e4d42c35e00db6359a72dca575edda671851489 GIT binary patch literal 611 zcmZ`$Jx{|h5Y1OV6sQDJ4tp!)O-ueC(ny3I_dg7!6(x zQ3D2L`51H}$%O1InUt)Z#979(&J!08aucVlv(4#>$cj_416iilvc1^~EzQ=_v`Y}F5GA?XFIh_BG&SHND11xR0C93mQKWG#H5J^*hSo;lMT5mkq?C2{t0bMk z!Rq8()1I54fCLl;`cwKBcv-CnBJub{bhpn zK_GYxQ+^AEA%=4luT~smh)GTg5)*84PVN*mrq-vqTX?YtzRSEfDE67p0$2lvBh(H5 zhTG7vi?u)Fs!pW@w;i32Re)sl)Lblw=|xAiQr_w5a9BM@#d>a{E(Higk=KW zU5N{g;?9L)d2%kfDAR(!zSS=k7dr+woMt=IO6=Tu?cDidXZG$FGyeRl8)^qWO*nut z-h6gAIv5O}B)!3?zkhrt{LHv0pK4cPjpw|fFu^gDiRI=lEk#<9Imh#d| zEz~w^yc{$x;7a-m!gq)aUq) zd6x5Y_;q5pllSwZXP)n&FBca&MZakJ+_=;!`{mK|ia#Ur!h2}>vtnKpMd=~)tCuZN z7L|vVsDyL-wm*;djF?4xHk?QM1lmx4s-5@}>$Pf3d!nI26?MWoHJV9p zrxnE^>1m@Isc4t<+?7sao0R#@{#M$QQBOrlcQfhBX4vF3)950dOmY!-T7?)`ZL4KP z)`0z${f^_?nggB-Tz)W2dY*U5C@gD6!& za;KIvpVK@^wbyCf2}LAR?erSzw$4NEHacO=^^0Wkz*s`g;rp^cg3PB2<`Iq4)ugX_ zeHEA6D1n!1}%Cv4?Q)gWkoi_9JavaDfBgQPhB=z7jUQVLxdJ#)#rlww&1`h z2R^y)3R^h9TVCXZi(hYqzk(=;BHF;9DC4&zDq;q|<**`VMfIVz&wy1z+MF&=f<{~Q zCK>$jBApgu5c2_)0UKBW%-@?)Y#)vY_`rs=2M#2R4vD+%3id5R7Z-e$e)IBuNYBW; z>9btWmXU|2VmM)(?7d}w6)X&4Ns?&>ouMgCbseP~4NK*h#)C$_} z%UC-iYARhY$V+-E)#Y9S5BFr!1V(isxfe7BIa)Lzwjm=FASgX0(NyE4$7qNZ$42f@(c$UO>NRT%t9PI@`gxCy% z=h2^%zb)pnGH(3YRl5v*_td1-mk-}~B3jA*#`ZB30H3uOqSb1fc-@09+T4Typnlqd zW)GNfNc%hoEw^eZyX_2(e%d-<>CfM?K4QwnGxsz=fZsgEP}>`|3-AjuL9s3?_@?HO z&}`Ya;f)d8$#%ZmkP1=cR}Xne1X+*HsYu14KYR4Rbky?lH0I09_8e)mBq*S?8>JK) zlEVBkm*1raCdiIiw$y9LGz<(Iy@Ek0v4zFw?dRD^yUMD3cX7(B6T_!0o5NdUO1XfB zvW8>}MmE~x29vPo&_1x%tY0pC!1wGuM>)Va+vfuye$~4B?0^q*2PT!kT6SHY#XLH% z8r$1ZcN?H7boy!34A9{h;-n!AGZ^UTT%^KInvsu_HtvOBqVc9RQ`*K`x&jeoRIGsC zW41UIOk$iqL*M-w7Lj@o7Pw`BRoU+IQx(J_jy0*Km#P=WqnI!5`&CzzU)z%KR{8F=@s-$Bc z`Gtm3GEh4`5D1MYD!KlcWkk;JZ`BG}T)&7tXRav_t*GQdC)W0Ms0>Y;h}TZi3%fdZ zU&4_!@5p!ZReE`a#LE!Ajd%SDeQlf;G;1Aq<=nJaFnk>^r^GiFB?ti9VVrqL82mr3 zyGZ=;-2ZVaCYW+5!F^05|3Cvd)3`{U92nq}xs`Dn(FmRqjbu*afIWx=y~wzzbKKFJ zUyR0sMyAHwD%W<0dAyUy6Z?SPA(96mhcBbNS=VWE`LNS1)@KrZudJ|-9{7}M3@jaY-a$y3|b+l>1#G*`-L0r>IYDit0<+7y%{4 zS;5){aIp+*Y>13^!2ZDRI-6M@uQ6G}2o%74`DM*Qu!Ny!L&2t)E3U@Ke$AxQJAW4= z|ApJyeQC$!c{NU&jW|6Il_d(K_X2h&JvlQ#v#31-<|xDf8XaU%S{T?w~Z8 z$8FNVdr-18iIl(<$`HiyqC$G?E}FKD_BrLC?J3l!`|KlzI0l!13rt}GNnB#e4@hu? z@E%ch4k~vuucE;1hF@E3Ve?;Cht*t3ET@Z`*KX9;E?&KMV@)sM6x0>XIO&VY{p$R! zYd5ancynE!7}cn|qvRN5Wu*f`TerlXyHN9{h2Kd$Zn* z=Rz)c70x7$YT}-{h$-kwX&r}Wl@O(;pEDc)7I^6S?#D>yexAxD`8Ebl0ga|0u+eY& zA%wCN($qd5a*ED9rsz;Ns?pk1(Q=#`%<%_0Kt<$2RV_i&Bp=kYsAY{lbVUJmNYOyA zG^%jRbzPvNb*dVjo-Xx!rb^oh8}bR_poMMH!iZ=MahxO`6c+GHF@S|-^ikvTdl=$- ziHuN>HDY6F^j<)(TqbeM>KD=XN9tdL*nREkcwL^TWS_<*esP>YQb|;v1<6lI`#!{% z|HU<#1Hmk{yw0Wl9x6-nMFt)sNnIg89vQ|Vi(qIkjuaech>(hNfBtZfM@jTC(}f@~ zXbXaDrv_TYrjKxEG=!vd(q2Fnv%pcFJ~comH;e*n1Mf<5fTVP1!d$*ZngKt}t}CPO zHp4hR{I)UmuV|cE1?}@UxKnbchp^00h{n`K2&C(qgZUkdo&q!3k+{f!nU;JU%o&jJ z-z5HPs<^w@A%VR=Oj9K>Fkvje0?@KKdo?Cj*3;@+Z_hN5a_BSas~yzo7rf zt-Q=U)F8WOo~anuDkOF+H^3y>itQZ+d3>A6W);esL#D(SeBNC7O{=pr$g^ wF3ufcQyGHHLdux&X!&EiX>T>7_VW~~hErfR#DkYf7_p3$;W@=ZHL1@25AG_@NB{r; literal 0 HcmV?d00001 diff --git a/ytdl_subscribe/enums.py b/ytdl_subscribe/enums.py new file mode 100644 index 00000000..49decf9b --- /dev/null +++ b/ytdl_subscribe/enums.py @@ -0,0 +1,10 @@ + + +class SubscriptionSource(object): + YOUTUBE = 'youtube' + SOUNDCLOUD = 'soundcloud' + +class YAMLSection(object): + CONFIG_KEY = 'config' + PRESET_KEY = 'presets' + SUBSCRIPTIONS_KEY = 'subscriptions' \ No newline at end of file diff --git a/ytdl_subscribe/main.py b/ytdl_subscribe/main.py new file mode 100644 index 00000000..8a4351b0 --- /dev/null +++ b/ytdl_subscribe/main.py @@ -0,0 +1,11 @@ + +from ytdl_subscribe.parse import parse_subscriptions + + + +if __name__ == "__main__": + # execute only if run as a script + subscriptions = parse_subscriptions('subscriptions.yaml') + for s in subscriptions: + s.extract_info() + print('hey') diff --git a/ytdl_subscribe/parse.py b/ytdl_subscribe/parse.py new file mode 100644 index 00000000..af244fd4 --- /dev/null +++ b/ytdl_subscribe/parse.py @@ -0,0 +1,33 @@ +import yaml +import time +from getpass import getpass +from ytdl_subscribe.subscriptions import Subscription + +from ytdl_subscribe.enums import YAMLSection + + + +def _set_config_variables(config): + Subscription.WORKING_DIRECTORY = config.get('working_directory', '') + Subscription.USERNAME = config.get('ytdl_username') + if Subscription.USERNAME: + print(f"Enter password for '{Subscription.USERNAME}': ") + time.sleep(0.2) + Subscription.PASSWORD = getpass() + +def parse_subscriptions(subscription_yaml_path): + with open(subscription_yaml_path, 'r') as f: + yaml_dict = yaml.safe_load(f) + + config = yaml_dict[YAMLSection.CONFIG_KEY] + _set_config_variables(config) + + presets = yaml_dict[YAMLSection.PRESET_KEY] + + subscriptions = [] + for name, subscription in yaml_dict[YAMLSection.SUBSCRIPTIONS_KEY].items(): + if subscription.get('preset') in presets: + preset = presets[subscription['preset']] + subscriptions.append(Subscription.from_dict(dict(preset, **subscription))) + + return subscriptions \ No newline at end of file diff --git a/ytdl_subscribe/subscriptions.py b/ytdl_subscribe/subscriptions.py new file mode 100644 index 00000000..1cf081e8 --- /dev/null +++ b/ytdl_subscribe/subscriptions.py @@ -0,0 +1,159 @@ +from datetime import datetime +import os +from shutil import copyfile +import urllib +import music_tag +from sanitize_filename import sanitize +import dicttoxml + +from PIL import Image +from ytdl_subscribe import SubscriptionSource + +import youtube_dl as ytdl + +def _f(value, entry): + return value.format(**entry) + + +def _ffile(file_value, path, entry, makedirs=False): + file_name = _f(file_value, entry) + output_file_path = f"{path}/{file_name}" + if makedirs: + os.makedirs(os.path.dirname(output_file_path), exist_ok=True) + + return output_file_path + + +class Subscription(object): + WORKING_DIRECTORY = '' + + source = None + + def __init__(self, url, ytdl_opts, post_process, overrides, output_path): + self.url = url + self.ytdl_opts = ytdl_opts or dict() + self.post_process = post_process + self.overrides = overrides + self.output_path = output_path + + # Always set outtmpl to the id and extension. Will be renamed using the subscription's output_path value + self.ytdl_opts['outtmpl'] = self.WORKING_DIRECTORY + '/%(id)s.%(ext)s' + + + def parse_entry(self, entry): + # Add overrides to the entry + entry = dict(entry, **self.overrides) + + # Add the file path to the entry, assert it exists + entry['file_path'] = f"{self.WORKING_DIRECTORY}/{entry['id']}.{entry['ext']}" + assert os.path.isfile(entry['file_path']) + + # Add sanitized values + entry['sanitized_artist'] = sanitize(entry['artist']) + entry['sanitized_title'] = sanitize(entry['title']) + + return entry + + def _post_process_tagging(self, entry): + t = music_tag.load_file(entry['file_path']) + for tag, tag_formatter in self.post_process['tagging'].items(): + t[tag] = _f(tag_formatter, entry) + t.save() + + def _post_process_nfo(self, entry): + nfo = {} + for tag, tag_formatter in self.post_process['nfo'].items(): + nfo[tag] = _f(tag_formatter, entry) + + xml = dicttoxml.dicttoxml( + obj=nfo, + root='nfo_root' in self.post_process, + custom_root=self.post_process.get('nfo_root'), + attr_type=False, + ) + nfo_file_path = _ffile(self.post_process['nfo_name'], self.output_path, entry, makedirs=True) + with open(nfo_file_path, 'wb') as f: + f.write(xml) + + def post_process_entry(self, entry): + if 'tagging' in self.post_process: + self._post_process_tagging(entry) + + # Move the file after all direct file modifications are complete + output_file_path = _ffile(self.post_process['file_name'], self.output_path, entry, makedirs=True) + copyfile(entry['file_path'], output_file_path) + + # Download the thumbnail if its present + if 'thumbnail_name' in self.post_process: + thumbnail_file_path = _ffile(self.post_process['thumbnail_name'], self.output_path, entry, makedirs=True) + urllib.request.urlretrieve(entry['thumbnail_url'], thumbnail_file_path) + if 'convert_thumbnail' in self.post_process: + # TODO: Clean with yaml definitions + if self.post_process['convert_thumbnail'] == 'jpg': + self.post_process['convert_thumbnail'] = 'jpeg' + im = Image.open(thumbnail_file_path).convert("RGB") + im.save(thumbnail_file_path, self.post_process['convert_thumbnail']) + + if 'nfo' in self.post_process: + self._post_process_nfo(entry) + + def extract_info(self): + """ + Extracts only the info of the source, does not download it + """ + with ytdl.YoutubeDL(self.ytdl_opts) as ytd: + info = ytd.extract_info(self.url) + + entries = [self.parse_entry(e) for e in info['entries']] + for e in entries: + self.post_process_entry(e) + + return + + @classmethod + def from_dict(cls, d): + source = d['source'] + if source == SubscriptionSource.SOUNDCLOUD: + dclass = SoundcloudSubscription + elif source == SubscriptionSource.YOUTUBE: + dclass = YoutubeSubscription + else: + raise ValueError('dne') + + return dclass( + url=d['url'], + ytdl_opts=d['ytdl_opts'], + post_process=d['post_process'], + overrides=d['overrides'], + output_path=d['output_path'] + ) + + +class SoundcloudSubscription(Subscription): + source = SubscriptionSource.SOUNDCLOUD + + def parse_entry(self, entry): + entry = super(SoundcloudSubscription, self).parse_entry(entry) + entry['upload_year'] = datetime.fromtimestamp(entry['timestamp']).year + + # Add thumbnail values + original_thumbnail = [t for t in entry['thumbnails'] if t['id'] == 'original'][0] + entry['thumbnail_url'] = original_thumbnail['url'] + entry['thumbnail_ext'] = original_thumbnail['url'].split('.')[-1] + + return entry + + +class YoutubeSubscription(Subscription): + source = SubscriptionSource.YOUTUBE + + def parse_entry(self, entry): + entry = super(YoutubeSubscription, self).parse_entry(entry) + entry['upload_year'] = entry['upload_date'][:4] + + entry['thumbnail_url'] = entry['thumbnail'] + entry['thumbnail_ext'] = 'webp' + + # Try to get the track, fall back on title + entry['sanitized_track'] = sanitize(entry.get('track', entry['title'])) + return entry \ No newline at end of file