danvics/ytdl-sub
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

goodbye kwargs

Browse source
This commit is contained in:
Jesse Bannon 2023-12-08 16:35:26 -08:00
parent f723ebca2c
commit 918e2bac4b
11 changed files with 72 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
src/ytdl_sub/downloaders/info_json/info_json_downloader.py
View file

@ -13,6 +13,7 @@ from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.entries.script.variable_scripts import ENTRY_INJECTED_VARIABLES
from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS
from ytdl_sub.entries.variables.kwargs import DOWNLOAD_INDEX
from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX
@ -101,27 +102,27 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]):
entries: List[Entry] = []
for download_mapping in self._enhanced_download_archive.mapping.entry_mappings.values():
entry = self._get_entry_from_download_mapping(download_mapping).initialize_script(
override_variables=self.overrides.dict_with_format_strings
)
entry = self._get_entry_from_download_mapping(download_mapping)
prior_variables = entry.kwargs_get(YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY, {})
# See if prior variables exist. If so, delete them from metadata
# to avoid saving them recursively on multiple updates
prior_variables = {}
if entry.kwargs_contains(YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY):
prior_variables = entry.kwargs(YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY)
del entry._kwargs[YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY]
entry.add(
entry.initialize_script(override_variables=self.overrides.dict_with_format_strings).add(
{
v.download_index.variable_name: prior_variables.get(
v.download_index.variable_name,
VARIABLE_SCRIPTS[v.download_index.variable_name],
),
v.upload_date_index.variable_name: prior_variables.get(
v.upload_date_index.variable_name,
VARIABLE_SCRIPTS[v.upload_date_index.variable_name],
),
inj.variable_name: prior_variables.get(
inj.variable_name,
VARIABLE_SCRIPTS[inj.variable_name],
)
for inj in ENTRY_INJECTED_VARIABLES
}
)
entries.append(entry)
for entry in sorted(entries, key=lambda ent: ent.get(v.download_index)):
for entry in sorted(entries, key=lambda ent: ent.get_str(v.download_index)):
# Remove each entry from the live download archive since it will get re-added
# unless it is filtered
self._enhanced_download_archive.mapping.remove_entry(entry.uid)

28
src/ytdl_sub/downloaders/url/downloader.py
View file

@ -24,13 +24,8 @@ from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.entry_parent import EntryParent
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.entries.variables.kwargs import COLLECTION_URL
from ytdl_sub.entries.variables.kwargs import COMMENTS
from ytdl_sub.entries.variables.kwargs import DOWNLOAD_INDEX
from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY
from ytdl_sub.entries.variables.kwargs import REQUESTED_SUBTITLES
from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY
from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS
from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX
from ytdl_sub.utils.file_handler import FileHandler
from ytdl_sub.utils.logger import Logger
from ytdl_sub.utils.thumbnail import ThumbnailTypes
@ -182,7 +177,7 @@ class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension):
"""
# COLLECTION_URL is a recent variable that may not exist for old entries when updating.
# Try to use source_webpage_url if it does not exist
entry_collection_url = entry.kwargs_get(COLLECTION_URL, entry.get(v.source_webpage_url))
entry_collection_url = entry.kwargs_get(COLLECTION_URL, entry.get_str(v.source_webpage_url))
# If the collection URL cannot find its mapping, use the last URL
collection_url = (
@ -504,18 +499,8 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
download_logger.info("Entry rejected by download match-filter, skipping ..")
return None
entry.add_kwargs(
{
# Subtitles are not downloaded in metadata run, only here, so move over
REQUESTED_SUBTITLES: download_entry.kwargs_get(REQUESTED_SUBTITLES),
# Same with sponsorblock chapters
SPONSORBLOCK_CHAPTERS: download_entry.kwargs_get(SPONSORBLOCK_CHAPTERS),
COMMENTS: download_entry.kwargs_get(COMMENTS),
}
)
upload_date_idx = self._enhanced_download_archive.mapping.get_num_entries_with_upload_date(
upload_date_standardized=entry.get(v.upload_date_standardized)
upload_date_standardized=entry.get_str(v.upload_date_standardized)
)
download_idx = self._enhanced_download_archive.num_entries
entry.add(
@ -524,6 +509,15 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]):
v.download_index.variable_name: download_idx + 1,
# Tracks number of entries with the same upload date to make them unique
v.upload_date_index.variable_name: upload_date_idx + 1,
v.requested_subtitles.variable_name: download_entry.kwargs_get(
v.requested_subtitles.metadata_key
),
v.sponsorblock_chapters.variable_name: download_entry.kwargs_get(
v.sponsorblock_chapters.metadata_key
),
v.comments.variable_name: download_entry.kwargs_get(
v.sponsorblock_chapters.metadata_key
),
}
)

18
src/ytdl_sub/entries/entry.py
View file

@ -4,6 +4,8 @@ import os
from pathlib import Path
from typing import Dict
from typing import Optional
from typing import Type
from typing import TypeVar
from typing import final
from ytdl_sub.entries.base_entry import BaseEntry
@ -15,6 +17,8 @@ from ytdl_sub.validators.audo_codec_validator import VIDEO_CODEC_EXTS
YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY: str = "ytdl_sub_entry_variables"
TType = TypeVar("TType")
class Entry(BaseEntry, Scriptable):
"""
@ -31,11 +35,15 @@ class Entry(BaseEntry, Scriptable):
self.update_script()
return self
def get(self, variable: Variable) -> str:
return self.script.resolve(unresolvable=self.unresolvable).get_str(variable.variable_name)
def get(self, variable: Variable, expected_type: Type[TType]) -> TType:
out = self.script.resolve(unresolvable=self.unresolvable).get_native(variable.variable_name)
return expected_type(out)
def get_str(self, variable: Variable) -> str:
return self.get(variable, str)
def get_int(self, variable: Variable) -> int:
return self.script.resolve(unresolvable=self.unresolvable).get_int(variable.variable_name)
return self.get(variable, int)
@property
def ext(self) -> str:
@ -44,7 +52,7 @@ class Entry(BaseEntry, Scriptable):
This is not reflected in the entry. See if the mkv file exists and return "mkv" if so,
otherwise, return the original extension.
"""
ext = self.get(VARIABLES.ext)
ext = self.get_str(VARIABLES.ext)
for possible_ext in [ext, "mkv"]:
file_path = str(Path(self.working_directory()) / f"{self.uid}.{possible_ext}")
if os.path.isfile(file_path):
@ -70,7 +78,7 @@ class Entry(BaseEntry, Scriptable):
-------
The download thumbnail's file name
"""
return f"{self.get(VARIABLES.uid)}.{self.get(VARIABLES.thumbnail_ext)}"
return f"{self.get_str(VARIABLES.uid)}.{self.get_str(VARIABLES.thumbnail_ext)}"
def get_download_thumbnail_path(self) -> str:
"""Returns the entry's thumbnail's file path to where it was downloaded"""

20
src/ytdl_sub/entries/script/variable_definitions.py
View file

@ -595,7 +595,19 @@ class _Variables:
return Variable("thumbnail_ext")
@property
def download_index(self) -> MetadataVariable:
def comments(self) -> MetadataVariable:
return MetadataVariable("comments", "comments")
@property
def sponsorblock_chapters(self) -> MetadataVariable:
return MetadataVariable("sponsorblock_chapters", "sponsorblock_chapters")
@property
def requested_subtitles(self) -> MetadataVariable:
return MetadataVariable("requested_subtitles", "requested_subtitles")
@property
def download_index(self) -> Variable:
"""
Returns
-------
@ -603,7 +615,7 @@ class _Variables:
The i'th entry downloaded. NOTE that this is fetched dynamically from the download
archive.
"""
return MetadataVariable(variable_name="download_index", metadata_key="download_index")
return Variable(variable_name="download_index")
@property
def download_index_padded6(self) -> Variable:
@ -616,14 +628,14 @@ class _Variables:
return Variable("download_index_padded6")
@property
def upload_date_index(self) -> MetadataVariable:
def upload_date_index(self) -> Variable:
"""
Returns
-------
int
The i'th entry downloaded with this upload date.
"""
return MetadataVariable(variable_name="upload_date_index", metadata_key="upload_date_index")
return Variable(variable_name="upload_date_index")
@property
def upload_date_index_padded(self) -> Variable:

3
src/ytdl_sub/entries/script/variable_scripts.py
View file

@ -134,6 +134,9 @@ ENTRY_INJECTED_VARIABLES: Dict[Variable, str] = {
v.download_index: "{%int(1)}",
v.upload_date_index: "{%int(1)}",
v.playlist_max_upload_year: f"{{{v.upload_year.variable_name}}}",
v.comments: "",
v.requested_subtitles: "",
v.sponsorblock_chapters: "",
}
ENTRY_DERIVED_VARIABLES: Dict[Variable, str] = {

4
src/ytdl_sub/plugins/chapters.py
View file

@ -9,7 +9,7 @@ from ytdl_sub.config.plugin import Plugin
from ytdl_sub.config.preset_options import OptionsDictValidator
from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder
from ytdl_sub.entries.entry import Entry
from ytdl_sub.entries.variables.kwargs import COMMENTS
from ytdl_sub.entries.script.variable_definitions import VARIABLES as v
from ytdl_sub.entries.variables.kwargs import YTDL_SUB_CUSTOM_CHAPTERS
from ytdl_sub.utils.chapters import Chapters
from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters
@ -305,7 +305,7 @@ class ChaptersPlugin(Plugin[ChaptersOptions]):
# If there are no embedded chapters, and comment chapters are allowed...
if not _contains_any_chapters(entry) and self.plugin_options.allow_chapters_from_comments:
# Try to get chapters from comments
for comment in entry.kwargs_get(COMMENTS, []):
for comment in entry.get(v.comments, list):
chapters = Chapters.from_string(comment.get("text", ""))
if chapters.contains_any_chapters():
break

2
src/ytdl_sub/thread/log_entries_downloaded_listener.py
View file

@ -42,7 +42,7 @@ class LogEntriesDownloadedListener(threading.Thread):
# swallow the error since this is only printing logs
return None
return file_json.get("title")
return file_json.get_str("title")
@classmethod
def _is_info_json(cls, path: Path) -> bool:

2
src/ytdl_sub/utils/scriptable.py
View file

@ -45,6 +45,8 @@ class Scriptable(ABC):
@classmethod
def to_script(cls, value: Any) -> str:
if value is None:
return ""
if isinstance(value, str):
return value
if isinstance(value, int):

4
src/ytdl_sub/ytdl_additions/enhanced_download_archive.py
View file

@ -72,8 +72,8 @@ class DownloadMapping:
DownloadMapping for the entry
"""
return DownloadMapping(
upload_date=entry.get(v.upload_date_standardized),
extractor=entry.get(v.extractor),
upload_date=entry.get_str(v.upload_date_standardized),
extractor=entry.get_str(v.extractor),
file_names=set(),
)

3
tests/unit/entries/conftest.py
View file

@ -69,6 +69,9 @@ def mock_entry_to_dict(
"title_sanitized": "entry ｛title｝",
"ext": ext,
"description": "",
"comments": "",
"requested_subtitles": "",
"sponsorblock_chapters": "",
"creator": "abc123",
"creator_sanitized": "abc123",
"channel": "abc123",

8
tests/unit/entries/test_entry.py
View file

@ -43,8 +43,8 @@ class TestEntry(object):
assert entry.get_int(v.upload_year_truncated_reversed) == year_rev
assert entry.get_int(v.upload_month_reversed) == month_rev
assert entry.get_int(v.upload_day_reversed) == day_rev
assert entry.get(v.upload_month_reversed_padded) == month_rev_pad
assert entry.get(v.upload_day_reversed_padded) == day_rev_pad
assert entry.get_str(v.upload_month_reversed_padded) == month_rev_pad
assert entry.get_str(v.upload_day_reversed_padded) == day_rev_pad
@pytest.mark.parametrize(
"upload_date, day_year, day_year_rev, day_year_pad, day_year_rev_pad",
@ -63,5 +63,5 @@ class TestEntry(object):
assert entry.get_int(v.upload_day_of_year) == day_year
assert entry.get_int(v.upload_day_of_year_reversed) == day_year_rev
assert entry.get(v.upload_day_of_year_padded) == day_year_pad
assert entry.get(v.upload_day_of_year_reversed_padded) == day_year_rev_pad
assert entry.get_str(v.upload_day_of_year_padded) == day_year_pad
assert entry.get_str(v.upload_day_of_year_reversed_padded) == day_year_rev_pad