diff --git a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py index 60983a1e..8d8276ed 100644 --- a/src/ytdl_sub/downloaders/info_json/info_json_downloader.py +++ b/src/ytdl_sub/downloaders/info_json/info_json_downloader.py @@ -13,6 +13,7 @@ from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.script.variable_definitions import VARIABLES as v +from ytdl_sub.entries.script.variable_scripts import ENTRY_INJECTED_VARIABLES from ytdl_sub.entries.script.variable_scripts import VARIABLE_SCRIPTS from ytdl_sub.entries.variables.kwargs import DOWNLOAD_INDEX from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX @@ -101,27 +102,27 @@ class InfoJsonDownloader(SourcePlugin[InfoJsonDownloaderOptions]): entries: List[Entry] = [] for download_mapping in self._enhanced_download_archive.mapping.entry_mappings.values(): - entry = self._get_entry_from_download_mapping(download_mapping).initialize_script( - override_variables=self.overrides.dict_with_format_strings - ) + entry = self._get_entry_from_download_mapping(download_mapping) - prior_variables = entry.kwargs_get(YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY, {}) + # See if prior variables exist. If so, delete them from metadata + # to avoid saving them recursively on multiple updates + prior_variables = {} + if entry.kwargs_contains(YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY): + prior_variables = entry.kwargs(YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY) + del entry._kwargs[YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY] - entry.add( + entry.initialize_script(override_variables=self.overrides.dict_with_format_strings).add( { - v.download_index.variable_name: prior_variables.get( - v.download_index.variable_name, - VARIABLE_SCRIPTS[v.download_index.variable_name], - ), - v.upload_date_index.variable_name: prior_variables.get( - v.upload_date_index.variable_name, - VARIABLE_SCRIPTS[v.upload_date_index.variable_name], - ), + inj.variable_name: prior_variables.get( + inj.variable_name, + VARIABLE_SCRIPTS[inj.variable_name], + ) + for inj in ENTRY_INJECTED_VARIABLES } ) entries.append(entry) - for entry in sorted(entries, key=lambda ent: ent.get(v.download_index)): + for entry in sorted(entries, key=lambda ent: ent.get_str(v.download_index)): # Remove each entry from the live download archive since it will get re-added # unless it is filtered self._enhanced_download_archive.mapping.remove_entry(entry.uid) diff --git a/src/ytdl_sub/downloaders/url/downloader.py b/src/ytdl_sub/downloaders/url/downloader.py index 53b1dc72..af24c2e9 100644 --- a/src/ytdl_sub/downloaders/url/downloader.py +++ b/src/ytdl_sub/downloaders/url/downloader.py @@ -24,13 +24,8 @@ from ytdl_sub.entries.entry import Entry from ytdl_sub.entries.entry_parent import EntryParent from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.entries.variables.kwargs import COLLECTION_URL -from ytdl_sub.entries.variables.kwargs import COMMENTS -from ytdl_sub.entries.variables.kwargs import DOWNLOAD_INDEX from ytdl_sub.entries.variables.kwargs import PLAYLIST_ENTRY -from ytdl_sub.entries.variables.kwargs import REQUESTED_SUBTITLES from ytdl_sub.entries.variables.kwargs import SOURCE_ENTRY -from ytdl_sub.entries.variables.kwargs import SPONSORBLOCK_CHAPTERS -from ytdl_sub.entries.variables.kwargs import UPLOAD_DATE_INDEX from ytdl_sub.utils.file_handler import FileHandler from ytdl_sub.utils.logger import Logger from ytdl_sub.utils.thumbnail import ThumbnailTypes @@ -182,7 +177,7 @@ class UrlDownloaderCollectionVariablePlugin(SourcePluginExtension): """ # COLLECTION_URL is a recent variable that may not exist for old entries when updating. # Try to use source_webpage_url if it does not exist - entry_collection_url = entry.kwargs_get(COLLECTION_URL, entry.get(v.source_webpage_url)) + entry_collection_url = entry.kwargs_get(COLLECTION_URL, entry.get_str(v.source_webpage_url)) # If the collection URL cannot find its mapping, use the last URL collection_url = ( @@ -504,18 +499,8 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): download_logger.info("Entry rejected by download match-filter, skipping ..") return None - entry.add_kwargs( - { - # Subtitles are not downloaded in metadata run, only here, so move over - REQUESTED_SUBTITLES: download_entry.kwargs_get(REQUESTED_SUBTITLES), - # Same with sponsorblock chapters - SPONSORBLOCK_CHAPTERS: download_entry.kwargs_get(SPONSORBLOCK_CHAPTERS), - COMMENTS: download_entry.kwargs_get(COMMENTS), - } - ) - upload_date_idx = self._enhanced_download_archive.mapping.get_num_entries_with_upload_date( - upload_date_standardized=entry.get(v.upload_date_standardized) + upload_date_standardized=entry.get_str(v.upload_date_standardized) ) download_idx = self._enhanced_download_archive.num_entries entry.add( @@ -524,6 +509,15 @@ class MultiUrlDownloader(SourcePlugin[MultiUrlValidator]): v.download_index.variable_name: download_idx + 1, # Tracks number of entries with the same upload date to make them unique v.upload_date_index.variable_name: upload_date_idx + 1, + v.requested_subtitles.variable_name: download_entry.kwargs_get( + v.requested_subtitles.metadata_key + ), + v.sponsorblock_chapters.variable_name: download_entry.kwargs_get( + v.sponsorblock_chapters.metadata_key + ), + v.comments.variable_name: download_entry.kwargs_get( + v.sponsorblock_chapters.metadata_key + ), } ) diff --git a/src/ytdl_sub/entries/entry.py b/src/ytdl_sub/entries/entry.py index f52050ec..f477c5f7 100644 --- a/src/ytdl_sub/entries/entry.py +++ b/src/ytdl_sub/entries/entry.py @@ -4,6 +4,8 @@ import os from pathlib import Path from typing import Dict from typing import Optional +from typing import Type +from typing import TypeVar from typing import final from ytdl_sub.entries.base_entry import BaseEntry @@ -15,6 +17,8 @@ from ytdl_sub.validators.audo_codec_validator import VIDEO_CODEC_EXTS YTDL_SUB_ENTRY_VARIABLES_KWARG_KEY: str = "ytdl_sub_entry_variables" +TType = TypeVar("TType") + class Entry(BaseEntry, Scriptable): """ @@ -31,11 +35,15 @@ class Entry(BaseEntry, Scriptable): self.update_script() return self - def get(self, variable: Variable) -> str: - return self.script.resolve(unresolvable=self.unresolvable).get_str(variable.variable_name) + def get(self, variable: Variable, expected_type: Type[TType]) -> TType: + out = self.script.resolve(unresolvable=self.unresolvable).get_native(variable.variable_name) + return expected_type(out) + + def get_str(self, variable: Variable) -> str: + return self.get(variable, str) def get_int(self, variable: Variable) -> int: - return self.script.resolve(unresolvable=self.unresolvable).get_int(variable.variable_name) + return self.get(variable, int) @property def ext(self) -> str: @@ -44,7 +52,7 @@ class Entry(BaseEntry, Scriptable): This is not reflected in the entry. See if the mkv file exists and return "mkv" if so, otherwise, return the original extension. """ - ext = self.get(VARIABLES.ext) + ext = self.get_str(VARIABLES.ext) for possible_ext in [ext, "mkv"]: file_path = str(Path(self.working_directory()) / f"{self.uid}.{possible_ext}") if os.path.isfile(file_path): @@ -70,7 +78,7 @@ class Entry(BaseEntry, Scriptable): ------- The download thumbnail's file name """ - return f"{self.get(VARIABLES.uid)}.{self.get(VARIABLES.thumbnail_ext)}" + return f"{self.get_str(VARIABLES.uid)}.{self.get_str(VARIABLES.thumbnail_ext)}" def get_download_thumbnail_path(self) -> str: """Returns the entry's thumbnail's file path to where it was downloaded""" diff --git a/src/ytdl_sub/entries/script/variable_definitions.py b/src/ytdl_sub/entries/script/variable_definitions.py index 110780a9..53e2ed48 100644 --- a/src/ytdl_sub/entries/script/variable_definitions.py +++ b/src/ytdl_sub/entries/script/variable_definitions.py @@ -595,7 +595,19 @@ class _Variables: return Variable("thumbnail_ext") @property - def download_index(self) -> MetadataVariable: + def comments(self) -> MetadataVariable: + return MetadataVariable("comments", "comments") + + @property + def sponsorblock_chapters(self) -> MetadataVariable: + return MetadataVariable("sponsorblock_chapters", "sponsorblock_chapters") + + @property + def requested_subtitles(self) -> MetadataVariable: + return MetadataVariable("requested_subtitles", "requested_subtitles") + + @property + def download_index(self) -> Variable: """ Returns ------- @@ -603,7 +615,7 @@ class _Variables: The i'th entry downloaded. NOTE that this is fetched dynamically from the download archive. """ - return MetadataVariable(variable_name="download_index", metadata_key="download_index") + return Variable(variable_name="download_index") @property def download_index_padded6(self) -> Variable: @@ -616,14 +628,14 @@ class _Variables: return Variable("download_index_padded6") @property - def upload_date_index(self) -> MetadataVariable: + def upload_date_index(self) -> Variable: """ Returns ------- int The i'th entry downloaded with this upload date. """ - return MetadataVariable(variable_name="upload_date_index", metadata_key="upload_date_index") + return Variable(variable_name="upload_date_index") @property def upload_date_index_padded(self) -> Variable: diff --git a/src/ytdl_sub/entries/script/variable_scripts.py b/src/ytdl_sub/entries/script/variable_scripts.py index 4525420f..0430ade3 100644 --- a/src/ytdl_sub/entries/script/variable_scripts.py +++ b/src/ytdl_sub/entries/script/variable_scripts.py @@ -134,6 +134,9 @@ ENTRY_INJECTED_VARIABLES: Dict[Variable, str] = { v.download_index: "{%int(1)}", v.upload_date_index: "{%int(1)}", v.playlist_max_upload_year: f"{{{v.upload_year.variable_name}}}", + v.comments: "", + v.requested_subtitles: "", + v.sponsorblock_chapters: "", } ENTRY_DERIVED_VARIABLES: Dict[Variable, str] = { diff --git a/src/ytdl_sub/plugins/chapters.py b/src/ytdl_sub/plugins/chapters.py index 456994f5..ae507248 100644 --- a/src/ytdl_sub/plugins/chapters.py +++ b/src/ytdl_sub/plugins/chapters.py @@ -9,7 +9,7 @@ from ytdl_sub.config.plugin import Plugin from ytdl_sub.config.preset_options import OptionsDictValidator from ytdl_sub.downloaders.ytdl_options_builder import YTDLOptionsBuilder from ytdl_sub.entries.entry import Entry -from ytdl_sub.entries.variables.kwargs import COMMENTS +from ytdl_sub.entries.script.variable_definitions import VARIABLES as v from ytdl_sub.entries.variables.kwargs import YTDL_SUB_CUSTOM_CHAPTERS from ytdl_sub.utils.chapters import Chapters from ytdl_sub.utils.ffmpeg import set_ffmpeg_metadata_chapters @@ -305,7 +305,7 @@ class ChaptersPlugin(Plugin[ChaptersOptions]): # If there are no embedded chapters, and comment chapters are allowed... if not _contains_any_chapters(entry) and self.plugin_options.allow_chapters_from_comments: # Try to get chapters from comments - for comment in entry.kwargs_get(COMMENTS, []): + for comment in entry.get(v.comments, list): chapters = Chapters.from_string(comment.get("text", "")) if chapters.contains_any_chapters(): break diff --git a/src/ytdl_sub/thread/log_entries_downloaded_listener.py b/src/ytdl_sub/thread/log_entries_downloaded_listener.py index ffa9c3b9..690c0468 100644 --- a/src/ytdl_sub/thread/log_entries_downloaded_listener.py +++ b/src/ytdl_sub/thread/log_entries_downloaded_listener.py @@ -42,7 +42,7 @@ class LogEntriesDownloadedListener(threading.Thread): # swallow the error since this is only printing logs return None - return file_json.get("title") + return file_json.get_str("title") @classmethod def _is_info_json(cls, path: Path) -> bool: diff --git a/src/ytdl_sub/utils/scriptable.py b/src/ytdl_sub/utils/scriptable.py index f1bb07f8..13371016 100644 --- a/src/ytdl_sub/utils/scriptable.py +++ b/src/ytdl_sub/utils/scriptable.py @@ -45,6 +45,8 @@ class Scriptable(ABC): @classmethod def to_script(cls, value: Any) -> str: + if value is None: + return "" if isinstance(value, str): return value if isinstance(value, int): diff --git a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py index 7cb9cdd7..4035abbd 100644 --- a/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py +++ b/src/ytdl_sub/ytdl_additions/enhanced_download_archive.py @@ -72,8 +72,8 @@ class DownloadMapping: DownloadMapping for the entry """ return DownloadMapping( - upload_date=entry.get(v.upload_date_standardized), - extractor=entry.get(v.extractor), + upload_date=entry.get_str(v.upload_date_standardized), + extractor=entry.get_str(v.extractor), file_names=set(), ) diff --git a/tests/unit/entries/conftest.py b/tests/unit/entries/conftest.py index a6b0e616..2fe03bde 100644 --- a/tests/unit/entries/conftest.py +++ b/tests/unit/entries/conftest.py @@ -69,6 +69,9 @@ def mock_entry_to_dict( "title_sanitized": "entry {title}", "ext": ext, "description": "", + "comments": "", + "requested_subtitles": "", + "sponsorblock_chapters": "", "creator": "abc123", "creator_sanitized": "abc123", "channel": "abc123", diff --git a/tests/unit/entries/test_entry.py b/tests/unit/entries/test_entry.py index be21ebd3..7edfcdd4 100644 --- a/tests/unit/entries/test_entry.py +++ b/tests/unit/entries/test_entry.py @@ -43,8 +43,8 @@ class TestEntry(object): assert entry.get_int(v.upload_year_truncated_reversed) == year_rev assert entry.get_int(v.upload_month_reversed) == month_rev assert entry.get_int(v.upload_day_reversed) == day_rev - assert entry.get(v.upload_month_reversed_padded) == month_rev_pad - assert entry.get(v.upload_day_reversed_padded) == day_rev_pad + assert entry.get_str(v.upload_month_reversed_padded) == month_rev_pad + assert entry.get_str(v.upload_day_reversed_padded) == day_rev_pad @pytest.mark.parametrize( "upload_date, day_year, day_year_rev, day_year_pad, day_year_rev_pad", @@ -63,5 +63,5 @@ class TestEntry(object): assert entry.get_int(v.upload_day_of_year) == day_year assert entry.get_int(v.upload_day_of_year_reversed) == day_year_rev - assert entry.get(v.upload_day_of_year_padded) == day_year_pad - assert entry.get(v.upload_day_of_year_reversed_padded) == day_year_rev_pad + assert entry.get_str(v.upload_day_of_year_padded) == day_year_pad + assert entry.get_str(v.upload_day_of_year_reversed_padded) == day_year_rev_pad