diff --git a/lib/pinchflat/media/media.ex b/lib/pinchflat/media/media.ex index b8a35fa..191b28f 100644 --- a/lib/pinchflat/media/media.ex +++ b/lib/pinchflat/media/media.ex @@ -41,9 +41,10 @@ defmodule Pinchflat.Media do Returns a list of media_items that are redownloadable based on the redownload delay of the media_profile their source belongs to. - The logic is that a media_item is past_redownload_delay if the media_item's - upload_date is at least redownload_delay_days ago AND - `media_downloaded_at` - `redownload_delay_days` is before the media_item's `upload_date`. + The logic is that a media_item is past_redownload_delay if the media_item's uploaded_at is + at least redownload_delay_days ago AND `media_downloaded_at` - `redownload_delay_days` + is before the media_item's `uploaded_at`. + This logic grabs media that we've recently downloaded AND is recently uploaded, but doesn't grab media that we've recently downloaded and was uploaded a long time ago. This also makes things work as expected when downloading media from a source for the @@ -135,7 +136,7 @@ defmodule Pinchflat.Media do Unlike `create_media_item`, this will attempt an update if the media_item already exists. This is so that future indexing can pick up attributes that - we may not have asked for in the past (eg: upload_date) + we may not have asked for in the past (eg: uploaded_at) Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}} """ diff --git a/lib/pinchflat/media/media_item.ex b/lib/pinchflat/media/media_item.ex index eab0ef5..bc4ec74 100644 --- a/lib/pinchflat/media/media_item.ex +++ b/lib/pinchflat/media/media_item.ex @@ -26,7 +26,7 @@ defmodule Pinchflat.Media.MediaItem do :livestream, :source_id, :short_form_content, - :upload_date, + :uploaded_at, :upload_date_index, :duration_seconds, # these fields are captured only on download @@ -51,7 +51,7 @@ defmodule Pinchflat.Media.MediaItem do livestream media_id source_id - upload_date + uploaded_at short_form_content )a @@ -69,7 +69,7 @@ defmodule Pinchflat.Media.MediaItem do field :short_form_content, :boolean, default: false field :media_downloaded_at, :utc_datetime field :media_redownloaded_at, :utc_datetime - field :upload_date, :date + field :uploaded_at, :utc_datetime field :upload_date_index, :integer, default: 0 field :duration_seconds, :integer @@ -130,7 +130,7 @@ defmodule Pinchflat.Media.MediaItem do ~w(__meta__ __struct__ metadata tasks media_items_search_index)a end - defp update_upload_date_index(%{changes: changes} = changeset) when is_map_key(changes, :upload_date) do + defp update_upload_date_index(%{changes: changes} = changeset) when is_map_key(changes, :uploaded_at) do source_id = get_field(changeset, :source_id) source = Sources.get_source!(source_id) # Channels should count down from 99, playlists should count up from 0 @@ -142,7 +142,7 @@ defmodule Pinchflat.Media.MediaItem do current_max = MediaQuery.new() - |> where(^dynamic([mi], mi.upload_date == ^changes.upload_date and ^MediaQuery.for_source(source))) + |> where(^dynamic([mi], ^MediaQuery.upload_date_matches(changes.uploaded_at) and ^MediaQuery.for_source(source))) |> Repo.aggregate(aggregator, :upload_date_index) case current_max do diff --git a/lib/pinchflat/media/media_query.ex b/lib/pinchflat/media/media_query.ex index 2eeb5f9..2adb0ce 100644 --- a/lib/pinchflat/media/media_query.ex +++ b/lib/pinchflat/media/media_query.ex @@ -35,9 +35,14 @@ defmodule Pinchflat.Media.MediaQuery do def culling_prevented, do: dynamic([mi], mi.prevent_culling == true) def culled, do: dynamic([mi], not is_nil(mi.culled_at)) def redownloaded, do: dynamic([mi], not is_nil(mi.media_redownloaded_at)) + def upload_date_matches(other_date), do: dynamic([mi], fragment("date(?) = date(?)", mi.uploaded_at, ^other_date)) def upload_date_after_source_cutoff do - dynamic([mi, source], is_nil(source.download_cutoff_date) or mi.upload_date >= source.download_cutoff_date) + dynamic( + [mi, source], + is_nil(source.download_cutoff_date) or + fragment("date(?) >= ?", mi.uploaded_at, source.download_cutoff_date) + ) end def format_matching_profile_preference do @@ -84,12 +89,12 @@ defmodule Pinchflat.Media.MediaQuery do def past_redownload_delay do dynamic( [mi, source, media_profile], - # Returns media items where the upload_date is at least redownload_delay_days ago AND + # Returns media items where the uploaded_at is at least redownload_delay_days ago AND # downloaded_at minus the redownload_delay_days is before the upload date fragment(""" IFNULL(redownload_delay_days, 0) > 0 AND - DATETIME('now', '-' || redownload_delay_days || ' day') > upload_date AND - DATETIME(media_downloaded_at, '-' || redownload_delay_days || ' day') < upload_date + DATETIME('now', '-' || redownload_delay_days || ' day') > uploaded_at AND + DATETIME(media_downloaded_at, '-' || redownload_delay_days || ' day') < uploaded_at """) ) end diff --git a/lib/pinchflat/metadata/metadata_file_helpers.ex b/lib/pinchflat/metadata/metadata_file_helpers.ex index f45d556..d54c803 100644 --- a/lib/pinchflat/metadata/metadata_file_helpers.ex +++ b/lib/pinchflat/metadata/metadata_file_helpers.ex @@ -88,14 +88,17 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do @doc """ Parses an upload date from the YYYYMMDD string returned in yt-dlp metadata - and returns a Date struct. + and returns a DateTime struct, appending a time of 00:00:00Z. - Returns Date.t() + Returns DateTime.t() """ def parse_upload_date(upload_date) do <> <> <> <> <> = upload_date - Date.from_iso8601!("#{year}-#{month}-#{day}") + case DateTime.from_iso8601("#{year}-#{month}-#{day}T00:00:00Z") do + {:ok, datetime, _} -> datetime + _ -> raise "Invalid upload date: #{upload_date}" + end end @doc """ diff --git a/lib/pinchflat/metadata/metadata_parser.ex b/lib/pinchflat/metadata/metadata_parser.ex index ef048b8..60456d8 100644 --- a/lib/pinchflat/metadata/metadata_parser.ex +++ b/lib/pinchflat/metadata/metadata_parser.ex @@ -8,6 +8,8 @@ defmodule Pinchflat.Metadata.MetadataParser do and not have it, ya know? """ + alias Pinchflat.YtDlp.Media, as: YtDlpMedia + @doc """ Parses the given JSON response from yt-dlp and returns a map of the needful media_item attributes, along with anything needed for @@ -24,15 +26,12 @@ defmodule Pinchflat.Metadata.MetadataParser do end defp parse_media_metadata(metadata) do - %{ - media_id: metadata["id"], - title: metadata["title"], - original_url: metadata["original_url"], - description: metadata["description"], - media_filepath: metadata["filepath"], - livestream: !!metadata["was_live"], - duration_seconds: metadata["duration"] && round(metadata["duration"]) - } + Map.merge( + Map.from_struct(YtDlpMedia.response_to_struct(metadata)), + %{ + media_filepath: metadata["filepath"] + } + ) end defp parse_subtitle_metadata(metadata) do diff --git a/lib/pinchflat/podcasts/podcast_helpers.ex b/lib/pinchflat/podcasts/podcast_helpers.ex index a40e63b..041107c 100644 --- a/lib/pinchflat/podcasts/podcast_helpers.ex +++ b/lib/pinchflat/podcasts/podcast_helpers.ex @@ -28,7 +28,7 @@ defmodule Pinchflat.Podcasts.PodcastHelpers do MediaQuery.new() |> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.downloaded())) - |> order_by(desc: :upload_date) + |> order_by(desc: :uploaded_at) |> Repo.maybe_limit(limit) |> Repo.all() |> Enum.filter(fn media_item -> File.exists?(media_item.media_filepath) end) diff --git a/lib/pinchflat/podcasts/rss_feed_builder.ex b/lib/pinchflat/podcasts/rss_feed_builder.ex index 34156fe..d5a5aea 100644 --- a/lib/pinchflat/podcasts/rss_feed_builder.ex +++ b/lib/pinchflat/podcasts/rss_feed_builder.ex @@ -7,7 +7,6 @@ defmodule Pinchflat.Podcasts.RssFeedBuilder do import Pinchflat.Utils.XmlUtils, only: [safe: 1] - alias Pinchflat.Utils.DatetimeUtils alias Pinchflat.Podcasts.PodcastHelpers alias PinchflatWeb.Router.Helpers, as: Routes @@ -83,7 +82,7 @@ defmodule Pinchflat.Podcasts.RssFeedBuilder do #{safe(media_item.title)} #{safe(media_item.original_url)} #{safe(media_item.description)} - #{generate_upload_date(media_item)} + #{Calendar.strftime(media_item.uploaded_at, @datetime_format)} #{media_item.duration_seconds} DatetimeUtils.date_to_datetime() - |> Calendar.strftime(@datetime_format) - end - defp podcast_route(action, params) do Routes.podcast_path(PinchflatWeb.Endpoint, action, params) end diff --git a/lib/pinchflat/utils/datetime_utils.ex b/lib/pinchflat/utils/datetime_utils.ex deleted file mode 100644 index 7fce7c6..0000000 --- a/lib/pinchflat/utils/datetime_utils.ex +++ /dev/null @@ -1,17 +0,0 @@ -defmodule Pinchflat.Utils.DatetimeUtils do - @moduledoc """ - Utility methods for working with dates and datetimes - """ - - @doc """ - Converts a Date to a DateTime - - Returns %DateTime{} - """ - def date_to_datetime(date) do - date - |> Date.to_gregorian_days() - |> Kernel.*(86_400) - |> DateTime.from_gregorian_seconds() - end -end diff --git a/lib/pinchflat/yt_dlp/media.ex b/lib/pinchflat/yt_dlp/media.ex index 70a3e97..106d6d0 100644 --- a/lib/pinchflat/yt_dlp/media.ex +++ b/lib/pinchflat/yt_dlp/media.ex @@ -10,7 +10,7 @@ defmodule Pinchflat.YtDlp.Media do :original_url, :livestream, :short_form_content, - :upload_date, + :uploaded_at, :duration_seconds ] @@ -21,7 +21,7 @@ defmodule Pinchflat.YtDlp.Media do :original_url, :livestream, :short_form_content, - :upload_date, + :uploaded_at, :duration_seconds ] @@ -72,7 +72,7 @@ defmodule Pinchflat.YtDlp.Media do Returns the output template for yt-dlp's indexing command. """ def indexing_output_template do - "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date})j" + "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date,timestamp})j" end @doc """ @@ -90,7 +90,7 @@ defmodule Pinchflat.YtDlp.Media do livestream: !!response["was_live"], duration_seconds: response["duration"] && round(response["duration"]), short_form_content: response["webpage_url"] && short_form_content?(response), - upload_date: response["upload_date"] && MetadataFileHelpers.parse_upload_date(response["upload_date"]) + uploaded_at: response["upload_date"] && parse_uploaded_at(response) } end @@ -110,6 +110,18 @@ defmodule Pinchflat.YtDlp.Media do end end + defp parse_uploaded_at(%{"timestamp" => ts} = response) when is_number(ts) do + case DateTime.from_unix(ts) do + {:ok, datetime} -> datetime + _ -> MetadataFileHelpers.parse_upload_date(response["upload_date"]) + end + end + + # This field is needed before inserting into the database, but absence + # of this field should fail at insert-time rather than here + defp parse_uploaded_at(%{"upload_date" => nil}), do: nil + defp parse_uploaded_at(response), do: MetadataFileHelpers.parse_upload_date(response["upload_date"]) + defp backend_runner do # This approach lets us mock the command for testing Application.get_env(:pinchflat, :yt_dlp_runner) diff --git a/lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex b/lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex index 1f4d5bf..372fe56 100644 --- a/lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex +++ b/lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex @@ -31,7 +31,7 @@ <.media_preview media_item={@media_item} />