danvics/pinchflat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[Housekeeping] Store the upload event as a datetime rather than a date (#269)

Browse source 
* [WIP] renamed and migrated upload_date column

* Refactored yt-dlp media module

* Refactored parse_upload_date

* Refactored media item upload_date_index

* Got media tests running

* Refactored media item table live

* Cleaned up the stragglers

* Fixed old oversight re: original_url
This commit is contained in:
Kieran 2024-05-28 13:01:40 -07:00 committed by GitHub
parent 7603ba017f
commit 22439e0273
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 298 additions and 280 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
lib/pinchflat/media/media.ex
View file

@ -41,9 +41,10 @@ defmodule Pinchflat.Media do
Returns a list of media_items that are redownloadable based on the redownload delay
of the media_profile their source belongs to.
The logic is that a media_item is past_redownload_delay if the media_item's
upload_date is at least redownload_delay_days ago AND
`media_downloaded_at` - `redownload_delay_days` is before the media_item's `upload_date`.
The logic is that a media_item is past_redownload_delay if the media_item's uploaded_at is
at least redownload_delay_days ago AND `media_downloaded_at` - `redownload_delay_days`
is before the media_item's `uploaded_at`.
This logic grabs media that we've recently downloaded AND is recently uploaded, but
doesn't grab media that we've recently downloaded and was uploaded a long time ago.
This also makes things work as expected when downloading media from a source for the
@ -135,7 +136,7 @@ defmodule Pinchflat.Media do
Unlike `create_media_item`, this will attempt an update if the media_item
already exists. This is so that future indexing can pick up attributes that
we may not have asked for in the past (eg: upload_date)
we may not have asked for in the past (eg: uploaded_at)
Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}
"""

10
lib/pinchflat/media/media_item.ex
View file

@ -26,7 +26,7 @@ defmodule Pinchflat.Media.MediaItem do
:livestream,
:source_id,
:short_form_content,
:upload_date,
:uploaded_at,
:upload_date_index,
:duration_seconds,
# these fields are captured only on download
@ -51,7 +51,7 @@ defmodule Pinchflat.Media.MediaItem do
livestream
media_id
source_id
upload_date
uploaded_at
short_form_content
)a
@ -69,7 +69,7 @@ defmodule Pinchflat.Media.MediaItem do
field :short_form_content, :boolean, default: false
field :media_downloaded_at, :utc_datetime
field :media_redownloaded_at, :utc_datetime
field :upload_date, :date
field :uploaded_at, :utc_datetime
field :upload_date_index, :integer, default: 0
field :duration_seconds, :integer
@ -130,7 +130,7 @@ defmodule Pinchflat.Media.MediaItem do
~w(__meta__ __struct__ metadata tasks media_items_search_index)a
end
defp update_upload_date_index(%{changes: changes} = changeset) when is_map_key(changes, :upload_date) do
defp update_upload_date_index(%{changes: changes} = changeset) when is_map_key(changes, :uploaded_at) do
source_id = get_field(changeset, :source_id)
source = Sources.get_source!(source_id)
# Channels should count down from 99, playlists should count up from 0
@ -142,7 +142,7 @@ defmodule Pinchflat.Media.MediaItem do
current_max =
MediaQuery.new()
|> where(^dynamic([mi], mi.upload_date == ^changes.upload_date and ^MediaQuery.for_source(source)))
|> where(^dynamic([mi], ^MediaQuery.upload_date_matches(changes.uploaded_at) and ^MediaQuery.for_source(source)))
|> Repo.aggregate(aggregator, :upload_date_index)
case current_max do

13
lib/pinchflat/media/media_query.ex
View file

@ -35,9 +35,14 @@ defmodule Pinchflat.Media.MediaQuery do
def culling_prevented, do: dynamic([mi], mi.prevent_culling == true)
def culled, do: dynamic([mi], not is_nil(mi.culled_at))
def redownloaded, do: dynamic([mi], not is_nil(mi.media_redownloaded_at))
def upload_date_matches(other_date), do: dynamic([mi], fragment("date(?) = date(?)", mi.uploaded_at, ^other_date))
def upload_date_after_source_cutoff do
dynamic([mi, source], is_nil(source.download_cutoff_date) or mi.upload_date >= source.download_cutoff_date)
dynamic(
[mi, source],
is_nil(source.download_cutoff_date) or
fragment("date(?) >= ?", mi.uploaded_at, source.download_cutoff_date)
)
end
def format_matching_profile_preference do
@ -84,12 +89,12 @@ defmodule Pinchflat.Media.MediaQuery do
def past_redownload_delay do
dynamic(
[mi, source, media_profile],
# Returns media items where the upload_date is at least redownload_delay_days ago AND
# Returns media items where the uploaded_at is at least redownload_delay_days ago AND
# downloaded_at minus the redownload_delay_days is before the upload date
fragment("""
IFNULL(redownload_delay_days, 0) > 0 AND
DATETIME('now', '-' || redownload_delay_days || ' day') > upload_date AND
DATETIME(media_downloaded_at, '-' || redownload_delay_days || ' day') < upload_date
DATETIME('now', '-' || redownload_delay_days || ' day') > uploaded_at AND
DATETIME(media_downloaded_at, '-' || redownload_delay_days || ' day') < uploaded_at
""")
)
end

9
lib/pinchflat/metadata/metadata_file_helpers.ex
View file

@ -88,14 +88,17 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
@doc """
Parses an upload date from the YYYYMMDD string returned in yt-dlp metadata
and returns a Date struct.
and returns a DateTime struct, appending a time of 00:00:00Z.
Returns Date.t()
Returns DateTime.t()
"""
def parse_upload_date(upload_date) do
<<year::binary-size(4)>> <> <<month::binary-size(2)>> <> <<day::binary-size(2)>> = upload_date
Date.from_iso8601!("#{year}-#{month}-#{day}")
case DateTime.from_iso8601("#{year}-#{month}-#{day}T00:00:00Z") do
{:ok, datetime, _} -> datetime
_ -> raise "Invalid upload date: #{upload_date}"
end
end
@doc """

17
lib/pinchflat/metadata/metadata_parser.ex
View file

@ -8,6 +8,8 @@ defmodule Pinchflat.Metadata.MetadataParser do
and not have it, ya know?
"""
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
@doc """
Parses the given JSON response from yt-dlp and returns a map of
the needful media_item attributes, along with anything needed for
@ -24,15 +26,12 @@ defmodule Pinchflat.Metadata.MetadataParser do
end
defp parse_media_metadata(metadata) do
%{
media_id: metadata["id"],
title: metadata["title"],
original_url: metadata["original_url"],
description: metadata["description"],
media_filepath: metadata["filepath"],
livestream: !!metadata["was_live"],
duration_seconds: metadata["duration"] && round(metadata["duration"])
}
Map.merge(
Map.from_struct(YtDlpMedia.response_to_struct(metadata)),
%{
media_filepath: metadata["filepath"]
}
)
end
defp parse_subtitle_metadata(metadata) do

2
lib/pinchflat/podcasts/podcast_helpers.ex
View file

@ -28,7 +28,7 @@ defmodule Pinchflat.Podcasts.PodcastHelpers do
MediaQuery.new()
|> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.downloaded()))
|> order_by(desc: :upload_date)
|> order_by(desc: :uploaded_at)
|> Repo.maybe_limit(limit)
|> Repo.all()
|> Enum.filter(fn media_item -> File.exists?(media_item.media_filepath) end)

9
lib/pinchflat/podcasts/rss_feed_builder.ex
View file

@ -7,7 +7,6 @@ defmodule Pinchflat.Podcasts.RssFeedBuilder do
import Pinchflat.Utils.XmlUtils, only: [safe: 1]
alias Pinchflat.Utils.DatetimeUtils
alias Pinchflat.Podcasts.PodcastHelpers
alias PinchflatWeb.Router.Helpers, as: Routes
@ -83,7 +82,7 @@ defmodule Pinchflat.Podcasts.RssFeedBuilder do
<title>#{safe(media_item.title)}</title>
<link>#{safe(media_item.original_url)}</link>
<description>#{safe(media_item.description)}</description>
<pubDate>#{generate_upload_date(media_item)}</pubDate>
<pubDate>#{Calendar.strftime(media_item.uploaded_at, @datetime_format)}</pubDate>
<itunes:duration>#{media_item.duration_seconds}</itunes:duration>
<enclosure
url="#{media_stream_path(url_base, media_item)}"
@ -133,12 +132,6 @@ defmodule Pinchflat.Podcasts.RssFeedBuilder do
end
end
defp generate_upload_date(media_item) do
media_item.upload_date
|> DatetimeUtils.date_to_datetime()
|> Calendar.strftime(@datetime_format)
end
defp podcast_route(action, params) do
Routes.podcast_path(PinchflatWeb.Endpoint, action, params)
end

17
lib/pinchflat/utils/datetime_utils.ex
View file

@ -1,17 +0,0 @@
defmodule Pinchflat.Utils.DatetimeUtils do
@moduledoc """
Utility methods for working with dates and datetimes
"""
@doc """
Converts a Date to a DateTime
Returns %DateTime{}
"""
def date_to_datetime(date) do
date
|> Date.to_gregorian_days()
|> Kernel.*(86_400)
|> DateTime.from_gregorian_seconds()
end
end

20
lib/pinchflat/yt_dlp/media.ex
View file

@ -10,7 +10,7 @@ defmodule Pinchflat.YtDlp.Media do
:original_url,
:livestream,
:short_form_content,
:upload_date,
:uploaded_at,
:duration_seconds
]
@ -21,7 +21,7 @@ defmodule Pinchflat.YtDlp.Media do
:original_url,
:livestream,
:short_form_content,
:upload_date,
:uploaded_at,
:duration_seconds
]
@ -72,7 +72,7 @@ defmodule Pinchflat.YtDlp.Media do
Returns the output template for yt-dlp's indexing command.
"""
def indexing_output_template do
"%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date})j"
"%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date,timestamp})j"
end
@doc """
@ -90,7 +90,7 @@ defmodule Pinchflat.YtDlp.Media do
livestream: !!response["was_live"],
duration_seconds: response["duration"] && round(response["duration"]),
short_form_content: response["webpage_url"] && short_form_content?(response),
upload_date: response["upload_date"] && MetadataFileHelpers.parse_upload_date(response["upload_date"])
uploaded_at: response["upload_date"] && parse_uploaded_at(response)
}
end
@ -110,6 +110,18 @@ defmodule Pinchflat.YtDlp.Media do
end
end
defp parse_uploaded_at(%{"timestamp" => ts} = response) when is_number(ts) do
case DateTime.from_unix(ts) do
{:ok, datetime} -> datetime
_ -> MetadataFileHelpers.parse_upload_date(response["upload_date"])
end
end
# This field is needed before inserting into the database, but absence
# of this field should fail at insert-time rather than here
defp parse_uploaded_at(%{"upload_date" => nil}), do: nil
defp parse_uploaded_at(response), do: MetadataFileHelpers.parse_upload_date(response["upload_date"])
defp backend_runner do
# This approach lets us mock the command for testing
Application.get_env(:pinchflat, :yt_dlp_runner)

2
lib/pinchflat_web/controllers/media_items/media_item_html/show.html.heex
View file

@ -31,7 +31,7 @@
<.media_preview media_item={@media_item} />
</div>
<aside class="mt-4 xl:mt-0">
<div>Uploaded: <%= @media_item.upload_date %></div>
<div>Uploaded: <%= DateTime.to_date(@media_item.uploaded_at) %></div>
<div>
<span :if={URI.parse(@media_item.original_url).scheme =~ "http"}>
<.subtle_link href={@media_item.original_url} target="_blank">Open Original</.subtle_link>

2
lib/pinchflat_web/controllers/pages/page_html/history_table_live.ex
View file

@ -32,7 +32,7 @@ defmodule Pinchflat.Pages.HistoryTableLive do
</.subtle_link>
</:col>
<:col :let={media_item} label="Upload Date">
<%= media_item.upload_date %>
<%= DateTime.to_date(media_item.uploaded_at) %>
</:col>
<:col :let={media_item} label="Indexed At">
<%= format_datetime(media_item.inserted_at) %>

8
lib/pinchflat_web/controllers/sources/source_html/media_item_table_live.ex
View file

@ -50,7 +50,7 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
</.subtle_link>
</:col>
<:col :let={media_item} label="Upload Date">
<%= media_item.upload_date %>
<%= DateTime.to_date(media_item.uploaded_at) %>
</:col>
<:col :let={media_item} :if={@media_state == "other"} label="Manually Ignored?">
<.icon name={if media_item.prevent_download, do: "hero-check", else: "hero-x-mark"} />
@ -150,14 +150,14 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
|> MediaQuery.require_assoc(:media_profile)
|> MediaQuery.require_assoc(:media_items_search_index)
|> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.pending()))
|> order_by(desc: fragment("rank"), desc: :upload_date)
|> order_by(desc: fragment("rank"), desc: :uploaded_at)
end
defp generate_base_query(source, "downloaded") do
MediaQuery.new()
|> MediaQuery.require_assoc(:media_items_search_index)
|> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.downloaded()))
|> order_by(desc: fragment("rank"), desc: :upload_date)
|> order_by(desc: fragment("rank"), desc: :uploaded_at)
end
defp generate_base_query(source, "other") do
@ -170,7 +170,7 @@ defmodule Pinchflat.Sources.MediaItemTableLive do
(not (^MediaQuery.downloaded()) and not (^MediaQuery.pending()))
)
)
|> order_by(desc: fragment("rank"), desc: :upload_date)
|> order_by(desc: fragment("rank"), desc: :uploaded_at)
end
defp filter_base_query(base_query, search_term) do

BIN
priv/repo/erd.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 449 KiB

After

Width:  |  Height:  |  Size: 448 KiB

21
priv/repo/migrations/20240528180212_rename_upload_date_to_uploaded_at.exs Normal file
View file

@ -0,0 +1,21 @@
defmodule Pinchflat.Repo.Migrations.RenameUploadDateToUploadedAt do
use Ecto.Migration
def up do
rename table(:media_items), :upload_date, to: :uploaded_at
execute """
UPDATE media_items
SET uploaded_at = uploaded_at || 'T00:00:00'
"""
end
def down do
rename table(:media_items), :uploaded_at, to: :upload_date
execute """
UPDATE media_items
SET upload_date = DATE(upload_date)
"""
end
end

4
test/pinchflat/downloading/media_quality_upgrade_worker_test.exs
View file

@ -16,7 +16,7 @@ defmodule Pinchflat.Downloading.MediaQualityUpgradeWorkerTest do
media_item =
media_item_fixture(%{
source_id: source.id,
upload_date: now_minus(6, :days),
uploaded_at: now_minus(6, :days),
media_downloaded_at: now_minus(5, :days)
})
@ -32,7 +32,7 @@ defmodule Pinchflat.Downloading.MediaQualityUpgradeWorkerTest do
_media_item =
media_item_fixture(%{
source_id: source.id,
upload_date: now_minus(6, :days),
uploaded_at: now_minus(6, :days),
media_downloaded_at: now_minus(1, :day)
})

312
test/pinchflat/media_test.exs
View file

@ -34,150 +34,6 @@ defmodule Pinchflat.MediaTest do
end
end
describe "schema when testing upload_date_index and source is a channel" do
test "upload_date_index is set to 99 if it's the only video uploaded that day" do
upload_date = Date.utc_today()
source = source_fixture(%{collection_type: :channel})
media_item = media_item_fixture(%{source_id: source.id, upload_date: upload_date})
assert media_item.upload_date_index == 99
end
test "upload_date_index is set to 98 if it's the second video uploaded that day" do
upload_date = Date.utc_today()
source = source_fixture(%{collection_type: :channel})
media_item_one = media_item_fixture(%{source_id: source.id, upload_date: upload_date})
media_item_two = media_item_fixture(%{source_id: source.id, upload_date: upload_date})
assert media_item_one.upload_date_index == 99
assert media_item_two.upload_date_index == 98
end
test "upload_date_index doesn't decrement if the video is uploaded on a different day" do
today = Date.utc_today()
one_day_ago = Date.add(today, -1)
source = source_fixture(%{collection_type: :channel})
media_item_new = media_item_fixture(%{source_id: source.id, upload_date: today})
media_item_old = media_item_fixture(%{source_id: source.id, upload_date: one_day_ago})
assert media_item_new.upload_date_index == 99
assert media_item_old.upload_date_index == 99
end
test "recomputes upload_date_index if an upload_date is changed...somehow" do
today = Date.utc_today()
one_day_ago = Date.add(today, -1)
source = source_fixture(%{collection_type: :channel})
media_item_new = media_item_fixture(%{source_id: source.id, upload_date: today})
media_item_old = media_item_fixture(%{source_id: source.id, upload_date: one_day_ago})
{:ok, updated_media_item} = Media.update_media_item(media_item_old, %{upload_date: today})
assert media_item_new.upload_date_index == 99
assert updated_media_item.upload_date_index == 98
end
test "upload_date_index doesn't decrement if the video is for a different source" do
today = Date.utc_today()
source_one = source_fixture(%{collection_type: :channel})
source_two = source_fixture(%{collection_type: :channel})
media_item_one = media_item_fixture(%{source_id: source_one.id, upload_date: today})
media_item_two = media_item_fixture(%{source_id: source_two.id, upload_date: today})
assert media_item_one.upload_date_index == 99
assert media_item_two.upload_date_index == 99
end
test "upload_date_index doesn't decrement if the a video's upload_date is updated but doesn't change" do
today = Date.utc_today()
source = source_fixture(%{collection_type: :channel})
media_item_one = media_item_fixture(%{source_id: source.id, upload_date: today})
_media_item_two = media_item_fixture(%{source_id: source.id, upload_date: today})
{:ok, updated_media_item} = Media.update_media_item(media_item_one, %{upload_date: today, title: "New title"})
assert updated_media_item.upload_date_index == 99
end
end
describe "schema when testing upload_date_index and source is a playlist" do
test "upload_date_index is set to 0 if it's the only video uploaded that day" do
upload_date = Date.utc_today()
source = source_fixture(%{collection_type: :playlist})
media_item = media_item_fixture(%{source_id: source.id, upload_date: upload_date})
assert media_item.upload_date_index == 0
end
test "upload_date_index is set to 1 if it's the second video uploaded that day" do
upload_date = Date.utc_today()
source = source_fixture(%{collection_type: :playlist})
media_item_one = media_item_fixture(%{source_id: source.id, upload_date: upload_date})
media_item_two = media_item_fixture(%{source_id: source.id, upload_date: upload_date})
assert media_item_one.upload_date_index == 0
assert media_item_two.upload_date_index == 1
end
test "upload_date_index doesn't increment if the video is uploaded on a different day" do
today = Date.utc_today()
one_day_ago = Date.add(today, -1)
source = source_fixture(%{collection_type: :playlist})
media_item_new = media_item_fixture(%{source_id: source.id, upload_date: today})
media_item_old = media_item_fixture(%{source_id: source.id, upload_date: one_day_ago})
assert media_item_new.upload_date_index == 0
assert media_item_old.upload_date_index == 0
end
test "recomputes upload_date_index if an upload_date is changed...somehow" do
today = Date.utc_today()
one_day_ago = Date.add(today, -1)
source = source_fixture(%{collection_type: :playlist})
media_item_new = media_item_fixture(%{source_id: source.id, upload_date: today})
media_item_old = media_item_fixture(%{source_id: source.id, upload_date: one_day_ago})
{:ok, updated_media_item} = Media.update_media_item(media_item_old, %{upload_date: today})
assert media_item_new.upload_date_index == 0
assert updated_media_item.upload_date_index == 1
end
test "upload_date_index doesn't increment if the video is for a different source" do
today = Date.utc_today()
source_one = source_fixture(%{collection_type: :playlist})
source_two = source_fixture(%{collection_type: :playlist})
media_item_one = media_item_fixture(%{source_id: source_one.id, upload_date: today})
media_item_two = media_item_fixture(%{source_id: source_two.id, upload_date: today})
assert media_item_one.upload_date_index == 0
assert media_item_two.upload_date_index == 0
end
test "upload_date_index doesn't increment if the a video's upload_date is updated but doesn't change" do
today = Date.utc_today()
source = source_fixture(%{collection_type: :playlist})
media_item_one = media_item_fixture(%{source_id: source.id, upload_date: today})
_media_item_two = media_item_fixture(%{source_id: source.id, upload_date: today})
{:ok, updated_media_item} = Media.update_media_item(media_item_one, %{upload_date: today, title: "New title"})
assert updated_media_item.upload_date_index == 0
end
end
describe "list_media_items/0" do
test "it returns all media_items" do
media_item = media_item_fixture()
@ -289,7 +145,7 @@ defmodule Pinchflat.MediaTest do
media_item =
media_item_fixture(%{
source_id: source.id,
upload_date: now_minus(6, :days),
uploaded_at: now_minus(6, :days),
media_downloaded_at: now_minus(5, :days)
})
@ -300,7 +156,7 @@ defmodule Pinchflat.MediaTest do
media_item =
media_item_fixture(%{
source_id: source.id,
upload_date: now_minus(20, :days),
uploaded_at: now_minus(20, :days),
media_downloaded_at: now_minus(19, :days)
})
@ -311,7 +167,7 @@ defmodule Pinchflat.MediaTest do
_media_item =
media_item_fixture(%{
source_id: source.id,
upload_date: now_minus(5, :days),
uploaded_at: now_minus(5, :days),
media_downloaded_at: nil
})
@ -322,7 +178,7 @@ defmodule Pinchflat.MediaTest do
_media_item =
media_item_fixture(%{
source_id: source.id,
upload_date: now_minus(5, :days),
uploaded_at: now_minus(5, :days),
media_downloaded_at: now(),
prevent_download: true
})
@ -334,7 +190,7 @@ defmodule Pinchflat.MediaTest do
_media_item =
media_item_fixture(%{
source_id: source.id,
upload_date: now_minus(5, :days),
uploaded_at: now_minus(5, :days),
media_downloaded_at: now(),
culled_at: now()
})
@ -346,7 +202,7 @@ defmodule Pinchflat.MediaTest do
_media_item =
media_item_fixture(%{
source_id: source.id,
upload_date: now_minus(3, :days),
uploaded_at: now_minus(3, :days),
media_downloaded_at: now_minus(3, :days)
})
@ -357,7 +213,7 @@ defmodule Pinchflat.MediaTest do
_media_item =
media_item_fixture(%{
source_id: source.id,
upload_date: now_minus(5, :days),
uploaded_at: now_minus(5, :days),
media_downloaded_at: now(),
media_redownloaded_at: now()
})
@ -365,12 +221,12 @@ defmodule Pinchflat.MediaTest do
assert Media.list_redownloadable_media_items() == []
end
test "does not return media items that were first downloaded well after the upload_date", %{source: source} do
test "does not return media items that were first downloaded well after the uploaded_at", %{source: source} do
_media_item =
media_item_fixture(%{
source_id: source.id,
media_downloaded_at: now(),
upload_date: now_minus(20, :days)
uploaded_at: now_minus(20, :days)
})
assert Media.list_redownloadable_media_items() == []
@ -381,7 +237,7 @@ defmodule Pinchflat.MediaTest do
media_item_fixture(%{
source_id: source.id,
media_downloaded_at: now(),
upload_date: now_minus(2, :days)
uploaded_at: now_minus(2, :days)
})
assert Media.list_redownloadable_media_items() == []
@ -394,7 +250,7 @@ defmodule Pinchflat.MediaTest do
_media_item =
media_item_fixture(%{
source_id: source.id,
upload_date: now_minus(6, :days),
uploaded_at: now_minus(6, :days),
media_downloaded_at: now_minus(5, :days)
})
@ -552,9 +408,9 @@ defmodule Pinchflat.MediaTest do
source = source_fixture(%{download_cutoff_date: now_minus(1, :day)})
_old_media_item =
media_item_fixture(%{source_id: source.id, media_filepath: nil, upload_date: now_minus(2, :days)})
media_item_fixture(%{source_id: source.id, media_filepath: nil, uploaded_at: now_minus(2, :days)})
new_media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, upload_date: now()})
new_media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, uploaded_at: now()})
assert Media.list_pending_media_items_for(source) == [new_media_item]
end
@ -563,9 +419,9 @@ defmodule Pinchflat.MediaTest do
source = source_fixture(%{download_cutoff_date: nil})
old_media_item =
media_item_fixture(%{source_id: source.id, media_filepath: nil, upload_date: now_minus(2, :days)})
media_item_fixture(%{source_id: source.id, media_filepath: nil, uploaded_at: now_minus(2, :days)})
new_media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, upload_date: now()})
new_media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, uploaded_at: now()})
assert Media.list_pending_media_items_for(source) == [old_media_item, new_media_item]
end
@ -630,21 +486,21 @@ defmodule Pinchflat.MediaTest do
test "returns true if there is a cutoff date before the media's upload date" do
source = source_fixture(%{download_cutoff_date: now_minus(2, :days)})
media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, upload_date: now_minus(1, :day)})
media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, uploaded_at: now_minus(1, :day)})
assert Media.pending_download?(media_item)
end
test "returns false if there is a cutoff date after the media's upload date" do
source = source_fixture(%{download_cutoff_date: now_minus(1, :day)})
media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, upload_date: now_minus(2, :days)})
media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, uploaded_at: now_minus(2, :days)})
refute Media.pending_download?(media_item)
end
test "returns true if there is no cutoff date" do
source = source_fixture(%{download_cutoff_date: nil})
media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, upload_date: now_minus(1, :day)})
media_item = media_item_fixture(%{source_id: source.id, media_filepath: nil, uploaded_at: now_minus(1, :day)})
assert Media.pending_download?(media_item)
end
@ -755,7 +611,7 @@ defmodule Pinchflat.MediaTest do
media_filepath: "/video/#{Faker.File.file_name(:video)}",
source_id: source_fixture().id,
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
upload_date: Date.utc_today()
uploaded_at: now()
}
assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs)
@ -772,7 +628,7 @@ defmodule Pinchflat.MediaTest do
media_filepath: "/video/#{Faker.File.file_name(:video)}",
source_id: source_fixture().id,
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
upload_date: Date.utc_today()
uploaded_at: now()
}
assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs)
@ -787,7 +643,7 @@ defmodule Pinchflat.MediaTest do
media_filepath: "/video/#{Faker.File.file_name(:video)}",
source_id: source_fixture().id,
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
upload_date: Date.utc_today(),
uploaded_at: now(),
uuid: "some-uuid"
}
@ -1052,4 +908,130 @@ defmodule Pinchflat.MediaTest do
assert %Ecto.Changeset{} = Media.change_media_item(media_item)
end
end
describe "change_media_item/1 when testing upload_date_index and source is a channel" do
test "upload_date_index is set to 99 if it's the only video uploaded that day" do
source = source_fixture(%{collection_type: :channel})
media_item = media_item_fixture(%{source_id: source.id, uploaded_at: now()})
assert media_item.upload_date_index == 99
end
test "upload_date_index is set to 98 if it's the second video uploaded that day" do
source = source_fixture(%{collection_type: :channel})
media_item_one = media_item_fixture(%{source_id: source.id, uploaded_at: now()})
media_item_two = media_item_fixture(%{source_id: source.id, uploaded_at: now()})
assert media_item_one.upload_date_index == 99
assert media_item_two.upload_date_index == 98
end
test "upload_date_index doesn't decrement if the video is uploaded on a different day" do
source = source_fixture(%{collection_type: :channel})
media_item_new = media_item_fixture(%{source_id: source.id, uploaded_at: now()})
media_item_old = media_item_fixture(%{source_id: source.id, uploaded_at: now_minus(1, :day)})
assert media_item_new.upload_date_index == 99
assert media_item_old.upload_date_index == 99
end
test "recomputes upload_date_index if an upload_date is changed...somehow" do
source = source_fixture(%{collection_type: :channel})
media_item_new = media_item_fixture(%{source_id: source.id, uploaded_at: now()})
media_item_old = media_item_fixture(%{source_id: source.id, uploaded_at: now_minus(1, :day)})
{:ok, updated_media_item} = Media.update_media_item(media_item_old, %{uploaded_at: now()})
assert media_item_new.upload_date_index == 99
assert updated_media_item.upload_date_index == 98
end
test "upload_date_index doesn't decrement if the video is for a different source" do
source_one = source_fixture(%{collection_type: :channel})
source_two = source_fixture(%{collection_type: :channel})
media_item_one = media_item_fixture(%{source_id: source_one.id, uploaded_at: now()})
media_item_two = media_item_fixture(%{source_id: source_two.id, uploaded_at: now()})
assert media_item_one.upload_date_index == 99
assert media_item_two.upload_date_index == 99
end
test "upload_date_index doesn't decrement if the a video's upload_date is updated but doesn't change" do
source = source_fixture(%{collection_type: :channel})
media_item_one = media_item_fixture(%{source_id: source.id, uploaded_at: now()})
_media_item_two = media_item_fixture(%{source_id: source.id, uploaded_at: now()})
{:ok, updated_media_item} = Media.update_media_item(media_item_one, %{uploaded_at: now(), title: "New title"})
assert updated_media_item.upload_date_index == 99
end
end
describe "change_media_item/1 when testing upload_date_index and source is a playlist" do
test "upload_date_index is set to 0 if it's the only video uploaded that day" do
source = source_fixture(%{collection_type: :playlist})
media_item = media_item_fixture(%{source_id: source.id, uploaded_at: now()})
assert media_item.upload_date_index == 0
end
test "upload_date_index is set to 1 if it's the second video uploaded that day" do
source = source_fixture(%{collection_type: :playlist})
media_item_one = media_item_fixture(%{source_id: source.id, uploaded_at: now()})
media_item_two = media_item_fixture(%{source_id: source.id, uploaded_at: now()})
assert media_item_one.upload_date_index == 0
assert media_item_two.upload_date_index == 1
end
test "upload_date_index doesn't increment if the video is uploaded on a different day" do
source = source_fixture(%{collection_type: :playlist})
media_item_new = media_item_fixture(%{source_id: source.id, uploaded_at: now()})
media_item_old = media_item_fixture(%{source_id: source.id, uploaded_at: now_minus(1, :day)})
assert media_item_new.upload_date_index == 0
assert media_item_old.upload_date_index == 0
end
test "recomputes upload_date_index if an upload_date is changed...somehow" do
source = source_fixture(%{collection_type: :playlist})
media_item_new = media_item_fixture(%{source_id: source.id, uploaded_at: now()})
media_item_old = media_item_fixture(%{source_id: source.id, uploaded_at: now_minus(1, :day)})
{:ok, updated_media_item} = Media.update_media_item(media_item_old, %{uploaded_at: now()})
assert media_item_new.upload_date_index == 0
assert updated_media_item.upload_date_index == 1
end
test "upload_date_index doesn't increment if the video is for a different source" do
source_one = source_fixture(%{collection_type: :playlist})
source_two = source_fixture(%{collection_type: :playlist})
media_item_one = media_item_fixture(%{source_id: source_one.id, uploaded_at: now()})
media_item_two = media_item_fixture(%{source_id: source_two.id, uploaded_at: now()})
assert media_item_one.upload_date_index == 0
assert media_item_two.upload_date_index == 0
end
test "upload_date_index doesn't increment if the a video's upload_date is updated but doesn't change" do
source = source_fixture(%{collection_type: :playlist})
media_item_one = media_item_fixture(%{source_id: source.id, uploaded_at: now()})
_media_item_two = media_item_fixture(%{source_id: source.id, uploaded_at: now()})
{:ok, updated_media_item} = Media.update_media_item(media_item_one, %{uploaded_at: now(), title: "New title"})
assert updated_media_item.upload_date_index == 0
end
end
end

4
test/pinchflat/metadata/metadata_file_helpers_test.exs
View file

@ -132,10 +132,10 @@ defmodule Pinchflat.Metadata.MetadataFileHelpersTest do
end
describe "parse_upload_date/1" do
test "returns a date from the given metadata upload date" do
test "returns a datetime from the given metadata upload date" do
upload_date = "20210101"
assert Helpers.parse_upload_date(upload_date) == ~D[2021-01-01]
assert Helpers.parse_upload_date(upload_date) == ~U[2021-01-01 00:00:00Z]
end
end

6
test/pinchflat/podcasts/podcast_helpers_test.exs
View file

@ -26,9 +26,9 @@ defmodule Pinchflat.Podcasts.PodcastHelpersTest do
test "orders by upload date where newest is first" do
source = source_fixture()
oldest = media_item_with_attachments(%{source_id: source.id, upload_date: now_minus(2, :day)})
current = media_item_with_attachments(%{source_id: source.id, upload_date: now()})
older = media_item_with_attachments(%{source_id: source.id, upload_date: now_minus(1, :days)})
oldest = media_item_with_attachments(%{source_id: source.id, uploaded_at: now_minus(2, :day)})
current = media_item_with_attachments(%{source_id: source.id, uploaded_at: now()})
older = media_item_with_attachments(%{source_id: source.id, uploaded_at: now_minus(1, :days)})
assert [^current, ^older, ^oldest] = PodcastHelpers.persisted_media_items_for(source)
end

4
test/pinchflat/podcasts/rss_feed_builder_test.exs
View file

@ -117,8 +117,8 @@ defmodule Pinchflat.Podcasts.RssFeedBuilderTest do
assert String.contains?(item_xml, ~s(<itunes:summary><![CDATA[#{media_item.description}]]></itunes:summary>))
end
test "returns pubDate based off the media's upload_date", %{source: source} do
media_item_with_attachments(%{source_id: source.id, upload_date: ~D[2020-01-01]})
test "returns pubDate based off the media's uploaded_at", %{source: source} do
media_item_with_attachments(%{source_id: source.id, uploaded_at: ~U[2020-01-01 00:00:00Z]})
res = RssFeedBuilder.build(source)
[_before, item_xml, _after] = String.split(res, ~r(</?item>))

14
test/pinchflat/utils/datetime_utils_test.exs
View file

@ -1,14 +0,0 @@
defmodule Pinchflat.Utils.DatetimeUtilsTest do
use Pinchflat.DataCase
alias Pinchflat.Utils.DatetimeUtils
describe "date_to_datetime/1" do
test "converts a Date to a DateTime" do
date = ~D[2022-01-01]
datetime = DatetimeUtils.date_to_datetime(date)
assert datetime == ~U[2022-01-01 00:00:00Z]
end
end
end

88
test/pinchflat/yt_dlp/media_test.exs
View file

@ -79,7 +79,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
describe "indexing_output_template/0" do
test "contains all the greatest hits" do
assert "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date})j" ==
assert "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date,timestamp})j" ==
Media.indexing_output_template()
end
end
@ -94,7 +94,8 @@ defmodule Pinchflat.YtDlp.MediaTest do
"was_live" => false,
"aspect_ratio" => 1.0,
"duration" => 60,
"upload_date" => "20210101"
"upload_date" => "20210101",
"timestamp" => 1_600_000_000
}
assert %Media{
@ -104,7 +105,7 @@ defmodule Pinchflat.YtDlp.MediaTest do
original_url: "https://www.youtube.com/watch?v=TiZPUDkDYbk",
livestream: false,
short_form_content: false,
upload_date: Date.from_iso8601!("2021-01-01"),
uploaded_at: ~U[2020-09-13 12:26:40Z],
duration_seconds: 60
} == Media.response_to_struct(response)
end
@ -146,34 +147,11 @@ defmodule Pinchflat.YtDlp.MediaTest do
response = %{
"webpage_url" => nil,
"aspect_ratio" => nil,
"duration" => nil
}
assert %Media{short_form_content: nil} = Media.response_to_struct(response)
end
test "parses the upload date" do
response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0,
"duration" => 61,
"duration" => nil,
"upload_date" => "20210101"
}
expected_date = Date.from_iso8601!("2021-01-01")
assert %Media{upload_date: ^expected_date} = Media.response_to_struct(response)
end
test "doesn't blow up if upload date is missing" do
response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0,
"duration" => 61,
"upload_date" => nil
}
assert %Media{upload_date: nil} = Media.response_to_struct(response)
assert %Media{short_form_content: nil} = Media.response_to_struct(response)
end
test "parses the duration" do
@ -209,4 +187,58 @@ defmodule Pinchflat.YtDlp.MediaTest do
assert %Media{livestream: false} = Media.response_to_struct(response)
end
end
describe "response_to_struct/1 when testing uploaded_at" do
test "parses the upload date from the timestamp if present" do
response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0,
"duration" => 61,
"upload_date" => "20210101",
"timestamp" => 1_600_000_000
}
expected_date = ~U[2020-09-13 12:26:40Z]
assert %Media{uploaded_at: ^expected_date} = Media.response_to_struct(response)
end
test "parses the upload date from the uploaded_at if timestamp is present but nil" do
response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0,
"duration" => 61,
"upload_date" => "20210101",
"timestamp" => nil
}
expected_date = ~U[2021-01-01 00:00:00Z]
assert %Media{uploaded_at: ^expected_date} = Media.response_to_struct(response)
end
test "parses the upload date from the uploaded_at if timestamp absent" do
response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0,
"duration" => 61,
"upload_date" => "20210101"
}
expected_date = ~U[2021-01-01 00:00:00Z]
assert %Media{uploaded_at: ^expected_date} = Media.response_to_struct(response)
end
test "doesn't blow up if upload date is missing" do
response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0,
"duration" => 61,
"upload_date" => nil
}
assert %Media{uploaded_at: nil} = Media.response_to_struct(response)
end
end
end

2
test/support/files/media_metadata.json
View file

@ -9637,7 +9637,7 @@
"uploader_url": "https://www.youtube.com/@PinchflatTestChannel",
"upload_date": "20210720",
"availability": "public",
"original_url": "ABC123",
"original_url": "https://www.youtube.com/watch?v=ABC123",
"webpage_url_basename": "watch",
"webpage_url_domain": "youtube.com",
"extractor": "youtube",

5
test/support/fixtures/media_fixtures.ex
View file

@ -23,7 +23,7 @@ defmodule Pinchflat.MediaFixtures do
short_form_content: false,
media_filepath: "/video/#{Faker.File.file_name(:video)}",
source_id: SourcesFixtures.source_fixture().id,
upload_date: DateTime.utc_now()
uploaded_at: DateTime.utc_now()
})
|> Pinchflat.Media.create_media_item()
@ -97,7 +97,8 @@ defmodule Pinchflat.MediaFixtures do
description: "desc1",
aspect_ratio: 1.67,
duration: 123.45,
upload_date: "20210101"
upload_date: "20210101",
timestamp: 1_600_000_000
}
Phoenix.json_library().encode!(media_attributes)