danvics/pinchflat
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Updated indexer to pull upload date

Browse source
This commit is contained in:
Kieran Eglin 2024-03-10 16:49:08 -07:00
parent cc5b759531
commit c9f3f585c0
No known key found for this signature in database
GPG key ID: 193984967FCF432D
9 changed files with 47 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
lib/pinchflat/media/media_item.ex
View file

@ -20,7 +20,7 @@ defmodule Pinchflat.Media.MediaItem do
:livestream,
:source_id,
:short_form_content,
:uploaded_at,
:upload_date,
# these fields are captured only on download
:media_downloaded_at,
:media_filepath,
@ -36,7 +36,7 @@ defmodule Pinchflat.Media.MediaItem do
livestream
media_id
source_id
uploaded_at
upload_date
short_form_content
)a
@ -48,7 +48,7 @@ defmodule Pinchflat.Media.MediaItem do
field :livestream, :boolean, default: false
field :short_form_content, :boolean, default: false
field :media_downloaded_at, :utc_datetime
field :uploaded_at, :utc_datetime
field :upload_date, :date
field :media_filepath, :string
field :media_size_bytes, :integer

17
lib/pinchflat/yt_dlp/backend/media.ex
View file

@ -9,7 +9,8 @@ defmodule Pinchflat.YtDlp.Backend.Media do
:description,
:original_url,
:livestream,
:short_form_content
:short_form_content,
:upload_date
]
defstruct [
@ -18,7 +19,8 @@ defmodule Pinchflat.YtDlp.Backend.Media do
:description,
:original_url,
:livestream,
:short_form_content
:short_form_content,
:upload_date
]
alias __MODULE__
@ -67,7 +69,7 @@ defmodule Pinchflat.YtDlp.Backend.Media do
Returns the output template for yt-dlp's indexing command.
"""
def indexing_output_template do
"%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration})j"
"%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date})j"
end
@doc """
@ -83,7 +85,8 @@ defmodule Pinchflat.YtDlp.Backend.Media do
description: response["description"],
original_url: response["webpage_url"],
livestream: response["was_live"],
short_form_content: short_form_content?(response)
short_form_content: short_form_content?(response),
upload_date: parse_upload_date(response["upload_date"])
}
end
@ -100,6 +103,12 @@ defmodule Pinchflat.YtDlp.Backend.Media do
end
end
defp parse_upload_date(upload_date) do
<<year::binary-size(4)>> <> <<month::binary-size(2)>> <> <<day::binary-size(2)>> = upload_date
Date.from_iso8601!("#{year}-#{month}-#{day}")
end
defp backend_runner do
# This approach lets us mock the command for testing
Application.get_env(:pinchflat, :yt_dlp_runner)

5
priv/repo/migrations/20240310230713_add_uploaded_at_to_media_items.exs
View file

@ -4,8 +4,9 @@ defmodule Pinchflat.Repo.Migrations.AddUploadedAtToMediaItems do
def change do
alter table(:media_items) do
# Setting default to unix epoch so I can enforce not null BUT also easily
# identify records that were created before this column was added
add :uploaded_at, :utc_datetime, default: "1970-01-01T00:00:00", null: false
# identify records that were created before this column was added.
# Not a DateTime because yt-dlp only returns the date
add :upload_date, :date, default: "1970-01-01", null: false
end
end
end

3
test/pinchflat/media_test.exs
View file

@ -373,7 +373,8 @@ defmodule Pinchflat.MediaTest do
title: Faker.Commerce.product_name(),
media_filepath: "/video/#{Faker.File.file_name(:video)}",
source_id: source_fixture().id,
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}"
original_url: "https://www.youtube.com/channel/#{Faker.String.base64(12)}",
upload_date: Date.utc_today()
}
assert {:ok, %MediaItem{} = media_item} = Media.create_media_item(valid_attrs)

3
test/pinchflat/tasks/media_items_tasks_test.exs
View file

@ -88,7 +88,8 @@ defmodule Pinchflat.Tasks.MediaItemTasksTest do
was_live: true,
description: "desc2",
aspect_ratio: 1.67,
duration: 345.67
duration: 345.67,
upload_date: "20210101"
})
{:ok, output}

3
test/pinchflat/tasks/source_tasks_test.exs
View file

@ -282,7 +282,8 @@ defmodule Pinchflat.Tasks.SourceTasksTest do
was_live: true,
description: "desc2",
aspect_ratio: 1.67,
duration: 345.67
duration: 345.67,
upload_date: "20210101"
})
File.write(filepath, contents)

19
test/pinchflat/yt_dlp/backend/media_test.exs
View file

@ -79,7 +79,7 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
describe "indexing_output_template/0" do
test "contains all the greatest hits" do
assert "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration})j" ==
assert "%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date})j" ==
Media.indexing_output_template()
end
end
@ -93,7 +93,8 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"was_live" => false,
"aspect_ratio" => 1.0,
"duration" => 60
"duration" => 60,
"upload_date" => "20210101"
}
assert %Media{
@ -102,15 +103,17 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
description: "I'm not sure what I expected.",
original_url: "https://www.youtube.com/watch?v=TiZPUDkDYbk",
livestream: false,
short_form_content: false
} = Media.response_to_struct(response)
short_form_content: false,
upload_date: Date.from_iso8601!("2021-01-01")
} == Media.response_to_struct(response)
end
test "sets short_form_content to true if the URL contains /shorts/" do
response = %{
"webpage_url" => "https://www.youtube.com/shorts/TiZPUDkDYbk",
"aspect_ratio" => 1.0,
"duration" => 61
"duration" => 61,
"upload_date" => "20210101"
}
assert %Media{short_form_content: true} = Media.response_to_struct(response)
@ -120,7 +123,8 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 0.5,
"duration" => 59
"duration" => 59,
"upload_date" => "20210101"
}
assert %Media{short_form_content: true} = Media.response_to_struct(response)
@ -130,7 +134,8 @@ defmodule Pinchflat.YtDlp.Backend.MediaTest do
response = %{
"webpage_url" => "https://www.youtube.com/watch?v=TiZPUDkDYbk",
"aspect_ratio" => 1.0,
"duration" => 61
"duration" => 61,
"upload_date" => "20210101"
}
assert %Media{short_form_content: false} = Media.response_to_struct(response)

6
test/support/fixtures/media_fixtures.ex
View file

@ -21,7 +21,8 @@ defmodule Pinchflat.MediaFixtures do
livestream: false,
short_form_content: false,
media_filepath: "/video/#{Faker.File.file_name(:video)}",
source_id: SourcesFixtures.source_fixture().id
source_id: SourcesFixtures.source_fixture().id,
upload_date: DateTime.utc_now()
})
|> Pinchflat.Media.create_media_item()
@ -75,7 +76,8 @@ defmodule Pinchflat.MediaFixtures do
was_live: false,
description: "desc1",
aspect_ratio: 1.67,
duration: 123.45
duration: 123.45,
upload_date: "20210101"
}
Phoenix.json_library().encode!(media_attributes)

9
test/support/fixtures/sources_fixtures.ex
View file

@ -39,7 +39,8 @@ defmodule Pinchflat.SourcesFixtures do
was_live: false,
description: "desc1",
aspect_ratio: 1.67,
duration: 12.34
duration: 12.34,
upload_date: "20210101"
},
%{
id: "video2",
@ -48,7 +49,8 @@ defmodule Pinchflat.SourcesFixtures do
was_live: true,
description: "desc2",
aspect_ratio: 1.67,
duration: 345.67
duration: 345.67,
upload_date: "20220202"
},
%{
id: "video3",
@ -57,7 +59,8 @@ defmodule Pinchflat.SourcesFixtures do
was_live: false,
description: "desc3",
aspect_ratio: 1.0,
duration: 678.90
duration: 678.90,
upload_date: "20230303"
}
]