Compare commits
3 commits
master
...
ke/add-pos
| Author | SHA1 | Date | |
|---|---|---|---|
|
c5bf51dfb3 | ||
|
|
5d624f545b | ||
|
|
1305a485a0 |
236 changed files with 2731 additions and 18653 deletions
|
|
@ -16,10 +16,8 @@
|
|||
{:compiler, env: %{"MIX_ENV" => "test"}},
|
||||
{:formatter, env: %{"MIX_ENV" => "test"}},
|
||||
{:sobelow, "mix sobelow --config"},
|
||||
{:prettier_formatting, "yarn run lint:check", fix: "yarn run lint:fix"},
|
||||
{:npm_test, false},
|
||||
{:gettext, false},
|
||||
{:ex_unit, env: %{"MIX_ENV" => "test", "EX_CHECK" => "1"}}
|
||||
{:prettier_formatting, "yarn run prettier . --check", fix: "yarn run prettier . --write"},
|
||||
{:npm_test, false}
|
||||
|
||||
## curated tools may be disabled (e.g. the check for compilation warnings)
|
||||
# {:compiler, false},
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
{
|
||||
"name": "Pinchflat Dev",
|
||||
"dockerComposeFile": "../docker-compose.yml",
|
||||
"service": "phx",
|
||||
"workspaceFolder": "/app",
|
||||
"shutdownAction": "stopCompose",
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
"extensions": ["phoenixframework.phoenix", "JakeBecker.elixir-ls", "esbenp.prettier-vscode"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,5 +1,3 @@
|
|||
# TODO: figure out why my vscode extension doesn't respect the formatter.exs file
|
||||
# if it's in a subdirectory
|
||||
[
|
||||
import_deps: [:ecto, :ecto_sql, :phoenix],
|
||||
subdirectories: ["priv/*/migrations"],
|
||||
|
|
|
|||
13
.github/workflows/docker_release.yml
vendored
13
.github/workflows/docker_release.yml
vendored
|
|
@ -11,12 +11,6 @@ on:
|
|||
options:
|
||||
- 'linux/amd64'
|
||||
- 'linux/amd64,linux/arm64'
|
||||
docker_tags:
|
||||
type: string
|
||||
description: 'Docker Tags'
|
||||
required: true
|
||||
default: 'dev'
|
||||
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
|
@ -47,8 +41,7 @@ jobs:
|
|||
# All non-release actions will be tagged as `dev` (ie: push, workflow_dispatch)
|
||||
tags: |
|
||||
type=ref,event=tag
|
||||
type=raw,value=dev,enable=${{ github.event_name != 'release' && github.event_name != 'workflow_dispatch' }}
|
||||
type=raw,value=${{ inputs.docker_tags }},enable=${{ github.event_name == 'workflow_dispatch' }}
|
||||
type=raw,value=dev,enable=${{ github.event_name != 'release' }}
|
||||
flavor: |
|
||||
latest=auto
|
||||
|
||||
|
|
@ -72,10 +65,10 @@ jobs:
|
|||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and Push
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ./docker/selfhosted.Dockerfile
|
||||
file: ./selfhosted.Dockerfile
|
||||
# If the event is a release, use the release_arch, otherwise use the
|
||||
# platforms input if present, falling back to dev_arch
|
||||
platforms: ${{ github.event_name == 'release' && env.release_arch || (github.event.inputs.platforms || env.dev_arch) }}
|
||||
|
|
|
|||
25
.github/workflows/lint_and_test.yml
vendored
25
.github/workflows/lint_and_test.yml
vendored
|
|
@ -16,28 +16,24 @@ jobs:
|
|||
if: "! contains(toJSON(github.event.commits.*.message), '[skip ci]')"
|
||||
env:
|
||||
COMPOSE_FILE: ./docker-compose.ci.yml
|
||||
MIX_ENV: test
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Pull prebuilt images
|
||||
run: docker compose pull
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build docker image
|
||||
uses: docker/build-push-action@v6
|
||||
- name: Setup Docker layer caching
|
||||
uses: jpribyl/action-docker-layer-caching@v0.1.1
|
||||
continue-on-error: true
|
||||
with:
|
||||
context: .
|
||||
file: ./docker/dev.Dockerfile
|
||||
load: true
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
key: ci-docker-cache-{hash}
|
||||
restore-keys: |
|
||||
ci-docker-cache-
|
||||
layer-ci-docker-cache-
|
||||
|
||||
- name: Run Docker image
|
||||
- name: Build and Run Docker image
|
||||
run: docker compose up --detach
|
||||
|
||||
# NOTE: All exec commands use the -T flag to compensate for
|
||||
|
|
@ -47,7 +43,8 @@ jobs:
|
|||
# See https://github.com/actions/runner/issues/241 and https://github.com/docker/compose/issues/8537
|
||||
- name: Install Elixir and JS deps
|
||||
run: |
|
||||
docker compose exec -T phx mix deps.get && yarn install && cd assets && yarn install && cd ..
|
||||
docker compose exec -T phx yarn install && cd assets && yarn install && cd ..
|
||||
docker compose exec -T phx mix deps.get
|
||||
|
||||
- name: Create and Migrate database
|
||||
run: |
|
||||
|
|
|
|||
8
.iex.exs
8
.iex.exs
|
|
@ -23,11 +23,3 @@ alias Pinchflat.Metadata.MetadataFileHelpers
|
|||
alias Pinchflat.SlowIndexing.FileFollowerServer
|
||||
|
||||
Pinchflat.Release.check_file_permissions()
|
||||
|
||||
defmodule IexHelpers do
|
||||
def restart do
|
||||
:init.restart()
|
||||
end
|
||||
end
|
||||
|
||||
import IexHelpers
|
||||
|
|
|
|||
|
|
@ -1 +1,3 @@
|
|||
assets/vendor/
|
||||
deps/
|
||||
_build/
|
||||
|
|
|
|||
71
README.md
71
README.md
|
|
@ -1,6 +1,3 @@
|
|||
> [!IMPORTANT]
|
||||
> (2025-02-14) [zakkarry](https://github.com/sponsors/zakkarry), who is a collaborator on [cross-seed](https://github.com/cross-seed/cross-seed) and an extremely helpful community member in general, is facing hard times due to medical debt and family illness. If you're able, please consider [sponsoring him on GitHub](https://github.com/sponsors/zakkarry) or donating via [buymeacoffee](https://tip.ary.dev). Tell him I sent you!
|
||||
|
||||
<p align="center">
|
||||
<img
|
||||
src="priv/static/images/originals/logo-white-wordmark-with-background.png"
|
||||
|
|
@ -18,10 +15,8 @@
|
|||
<div align="center">
|
||||
|
||||
[](LICENSE)
|
||||
[](https://github.com/kieraneglin/pinchflat/releases)
|
||||
[](https://discord.gg/j7T6dCuwU4)
|
||||
[](https://github.com/kieraneglin/pinchflat/releases)
|
||||
[](#)
|
||||
[](https://vscode.dev/redirect?url=vscode://ms-vscode-remote.remote-containers/cloneInVolume?url=https://github.com/kieraneglin/pinchflat)
|
||||
|
||||
</div>
|
||||
|
||||
|
|
@ -37,7 +32,6 @@
|
|||
- [Portainer](#portainer)
|
||||
- [Docker](#docker)
|
||||
- [Environment Variables](#environment-variables)
|
||||
- [A note on reverse proxies](#reverse-proxies)
|
||||
- [Username and Password (authentication)](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password)
|
||||
- [Frequently asked questions](https://github.com/kieraneglin/pinchflat/wiki/Frequently-Asked-Questions)
|
||||
- [Documentation](https://github.com/kieraneglin/pinchflat/wiki)
|
||||
|
|
@ -58,7 +52,7 @@ If it doesn't work for your use case, please make a feature request! You can als
|
|||
- Self-contained - just one Docker container with no external dependencies
|
||||
- Powerful naming system so content is stored where and how you want it
|
||||
- Easy-to-use web interface with presets to get you started right away
|
||||
- First-class support for media center apps like Plex, Jellyfin, and Kodi ([docs](https://github.com/kieraneglin/pinchflat/wiki/Frequently-Asked-Questions#how-do-i-get-media-into-plexjellyfinkodi))
|
||||
- First-class support for media center apps like Plex, Jellyfin, and Kodi
|
||||
- Supports serving RSS feeds to your favourite podcast app ([docs](https://github.com/kieraneglin/pinchflat/wiki/Podcast-RSS-Feeds))
|
||||
- Automatically downloads new content from channels and playlists
|
||||
- Uses a novel approach to download new content more quickly than other apps
|
||||
|
|
@ -68,7 +62,7 @@ If it doesn't work for your use case, please make a feature request! You can als
|
|||
- Allows automatically redownloading new media after a set period
|
||||
- This can help improve the download quality of new content or improve SponsorBlock tags
|
||||
- Optionally automatically delete old content ([docs](https://github.com/kieraneglin/pinchflat/wiki/Automatically-Delete-Media))
|
||||
- Advanced options like setting cutoff dates and filtering by title ([docs](https://github.com/kieraneglin/pinchflat/wiki/Frequently-Asked-Questions#i-only-want-certain-videos-from-a-source---how-can-i-only-download-those))
|
||||
- Advanced options like setting cutoff dates and filtering by title
|
||||
- Reliable hands-off operation
|
||||
- Can pass cookies to YouTube to download your private playlists ([docs](https://github.com/kieraneglin/pinchflat/wiki/YouTube-Cookies))
|
||||
- Sponsorblock integration
|
||||
|
|
@ -77,8 +71,8 @@ If it doesn't work for your use case, please make a feature request! You can als
|
|||
|
||||
## Screenshots
|
||||
|
||||
<img src="priv/static/images/app-form-screenshot.jpg" alt="Pinchflat screenshot" width="700" />
|
||||
<img src="priv/static/images/app-screenshot.jpg" alt="Pinchflat screenshot" width="700" />
|
||||
<img src="priv/static/images/app-form-screenshot.png" alt="Pinchflat screenshot" width="700" />
|
||||
<img src="priv/static/images/app-screenshot.png" alt="Pinchflat screenshot" width="700" />
|
||||
|
||||
## Installation
|
||||
|
||||
|
|
@ -97,7 +91,7 @@ Docker Compose file:
|
|||
version: '3'
|
||||
services:
|
||||
pinchflat:
|
||||
image: ghcr.io/kieraneglin/pinchflat:latest
|
||||
image: keglin/pinchflat:latest
|
||||
environment:
|
||||
# Set the timezone to your local timezone
|
||||
- TZ=America/New_York
|
||||
|
|
@ -114,7 +108,7 @@ services:
|
|||
2. Prepare the docker image in one of the two ways below:
|
||||
- **From GHCR:** `docker pull ghcr.io/kieraneglin/pinchflat:latest`
|
||||
- NOTE: also available on Docker Hub at `keglin/pinchflat:latest`
|
||||
- **Building locally:** `docker build . --file docker/selfhosted.Dockerfile -t ghcr.io/kieraneglin/pinchflat:latest`
|
||||
- **Building locally:** `docker build . --file selfhosted.Dockerfile -t ghcr.io/kieraneglin/pinchflat:latest`
|
||||
3. Run the container:
|
||||
|
||||
```bash
|
||||
|
|
@ -129,23 +123,6 @@ docker run \
|
|||
ghcr.io/kieraneglin/pinchflat:latest
|
||||
```
|
||||
|
||||
### Podman
|
||||
|
||||
The Podman setup is similar to Docker but changes a few flags to run under a User Namespace instead of root. To run Pinchflat under Podman and use the current user's UID/GID for file access run this:
|
||||
|
||||
```
|
||||
podman run \
|
||||
--security-opt label=disable \
|
||||
--userns=keep-id --user=$UID \
|
||||
-e TZ=America/Los_Angeles \
|
||||
-p 8945:8945 \
|
||||
-v /host/path/to/config:/config:rw \
|
||||
-v /host/path/to/downloads/:/downloads:rw \
|
||||
ghcr.io/kieraneglin/pinchflat:latest
|
||||
```
|
||||
|
||||
Using this setup consider creating a new `pinchflat` user and giving that user ownership to the config and download directory. See [Podman --userns](https://docs.podman.io/en/v4.6.1/markdown/options/userns.container.html) docs.
|
||||
|
||||
### IMPORTANT: File permissions
|
||||
|
||||
You _must_ ensure the host directories you've mounted are writable by the user running the Docker container. If you get a permission error follow the steps it suggests. See [#106](https://github.com/kieraneglin/pinchflat/issues/106) for more.
|
||||
|
|
@ -153,6 +130,9 @@ You _must_ ensure the host directories you've mounted are writable by the user r
|
|||
> [!IMPORTANT]
|
||||
> It's not recommended to run the container as root. Doing so can create permission issues if other apps need to work with the downloaded media.
|
||||
|
||||
> [!TIP]
|
||||
> If you need to run any command as root, you can run `su` from the container's shell as there is no password set for the root user.
|
||||
|
||||
### ADVANCED: Storing Pinchflat config directory on a network share
|
||||
|
||||
As pointed out in [#137](https://github.com/kieraneglin/pinchflat/issues/137), SQLite doesn't like being run in WAL mode on network shares. If you're running Pinchflat on a network share, you can disable WAL mode by setting the `JOURNAL_MODE` environment variable to `delete`. This will make Pinchflat run in rollback journal mode which is less performant but should work on network shares.
|
||||
|
|
@ -164,24 +144,15 @@ If you change this setting and it works well for you, please leave a comment on
|
|||
|
||||
### Environment variables
|
||||
|
||||
| Name | Required? | Default | Notes |
|
||||
| --------------------------- | --------- | ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `TZ` | No | `UTC` | Must follow IANA TZ format |
|
||||
| `LOG_LEVEL` | No | `debug` | Can be set to `info` but `debug` is strongly recommended |
|
||||
| `UMASK` | No | `022` | Unraid users may want to set this to `000` |
|
||||
| `BASIC_AUTH_USERNAME` | No | | See [authentication docs](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password) |
|
||||
| `BASIC_AUTH_PASSWORD` | No | | See [authentication docs](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password) |
|
||||
| `EXPOSE_FEED_ENDPOINTS` | No | `false` | See [RSS feed docs](https://github.com/kieraneglin/pinchflat/wiki/Podcast-RSS-Feeds) |
|
||||
| `ENABLE_IPV6` | No | `false` | Setting to _any_ non-blank value will enable IPv6 |
|
||||
| `JOURNAL_MODE` | No | `wal` | Set to `delete` if your config directory is stored on a network share (not recommended) |
|
||||
| `TZ_DATA_DIR` | No | `/etc/elixir_tzdata_data` | The container path where the timezone database is stored |
|
||||
| `BASE_ROUTE_PATH` | No | `/` | The base path for route generation. Useful when running behind certain reverse proxies - prefixes must be stripped. |
|
||||
| `YT_DLP_WORKER_CONCURRENCY` | No | `2` | The number of concurrent workers that use `yt-dlp` _per queue_. Set to 1 if you're getting IP limited, otherwise don't touch it |
|
||||
| `ENABLE_PROMETHEUS` | No | `false` | Setting to _any_ non-blank value will enable Prometheus. See [docs](https://github.com/kieraneglin/pinchflat/wiki/Prometheus-and-Grafana) |
|
||||
|
||||
### Reverse Proxies
|
||||
|
||||
Pinchflat makes heavy use of websockets for real-time updates. If you're running Pinchflat behind a reverse proxy then you'll need to make sure it's configured to support websockets.
|
||||
| Name | Required? | Default | Notes |
|
||||
| --------------------- | --------- | ------------------------- | ---------------------------------------------------------------------------------------------- |
|
||||
| TZ | No | `UTC` | Must follow IANA TZ format |
|
||||
| LOG_LEVEL | No | `debug` | Can be set to `info` |
|
||||
| BASIC_AUTH_USERNAME | No | | See [authentication docs](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password) |
|
||||
| BASIC_AUTH_PASSWORD | No | | See [authentication docs](https://github.com/kieraneglin/pinchflat/wiki/Username-and-Password) |
|
||||
| EXPOSE_FEED_ENDPOINTS | No | | See [RSS feed docs](https://github.com/kieraneglin/pinchflat/wiki/Podcast-RSS-Feeds) |
|
||||
| JOURNAL_MODE | No | `wal` | Set to `delete` if your config directory is stored on a network share (not recommended) |
|
||||
| TZ_DATA_DIR | No | `/etc/elixir_tzdata_data` | The container path where the timezone database is stored |
|
||||
|
||||
## EFF donations
|
||||
|
||||
|
|
@ -189,9 +160,9 @@ Prior to 2024-05-10, a portion of all donations were given to the [Electronic Fr
|
|||
|
||||
The EFF defends your online liberties and [backed](https://github.com/github/dmca/blob/9a85e0f021f7967af80e186b890776a50443f06c/2020/11/2020-11-16-RIAA-reversal-effletter.pdf) `youtube-dl` when Google took them down.
|
||||
|
||||
## Stability disclaimer
|
||||
## Pre-release disclaimer
|
||||
|
||||
This software is in active development and anything can break at any time. I make no guarantees about the stability of this software, forward-compatibility of updates, or integrity (both related to and independent of Pinchflat).
|
||||
This is pre-release software and anything can break at any time. I make not guarantees about the stability of this software, forward-compatibility of updates, or integrity (both related to and independent of Pinchflat). Essentially, use at your own risk and expect there will be rough edges for now.
|
||||
|
||||
## License
|
||||
|
||||
|
|
|
|||
|
|
@ -22,8 +22,8 @@
|
|||
|
||||
@font-face {
|
||||
font-family: 'Satoshi';
|
||||
src: url('../fonts/satoshi/Satoshi-Light.woff2'), url('../fonts/satoshi/Satoshi-Light.woff'),
|
||||
url('../fonts/satoshi/Satoshi-Light.ttf');
|
||||
src: url('/fonts/satoshi/Satoshi-Light.woff2'), url('/fonts/satoshi/Satoshi-Light.woff'),
|
||||
url('/fonts/satoshi/Satoshi-Light.ttf');
|
||||
font-weight: 300;
|
||||
font-display: swap;
|
||||
font-style: normal;
|
||||
|
|
@ -31,9 +31,8 @@
|
|||
|
||||
@font-face {
|
||||
font-family: 'Satoshi';
|
||||
src: url('../fonts/satoshi/Satoshi-LightItalic.woff2'),
|
||||
url('../fonts/satoshi/Satoshi-LightItalic.woff'),
|
||||
url('../fonts/satoshi/Satoshi-LightItalic.ttf');
|
||||
src: url('/fonts/satoshi/Satoshi-LightItalic.woff2'),
|
||||
url('/fonts/satoshi/Satoshi-LightItalic.woff'), url('/fonts/satoshi/Satoshi-LightItalic.ttf');
|
||||
font-weight: 300;
|
||||
font-display: swap;
|
||||
font-style: italic;
|
||||
|
|
@ -41,8 +40,8 @@
|
|||
|
||||
@font-face {
|
||||
font-family: 'Satoshi';
|
||||
src: url('../fonts/satoshi/Satoshi-Regular.woff2'), url('../fonts/satoshi/Satoshi-Regular.woff'),
|
||||
url('../fonts/satoshi/Satoshi-Regular.ttf');
|
||||
src: url('/fonts/satoshi/Satoshi-Regular.woff2'), url('/fonts/satoshi/Satoshi-Regular.woff'),
|
||||
url('/fonts/satoshi/Satoshi-Regular.ttf');
|
||||
font-weight: 400;
|
||||
font-display: swap;
|
||||
font-style: normal;
|
||||
|
|
@ -50,8 +49,8 @@
|
|||
|
||||
@font-face {
|
||||
font-family: 'Satoshi';
|
||||
src: url('../fonts/satoshi/Satoshi-Italic.woff2'), url('../fonts/satoshi/Satoshi-Italic.woff'),
|
||||
url('../fonts/satoshi/Satoshi-Italic.ttf');
|
||||
src: url('/fonts/satoshi/Satoshi-Italic.woff2'), url('/fonts/satoshi/Satoshi-Italic.woff'),
|
||||
url('/fonts/satoshi/Satoshi-Italic.ttf');
|
||||
font-weight: 400;
|
||||
font-display: swap;
|
||||
font-style: italic;
|
||||
|
|
@ -59,8 +58,8 @@
|
|||
|
||||
@font-face {
|
||||
font-family: 'Satoshi';
|
||||
src: url('../fonts/satoshi/Satoshi-Medium.woff2'), url('../fonts/satoshi/Satoshi-Medium.woff'),
|
||||
url('../fonts/satoshi/Satoshi-Medium.ttf');
|
||||
src: url('/fonts/satoshi/Satoshi-Medium.woff2'), url('/fonts/satoshi/Satoshi-Medium.woff'),
|
||||
url('/fonts/satoshi/Satoshi-Medium.ttf');
|
||||
font-weight: 500;
|
||||
font-display: swap;
|
||||
font-style: normal;
|
||||
|
|
@ -68,9 +67,8 @@
|
|||
|
||||
@font-face {
|
||||
font-family: 'Satoshi';
|
||||
src: url('../fonts/satoshi/Satoshi-MediumItalic.woff2'),
|
||||
url('../fonts/satoshi/Satoshi-MediumItalic.woff'),
|
||||
url('../fonts/satoshi/Satoshi-MediumItalic.ttf');
|
||||
src: url('/fonts/satoshi/Satoshi-MediumItalic.woff2'),
|
||||
url('/fonts/satoshi/Satoshi-MediumItalic.woff'), url('/fonts/satoshi/Satoshi-MediumItalic.ttf');
|
||||
font-weight: 500;
|
||||
font-display: swap;
|
||||
font-style: italic;
|
||||
|
|
@ -78,8 +76,8 @@
|
|||
|
||||
@font-face {
|
||||
font-family: 'Satoshi';
|
||||
src: url('../fonts/satoshi/Satoshi-Bold.woff2'), url('../fonts/satoshi/Satoshi-Bold.woff'),
|
||||
url('../fonts/satoshi/Satoshi-Bold.ttf');
|
||||
src: url('/fonts/satoshi/Satoshi-Bold.woff2'), url('/fonts/satoshi/Satoshi-Bold.woff'),
|
||||
url('/fonts/satoshi/Satoshi-Bold.ttf');
|
||||
font-weight: 700;
|
||||
font-display: swap;
|
||||
font-style: normal;
|
||||
|
|
@ -87,8 +85,8 @@
|
|||
|
||||
@font-face {
|
||||
font-family: 'Satoshi';
|
||||
src: url('../fonts/satoshi/Satoshi-BoldItalic.woff2'),
|
||||
url('../fonts/satoshi/Satoshi-BoldItalic.woff'), url('../fonts/satoshi/Satoshi-BoldItalic.ttf');
|
||||
src: url('/fonts/satoshi/Satoshi-BoldItalic.woff2'), url('/fonts/satoshi/Satoshi-BoldItalic.woff'),
|
||||
url('/fonts/satoshi/Satoshi-BoldItalic.ttf');
|
||||
font-weight: 700;
|
||||
font-display: swap;
|
||||
font-style: italic;
|
||||
|
|
@ -96,9 +94,8 @@
|
|||
|
||||
@font-face {
|
||||
font-family: 'Satoshi';
|
||||
src: url('../fonts/satoshi/Satoshi-BlackItalic.woff2'),
|
||||
url('../fonts/satoshi/Satoshi-BlackItalic.woff'),
|
||||
url('../fonts/satoshi/Satoshi-BlackItalic.ttf');
|
||||
src: url('/fonts/satoshi/Satoshi-BlackItalic.woff2'),
|
||||
url('/fonts/satoshi/Satoshi-BlackItalic.woff'), url('/fonts/satoshi/Satoshi-BlackItalic.ttf');
|
||||
font-weight: 900;
|
||||
font-display: swap;
|
||||
font-style: italic;
|
||||
|
|
|
|||
|
|
@ -35,14 +35,3 @@ window.markVersionAsSeen = (versionString) => {
|
|||
window.isVersionSeen = (versionString) => {
|
||||
return localStorage.getItem('seenVersion') === versionString
|
||||
}
|
||||
|
||||
window.dispatchFor = (elementOrId, eventName, detail = {}) => {
|
||||
const element =
|
||||
typeof elementOrId === 'string' ? document.getElementById(elementOrId) : elementOrId
|
||||
|
||||
// This is needed to ensure the DOM has updated before dispatching the event.
|
||||
// Doing so ensures that the latest DOM state is what's sent to the server
|
||||
setTimeout(() => {
|
||||
element.dispatchEvent(new Event(eventName, { bubbles: true, detail }))
|
||||
}, 0)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ window.Alpine = Alpine
|
|||
Alpine.start()
|
||||
|
||||
let csrfToken = document.querySelector("meta[name='csrf-token']").getAttribute('content')
|
||||
let liveSocket = new LiveSocket(document.body.dataset.socketPath, Socket, {
|
||||
let liveSocket = new LiveSocket('/live', Socket, {
|
||||
params: { _csrf_token: csrfToken },
|
||||
dom: {
|
||||
onBeforeElUpdated(from, to) {
|
||||
|
|
@ -39,7 +39,7 @@ let liveSocket = new LiveSocket(document.body.dataset.socketPath, Socket, {
|
|||
}
|
||||
},
|
||||
hooks: {
|
||||
'supress-enter-submission': {
|
||||
supressEnterSubmission: {
|
||||
mounted() {
|
||||
this.el.addEventListener('keypress', (event) => {
|
||||
if (event.key === 'Enter') {
|
||||
|
|
|
|||
|
|
@ -347,38 +347,6 @@ module.exports = {
|
|||
},
|
||||
{ values }
|
||||
)
|
||||
}),
|
||||
plugin(function ({ matchComponents, theme }) {
|
||||
let iconsDir = path.join(__dirname, './vendor/simple-icons')
|
||||
let values = {}
|
||||
|
||||
fs.readdirSync(iconsDir).forEach((file) => {
|
||||
let name = path.basename(file, '.svg')
|
||||
values[name] = { name, fullPath: path.join(iconsDir, file) }
|
||||
})
|
||||
|
||||
matchComponents(
|
||||
{
|
||||
si: ({ name, fullPath }) => {
|
||||
let content = fs
|
||||
.readFileSync(fullPath)
|
||||
.toString()
|
||||
.replace(/\r?\n|\r/g, '')
|
||||
return {
|
||||
[`--si-${name}`]: `url('data:image/svg+xml;utf8,${content}')`,
|
||||
'-webkit-mask': `var(--si-${name})`,
|
||||
mask: `var(--si-${name})`,
|
||||
'mask-repeat': 'no-repeat',
|
||||
'background-color': 'currentColor',
|
||||
'vertical-align': 'middle',
|
||||
display: 'inline-block',
|
||||
width: theme('spacing.5'),
|
||||
height: theme('spacing.5')
|
||||
}
|
||||
}
|
||||
},
|
||||
{ values }
|
||||
)
|
||||
})
|
||||
]
|
||||
}
|
||||
|
|
|
|||
1
assets/vendor/simple-icons/discord.svg
vendored
1
assets/vendor/simple-icons/discord.svg
vendored
|
|
@ -1 +0,0 @@
|
|||
<svg role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>Discord</title><path d="M20.317 4.3698a19.7913 19.7913 0 00-4.8851-1.5152.0741.0741 0 00-.0785.0371c-.211.3753-.4447.8648-.6083 1.2495-1.8447-.2762-3.68-.2762-5.4868 0-.1636-.3933-.4058-.8742-.6177-1.2495a.077.077 0 00-.0785-.037 19.7363 19.7363 0 00-4.8852 1.515.0699.0699 0 00-.0321.0277C.5334 9.0458-.319 13.5799.0992 18.0578a.0824.0824 0 00.0312.0561c2.0528 1.5076 4.0413 2.4228 5.9929 3.0294a.0777.0777 0 00.0842-.0276c.4616-.6304.8731-1.2952 1.226-1.9942a.076.076 0 00-.0416-.1057c-.6528-.2476-1.2743-.5495-1.8722-.8923a.077.077 0 01-.0076-.1277c.1258-.0943.2517-.1923.3718-.2914a.0743.0743 0 01.0776-.0105c3.9278 1.7933 8.18 1.7933 12.0614 0a.0739.0739 0 01.0785.0095c.1202.099.246.1981.3728.2924a.077.077 0 01-.0066.1276 12.2986 12.2986 0 01-1.873.8914.0766.0766 0 00-.0407.1067c.3604.698.7719 1.3628 1.225 1.9932a.076.076 0 00.0842.0286c1.961-.6067 3.9495-1.5219 6.0023-3.0294a.077.077 0 00.0313-.0552c.5004-5.177-.8382-9.6739-3.5485-13.6604a.061.061 0 00-.0312-.0286zM8.02 15.3312c-1.1825 0-2.1569-1.0857-2.1569-2.419 0-1.3332.9555-2.4189 2.157-2.4189 1.2108 0 2.1757 1.0952 2.1568 2.419 0 1.3332-.9555 2.4189-2.1569 2.4189zm7.9748 0c-1.1825 0-2.1569-1.0857-2.1569-2.419 0-1.3332.9554-2.4189 2.1569-2.4189 1.2108 0 2.1757 1.0952 2.1568 2.419 0 1.3332-.946 2.4189-2.1568 2.4189Z"/></svg>
|
||||
|
Before Width: | Height: | Size: 1.3 KiB |
1
assets/vendor/simple-icons/github.svg
vendored
1
assets/vendor/simple-icons/github.svg
vendored
|
|
@ -1 +0,0 @@
|
|||
<svg role="img" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><title>GitHub</title><path d="M12 .297c-6.63 0-12 5.373-12 12 0 5.303 3.438 9.8 8.205 11.385.6.113.82-.258.82-.577 0-.285-.01-1.04-.015-2.04-3.338.724-4.042-1.61-4.042-1.61C4.422 18.07 3.633 17.7 3.633 17.7c-1.087-.744.084-.729.084-.729 1.205.084 1.838 1.236 1.838 1.236 1.07 1.835 2.809 1.305 3.495.998.108-.776.417-1.305.76-1.605-2.665-.3-5.466-1.332-5.466-5.93 0-1.31.465-2.38 1.235-3.22-.135-.303-.54-1.523.105-3.176 0 0 1.005-.322 3.3 1.23.96-.267 1.98-.399 3-.405 1.02.006 2.04.138 3 .405 2.28-1.552 3.285-1.23 3.285-1.23.645 1.653.24 2.873.12 3.176.765.84 1.23 1.91 1.23 3.22 0 4.61-2.805 5.625-5.475 5.92.42.36.81 1.096.81 2.22 0 1.606-.015 2.896-.015 3.286 0 .315.21.69.825.57C20.565 22.092 24 17.592 24 12.297c0-6.627-5.373-12-12-12"/></svg>
|
||||
|
Before Width: | Height: | Size: 823 B |
|
|
@ -10,7 +10,6 @@ import Config
|
|||
config :pinchflat,
|
||||
ecto_repos: [Pinchflat.Repo],
|
||||
generators: [timestamp_type: :utc_datetime],
|
||||
env: config_env(),
|
||||
# Specifying backend data here makes mocking and local testing SUPER easy
|
||||
yt_dlp_executable: System.find_executable("yt-dlp"),
|
||||
apprise_executable: System.find_executable("apprise"),
|
||||
|
|
@ -27,8 +26,7 @@ config :pinchflat,
|
|||
basic_auth_password: "",
|
||||
expose_feed_endpoints: false,
|
||||
file_watcher_poll_interval: 1000,
|
||||
timezone: "UTC",
|
||||
base_route_path: "/"
|
||||
timezone: "UTC"
|
||||
|
||||
config :pinchflat, Pinchflat.Repo,
|
||||
journal_mode: :wal,
|
||||
|
|
@ -42,15 +40,33 @@ config :pinchflat, PinchflatWeb.Endpoint,
|
|||
adapter: Phoenix.Endpoint.Cowboy2Adapter,
|
||||
render_errors: [
|
||||
formats: [html: PinchflatWeb.ErrorHTML, json: PinchflatWeb.ErrorJSON],
|
||||
root_layout: {PinchflatWeb.Layouts, :root},
|
||||
layout: {PinchflatWeb.Layouts, :app}
|
||||
layout: false
|
||||
],
|
||||
pubsub_server: Pinchflat.PubSub,
|
||||
live_view: [signing_salt: "/t5878kO"]
|
||||
|
||||
config :pinchflat, Oban,
|
||||
engine: Oban.Engines.Lite,
|
||||
repo: Pinchflat.Repo
|
||||
repo: Pinchflat.Repo,
|
||||
# Keep old jobs for 30 days for display in the UI
|
||||
plugins: [
|
||||
{Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60},
|
||||
{Oban.Plugins.Cron,
|
||||
crontab: [
|
||||
{"0 1 * * *", Pinchflat.Downloading.MediaRetentionWorker},
|
||||
{"0 2 * * *", Pinchflat.Downloading.MediaQualityUpgradeWorker}
|
||||
]}
|
||||
],
|
||||
# TODO: consider making this an env var or something?
|
||||
queues: [
|
||||
default: 10,
|
||||
fast_indexing: 6,
|
||||
media_indexing: 2,
|
||||
media_collection_indexing: 2,
|
||||
media_fetching: 2,
|
||||
local_metadata: 8,
|
||||
remote_metadata: 4
|
||||
]
|
||||
|
||||
# Configures the mailer
|
||||
#
|
||||
|
|
@ -84,19 +100,13 @@ config :tailwind,
|
|||
]
|
||||
|
||||
# Configures Elixir's Logger
|
||||
config :logger, :default_formatter,
|
||||
format: "$date $time $metadata[$level] | $message\n",
|
||||
config :logger, :console,
|
||||
format: "$time $metadata[$level] $message\n",
|
||||
metadata: [:request_id]
|
||||
|
||||
# Use Jason for JSON parsing in Phoenix
|
||||
config :phoenix, :json_library, Jason
|
||||
|
||||
config :pinchflat, Pinchflat.PromEx,
|
||||
disabled: true,
|
||||
manual_metrics_start_delay: :no_delay,
|
||||
drop_metrics_groups: [],
|
||||
metrics_server: :disabled
|
||||
|
||||
# Import environment specific config. This must remain at the bottom
|
||||
# of this file so it overrides the configuration defined above.
|
||||
import_config "#{config_env()}.exs"
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ config :pinchflat, PinchflatWeb.Endpoint,
|
|||
config :pinchflat, dev_routes: true
|
||||
|
||||
# Do not include metadata nor timestamps in development logs
|
||||
config :logger, :default_formatter, format: "[$level] $message\n"
|
||||
config :logger, :console, format: "[$level] $message\n"
|
||||
|
||||
# Set a higher stacktrace during development. Avoid configuring such
|
||||
# in production as building large stacktraces may be expensive.
|
||||
|
|
@ -81,5 +81,3 @@ config :phoenix_live_view, :debug_heex_annotations, true
|
|||
|
||||
# Disable swoosh api client as it is only required for production adapters.
|
||||
config :swoosh, :api_client, false
|
||||
|
||||
config :pinchflat, Pinchflat.PromEx, disabled: false
|
||||
|
|
|
|||
|
|
@ -40,79 +40,39 @@ config :pinchflat, Pinchflat.Repo,
|
|||
Path.join([:code.priv_dir(:pinchflat), "repo", "extensions", "sqlean-linux-#{system_arch}", "sqlean"])
|
||||
]
|
||||
|
||||
# Some users may want to increase the number of workers that use yt-dlp to improve speeds
|
||||
# Others may want to decrease the number of these workers to lessen the chance of an IP ban
|
||||
{yt_dlp_worker_count, _} = Integer.parse(System.get_env("YT_DLP_WORKER_CONCURRENCY", "2"))
|
||||
# Used to set the cron for the yt-dlp update worker. The reason for this is
|
||||
# to avoid all instances of PF updating yt-dlp at the same time, which 1)
|
||||
# could result in rate limiting and 2) gives me time to react if an update
|
||||
# breaks something
|
||||
%{hour: current_hour, minute: current_minute} = DateTime.utc_now()
|
||||
|
||||
config :pinchflat, Oban,
|
||||
queues: [
|
||||
default: 10,
|
||||
fast_indexing: yt_dlp_worker_count,
|
||||
media_collection_indexing: yt_dlp_worker_count,
|
||||
media_fetching: yt_dlp_worker_count,
|
||||
remote_metadata: yt_dlp_worker_count,
|
||||
local_data: 8
|
||||
],
|
||||
plugins: [
|
||||
# Keep old jobs for 30 days for display in the UI
|
||||
{Oban.Plugins.Pruner, max_age: 30 * 24 * 60 * 60},
|
||||
{Oban.Plugins.Cron,
|
||||
crontab: [
|
||||
{"#{current_minute} #{current_hour} * * *", Pinchflat.YtDlp.UpdateWorker},
|
||||
{"0 1 * * *", Pinchflat.Downloading.MediaRetentionWorker},
|
||||
{"0 2 * * *", Pinchflat.Downloading.MediaQualityUpgradeWorker}
|
||||
]}
|
||||
]
|
||||
|
||||
if config_env() == :prod do
|
||||
# Various paths. These ones shouldn't be tweaked if running in Docker
|
||||
media_path = System.get_env("MEDIA_PATH", "/downloads")
|
||||
config_path = System.get_env("CONFIG_PATH", "/config")
|
||||
config_path = "/config"
|
||||
db_path = System.get_env("DATABASE_PATH", Path.join([config_path, "db", "pinchflat.db"]))
|
||||
log_path = System.get_env("LOG_PATH", Path.join([config_path, "logs", "pinchflat.log"]))
|
||||
metadata_path = System.get_env("METADATA_PATH", Path.join([config_path, "metadata"]))
|
||||
extras_path = System.get_env("EXTRAS_PATH", Path.join([config_path, "extras"]))
|
||||
tmpfile_path = System.get_env("TMPFILE_PATH", Path.join([System.tmp_dir!(), "pinchflat", "data"]))
|
||||
# This one can be changed if you want
|
||||
tz_data_path = System.get_env("TZ_DATA_PATH", Path.join([extras_path, "elixir_tz_data"]))
|
||||
|
||||
# For running PF as a podcast host on self-hosted environments
|
||||
expose_feed_endpoints = String.length(System.get_env("EXPOSE_FEED_ENDPOINTS", "")) > 0
|
||||
|
||||
# For testing alternate journal modes (see issue #137)
|
||||
journal_mode = String.to_existing_atom(System.get_env("JOURNAL_MODE", "wal"))
|
||||
# For running PF in a subdirectory via a reverse proxy
|
||||
base_route_path = System.get_env("BASE_ROUTE_PATH", "/")
|
||||
enable_ipv6 = String.length(System.get_env("ENABLE_IPV6", "")) > 0
|
||||
enable_prometheus = String.length(System.get_env("ENABLE_PROMETHEUS", "")) > 0
|
||||
|
||||
config :logger, level: String.to_existing_atom(System.get_env("LOG_LEVEL", "debug"))
|
||||
|
||||
config :pinchflat,
|
||||
yt_dlp_executable: System.find_executable("yt-dlp"),
|
||||
apprise_executable: System.find_executable("apprise"),
|
||||
media_directory: media_path,
|
||||
media_directory: "/downloads",
|
||||
metadata_directory: metadata_path,
|
||||
extras_directory: extras_path,
|
||||
tmpfile_directory: tmpfile_path,
|
||||
tmpfile_directory: Path.join([System.tmp_dir!(), "pinchflat", "data"]),
|
||||
dns_cluster_query: System.get_env("DNS_CLUSTER_QUERY"),
|
||||
expose_feed_endpoints: expose_feed_endpoints,
|
||||
# This is configured in application.ex
|
||||
timezone: "UTC",
|
||||
log_path: log_path,
|
||||
base_route_path: base_route_path
|
||||
timezone: System.get_env("TIMEZONE") || System.get_env("TZ") || "UTC",
|
||||
log_path: log_path
|
||||
|
||||
config :tzdata, :data_dir, tz_data_path
|
||||
config :tzdata, :data_dir, System.get_env("TZ_DATA_DIR", "/etc/elixir_tzdata_data")
|
||||
|
||||
config :pinchflat, Pinchflat.Repo,
|
||||
database: db_path,
|
||||
journal_mode: journal_mode
|
||||
|
||||
config :pinchflat, Pinchflat.PromEx, disabled: !enable_prometheus
|
||||
|
||||
# The secret key base is used to sign/encrypt cookies and other secrets.
|
||||
# A default value is used in config/dev.exs and config/test.exs but you
|
||||
# want to use a different value for prod and you most likely don't want
|
||||
|
|
@ -145,10 +105,9 @@ if config_env() == :prod do
|
|||
# Set it to {0, 0, 0, 0, 0, 0, 0, 1} for local network only access.
|
||||
# See the documentation on https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html
|
||||
# for details about using IPv6 vs IPv4 and loopback vs public addresses.
|
||||
ip: if(enable_ipv6, do: {0, 0, 0, 0, 0, 0, 0, 0}, else: {0, 0, 0, 0}),
|
||||
ip: {0, 0, 0, 0},
|
||||
port: String.to_integer(System.get_env("PORT") || "4000")
|
||||
],
|
||||
url: [path: base_route_path],
|
||||
secret_key_base: secret_key_base
|
||||
|
||||
config :pinchflat, :logger, [
|
||||
|
|
|
|||
65
dev.Dockerfile
Normal file
65
dev.Dockerfile
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
ARG ELIXIR_VERSION=1.16.2
|
||||
ARG OTP_VERSION=26.2.2
|
||||
ARG DEBIAN_VERSION=bookworm-20240130
|
||||
ARG DEV_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
|
||||
|
||||
FROM ${DEV_IMAGE}
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
RUN echo "Building for ${TARGETPLATFORM:?}"
|
||||
|
||||
# Install debian packages
|
||||
RUN apt-get update -qq
|
||||
RUN apt-get install -y inotify-tools curl git openssh-client jq \
|
||||
python3 python3-setuptools python3-wheel python3-dev pipx \
|
||||
python3-mutagen locales procps build-essential graphviz
|
||||
|
||||
# Install ffmpeg
|
||||
RUN export FFMPEG_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \
|
||||
"linux/amd64") echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz" ;; \
|
||||
"linux/arm64") echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz" ;; \
|
||||
*) echo "" ;; esac) && \
|
||||
curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg.tar.xz && \
|
||||
tar -xf /tmp/ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
|
||||
tar -xf /tmp/ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe"
|
||||
|
||||
# Install nodejs
|
||||
RUN curl -sL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh
|
||||
RUN bash nodesource_setup.sh
|
||||
RUN apt-get install nodejs
|
||||
RUN npm install -g yarn
|
||||
|
||||
# Install baseline Elixir packages
|
||||
RUN mix local.hex --force
|
||||
RUN mix local.rebar --force
|
||||
|
||||
# Download and update YT-DLP
|
||||
RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp
|
||||
RUN chmod a+rx /usr/local/bin/yt-dlp
|
||||
RUN yt-dlp -U
|
||||
|
||||
# Install Apprise
|
||||
RUN export PIPX_HOME=/opt/pipx && \
|
||||
export PIPX_BIN_DIR=/usr/local/bin && \
|
||||
pipx install apprise
|
||||
|
||||
# Set the locale
|
||||
RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
|
||||
ENV LANG en_US.UTF-8
|
||||
ENV LANGUAGE en_US:en
|
||||
ENV LC_ALL en_US.UTF-8
|
||||
|
||||
# Create app directory and copy the Elixir projects into it.
|
||||
WORKDIR /app
|
||||
COPY . ./
|
||||
|
||||
# Needs permissions to be updated AFTER the copy step
|
||||
RUN chmod +x ./docker-run.dev.sh
|
||||
|
||||
# Install Elixir deps
|
||||
# RUN mix archive.install github hexpm/hex branch latest
|
||||
RUN mix deps.get
|
||||
# Gives us iex shell history
|
||||
ENV ERL_AFLAGS="-kernel shell_history enabled"
|
||||
|
||||
EXPOSE 4008
|
||||
|
|
@ -1,15 +1,13 @@
|
|||
version: '3'
|
||||
services:
|
||||
phx:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: ./docker/dev.Dockerfile
|
||||
dockerfile: dev.Dockerfile
|
||||
environment:
|
||||
- MIX_ENV=test
|
||||
volumes:
|
||||
- '.:/app'
|
||||
# These lines ensure the deps can be saved as build artifacts for caching
|
||||
- '/app/deps'
|
||||
- '/app/_build'
|
||||
ports:
|
||||
- '4008:4008'
|
||||
command: tail -F /dev/null
|
||||
|
|
|
|||
|
|
@ -1,12 +1,14 @@
|
|||
version: '3'
|
||||
services:
|
||||
phx:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: ./docker/dev.Dockerfile
|
||||
dockerfile: dev.Dockerfile
|
||||
volumes:
|
||||
- '.:/app'
|
||||
ports:
|
||||
- '4008:4008'
|
||||
command: bash -c "chmod +x docker/docker-run.dev.sh && docker/docker-run.dev.sh"
|
||||
command:
|
||||
- ./docker-run.dev.sh
|
||||
stdin_open: true
|
||||
tty: true
|
||||
|
|
|
|||
|
|
@ -2,12 +2,12 @@
|
|||
|
||||
set -e
|
||||
|
||||
echo "\nInstalling Elixir deps..."
|
||||
# Ensure the app's deps are installed
|
||||
mix deps.get
|
||||
|
||||
# Install both project-level and assets-level JS dependencies
|
||||
echo "\nInstalling JS deps..."
|
||||
yarn install && cd assets && yarn install
|
||||
# Install JS deps
|
||||
echo "\nInstalling JS..."
|
||||
cd assets && yarn install
|
||||
cd ..
|
||||
|
||||
# Potentially Set up the database
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
ARG ELIXIR_VERSION=1.18.4
|
||||
ARG OTP_VERSION=27.2.4
|
||||
ARG DEBIAN_VERSION=bookworm-20250428-slim
|
||||
|
||||
ARG DEV_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
|
||||
|
||||
FROM ${DEV_IMAGE}
|
||||
|
||||
ARG TARGETPLATFORM
|
||||
RUN echo "Building for ${TARGETPLATFORM:?}"
|
||||
|
||||
# Install debian packages
|
||||
RUN apt-get update -qq && \
|
||||
apt-get install -y inotify-tools curl git openssh-client jq \
|
||||
python3 python3-setuptools python3-wheel python3-dev pipx \
|
||||
python3-mutagen locales procps build-essential graphviz zsh unzip
|
||||
|
||||
# Install ffmpeg
|
||||
RUN export FFMPEG_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \
|
||||
"linux/amd64") echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz" ;; \
|
||||
"linux/arm64") echo "https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linuxarm64-gpl.tar.xz" ;; \
|
||||
*) echo "" ;; esac) && \
|
||||
curl -L ${FFMPEG_DOWNLOAD} --output /tmp/ffmpeg.tar.xz && \
|
||||
tar -xf /tmp/ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffmpeg" && \
|
||||
tar -xf /tmp/ffmpeg.tar.xz --strip-components=2 --no-anchored -C /usr/bin/ "ffprobe"
|
||||
|
||||
# Install nodejs and Yarn
|
||||
RUN curl -sL https://deb.nodesource.com/setup_20.x -o nodesource_setup.sh && \
|
||||
bash nodesource_setup.sh && \
|
||||
apt-get install -y nodejs && \
|
||||
npm install -g yarn && \
|
||||
# Install baseline Elixir packages
|
||||
mix local.hex --force && \
|
||||
mix local.rebar --force && \
|
||||
# Install Deno - required for YouTube downloads (See yt-dlp#14404)
|
||||
curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh -s -- -y --no-modify-path && \
|
||||
# Download and update YT-DLP
|
||||
export YT_DLP_DOWNLOAD=$(case ${TARGETPLATFORM:-linux/amd64} in \
|
||||
"linux/amd64") echo "https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux" ;; \
|
||||
"linux/arm64") echo "https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux_aarch64" ;; \
|
||||
*) echo "https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux" ;; esac) && \
|
||||
curl -L ${YT_DLP_DOWNLOAD} -o /usr/local/bin/yt-dlp && \
|
||||
chmod a+rx /usr/local/bin/yt-dlp && \
|
||||
yt-dlp -U && \
|
||||
# Install Apprise
|
||||
export PIPX_HOME=/opt/pipx && \
|
||||
export PIPX_BIN_DIR=/usr/local/bin && \
|
||||
pipx install apprise && \
|
||||
# Set up ZSH tools
|
||||
chsh -s $(which zsh) && \
|
||||
sh -c "$(curl -fsSL https://raw.githubusercontent.com/ohmyzsh/ohmyzsh/master/tools/install.sh)"
|
||||
|
||||
# Set the locale
|
||||
RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen
|
||||
ENV LANG=en_US.UTF-8
|
||||
ENV LANGUAGE=en_US:en
|
||||
ENV LC_ALL=en_US.UTF-8
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY mix.exs mix.lock ./
|
||||
# Install Elixir deps
|
||||
# NOTE: this has to be before the bulk copy to ensure that deps are cached
|
||||
RUN MIX_ENV=dev mix deps.get && MIX_ENV=dev mix deps.compile
|
||||
RUN MIX_ENV=test mix deps.get && MIX_ENV=test mix deps.compile
|
||||
|
||||
COPY . ./
|
||||
|
||||
# Gives us iex shell history
|
||||
ENV ERL_AFLAGS="-kernel shell_history enabled"
|
||||
|
||||
EXPOSE 4008
|
||||
|
|
@ -4,18 +4,10 @@ defmodule Pinchflat.Application do
|
|||
@moduledoc false
|
||||
|
||||
use Application
|
||||
require Logger
|
||||
|
||||
@impl true
|
||||
def start(_type, _args) do
|
||||
check_and_update_timezone()
|
||||
attach_oban_telemetry()
|
||||
Logger.add_handlers(:pinchflat)
|
||||
|
||||
# See https://hexdocs.pm/elixir/Supervisor.html
|
||||
# for other strategies and supported options
|
||||
[
|
||||
Pinchflat.PromEx,
|
||||
children = [
|
||||
PinchflatWeb.Telemetry,
|
||||
Pinchflat.Repo,
|
||||
# Must be before startup tasks
|
||||
|
|
@ -28,11 +20,17 @@ defmodule Pinchflat.Application do
|
|||
{Finch, name: Pinchflat.Finch},
|
||||
# Start a worker by calling: Pinchflat.Worker.start_link(arg)
|
||||
# {Pinchflat.Worker, arg},
|
||||
# Start to serve requests, typically the last entry (except for the post-boot tasks)
|
||||
PinchflatWeb.Endpoint,
|
||||
Pinchflat.Boot.PostBootStartupTasks
|
||||
# Start to serve requests, typically the last entry
|
||||
PinchflatWeb.Endpoint
|
||||
]
|
||||
|> Supervisor.start_link(strategy: :one_for_one, name: Pinchflat.Supervisor)
|
||||
|
||||
attach_oban_telemetry()
|
||||
Logger.add_handlers(:pinchflat)
|
||||
|
||||
# See https://hexdocs.pm/elixir/Supervisor.html
|
||||
# for other strategies and supported options
|
||||
opts = [strategy: :one_for_one, name: Pinchflat.Supervisor]
|
||||
Supervisor.start_link(children, opts)
|
||||
end
|
||||
|
||||
# Tell Phoenix to update the endpoint configuration
|
||||
|
|
@ -49,20 +47,4 @@ defmodule Pinchflat.Application do
|
|||
:ok = Oban.Telemetry.attach_default_logger()
|
||||
:telemetry.attach_many("job-telemetry-broadcast", events, &PinchflatWeb.Telemetry.job_state_change_broadcast/4, [])
|
||||
end
|
||||
|
||||
# This has to be here (rather than runtime.exs) since the `tzdata` application
|
||||
# has to be started before we can check the timezone
|
||||
defp check_and_update_timezone do
|
||||
attempted_timezone = System.get_env("TIMEZONE") || System.get_env("TZ") || "UTC"
|
||||
|
||||
valid_timezone =
|
||||
if Tzdata.zone_exists?(attempted_timezone) do
|
||||
attempted_timezone
|
||||
else
|
||||
Logger.warning("Invalid timezone #{attempted_timezone}, defaulting to UTC")
|
||||
"UTC"
|
||||
end
|
||||
|
||||
Application.put_env(:pinchflat, :timezone, valid_timezone)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,46 +0,0 @@
|
|||
defmodule Pinchflat.Boot.PostBootStartupTasks do
|
||||
@moduledoc """
|
||||
This module is responsible for running startup tasks on app boot
|
||||
AFTER all other boot steps have taken place and the app is ready to serve requests.
|
||||
|
||||
It's a GenServer because that plays REALLY nicely with the existing
|
||||
Phoenix supervision tree.
|
||||
"""
|
||||
|
||||
alias Pinchflat.YtDlp.UpdateWorker, as: YtDlpUpdateWorker
|
||||
|
||||
# restart: :temporary means that this process will never be restarted (ie: will run once and then die)
|
||||
use GenServer, restart: :temporary
|
||||
import Ecto.Query, warn: false
|
||||
|
||||
def start_link(opts \\ []) do
|
||||
GenServer.start_link(__MODULE__, %{env: Application.get_env(:pinchflat, :env)}, opts)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Runs post-boot application startup tasks.
|
||||
|
||||
Any code defined here will run every time the application starts. You must
|
||||
make sure that the code is idempotent and safe to run multiple times.
|
||||
|
||||
This is a good place to set up default settings, create initial records, stuff like that.
|
||||
Should be fast - anything with the potential to be slow should be kicked off as a job instead.
|
||||
"""
|
||||
@impl true
|
||||
def init(%{env: :test} = state) do
|
||||
# Do nothing _as part of the app bootup process_.
|
||||
# Since bootup calls `start_link` and that's where the `env` state is injected,
|
||||
# you can still call `.init()` manually to run these tasks for testing purposes
|
||||
{:ok, state}
|
||||
end
|
||||
|
||||
def init(state) do
|
||||
update_yt_dlp()
|
||||
|
||||
{:ok, state}
|
||||
end
|
||||
|
||||
defp update_yt_dlp do
|
||||
YtDlpUpdateWorker.kickoff()
|
||||
end
|
||||
end
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
defmodule Pinchflat.Boot.PostJobStartupTasks do
|
||||
@moduledoc """
|
||||
This module is responsible for running startup tasks on app boot
|
||||
AFTER the job runner has initialized.
|
||||
AFTER the job runner has initiallized.
|
||||
|
||||
It's a GenServer because that plays REALLY nicely with the existing
|
||||
Phoenix supervision tree.
|
||||
|
|
@ -12,7 +12,7 @@ defmodule Pinchflat.Boot.PostJobStartupTasks do
|
|||
import Ecto.Query, warn: false
|
||||
|
||||
def start_link(opts \\ []) do
|
||||
GenServer.start_link(__MODULE__, %{env: Application.get_env(:pinchflat, :env)}, opts)
|
||||
GenServer.start_link(__MODULE__, %{}, opts)
|
||||
end
|
||||
|
||||
@doc """
|
||||
|
|
@ -25,13 +25,6 @@ defmodule Pinchflat.Boot.PostJobStartupTasks do
|
|||
Should be fast - anything with the potential to be slow should be kicked off as a job instead.
|
||||
"""
|
||||
@impl true
|
||||
def init(%{env: :test} = state) do
|
||||
# Do nothing _as part of the app bootup process_.
|
||||
# Since bootup calls `start_link` and that's where the `env` state is injected,
|
||||
# you can still call `.init()` manually to run these tasks for testing purposes
|
||||
{:ok, state}
|
||||
end
|
||||
|
||||
def init(state) do
|
||||
# Nothing at the moment!
|
||||
|
||||
|
|
|
|||
|
|
@ -16,10 +16,8 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
|
|||
alias Pinchflat.Settings
|
||||
alias Pinchflat.Utils.FilesystemUtils
|
||||
|
||||
alias Pinchflat.Lifecycle.UserScripts.CommandRunner, as: UserScriptRunner
|
||||
|
||||
def start_link(opts \\ []) do
|
||||
GenServer.start_link(__MODULE__, %{env: Application.get_env(:pinchflat, :env)}, opts)
|
||||
GenServer.start_link(__MODULE__, %{}, opts)
|
||||
end
|
||||
|
||||
@doc """
|
||||
|
|
@ -32,20 +30,12 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
|
|||
Should be fast - anything with the potential to be slow should be kicked off as a job instead.
|
||||
"""
|
||||
@impl true
|
||||
def init(%{env: :test} = state) do
|
||||
# Do nothing _as part of the app bootup process_.
|
||||
# Since bootup calls `start_link` and that's where the `env` state is injected,
|
||||
# you can still call `.init()` manually to run these tasks for testing purposes
|
||||
{:ok, state}
|
||||
end
|
||||
|
||||
def init(state) do
|
||||
ensure_tmpfile_directory()
|
||||
reset_executing_jobs()
|
||||
create_blank_yt_dlp_files()
|
||||
create_blank_user_script_file()
|
||||
apply_default_settings()
|
||||
run_app_init_script()
|
||||
|
||||
{:ok, state}
|
||||
end
|
||||
|
|
@ -105,12 +95,6 @@ defmodule Pinchflat.Boot.PreJobStartupTasks do
|
|||
Settings.set(apprise_version: apprise_version)
|
||||
end
|
||||
|
||||
defp run_app_init_script do
|
||||
runner = Application.get_env(:pinchflat, :user_script_runner, UserScriptRunner)
|
||||
|
||||
runner.run(:app_init, %{})
|
||||
end
|
||||
|
||||
defp yt_dlp_runner do
|
||||
Application.get_env(:pinchflat, :yt_dlp_runner)
|
||||
end
|
||||
|
|
|
|||
82
lib/pinchflat/downloading/codec_parser.ex
Normal file
82
lib/pinchflat/downloading/codec_parser.ex
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
defmodule Pinchflat.Downloading.CodecParser do
|
||||
@moduledoc """
|
||||
Functions for generating yt-dlp codec strings
|
||||
"""
|
||||
|
||||
alias Pinchflat.Settings
|
||||
|
||||
@doc """
|
||||
Generate a video codec string based on the value of the video_codec_preference setting.
|
||||
|
||||
Returns binary()
|
||||
"""
|
||||
def generate_vcodec_string_from_settings do
|
||||
generate_vcodec_string(Settings.get!(:video_codec_preference))
|
||||
end
|
||||
|
||||
@doc """
|
||||
Generate an audio codec string based on the value of the audio_codec_preference setting.
|
||||
|
||||
Returns binary()
|
||||
"""
|
||||
def generate_acodec_string_from_settings do
|
||||
generate_acodec_string(Settings.get!(:audio_codec_preference))
|
||||
end
|
||||
|
||||
@doc """
|
||||
Generate a video codec string from a list of video codecs.
|
||||
|
||||
If the list is nil or empty, the default video codec is AVC.
|
||||
|
||||
Returns binary()
|
||||
"""
|
||||
def generate_vcodec_string(nil), do: "bestvideo[vcodec~='^avc']/bestvideo"
|
||||
def generate_vcodec_string([]), do: generate_vcodec_string(nil)
|
||||
|
||||
def generate_vcodec_string(video_codecs) do
|
||||
video_codecs
|
||||
|> Enum.map(&video_codec_map()[&1])
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|> Enum.map(&"bestvideo[vcodec~='^#{&1}']")
|
||||
|> Enum.concat(["bestvideo"])
|
||||
|> Enum.join("/")
|
||||
end
|
||||
|
||||
@doc """
|
||||
Generate an audio codec string from a list of audio codecs.
|
||||
|
||||
If the list is nil or empty, the default audio codec is MP4A.
|
||||
|
||||
Returns binary()
|
||||
"""
|
||||
def generate_acodec_string(nil), do: "bestaudio[acodec~='^mp4a']/bestaudio"
|
||||
def generate_acodec_string([]), do: generate_acodec_string(nil)
|
||||
|
||||
def generate_acodec_string(audio_codecs) do
|
||||
audio_codecs
|
||||
|> Enum.map(&audio_codec_map()[&1])
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|> Enum.map(&"bestaudio[acodec~='^#{&1}']")
|
||||
|> Enum.concat(["bestaudio"])
|
||||
|> Enum.join("/")
|
||||
end
|
||||
|
||||
@doc false
|
||||
def video_codec_map do
|
||||
%{
|
||||
"av01" => "av01",
|
||||
"avc" => "avc",
|
||||
"vp9" => "vp0?9"
|
||||
}
|
||||
end
|
||||
|
||||
@doc false
|
||||
def audio_codec_map do
|
||||
%{
|
||||
"aac" => "aac",
|
||||
"mp4a" => "mp4a",
|
||||
"mp3" => "mp3",
|
||||
"opus" => "opus"
|
||||
}
|
||||
end
|
||||
end
|
||||
|
|
@ -6,8 +6,8 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
|
|||
alias Pinchflat.Sources
|
||||
alias Pinchflat.Sources.Source
|
||||
alias Pinchflat.Media.MediaItem
|
||||
alias Pinchflat.Downloading.CodecParser
|
||||
alias Pinchflat.Downloading.OutputPathBuilder
|
||||
alias Pinchflat.Downloading.QualityOptionBuilder
|
||||
|
||||
alias Pinchflat.Utils.FilesystemUtils, as: FSUtils
|
||||
|
||||
|
|
@ -34,38 +34,21 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
|
|||
|
||||
@doc """
|
||||
Builds the output path for yt-dlp to download media based on the given source's
|
||||
or media_item's media profile. Uses the source's override output path template if it exists.
|
||||
media profile. Uses the source's override output path template if it exists.
|
||||
|
||||
Accepts a %MediaItem{} or %Source{} struct. If a %Source{} struct is passed, it
|
||||
will use a default %MediaItem{} struct with the given source.
|
||||
|
||||
Returns binary()
|
||||
"""
|
||||
def build_output_path_for(%Source{} = source_with_preloads) do
|
||||
build_output_path_for(%MediaItem{source: source_with_preloads})
|
||||
end
|
||||
|
||||
def build_output_path_for(%MediaItem{} = media_item_with_preloads) do
|
||||
output_path_template = Sources.output_path_template(media_item_with_preloads.source)
|
||||
|
||||
build_output_path(output_path_template, media_item_with_preloads)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Builds the quality options for yt-dlp to download media based on the given source's
|
||||
or media_item's media profile. Useful for helping predict final filepath of downloaded
|
||||
media.
|
||||
|
||||
returns [Keyword.t()]
|
||||
"""
|
||||
def build_quality_options_for(%Source{} = source_with_preloads) do
|
||||
build_quality_options_for(%MediaItem{source: source_with_preloads})
|
||||
end
|
||||
|
||||
def build_quality_options_for(%MediaItem{} = media_item_with_preloads) do
|
||||
media_profile = media_item_with_preloads.source.media_profile
|
||||
|
||||
quality_options(media_profile)
|
||||
def build_output_path_for(%Source{} = source_with_preloads) do
|
||||
build_output_path_for(%MediaItem{source: source_with_preloads})
|
||||
end
|
||||
|
||||
defp default_options(override_opts) do
|
||||
|
|
@ -91,9 +74,6 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
|
|||
{{:download_auto_subs, true}, %{download_subs: true}} ->
|
||||
acc ++ [:write_auto_subs]
|
||||
|
||||
{{:download_auto_subs, true}, %{embed_subs: true}} ->
|
||||
acc ++ [:write_auto_subs]
|
||||
|
||||
{{:embed_subs, true}, %{preferred_resolution: pr}} when pr != :audio ->
|
||||
acc ++ [:embed_subs]
|
||||
|
||||
|
|
@ -142,7 +122,27 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
|
|||
end
|
||||
|
||||
defp quality_options(media_profile) do
|
||||
QualityOptionBuilder.build(media_profile)
|
||||
vcodec_string = CodecParser.generate_vcodec_string_from_settings()
|
||||
acodec_string = CodecParser.generate_acodec_string_from_settings()
|
||||
|
||||
case media_profile.preferred_resolution do
|
||||
# Also be aware that :audio disabled all embedding options for subtitles
|
||||
:audio ->
|
||||
[:extract_audio, format: "#{acodec_string}/best"]
|
||||
|
||||
resolution_atom ->
|
||||
{resolution_string, _} =
|
||||
resolution_atom
|
||||
|> Atom.to_string()
|
||||
|> Integer.parse()
|
||||
|
||||
[
|
||||
format_sort: "res:#{resolution_string}",
|
||||
# Since Plex doesn't support reading metadata from MKV
|
||||
remux_video: "mp4",
|
||||
format: "((#{vcodec_string})+(#{acodec_string}))/best"
|
||||
]
|
||||
end
|
||||
end
|
||||
|
||||
defp sponsorblock_options(media_profile) do
|
||||
|
|
@ -152,7 +152,6 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
|
|||
case {behaviour, categories} do
|
||||
{_, []} -> []
|
||||
{:remove, _} -> [sponsorblock_remove: Enum.join(categories, ",")]
|
||||
{:mark, _} -> [sponsorblock_mark: Enum.join(categories, ",")]
|
||||
{:disabled, _} -> []
|
||||
end
|
||||
end
|
||||
|
|
@ -201,15 +200,14 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
|
|||
source = media_item_with_preloads.source
|
||||
|
||||
%{
|
||||
"media_item_id" => to_string(media_item_with_preloads.id),
|
||||
"source_id" => to_string(source.id),
|
||||
"media_profile_id" => to_string(source.media_profile_id),
|
||||
"source_custom_name" => source.custom_name,
|
||||
"source_collection_id" => source.collection_id,
|
||||
"source_collection_name" => source.collection_name,
|
||||
"source_collection_type" => to_string(source.collection_type),
|
||||
"media_playlist_index" => pad_int(media_item_with_preloads.playlist_index),
|
||||
"media_upload_date_index" => pad_int(media_item_with_preloads.upload_date_index)
|
||||
"media_upload_date_index" =>
|
||||
media_item_with_preloads.upload_date_index
|
||||
|> to_string()
|
||||
|> String.pad_leading(2, "0")
|
||||
}
|
||||
end
|
||||
|
||||
|
|
@ -226,12 +224,6 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do
|
|||
|> build_output_path(media_item_with_preloads)
|
||||
end
|
||||
|
||||
defp pad_int(integer, count \\ 2, padding \\ "0") do
|
||||
integer
|
||||
|> to_string()
|
||||
|> String.pad_leading(count, padding)
|
||||
end
|
||||
|
||||
defp base_directory do
|
||||
Application.get_env(:pinchflat, :media_directory)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -16,10 +16,14 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
|
|||
alias Pinchflat.Media.MediaItem
|
||||
alias Pinchflat.Downloading.MediaDownloadWorker
|
||||
|
||||
alias Pinchflat.Lifecycle.UserScripts.CommandRunner, as: UserScriptRunner
|
||||
|
||||
@doc """
|
||||
Starts tasks for downloading media for any of a sources _pending_ media items.
|
||||
Jobs are not enqueued if the source is set to not download media. This will return :ok.
|
||||
|
||||
You can optionally set the `kickoff_delay` option to delay when the jobs are enqueued.
|
||||
|
||||
NOTE: this starts a download for each media item that is pending,
|
||||
not just the ones that were indexed in this job run. This should ensure
|
||||
that any stragglers are caught if, for some reason, they weren't enqueued
|
||||
|
|
@ -27,15 +31,17 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
|
|||
|
||||
Returns :ok
|
||||
"""
|
||||
def enqueue_pending_download_tasks(source, job_opts \\ [])
|
||||
def enqueue_pending_download_tasks(source, opts \\ [])
|
||||
|
||||
def enqueue_pending_download_tasks(%Source{download_media: true} = source, opts) do
|
||||
kickoff_delay = Keyword.get(opts, :kickoff_delay, 0)
|
||||
|
||||
def enqueue_pending_download_tasks(%Source{download_media: true} = source, job_opts) do
|
||||
source
|
||||
|> Media.list_pending_media_items_for()
|
||||
|> Enum.each(&MediaDownloadWorker.kickoff_with_task(&1, %{}, job_opts))
|
||||
|> Enum.each(&MediaDownloadWorker.kickoff_with_task(&1, %{}, schedule_in: kickoff_delay))
|
||||
end
|
||||
|
||||
def enqueue_pending_download_tasks(%Source{download_media: false}, _job_opts) do
|
||||
def enqueue_pending_download_tasks(%Source{download_media: false}, _opts) do
|
||||
:ok
|
||||
end
|
||||
|
||||
|
|
@ -55,15 +61,18 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
|
|||
downloaded, based on the source's download settings and whether media is
|
||||
considered pending.
|
||||
|
||||
You can optionally set the `kickoff_delay` option to delay when the jobs are enqueued.
|
||||
|
||||
Returns {:ok, %Task{}} | {:error, :should_not_download} | {:error, any()}
|
||||
"""
|
||||
def kickoff_download_if_pending(%MediaItem{} = media_item, job_opts \\ []) do
|
||||
def kickoff_download_if_pending(%MediaItem{} = media_item, opts \\ []) do
|
||||
kickoff_delay = Keyword.get(opts, :kickoff_delay, 0)
|
||||
media_item = Repo.preload(media_item, :source)
|
||||
|
||||
if media_item.source.download_media && Media.pending_download?(media_item) do
|
||||
Logger.info("Kicking off download for media item ##{media_item.id} (#{media_item.media_id})")
|
||||
|
||||
MediaDownloadWorker.kickoff_with_task(media_item, %{}, job_opts)
|
||||
MediaDownloadWorker.kickoff_with_task(media_item, %{}, schedule_in: kickoff_delay)
|
||||
else
|
||||
{:error, :should_not_download}
|
||||
end
|
||||
|
|
@ -93,10 +102,39 @@ defmodule Pinchflat.Downloading.DownloadingHelpers do
|
|||
[m, s, mp],
|
||||
^MediaQuery.for_source(source) and
|
||||
^MediaQuery.downloaded() and
|
||||
not (^MediaQuery.download_prevented())
|
||||
not (^MediaQuery.download_prevented()) and
|
||||
not (^MediaQuery.culled())
|
||||
)
|
||||
)
|
||||
|> Repo.all()
|
||||
|> Enum.map(&MediaDownloadWorker.kickoff_with_task/1)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Creates a media item from the attributes returned by the video backend
|
||||
(read: yt-dlp) and runs the user script with a `media_indexed` event type.
|
||||
|
||||
Only runs the user script if the media item was created successfully and the media item
|
||||
doesn't already exist in the database.
|
||||
|
||||
Returns {:ok, %MediaItem{}} | {:error, any()}
|
||||
"""
|
||||
def create_media_item_and_run_script(%Source{} = source, media_attrs_struct) do
|
||||
media_already_exists =
|
||||
MediaQuery.new()
|
||||
|> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.media_id(media_attrs_struct.media_id)))
|
||||
|> Repo.exists?()
|
||||
|
||||
case Media.create_media_item_from_backend_attrs(source, media_attrs_struct) do
|
||||
{:ok, media_item} ->
|
||||
if !media_already_exists do
|
||||
UserScriptRunner.run(:media_indexed, media_item)
|
||||
end
|
||||
|
||||
{:ok, media_item}
|
||||
|
||||
err ->
|
||||
err
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
|
|||
|
||||
use Oban.Worker,
|
||||
queue: :media_fetching,
|
||||
priority: 5,
|
||||
unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
|
||||
tags: ["media_item", "media_fetching", "show_in_dashboard"]
|
||||
|
||||
|
|
@ -13,7 +12,6 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
|
|||
alias Pinchflat.Tasks
|
||||
alias Pinchflat.Repo
|
||||
alias Pinchflat.Media
|
||||
alias Pinchflat.Media.FileSyncing
|
||||
alias Pinchflat.Downloading.MediaDownloader
|
||||
|
||||
alias Pinchflat.Lifecycle.UserScripts.CommandRunner, as: UserScriptRunner
|
||||
|
|
@ -41,16 +39,20 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
|
|||
- `quality_upgrade?`: re-downloads media, including the video. Does not force download
|
||||
if the source is set to not download media
|
||||
|
||||
Returns :ok | {:error, any, ...any}
|
||||
Returns :ok | {:ok, %MediaItem{}} | {:error, any, ...any}
|
||||
"""
|
||||
@impl Oban.Worker
|
||||
def perform(%Oban.Job{args: %{"id" => media_item_id} = args}) do
|
||||
should_force = Map.get(args, "force", false)
|
||||
is_quality_upgrade = Map.get(args, "quality_upgrade?", false)
|
||||
|
||||
media_item = fetch_and_run_prevent_download_user_script(media_item_id)
|
||||
media_item =
|
||||
media_item_id
|
||||
|> Media.get_media_item!()
|
||||
|> Repo.preload(:source)
|
||||
|
||||
if should_download_media?(media_item, should_force, is_quality_upgrade) do
|
||||
# If the source or media item is set to not download media, perform a no-op unless forced
|
||||
if (media_item.source.download_media && !media_item.prevent_download) || should_force do
|
||||
download_media_and_schedule_jobs(media_item, is_quality_upgrade, should_force)
|
||||
else
|
||||
:ok
|
||||
|
|
@ -60,34 +62,6 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
|
|||
Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: media item #{media_item_id} stale")
|
||||
end
|
||||
|
||||
# If this is a quality upgrade, only check if the source is set to download media
|
||||
# or that the media item's download hasn't been prevented
|
||||
defp should_download_media?(media_item, should_force, true = _is_quality_upgrade) do
|
||||
(media_item.source.download_media && !media_item.prevent_download) || should_force
|
||||
end
|
||||
|
||||
# If it's not a quality upgrade, additionally check if the media item is pending download
|
||||
defp should_download_media?(media_item, should_force, _is_quality_upgrade) do
|
||||
source = media_item.source
|
||||
is_pending = Media.pending_download?(media_item)
|
||||
|
||||
(is_pending && source.download_media && !media_item.prevent_download) || should_force
|
||||
end
|
||||
|
||||
# If a user script exists and, when run, returns a non-zero exit code, prevent this and all future downloads
|
||||
# of the media item.
|
||||
defp fetch_and_run_prevent_download_user_script(media_item_id) do
|
||||
media_item = Media.get_media_item!(media_item_id)
|
||||
|
||||
{:ok, media_item} =
|
||||
case run_user_script(:media_pre_download, media_item) do
|
||||
{:ok, _, exit_code} when exit_code != 0 -> Media.update_media_item(media_item, %{prevent_download: true})
|
||||
_ -> {:ok, media_item}
|
||||
end
|
||||
|
||||
Repo.preload(media_item, :source)
|
||||
end
|
||||
|
||||
defp download_media_and_schedule_jobs(media_item, is_quality_upgrade, should_force) do
|
||||
overwrite_behaviour = if should_force || is_quality_upgrade, do: :force_overwrites, else: :no_force_overwrites
|
||||
override_opts = [overwrite_behaviour: overwrite_behaviour]
|
||||
|
|
@ -100,18 +74,14 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
|
|||
media_redownloaded_at: get_redownloaded_at(is_quality_upgrade)
|
||||
})
|
||||
|
||||
:ok = FileSyncing.delete_outdated_files(media_item, updated_media_item)
|
||||
run_user_script(:media_downloaded, updated_media_item)
|
||||
:ok = run_user_script(updated_media_item)
|
||||
|
||||
:ok
|
||||
{:ok, updated_media_item}
|
||||
|
||||
{:recovered, _media_item, _message} ->
|
||||
{:recovered, _} ->
|
||||
{:error, :retry}
|
||||
|
||||
{:error, :unsuitable_for_download, _message} ->
|
||||
{:ok, :non_retry}
|
||||
|
||||
{:error, _error_atom, message} ->
|
||||
{:error, message} ->
|
||||
action_on_error(message)
|
||||
end
|
||||
end
|
||||
|
|
@ -129,11 +99,7 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
|
|||
defp action_on_error(message) do
|
||||
# This will attempt re-download at the next indexing, but it won't be retried
|
||||
# immediately as part of job failure logic
|
||||
non_retryable_errors = [
|
||||
"Video unavailable",
|
||||
"Sign in to confirm",
|
||||
"This video is available to this channel's members"
|
||||
]
|
||||
non_retryable_errors = ["Video unavailable"]
|
||||
|
||||
if String.contains?(to_string(message), non_retryable_errors) do
|
||||
Logger.error("yt-dlp download will not be retried: #{inspect(message)}")
|
||||
|
|
@ -146,9 +112,9 @@ defmodule Pinchflat.Downloading.MediaDownloadWorker do
|
|||
|
||||
# NOTE: I like this pattern of using the default value so that I don't have to
|
||||
# define it in config.exs (and friends). Consider using this elsewhere.
|
||||
defp run_user_script(event, media_item) do
|
||||
defp run_user_script(media_item) do
|
||||
runner = Application.get_env(:pinchflat, :user_script_runner, UserScriptRunner)
|
||||
|
||||
runner.run(event, media_item)
|
||||
runner.run(:media_downloaded, media_item)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -9,9 +9,7 @@ defmodule Pinchflat.Downloading.MediaDownloader do
|
|||
|
||||
alias Pinchflat.Repo
|
||||
alias Pinchflat.Media
|
||||
alias Pinchflat.Sources
|
||||
alias Pinchflat.Media.MediaItem
|
||||
alias Pinchflat.Utils.StringUtils
|
||||
alias Pinchflat.Metadata.NfoBuilder
|
||||
alias Pinchflat.Metadata.MetadataParser
|
||||
alias Pinchflat.Metadata.MetadataFileHelpers
|
||||
|
|
@ -22,57 +20,16 @@ defmodule Pinchflat.Downloading.MediaDownloader do
|
|||
|
||||
@doc """
|
||||
Downloads media for a media item, updating the media item based on the metadata
|
||||
returned by yt-dlp. Encountered errors are saved to the Media Item record. Saves
|
||||
the entire metadata response to the associated media_metadata record.
|
||||
returned by yt-dlp. Also saves the entire metadata response to the associated
|
||||
media_metadata record.
|
||||
|
||||
NOTE: related methods (like the download worker) won't download if Pthe media item's source
|
||||
NOTE: related methods (like the download worker) won't download if the media item's source
|
||||
is set to not download media. However, I'm not enforcing that here since I need this for testing.
|
||||
This may change in the future but I'm not stressed.
|
||||
|
||||
Returns {:ok, %MediaItem{}} | {:error, atom(), String.t()} | {:recovered, %MediaItem{}, String.t()}
|
||||
Returns {:ok, %MediaItem{}} | {:error, any, ...any}
|
||||
"""
|
||||
def download_for_media_item(%MediaItem{} = media_item, override_opts \\ []) do
|
||||
case attempt_download_and_update_for_media_item(media_item, override_opts) do
|
||||
{:ok, media_item} ->
|
||||
# Returns {:ok, %MediaItem{}}
|
||||
Media.update_media_item(media_item, %{last_error: nil})
|
||||
|
||||
{:error, error_atom, message} ->
|
||||
Media.update_media_item(media_item, %{last_error: StringUtils.wrap_string(message)})
|
||||
|
||||
{:error, error_atom, message}
|
||||
|
||||
{:recovered, media_item, message} ->
|
||||
{:ok, updated_media_item} = Media.update_media_item(media_item, %{last_error: StringUtils.wrap_string(message)})
|
||||
|
||||
{:recovered, updated_media_item, message}
|
||||
end
|
||||
end
|
||||
|
||||
# Looks complicated, but here's the key points:
|
||||
# - download_with_options runs a pre-check to see if the media item is suitable for download.
|
||||
# - If the media item fails the precheck, it returns {:error, :unsuitable_for_download, message}
|
||||
# - However, if the precheck fails in a way that we think can be fixed by using cookies, we retry with cookies
|
||||
# and return the result of that
|
||||
# - If the precheck passes but the download fails, it normally returns {:error, :download_failed, message}
|
||||
# - However, there are some errors we can recover from (eg: failure to communicate with SponsorBlock).
|
||||
# In this case, we attempt the download anyway and update the media item with what details we do have.
|
||||
# This case returns {:recovered, updated_media_item, message}
|
||||
# - If we attempt a retry but it fails, we return {:error, :unrecoverable, message}
|
||||
# - If there is an unknown error unrelated to the above, we return {:error, :unknown, message}
|
||||
# - Finally, if there is no error, we update the media item with the parsed JSON and return {:ok, updated_media_item}
|
||||
#
|
||||
# Restated, here are the return values for each case:
|
||||
# - On success: {:ok, updated_media_item}
|
||||
# - On initial failure but successfully recovered: {:recovered, updated_media_item, message}
|
||||
# - On error: {:error, error_atom, message} where error_atom is one of:
|
||||
# - `:unsuitable_for_download` if the media item fails the precheck
|
||||
# - `:unrecoverable` if there was an initial failure and the recovery attempt failed
|
||||
# - `:download_failed` for all other yt-dlp-related downloading errors
|
||||
# - `:unknown` for any other errors, including those not related to yt-dlp
|
||||
# - If we retry using cookies, all of the above return values apply. The cookie retry
|
||||
# logic is handled transparently as far as the caller is concerned
|
||||
defp attempt_download_and_update_for_media_item(media_item, override_opts) do
|
||||
output_filepath = FilesystemUtils.generate_metadata_tmpfile(:json)
|
||||
media_with_preloads = Repo.preload(media_item, [:metadata, source: :media_profile])
|
||||
|
||||
|
|
@ -80,31 +37,25 @@ defmodule Pinchflat.Downloading.MediaDownloader do
|
|||
{:ok, parsed_json} ->
|
||||
update_media_item_from_parsed_json(media_with_preloads, parsed_json)
|
||||
|
||||
{:error, :unsuitable_for_download} ->
|
||||
message =
|
||||
"Media item ##{media_with_preloads.id} isn't suitable for download yet. May be an active or processing live stream"
|
||||
|
||||
Logger.warning(message)
|
||||
|
||||
{:error, :unsuitable_for_download, message}
|
||||
|
||||
{:error, message, _exit_code} ->
|
||||
Logger.error("yt-dlp download error for media item ##{media_with_preloads.id}: #{inspect(message)}")
|
||||
|
||||
if String.contains?(to_string(message), recoverable_errors()) do
|
||||
attempt_recovery_from_error(media_with_preloads, output_filepath, message)
|
||||
attempt_update_media_item(media_with_preloads, output_filepath)
|
||||
|
||||
{:recovered, message}
|
||||
else
|
||||
{:error, :download_failed, message}
|
||||
{:error, message}
|
||||
end
|
||||
|
||||
err ->
|
||||
Logger.error("Unknown error downloading media item ##{media_with_preloads.id}: #{inspect(err)}")
|
||||
|
||||
{:error, :unknown, "Unknown error: #{inspect(err)}"}
|
||||
{:error, "Unknown error: #{inspect(err)}"}
|
||||
end
|
||||
end
|
||||
|
||||
defp attempt_recovery_from_error(media_with_preloads, output_filepath, error_message) do
|
||||
defp attempt_update_media_item(media_with_preloads, output_filepath) do
|
||||
with {:ok, contents} <- File.read(output_filepath),
|
||||
{:ok, parsed_json} <- Phoenix.json_library().decode(contents) do
|
||||
Logger.info("""
|
||||
|
|
@ -113,13 +64,12 @@ defmodule Pinchflat.Downloading.MediaDownloader do
|
|||
anyway
|
||||
""")
|
||||
|
||||
{:ok, updated_media_item} = update_media_item_from_parsed_json(media_with_preloads, parsed_json)
|
||||
{:recovered, updated_media_item, error_message}
|
||||
update_media_item_from_parsed_json(media_with_preloads, parsed_json)
|
||||
else
|
||||
err ->
|
||||
Logger.error("Unable to recover error for media item ##{media_with_preloads.id}: #{inspect(err)}")
|
||||
|
||||
{:error, :unrecoverable, error_message}
|
||||
{:error, :retry_failed}
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -129,13 +79,12 @@ defmodule Pinchflat.Downloading.MediaDownloader do
|
|||
|> MetadataParser.parse_for_media_item()
|
||||
|> Map.merge(%{
|
||||
media_downloaded_at: DateTime.utc_now(),
|
||||
culled_at: nil,
|
||||
nfo_filepath: determine_nfo_filepath(media_with_preloads, parsed_json),
|
||||
metadata: %{
|
||||
# IDEA: might be worth kicking off a job for this since thumbnail fetching
|
||||
# could fail and I want to handle that in isolation
|
||||
metadata_filepath: MetadataFileHelpers.compress_and_store_metadata_for(media_with_preloads, parsed_json),
|
||||
thumbnail_filepath: MetadataFileHelpers.download_and_store_thumbnail_for(media_with_preloads)
|
||||
thumbnail_filepath: MetadataFileHelpers.download_and_store_thumbnail_for(media_with_preloads, parsed_json)
|
||||
}
|
||||
})
|
||||
|
||||
|
|
@ -156,49 +105,8 @@ defmodule Pinchflat.Downloading.MediaDownloader do
|
|||
|
||||
defp download_with_options(url, item_with_preloads, output_filepath, override_opts) do
|
||||
{:ok, options} = DownloadOptionBuilder.build(item_with_preloads, override_opts)
|
||||
force_use_cookies = Keyword.get(override_opts, :force_use_cookies, false)
|
||||
source_uses_cookies = Sources.use_cookies?(item_with_preloads.source, :downloading)
|
||||
should_use_cookies = force_use_cookies || source_uses_cookies
|
||||
|
||||
runner_opts = [output_filepath: output_filepath, use_cookies: should_use_cookies]
|
||||
|
||||
case {YtDlpMedia.get_downloadable_status(url, use_cookies: should_use_cookies), should_use_cookies} do
|
||||
{{:ok, :downloadable}, _} ->
|
||||
YtDlpMedia.download(url, options, runner_opts)
|
||||
|
||||
{{:ok, :ignorable}, _} ->
|
||||
{:error, :unsuitable_for_download}
|
||||
|
||||
{{:error, _message, _exit_code} = err, false} ->
|
||||
# If there was an error and we don't have cookies, this method will retry with cookies
|
||||
# if doing so would help AND the source allows. Otherwise, it will return the error as-is
|
||||
maybe_retry_with_cookies(url, item_with_preloads, output_filepath, override_opts, err)
|
||||
|
||||
# This gets hit if cookies are enabled which, importantly, also covers the case where we
|
||||
# retry a download with cookies and it fails again
|
||||
{{:error, message, exit_code}, true} ->
|
||||
{:error, message, exit_code}
|
||||
|
||||
{err, _} ->
|
||||
err
|
||||
end
|
||||
end
|
||||
|
||||
defp maybe_retry_with_cookies(url, item_with_preloads, output_filepath, override_opts, err) do
|
||||
{:error, message, _} = err
|
||||
source = item_with_preloads.source
|
||||
message_contains_cookie_error = String.contains?(to_string(message), recoverable_cookie_errors())
|
||||
|
||||
if Sources.use_cookies?(source, :error_recovery) && message_contains_cookie_error do
|
||||
download_with_options(
|
||||
url,
|
||||
item_with_preloads,
|
||||
output_filepath,
|
||||
Keyword.put(override_opts, :force_use_cookies, true)
|
||||
)
|
||||
else
|
||||
err
|
||||
end
|
||||
YtDlpMedia.download(url, options, output_filepath: output_filepath)
|
||||
end
|
||||
|
||||
defp recoverable_errors do
|
||||
|
|
@ -206,11 +114,4 @@ defmodule Pinchflat.Downloading.MediaDownloader do
|
|||
"Unable to communicate with SponsorBlock"
|
||||
]
|
||||
end
|
||||
|
||||
defp recoverable_cookie_errors do
|
||||
[
|
||||
"Sign in to confirm",
|
||||
"This video is available to this channel's members"
|
||||
]
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -23,10 +23,10 @@ defmodule Pinchflat.Downloading.MediaQualityUpgradeWorker do
|
|||
"""
|
||||
@impl Oban.Worker
|
||||
def perform(%Oban.Job{}) do
|
||||
upgradable_media = Media.list_upgradeable_media_items()
|
||||
Logger.info("Redownloading #{length(upgradable_media)} media items")
|
||||
redownloadable_media = Media.list_redownloadable_media_items()
|
||||
Logger.info("Redownloading #{length(redownloadable_media)} media items")
|
||||
|
||||
Enum.each(upgradable_media, fn media_item ->
|
||||
Enum.each(redownloadable_media, fn media_item ->
|
||||
MediaDownloadWorker.kickoff_with_task(media_item, %{quality_upgrade?: true})
|
||||
end)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -2,15 +2,12 @@ defmodule Pinchflat.Downloading.MediaRetentionWorker do
|
|||
@moduledoc false
|
||||
|
||||
use Oban.Worker,
|
||||
queue: :local_data,
|
||||
queue: :local_metadata,
|
||||
unique: [period: :infinity, states: [:available, :scheduled, :retryable, :executing]],
|
||||
tags: ["media_item", "local_data"]
|
||||
|
||||
use Pinchflat.Media.MediaQuery
|
||||
tags: ["media_item", "local_metadata"]
|
||||
|
||||
require Logger
|
||||
|
||||
alias Pinchflat.Repo
|
||||
alias Pinchflat.Media
|
||||
|
||||
@doc """
|
||||
|
|
@ -23,53 +20,14 @@ defmodule Pinchflat.Downloading.MediaRetentionWorker do
|
|||
"""
|
||||
@impl Oban.Worker
|
||||
def perform(%Oban.Job{}) do
|
||||
cull_cullable_media_items()
|
||||
delete_media_items_from_before_cutoff()
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
defp cull_cullable_media_items do
|
||||
cullable_media =
|
||||
MediaQuery.new()
|
||||
|> MediaQuery.require_assoc(:source)
|
||||
|> where(^MediaQuery.cullable())
|
||||
|> Repo.all()
|
||||
|
||||
cullable_media = Media.list_cullable_media_items()
|
||||
Logger.info("Culling #{length(cullable_media)} media items past their retention date")
|
||||
|
||||
Enum.each(cullable_media, fn media_item ->
|
||||
# Setting `prevent_download` does what it says on the tin, but `culled_at` is purely informational.
|
||||
# We don't actually do anything with that in terms of queries and it gets set to nil if the media item
|
||||
# gets re-downloaded.
|
||||
Media.delete_media_files(media_item, %{
|
||||
prevent_download: true,
|
||||
culled_at: DateTime.utc_now()
|
||||
})
|
||||
end)
|
||||
end
|
||||
|
||||
# NOTE: Since this is a date and not a datetime, we can't add logic to have to-the-minute
|
||||
# comparison like we can with retention periods. We can only compare to the day.
|
||||
defp delete_media_items_from_before_cutoff do
|
||||
deletable_media =
|
||||
MediaQuery.new()
|
||||
|> MediaQuery.require_assoc(:source)
|
||||
|> where(^MediaQuery.deletable_based_on_source_cutoff())
|
||||
|> Repo.all()
|
||||
|
||||
Logger.info("Deleting #{length(deletable_media)} media items that are from before the source cutoff")
|
||||
|
||||
Enum.each(deletable_media, fn media_item ->
|
||||
# Note that I'm not setting `prevent_download` on the media_item here.
|
||||
# That's because cutoff_date can easily change and it's a valid behavior to re-download older
|
||||
# media items if the cutoff_date changes.
|
||||
# Download is ultimately prevented because `MediaQuery.pending()` only returns media items
|
||||
# from after the cutoff date (among other things), so it's not like the media will just immediately
|
||||
# be re-downloaded.
|
||||
Media.delete_media_files(media_item, %{
|
||||
culled_at: DateTime.utc_now()
|
||||
})
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -54,12 +54,7 @@ defmodule Pinchflat.Downloading.OutputPathBuilder do
|
|||
"season_from_date" => "%(upload_date>%Y)S",
|
||||
"season_episode_from_date" => "s%(upload_date>%Y)Se%(upload_date>%m%d)S",
|
||||
"season_episode_index_from_date" => "s%(upload_date>%Y)Se%(upload_date>%m%d)S{{ media_upload_date_index }}",
|
||||
"artist_name" => "%(artist,creator,uploader,uploader_id)S",
|
||||
"static_season__episode_by_index" => "Season 1/s01e{{ media_playlist_index }}",
|
||||
"static_season__episode_by_date" => "Season 1/s01e%(upload_date>%y%m%d)S",
|
||||
"season_by_year__episode_by_date" => "Season %(upload_date>%Y)S/s%(upload_date>%Y)Se%(upload_date>%m%d)S",
|
||||
"season_by_year__episode_by_date_and_index" =>
|
||||
"Season %(upload_date>%Y)S/s%(upload_date>%Y)Se%(upload_date>%m%d)S{{ media_upload_date_index }}"
|
||||
"artist_name" => "%(artist,creator,uploader,uploader_id)S"
|
||||
}
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,66 +0,0 @@
|
|||
defmodule Pinchflat.Downloading.QualityOptionBuilder do
|
||||
@moduledoc """
|
||||
A standalone builder module for building quality-related options for yt-dlp to download media.
|
||||
|
||||
Currently exclusively used in DownloadOptionBuilder since this logic is too complex to just
|
||||
place in the main module.
|
||||
"""
|
||||
|
||||
alias Pinchflat.Settings
|
||||
alias Pinchflat.Profiles.MediaProfile
|
||||
|
||||
@doc """
|
||||
Builds the quality-related options for yt-dlp to download media based on the given media profile
|
||||
|
||||
Includes things like container, preferred format/codec, and audio track options.
|
||||
"""
|
||||
def build(%MediaProfile{preferred_resolution: :audio, media_container: container} = media_profile) do
|
||||
acodec = Settings.get!(:audio_codec_preference)
|
||||
|
||||
[
|
||||
:extract_audio,
|
||||
format_sort: "+acodec:#{acodec}",
|
||||
audio_format: container || "best",
|
||||
format: build_format_string(media_profile)
|
||||
]
|
||||
end
|
||||
|
||||
def build(%MediaProfile{preferred_resolution: resolution_atom, media_container: container} = media_profile) do
|
||||
vcodec = Settings.get!(:video_codec_preference)
|
||||
acodec = Settings.get!(:audio_codec_preference)
|
||||
{resolution_string, _} = resolution_atom |> Atom.to_string() |> Integer.parse()
|
||||
|
||||
[
|
||||
# Since Plex doesn't support reading metadata from MKV
|
||||
remux_video: container || "mp4",
|
||||
format_sort: "res:#{resolution_string},+codec:#{vcodec}:#{acodec}",
|
||||
format: build_format_string(media_profile)
|
||||
]
|
||||
end
|
||||
|
||||
defp build_format_string(%MediaProfile{preferred_resolution: :audio, audio_track: audio_track}) do
|
||||
if audio_track do
|
||||
"bestaudio[#{build_format_modifier(audio_track)}]/bestaudio/best"
|
||||
else
|
||||
"bestaudio/best"
|
||||
end
|
||||
end
|
||||
|
||||
defp build_format_string(%MediaProfile{audio_track: audio_track}) do
|
||||
if audio_track do
|
||||
"bestvideo+bestaudio[#{build_format_modifier(audio_track)}]/bestvideo*+bestaudio/best"
|
||||
else
|
||||
"bestvideo*+bestaudio/best"
|
||||
end
|
||||
end
|
||||
|
||||
# Reminder to self: this conflicts with `--extractor-args "youtube:lang=<LANG>"`
|
||||
# since that will translate the format_notes as well, which means they may not match.
|
||||
# At least that's what happens now - worth a re-check if I have to come back to this
|
||||
defp build_format_modifier("original"), do: "format_note*=original"
|
||||
defp build_format_modifier("default"), do: "format_note*='(default)'"
|
||||
# This uses the carat to anchor the language to the beginning of the string
|
||||
# since that's what's needed to match `en` to `en-US` and `en-GB`, etc. The user
|
||||
# can always specify the full language code if they want.
|
||||
defp build_format_modifier(language_code), do: "language^=#{language_code}"
|
||||
end
|
||||
|
|
@ -10,43 +10,22 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
|
|||
use Pinchflat.Media.MediaQuery
|
||||
|
||||
alias Pinchflat.Repo
|
||||
alias Pinchflat.Media
|
||||
alias Pinchflat.Tasks
|
||||
alias Pinchflat.Sources
|
||||
alias Pinchflat.Sources.Source
|
||||
alias Pinchflat.FastIndexing.YoutubeRss
|
||||
alias Pinchflat.FastIndexing.YoutubeApi
|
||||
alias Pinchflat.Downloading.DownloadingHelpers
|
||||
alias Pinchflat.FastIndexing.FastIndexingWorker
|
||||
alias Pinchflat.Downloading.DownloadOptionBuilder
|
||||
|
||||
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
|
||||
|
||||
@doc """
|
||||
Kicks off a new fast indexing task for a source. This will delete any existing fast indexing
|
||||
tasks for the source before starting a new one.
|
||||
|
||||
Returns {:ok, %Task{}}
|
||||
"""
|
||||
def kickoff_indexing_task(%Source{} = source) do
|
||||
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker", include_executing: true)
|
||||
FastIndexingWorker.kickoff_with_task(source)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Fetches new media IDs for a source from YT's API or RSS, indexes them, and kicks off downloading
|
||||
Fetches new media IDs from a source's YouTube RSS feed, indexes them, and kicks off downloading
|
||||
tasks for any pending media items. See comments in `FastIndexingWorker` for more info on the
|
||||
order of operations and how this fits into the indexing process.
|
||||
|
||||
Returns [%MediaItem{}] where each item is a new media item that was created _but not necessarily
|
||||
downloaded_.
|
||||
"""
|
||||
def index_and_kickoff_downloads(%Source{} = source) do
|
||||
# The media_profile is needed to determine the quality options to _then_ determine a more
|
||||
# accurate predicted filepath
|
||||
source = Repo.preload(source, [:media_profile])
|
||||
|
||||
{:ok, media_ids} = get_recent_media_ids(source)
|
||||
def kickoff_download_tasks_from_youtube_rss_feed(%Source{} = source) do
|
||||
{:ok, media_ids} = YoutubeRss.get_recent_media_ids_from_rss(source)
|
||||
existing_media_items = list_media_items_by_media_id_for(source, media_ids)
|
||||
new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id)
|
||||
|
||||
|
|
@ -54,7 +33,6 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
|
|||
Enum.map(new_media_ids, fn media_id ->
|
||||
case create_media_item_from_media_id(source, media_id) do
|
||||
{:ok, media_item} ->
|
||||
DownloadingHelpers.kickoff_download_if_pending(media_item, priority: 0)
|
||||
media_item
|
||||
|
||||
err ->
|
||||
|
|
@ -63,24 +41,12 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
|
|||
end
|
||||
end)
|
||||
|
||||
# Pick up any stragglers. Intentionally has a lower priority than the per-media item
|
||||
# kickoff above
|
||||
DownloadingHelpers.enqueue_pending_download_tasks(source, priority: 1)
|
||||
# Wait 5s before enqueuing downloads to give the post-indexing user script a chance to run
|
||||
DownloadingHelpers.enqueue_pending_download_tasks(source, kickoff_delay: 5)
|
||||
|
||||
Enum.filter(maybe_new_media_items, & &1)
|
||||
end
|
||||
|
||||
# If possible, use the YouTube API to fetch media IDs. If that fails, fall back to the RSS feed.
|
||||
# If the YouTube API isn't set up, just use the RSS feed.
|
||||
defp get_recent_media_ids(source) do
|
||||
with true <- YoutubeApi.enabled?(),
|
||||
{:ok, media_ids} <- YoutubeApi.get_recent_media_ids(source) do
|
||||
{:ok, media_ids}
|
||||
else
|
||||
_ -> YoutubeRss.get_recent_media_ids(source)
|
||||
end
|
||||
end
|
||||
|
||||
defp list_media_items_by_media_id_for(source, media_ids) do
|
||||
MediaQuery.new()
|
||||
|> where(^dynamic([mi], ^MediaQuery.for_source(source) and mi.media_id in ^media_ids))
|
||||
|
|
@ -89,18 +55,10 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do
|
|||
|
||||
defp create_media_item_from_media_id(source, media_id) do
|
||||
url = "https://www.youtube.com/watch?v=#{media_id}"
|
||||
# This is set to :metadata instead of :indexing since this happens _after_ the
|
||||
# actual indexing process. In reality, slow indexing is the only thing that
|
||||
# should be using :indexing.
|
||||
should_use_cookies = Sources.use_cookies?(source, :metadata)
|
||||
|
||||
command_opts =
|
||||
[output: DownloadOptionBuilder.build_output_path_for(source)] ++
|
||||
DownloadOptionBuilder.build_quality_options_for(source)
|
||||
|
||||
case YtDlpMedia.get_media_attributes(url, command_opts, use_cookies: should_use_cookies) do
|
||||
{:ok, media_attrs} ->
|
||||
Media.create_media_item_from_backend_attrs(source, media_attrs)
|
||||
case YtDlpMedia.get_media_attributes(url) do
|
||||
{:ok, media_attrs_struct} ->
|
||||
DownloadingHelpers.create_media_item_and_run_script(source, media_attrs_struct)
|
||||
|
||||
err ->
|
||||
err
|
||||
|
|
|
|||
|
|
@ -38,8 +38,8 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
|
|||
|
||||
Order of operations:
|
||||
1. FastIndexingWorker (this module) periodically checks the YouTube RSS feed for new media.
|
||||
with `FastIndexingHelpers.index_and_kickoff_downloads`
|
||||
2. If the above `index_and_kickoff_downloads` finds new media items in the RSS feed,
|
||||
with `FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed`
|
||||
2. If the above `kickoff_download_tasks_from_youtube_rss_feed` finds new media items in the RSS feed,
|
||||
it indexes them with a yt-dlp call to create the media item records then kicks off downloading
|
||||
tasks (MediaDownloadWorker) for any new media items _that should be downloaded_.
|
||||
3. Once downloads are kicked off, this worker sends a notification to the apprise server if applicable
|
||||
|
|
@ -67,7 +67,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingWorker do
|
|||
|
||||
new_media_items =
|
||||
source
|
||||
|> FastIndexingHelpers.index_and_kickoff_downloads()
|
||||
|> FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed()
|
||||
|> Enum.filter(&Media.pending_download?(&1))
|
||||
|
||||
if source.download_media do
|
||||
|
|
|
|||
|
|
@ -1,131 +0,0 @@
|
|||
defmodule Pinchflat.FastIndexing.YoutubeApi do
|
||||
@moduledoc """
|
||||
Methods for interacting with the YouTube API for fast indexing
|
||||
"""
|
||||
|
||||
require Logger
|
||||
|
||||
alias Pinchflat.Settings
|
||||
alias Pinchflat.Sources.Source
|
||||
alias Pinchflat.Utils.FunctionUtils
|
||||
alias Pinchflat.FastIndexing.YoutubeBehaviour
|
||||
|
||||
@behaviour YoutubeBehaviour
|
||||
|
||||
@agent_name {:global, __MODULE__.KeyIndex}
|
||||
|
||||
@doc """
|
||||
Determines if the YouTube API is enabled for fast indexing by checking
|
||||
if the user has an API key set
|
||||
|
||||
Returns boolean()
|
||||
"""
|
||||
@impl YoutubeBehaviour
|
||||
def enabled?, do: Enum.any?(api_keys())
|
||||
|
||||
@doc """
|
||||
Fetches the recent media IDs from the YouTube API for a given source.
|
||||
|
||||
Returns {:ok, [binary()]} | {:error, binary()}
|
||||
"""
|
||||
@impl YoutubeBehaviour
|
||||
def get_recent_media_ids(%Source{} = source) do
|
||||
api_response =
|
||||
source
|
||||
|> determine_playlist_id()
|
||||
|> do_api_request()
|
||||
|
||||
case api_response do
|
||||
{:ok, parsed_json} -> get_media_ids_from_response(parsed_json)
|
||||
{:error, reason} -> {:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
# The UC prefix is for channels which won't work with this API endpoint. Swapping
|
||||
# the prefix to UU will get us the playlist that represents the channel's uploads
|
||||
defp determine_playlist_id(%{collection_id: c_id}) do
|
||||
String.replace_prefix(c_id, "UC", "UU")
|
||||
end
|
||||
|
||||
defp do_api_request(playlist_id) do
|
||||
Logger.debug("Fetching recent media IDs from YouTube API for playlist: #{playlist_id}")
|
||||
|
||||
playlist_id
|
||||
|> construct_api_endpoint()
|
||||
|> http_client().get(accept: "application/json")
|
||||
|> case do
|
||||
{:ok, response} ->
|
||||
Phoenix.json_library().decode(response)
|
||||
|
||||
{:error, reason} ->
|
||||
Logger.error("Failed to fetch YouTube API: #{inspect(reason)}")
|
||||
{:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
defp get_media_ids_from_response(parsed_json) do
|
||||
parsed_json
|
||||
|> Map.get("items", [])
|
||||
|> Enum.map(fn item ->
|
||||
item
|
||||
|> Map.get("contentDetails", %{})
|
||||
|> Map.get("videoId", nil)
|
||||
end)
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|> Enum.uniq()
|
||||
|> FunctionUtils.wrap_ok()
|
||||
end
|
||||
|
||||
defp api_keys do
|
||||
case Settings.get!(:youtube_api_key) do
|
||||
nil ->
|
||||
[]
|
||||
|
||||
keys ->
|
||||
keys
|
||||
|> String.split(",")
|
||||
|> Enum.map(&String.trim/1)
|
||||
|> Enum.reject(&(&1 == ""))
|
||||
end
|
||||
end
|
||||
|
||||
defp get_or_start_api_key_agent do
|
||||
case Agent.start(fn -> 0 end, name: @agent_name) do
|
||||
{:ok, pid} -> pid
|
||||
{:error, {:already_started, pid}} -> pid
|
||||
end
|
||||
end
|
||||
|
||||
# Gets the next API key in round-robin fashion
|
||||
defp next_api_key do
|
||||
keys = api_keys()
|
||||
|
||||
case keys do
|
||||
[] ->
|
||||
nil
|
||||
|
||||
keys ->
|
||||
pid = get_or_start_api_key_agent()
|
||||
|
||||
current_index =
|
||||
Agent.get_and_update(pid, fn current ->
|
||||
{current, rem(current + 1, length(keys))}
|
||||
end)
|
||||
|
||||
Logger.debug("Using YouTube API key: #{Enum.at(keys, current_index)}")
|
||||
Enum.at(keys, current_index)
|
||||
end
|
||||
end
|
||||
|
||||
defp construct_api_endpoint(playlist_id) do
|
||||
api_base = "https://youtube.googleapis.com/youtube/v3/playlistItems"
|
||||
property_type = "contentDetails"
|
||||
max_results = 50
|
||||
|
||||
"#{api_base}?part=#{property_type}&maxResults=#{max_results}&playlistId=#{playlist_id}&key=#{next_api_key()}"
|
||||
end
|
||||
|
||||
defp http_client do
|
||||
Application.get_env(:pinchflat, :http_client, Pinchflat.HTTP.HTTPClient)
|
||||
end
|
||||
end
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
defmodule Pinchflat.FastIndexing.YoutubeBehaviour do
|
||||
@moduledoc """
|
||||
This module defines the behaviour for clients that interface with YouTube
|
||||
for the purpose of fast indexing.
|
||||
"""
|
||||
|
||||
alias Pinchflat.Sources.Source
|
||||
|
||||
@callback enabled?() :: boolean()
|
||||
@callback get_recent_media_ids(%Source{}) :: {:ok, [String.t()]} | {:error, String.t()}
|
||||
end
|
||||
|
|
@ -1,31 +1,18 @@
|
|||
defmodule Pinchflat.FastIndexing.YoutubeRss do
|
||||
@moduledoc """
|
||||
Methods for interacting with YouTube RSS feeds for fast indexing
|
||||
Methods for interacting with YouTube RSS feeds
|
||||
"""
|
||||
|
||||
require Logger
|
||||
|
||||
alias Pinchflat.Sources.Source
|
||||
alias Pinchflat.FastIndexing.YoutubeBehaviour
|
||||
|
||||
@behaviour YoutubeBehaviour
|
||||
|
||||
@doc """
|
||||
Determines if the YouTube RSS feed is enabled for fast indexing. Used to satisfy
|
||||
the `YoutubeBehaviour` behaviour.
|
||||
|
||||
Returns true
|
||||
"""
|
||||
@impl YoutubeBehaviour
|
||||
def enabled?(), do: true
|
||||
|
||||
@doc """
|
||||
Fetches the recent media IDs from a YouTube RSS feed for a given source.
|
||||
|
||||
Returns {:ok, [binary()]} | {:error, binary()}
|
||||
"""
|
||||
@impl YoutubeBehaviour
|
||||
def get_recent_media_ids(%Source{} = source) do
|
||||
def get_recent_media_ids_from_rss(%Source{} = source) do
|
||||
Logger.debug("Fetching recent media IDs from YouTube RSS feed for source: #{source.collection_id}")
|
||||
|
||||
case http_client().get(rss_url_for_source(source)) do
|
||||
|
|
|
|||
|
|
@ -21,11 +21,9 @@ defmodule Pinchflat.HTTP.HTTPClient do
|
|||
"""
|
||||
@impl HTTPBehaviour
|
||||
def get(url, headers \\ [], opts \\ []) do
|
||||
headers = parse_headers(headers)
|
||||
|
||||
case :httpc.request(:get, {url, headers}, [], opts) do
|
||||
{:ok, {{_version, 200, _reason_phrase}, _headers, body}} ->
|
||||
{:ok, to_string(body)}
|
||||
{:ok, body}
|
||||
|
||||
{:ok, {{_version, status_code, reason_phrase}, _headers, _body}} ->
|
||||
{:error, "HTTP request failed with status code #{status_code}: #{reason_phrase}"}
|
||||
|
|
@ -34,8 +32,4 @@ defmodule Pinchflat.HTTP.HTTPClient do
|
|||
{:error, "HTTP request failed: #{reason}"}
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_headers(headers) do
|
||||
Enum.map(headers, fn {k, v} -> {to_charlist(k), to_charlist(v)} end)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -12,8 +12,7 @@ defmodule Pinchflat.Lifecycle.UserScripts.CommandRunner do
|
|||
@behaviour UserScriptCommandRunner
|
||||
|
||||
@event_types [
|
||||
:app_init,
|
||||
:media_pre_download,
|
||||
:media_indexed,
|
||||
:media_downloaded,
|
||||
:media_deleted
|
||||
]
|
||||
|
|
@ -24,25 +23,24 @@ defmodule Pinchflat.Lifecycle.UserScripts.CommandRunner do
|
|||
|
||||
This function will succeed in almost all cases, even if the user script command
|
||||
failed - this is because I don't want bad scripts to stop the whole process.
|
||||
If something fails, it'll be logged and returned BUT the tuple will always
|
||||
start with {:ok, ...}.
|
||||
If something fails, it'll be logged.
|
||||
|
||||
The only things that can cause a true failure are passing in an invalid event
|
||||
type or if the passed data cannot be encoded into JSON - both indicative of
|
||||
failures in the development process.
|
||||
|
||||
Returns {:ok, :no_executable} | {:ok, output, exit_code}
|
||||
Returns :ok
|
||||
"""
|
||||
@impl UserScriptCommandRunner
|
||||
def run(event_type, encodable_data) when event_type in @event_types do
|
||||
case backend_executable() do
|
||||
{:ok, :no_executable} ->
|
||||
{:ok, :no_executable}
|
||||
:ok
|
||||
|
||||
{:ok, executable_path} ->
|
||||
{:ok, encoded_data} = Phoenix.json_library().encode(encodable_data)
|
||||
|
||||
{output, exit_code} =
|
||||
{_output, _exit_code} =
|
||||
CliUtils.wrap_cmd(
|
||||
executable_path,
|
||||
[to_string(event_type), encoded_data],
|
||||
|
|
@ -50,7 +48,7 @@ defmodule Pinchflat.Lifecycle.UserScripts.CommandRunner do
|
|||
logging_arg_override: "[suppressed]"
|
||||
)
|
||||
|
||||
{:ok, output, exit_code}
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -65,7 +63,7 @@ defmodule Pinchflat.Lifecycle.UserScripts.CommandRunner do
|
|||
if FilesystemUtils.exists_and_nonempty?(filepath) do
|
||||
{:ok, filepath}
|
||||
else
|
||||
Logger.info("User scripts lifecyle file either not present or is empty. Skipping.")
|
||||
Logger.warning("User scripts lifecyle file either not present or is empty. Skipping.")
|
||||
|
||||
{:ok, :no_executable}
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,93 +0,0 @@
|
|||
defmodule Pinchflat.Media.FileSyncing do
|
||||
@moduledoc """
|
||||
Functions for ensuring file state is accurately reflected in the database.
|
||||
"""
|
||||
|
||||
alias Pinchflat.Media
|
||||
alias Pinchflat.Utils.MapUtils
|
||||
alias Pinchflat.Media.MediaItem
|
||||
alias Pinchflat.Utils.FilesystemUtils, as: FSUtils
|
||||
|
||||
@doc """
|
||||
Deletes files that are no longer needed by a media item.
|
||||
|
||||
This means that if a media item has been updated, the old and new versions
|
||||
can be passed and any files that are no longer needed will be deleted.
|
||||
|
||||
An example is a video that gets its quality upgraded and its name changes
|
||||
between original download and re-download. The old file will exist on-disk
|
||||
with the old name but the database entry will point to the new file. This
|
||||
function can be used to delete the old file in this case.
|
||||
|
||||
Returns :ok
|
||||
"""
|
||||
def delete_outdated_files(old_media_item, new_media_item) do
|
||||
non_subtitle_keys = MediaItem.filepath_attributes() -- [:subtitle_filepaths]
|
||||
|
||||
old_non_subtitles = Map.take(old_media_item, non_subtitle_keys)
|
||||
old_subtitles = MapUtils.from_nested_list(old_media_item.subtitle_filepaths)
|
||||
new_non_subtitles = Map.take(new_media_item, non_subtitle_keys)
|
||||
new_subtitles = MapUtils.from_nested_list(new_media_item.subtitle_filepaths)
|
||||
|
||||
handle_file_deletion(old_non_subtitles, new_non_subtitles)
|
||||
handle_file_deletion(old_subtitles, new_subtitles)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
@doc """
|
||||
Nillifies any media item filepaths that don't exist on disk for a list of media items
|
||||
|
||||
returns [%MediaItem{}]
|
||||
"""
|
||||
def sync_file_presence_on_disk(media_items) do
|
||||
Enum.map(media_items, fn media_item ->
|
||||
new_attributes = sync_media_item_files(media_item)
|
||||
# Doing this one-by-one instead of batching since this process
|
||||
# can take time and a batch could let MediaItem state get out of sync
|
||||
{:ok, updated_media_item} = Media.update_media_item(media_item, new_attributes)
|
||||
|
||||
updated_media_item
|
||||
end)
|
||||
end
|
||||
|
||||
defp handle_file_deletion(old_attributes, new_attributes) do
|
||||
# The logic:
|
||||
# - A file should only be deleted if it exists and the new file is different
|
||||
# - The new attributes are the ones we're interested in keeping
|
||||
# - If the old attributes have a key that doesn't exist in the new attributes, don't touch it.
|
||||
# This is good for archiving but may be unpopular for other users so this may change.
|
||||
|
||||
Enum.each(new_attributes, fn {key, new_filepath} ->
|
||||
old_filepath = Map.get(old_attributes, key)
|
||||
files_have_changed = old_filepath && new_filepath && old_filepath != new_filepath
|
||||
files_exist_on_disk = files_have_changed && File.exists?(old_filepath) && File.exists?(new_filepath)
|
||||
|
||||
if files_exist_on_disk && !FSUtils.filepaths_reference_same_file?(old_filepath, new_filepath) do
|
||||
FSUtils.delete_file_and_remove_empty_directories(old_filepath)
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp sync_media_item_files(media_item) do
|
||||
non_subtitle_keys = MediaItem.filepath_attributes() -- [:subtitle_filepaths]
|
||||
subtitle_keys = MapUtils.from_nested_list(media_item.subtitle_filepaths)
|
||||
non_subtitles = Map.take(media_item, non_subtitle_keys)
|
||||
|
||||
# This one is checking for the negative (ie: only update if the file doesn't exist)
|
||||
new_non_subtitle_attrs =
|
||||
Enum.reduce(non_subtitles, %{}, fn {key, filepath}, acc ->
|
||||
if filepath && File.exists?(filepath), do: acc, else: Map.put(acc, key, nil)
|
||||
end)
|
||||
|
||||
# This one is checking for the positive (ie: only update if the file exists)
|
||||
# This is because subtitles, being an array type in the DB, are most easily updated
|
||||
# by a full replacement rather than finding the actual diff
|
||||
new_subtitle_attrs =
|
||||
Enum.reduce(subtitle_keys, [], fn {key, filepath}, acc ->
|
||||
if filepath && File.exists?(filepath), do: acc ++ [[key, filepath]], else: acc
|
||||
end)
|
||||
|
||||
Map.put(new_non_subtitle_attrs, :subtitle_filepaths, new_subtitle_attrs)
|
||||
end
|
||||
end
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
defmodule Pinchflat.Media.FileSyncingWorker do
|
||||
@moduledoc false
|
||||
|
||||
use Oban.Worker,
|
||||
queue: :local_data,
|
||||
tags: ["sources", "local_data"]
|
||||
|
||||
alias __MODULE__
|
||||
alias Pinchflat.Repo
|
||||
alias Pinchflat.Tasks
|
||||
alias Pinchflat.Sources
|
||||
alias Pinchflat.Media.FileSyncing
|
||||
|
||||
@doc """
|
||||
Starts the source file syncing worker.
|
||||
|
||||
Returns {:ok, %Task{}} | {:error, %Ecto.Changeset{}}
|
||||
"""
|
||||
def kickoff_with_task(source, opts \\ []) do
|
||||
%{id: source.id}
|
||||
|> FileSyncingWorker.new(opts)
|
||||
|> Tasks.create_job_with_task(source)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Deletes a profile and optionally deletes its files
|
||||
|
||||
Returns :ok
|
||||
"""
|
||||
@impl Oban.Worker
|
||||
def perform(%Oban.Job{args: %{"id" => source_id}}) do
|
||||
source = Repo.preload(Sources.get_source!(source_id), :media_items)
|
||||
|
||||
FileSyncing.sync_file_presence_on_disk(source.media_items)
|
||||
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
|
@ -15,9 +15,6 @@ defmodule Pinchflat.Media do
|
|||
|
||||
alias Pinchflat.Lifecycle.UserScripts.CommandRunner, as: UserScriptRunner
|
||||
|
||||
# Some fields should only be set on insert and not on update.
|
||||
@fields_to_drop_on_update [:playlist_index]
|
||||
|
||||
@doc """
|
||||
Returns the list of media_items.
|
||||
|
||||
|
|
@ -28,15 +25,25 @@ defmodule Pinchflat.Media do
|
|||
end
|
||||
|
||||
@doc """
|
||||
Returns a list of media_items that are upgradeable based on the redownload delay
|
||||
of the media_profile their source belongs to. In this context, upgradeable means
|
||||
that it's been long enough since upload that the video may be in a higher quality
|
||||
or have better sponsorblock segments (or similar).
|
||||
Returns a list of media_items that are cullable based on the retention period
|
||||
of the source they belong to.
|
||||
|
||||
The logic is that a media_item is past_redownload_delay if the media_item's uploaded_at is
|
||||
at least redownload_delay_days ago AND `media_downloaded_at` - `redownload_delay_days`
|
||||
is before the media_item's `uploaded_at`.
|
||||
Returns [%MediaItem{}, ...]
|
||||
"""
|
||||
def list_cullable_media_items do
|
||||
MediaQuery.new()
|
||||
|> MediaQuery.require_assoc(:source)
|
||||
|> where(^MediaQuery.cullable())
|
||||
|> Repo.all()
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns a list of media_items that are redownloadable based on the redownload delay
|
||||
of the media_profile their source belongs to.
|
||||
|
||||
The logic is that a media_item is past_redownload_delay if the media_item's
|
||||
upload_date is at least redownload_delay_days ago AND
|
||||
`media_downloaded_at` - `redownload_delay_days` is before the media_item's `upload_date`.
|
||||
This logic grabs media that we've recently downloaded AND is recently uploaded, but
|
||||
doesn't grab media that we've recently downloaded and was uploaded a long time ago.
|
||||
This also makes things work as expected when downloading media from a source for the
|
||||
|
|
@ -44,10 +51,10 @@ defmodule Pinchflat.Media do
|
|||
|
||||
Returns [%MediaItem{}, ...]
|
||||
"""
|
||||
def list_upgradeable_media_items do
|
||||
def list_redownloadable_media_items do
|
||||
MediaQuery.new()
|
||||
|> MediaQuery.require_assoc(:media_profile)
|
||||
|> where(^MediaQuery.upgradeable())
|
||||
|> where(^MediaQuery.redownloadable())
|
||||
|> Repo.all()
|
||||
end
|
||||
|
||||
|
|
@ -128,7 +135,7 @@ defmodule Pinchflat.Media do
|
|||
|
||||
Unlike `create_media_item`, this will attempt an update if the media_item
|
||||
already exists. This is so that future indexing can pick up attributes that
|
||||
we may not have asked for in the past (eg: uploaded_at)
|
||||
we may not have asked for in the past (eg: upload_date)
|
||||
|
||||
Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}
|
||||
"""
|
||||
|
|
@ -139,10 +146,7 @@ defmodule Pinchflat.Media do
|
|||
|> MediaItem.changeset(attrs)
|
||||
|> Repo.insert(
|
||||
on_conflict: [
|
||||
set:
|
||||
attrs
|
||||
|> Map.drop(@fields_to_drop_on_update)
|
||||
|> Map.to_list()
|
||||
set: Map.to_list(attrs)
|
||||
],
|
||||
conflict_target: [:source_id, :media_id]
|
||||
)
|
||||
|
|
@ -154,10 +158,8 @@ defmodule Pinchflat.Media do
|
|||
Returns {:ok, %MediaItem{}} | {:error, %Ecto.Changeset{}}
|
||||
"""
|
||||
def update_media_item(%MediaItem{} = media_item, attrs) do
|
||||
update_attrs = Map.drop(attrs, @fields_to_drop_on_update)
|
||||
|
||||
media_item
|
||||
|> MediaItem.changeset(update_attrs)
|
||||
|> MediaItem.changeset(attrs)
|
||||
|> Repo.update()
|
||||
end
|
||||
|
||||
|
|
@ -174,7 +176,7 @@ defmodule Pinchflat.Media do
|
|||
|
||||
if delete_files do
|
||||
{:ok, _} = do_delete_media_files(media_item)
|
||||
run_user_script(:media_deleted, media_item)
|
||||
:ok = run_user_script(:media_deleted, media_item)
|
||||
end
|
||||
|
||||
# Should delete these no matter what
|
||||
|
|
@ -197,7 +199,7 @@ defmodule Pinchflat.Media do
|
|||
|
||||
Tasks.delete_tasks_for(media_item)
|
||||
{:ok, _} = do_delete_media_files(media_item)
|
||||
run_user_script(:media_deleted, media_item)
|
||||
:ok = run_user_script(:media_deleted, media_item)
|
||||
|
||||
update_media_item(media_item, Map.merge(filepath_attrs, addl_attrs))
|
||||
end
|
||||
|
|
|
|||
|
|
@ -18,8 +18,6 @@ defmodule Pinchflat.Media.MediaItem do
|
|||
alias Pinchflat.Media.MediaItemsSearchIndex
|
||||
|
||||
@allowed_fields [
|
||||
# these fields are only captured on index
|
||||
:playlist_index,
|
||||
# these fields are captured on indexing (and again on download)
|
||||
:title,
|
||||
:media_id,
|
||||
|
|
@ -28,10 +26,9 @@ defmodule Pinchflat.Media.MediaItem do
|
|||
:livestream,
|
||||
:source_id,
|
||||
:short_form_content,
|
||||
:uploaded_at,
|
||||
:upload_date,
|
||||
:upload_date_index,
|
||||
:duration_seconds,
|
||||
:predicted_media_filepath,
|
||||
# these fields are captured only on download
|
||||
:media_downloaded_at,
|
||||
:media_filepath,
|
||||
|
|
@ -40,7 +37,6 @@ defmodule Pinchflat.Media.MediaItem do
|
|||
:thumbnail_filepath,
|
||||
:metadata_filepath,
|
||||
:nfo_filepath,
|
||||
:last_error,
|
||||
# These are user or system controlled fields
|
||||
:prevent_download,
|
||||
:prevent_culling,
|
||||
|
|
@ -55,7 +51,7 @@ defmodule Pinchflat.Media.MediaItem do
|
|||
livestream
|
||||
media_id
|
||||
source_id
|
||||
uploaded_at
|
||||
upload_date
|
||||
short_form_content
|
||||
)a
|
||||
|
||||
|
|
@ -73,12 +69,10 @@ defmodule Pinchflat.Media.MediaItem do
|
|||
field :short_form_content, :boolean, default: false
|
||||
field :media_downloaded_at, :utc_datetime
|
||||
field :media_redownloaded_at, :utc_datetime
|
||||
field :uploaded_at, :utc_datetime
|
||||
field :upload_date, :date
|
||||
field :upload_date_index, :integer, default: 0
|
||||
field :duration_seconds, :integer
|
||||
field :playlist_index, :integer, default: 0
|
||||
|
||||
field :predicted_media_filepath, :string
|
||||
field :media_filepath, :string
|
||||
field :media_size_bytes, :integer
|
||||
field :thumbnail_filepath, :string
|
||||
|
|
@ -89,7 +83,6 @@ defmodule Pinchflat.Media.MediaItem do
|
|||
# Will very likely revisit because I can't leave well-enough alone.
|
||||
field :subtitle_filepaths, {:array, {:array, :string}}, default: []
|
||||
|
||||
field :last_error, :string
|
||||
field :prevent_download, :boolean, default: false
|
||||
field :prevent_culling, :boolean, default: false
|
||||
field :culled_at, :utc_datetime
|
||||
|
|
@ -114,9 +107,6 @@ defmodule Pinchflat.Media.MediaItem do
|
|||
|> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end)
|
||||
|> update_upload_date_index()
|
||||
|> validate_required(@required_fields)
|
||||
# Validate that the title does NOT start with "youtube video #" since that indicates a restriction by YouTube.
|
||||
# See issue #549 for more information.
|
||||
|> validate_format(:title, ~r/^(?!youtube video #)/)
|
||||
|> unique_constraint([:media_id, :source_id])
|
||||
end
|
||||
|
||||
|
|
@ -140,31 +130,7 @@ defmodule Pinchflat.Media.MediaItem do
|
|||
~w(__meta__ __struct__ metadata tasks media_items_search_index)a
|
||||
end
|
||||
|
||||
# Run it on new records no matter what. The method we delegate to
|
||||
# will handle the case where `uploaded_at` is `nil`
|
||||
defp update_upload_date_index(%{data: %{id: nil}} = changeset) do
|
||||
do_update_upload_date_index(changeset)
|
||||
end
|
||||
|
||||
# For the update case, we only want to recalculate if the day itself has changed.
|
||||
# For instance, this is useful in the migration from `upload_date` to `uploaded_at`
|
||||
defp update_upload_date_index(%{changes: changes} = changeset) when is_map_key(changes, :uploaded_at) do
|
||||
old_uploaded_at = changeset.data.uploaded_at
|
||||
new_uploaded_at = get_change(changeset, :uploaded_at)
|
||||
upload_dates_match = DateTime.to_date(old_uploaded_at) == DateTime.to_date(new_uploaded_at)
|
||||
|
||||
if upload_dates_match do
|
||||
changeset
|
||||
else
|
||||
do_update_upload_date_index(changeset)
|
||||
end
|
||||
end
|
||||
|
||||
# If the record is persisted and the `uploaded_at` field is not being changed,
|
||||
# we don't need to recalculate the index.
|
||||
defp update_upload_date_index(changeset), do: changeset
|
||||
|
||||
defp do_update_upload_date_index(%{changes: changes} = changeset) when is_map_key(changes, :uploaded_at) do
|
||||
defp update_upload_date_index(%{changes: changes} = changeset) when is_map_key(changes, :upload_date) do
|
||||
source_id = get_field(changeset, :source_id)
|
||||
source = Sources.get_source!(source_id)
|
||||
# Channels should count down from 99, playlists should count up from 0
|
||||
|
|
@ -176,7 +142,7 @@ defmodule Pinchflat.Media.MediaItem do
|
|||
|
||||
current_max =
|
||||
MediaQuery.new()
|
||||
|> where(^dynamic([mi], ^MediaQuery.upload_date_matches(changes.uploaded_at) and ^MediaQuery.for_source(source)))
|
||||
|> where(^dynamic([mi], mi.upload_date == ^changes.upload_date and ^MediaQuery.for_source(source)))
|
||||
|> Repo.aggregate(aggregator, :upload_date_index)
|
||||
|
||||
case current_max do
|
||||
|
|
@ -185,7 +151,7 @@ defmodule Pinchflat.Media.MediaItem do
|
|||
end
|
||||
end
|
||||
|
||||
defp do_update_upload_date_index(changeset), do: changeset
|
||||
defp update_upload_date_index(changeset), do: changeset
|
||||
|
||||
defimpl Jason.Encoder, for: MediaItem do
|
||||
def encode(value, opts) do
|
||||
|
|
|
|||
|
|
@ -33,15 +33,13 @@ defmodule Pinchflat.Media.MediaQuery do
|
|||
def downloaded, do: dynamic([mi], not is_nil(mi.media_filepath))
|
||||
def download_prevented, do: dynamic([mi], mi.prevent_download == true)
|
||||
def culling_prevented, do: dynamic([mi], mi.prevent_culling == true)
|
||||
def culled, do: dynamic([mi], not is_nil(mi.culled_at))
|
||||
def redownloaded, do: dynamic([mi], not is_nil(mi.media_redownloaded_at))
|
||||
def upload_date_matches(other_date), do: dynamic([mi], fragment("date(?) = date(?)", mi.uploaded_at, ^other_date))
|
||||
def media_id(nil), do: dynamic(false)
|
||||
def media_id(media_id), do: dynamic([mi], mi.media_id == ^media_id)
|
||||
|
||||
def upload_date_after_source_cutoff do
|
||||
dynamic(
|
||||
[mi, source],
|
||||
is_nil(source.download_cutoff_date) or
|
||||
fragment("date(?) >= ?", mi.uploaded_at, source.download_cutoff_date)
|
||||
)
|
||||
dynamic([mi, source], is_nil(source.download_cutoff_date) or mi.upload_date >= source.download_cutoff_date)
|
||||
end
|
||||
|
||||
def format_matching_profile_preference do
|
||||
|
|
@ -75,20 +73,12 @@ defmodule Pinchflat.Media.MediaQuery do
|
|||
)
|
||||
end
|
||||
|
||||
def meets_min_and_max_duration do
|
||||
dynamic(
|
||||
[mi, source],
|
||||
(is_nil(source.min_duration_seconds) or fragment("duration_seconds >= ?", source.min_duration_seconds)) and
|
||||
(is_nil(source.max_duration_seconds) or fragment("duration_seconds <= ?", source.max_duration_seconds))
|
||||
)
|
||||
end
|
||||
|
||||
def past_retention_period do
|
||||
dynamic(
|
||||
[mi, source],
|
||||
fragment("""
|
||||
IFNULL(retention_period_days, 0) > 0 AND
|
||||
DATETIME(media_downloaded_at, '+' || retention_period_days || ' day') < DATETIME('now')
|
||||
DATETIME('now', '-' || retention_period_days || ' day') > media_downloaded_at
|
||||
""")
|
||||
)
|
||||
end
|
||||
|
|
@ -96,12 +86,12 @@ defmodule Pinchflat.Media.MediaQuery do
|
|||
def past_redownload_delay do
|
||||
dynamic(
|
||||
[mi, source, media_profile],
|
||||
# Returns media items where the uploaded_at is at least redownload_delay_days ago AND
|
||||
# Returns media items where the upload_date is at least redownload_delay_days ago AND
|
||||
# downloaded_at minus the redownload_delay_days is before the upload date
|
||||
fragment("""
|
||||
IFNULL(redownload_delay_days, 0) > 0 AND
|
||||
DATE('now', '-' || redownload_delay_days || ' day') > DATE(uploaded_at) AND
|
||||
DATE(media_downloaded_at, '-' || redownload_delay_days || ' day') < DATE(uploaded_at)
|
||||
DATETIME('now', '-' || redownload_delay_days || ' day') > upload_date AND
|
||||
DATETIME(media_downloaded_at, '-' || redownload_delay_days || ' day') < upload_date
|
||||
""")
|
||||
)
|
||||
end
|
||||
|
|
@ -115,15 +105,6 @@ defmodule Pinchflat.Media.MediaQuery do
|
|||
)
|
||||
end
|
||||
|
||||
def deletable_based_on_source_cutoff do
|
||||
dynamic(
|
||||
[mi, source],
|
||||
^downloaded() and
|
||||
not (^upload_date_after_source_cutoff()) and
|
||||
not (^culling_prevented())
|
||||
)
|
||||
end
|
||||
|
||||
def pending do
|
||||
dynamic(
|
||||
[mi],
|
||||
|
|
@ -131,16 +112,16 @@ defmodule Pinchflat.Media.MediaQuery do
|
|||
not (^download_prevented()) and
|
||||
^upload_date_after_source_cutoff() and
|
||||
^format_matching_profile_preference() and
|
||||
^matches_source_title_regex() and
|
||||
^meets_min_and_max_duration()
|
||||
^matches_source_title_regex()
|
||||
)
|
||||
end
|
||||
|
||||
def upgradeable do
|
||||
def redownloadable do
|
||||
dynamic(
|
||||
[mi, source],
|
||||
^downloaded() and
|
||||
not (^download_prevented()) and
|
||||
not (^culled()) and
|
||||
not (^redownloaded()) and
|
||||
^past_redownload_delay()
|
||||
)
|
||||
|
|
|
|||
|
|
@ -9,11 +9,8 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
|
|||
needed
|
||||
"""
|
||||
|
||||
alias Pinchflat.Sources
|
||||
alias Pinchflat.Utils.FilesystemUtils
|
||||
|
||||
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
|
||||
|
||||
@doc """
|
||||
Returns the directory where metadata for a database record should be stored.
|
||||
|
||||
|
|
@ -50,44 +47,55 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
|
|||
Returns {:ok, map()} | {:error, any}
|
||||
"""
|
||||
def read_compressed_metadata(filepath) do
|
||||
{:ok, json} = File.open(filepath, [:read, :compressed], &IO.read(&1, :eof))
|
||||
{:ok, json} = File.open(filepath, [:read, :compressed], &IO.read(&1, :all))
|
||||
|
||||
Phoenix.json_library().decode(json)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Downloads and stores a thumbnail for a media item, returning the filepath.
|
||||
Chooses the highest quality thumbnail available and converts it to a JPG
|
||||
|
||||
Returns nil if no thumbnail is available or if yt-dlp encounters an error
|
||||
Chooses the highest quality thumbnail available (preferring jpg). Returns
|
||||
nil if no thumbnails are available.
|
||||
|
||||
Returns binary() | nil
|
||||
"""
|
||||
def download_and_store_thumbnail_for(media_item_with_preloads) do
|
||||
yt_dlp_filepath = generate_filepath_for(media_item_with_preloads, "thumbnail.%(ext)s")
|
||||
real_filepath = generate_filepath_for(media_item_with_preloads, "thumbnail.jpg")
|
||||
command_opts = [output: yt_dlp_filepath]
|
||||
addl_opts = [use_cookies: Sources.use_cookies?(media_item_with_preloads.source, :metadata)]
|
||||
def download_and_store_thumbnail_for(database_record, metadata_map) do
|
||||
thumbnails =
|
||||
(metadata_map["thumbnails"] || [])
|
||||
# Give it a low preference if the `preference` key doesn't exist
|
||||
|> Enum.map(&Map.put_new(&1, "preference", -1000))
|
||||
# Give it a low preference if image isn't a jpg
|
||||
|> Enum.map(fn t ->
|
||||
preference_weight = if String.ends_with?(t["url"], ".jpg"), do: t["preference"], else: t["preference"] - 1000
|
||||
|
||||
case YtDlpMedia.download_thumbnail(media_item_with_preloads.original_url, command_opts, addl_opts) do
|
||||
{:ok, _} -> real_filepath
|
||||
_ -> nil
|
||||
Map.put(t, "preference", preference_weight)
|
||||
end)
|
||||
|
||||
case Enum.sort_by(thumbnails, & &1["preference"], :desc) do
|
||||
[thumbnail_map | _] ->
|
||||
thumbnail_url = thumbnail_map["url"]
|
||||
filepath = generate_filepath_for(database_record, Path.basename(thumbnail_url))
|
||||
thumbnail_blob = fetch_thumbnail_from_url(thumbnail_url)
|
||||
|
||||
:ok = FilesystemUtils.write_p!(filepath, thumbnail_blob)
|
||||
|
||||
filepath
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Parses an upload date from the YYYYMMDD string returned in yt-dlp metadata
|
||||
and returns a DateTime struct, appending a time of 00:00:00Z.
|
||||
and returns a Date struct.
|
||||
|
||||
Returns DateTime.t()
|
||||
Returns Date.t()
|
||||
"""
|
||||
def parse_upload_date(upload_date) do
|
||||
<<year::binary-size(4)>> <> <<month::binary-size(2)>> <> <<day::binary-size(2)>> = upload_date
|
||||
|
||||
case DateTime.from_iso8601("#{year}-#{month}-#{day}T00:00:00Z") do
|
||||
{:ok, datetime, _} -> datetime
|
||||
_ -> raise "Invalid upload date: #{upload_date}"
|
||||
end
|
||||
Date.from_iso8601!("#{year}-#{month}-#{day}")
|
||||
end
|
||||
|
||||
@doc """
|
||||
|
|
@ -127,19 +135,11 @@ defmodule Pinchflat.Metadata.MetadataFileHelpers do
|
|||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Attempts to determine the season and episode number from a media filepath.
|
||||
defp fetch_thumbnail_from_url(url) do
|
||||
http_client = Application.get_env(:pinchflat, :http_client, Pinchflat.HTTP.HTTPClient)
|
||||
{:ok, body} = http_client.get(url, [], body_format: :binary)
|
||||
|
||||
Returns {:ok, {binary(), binary()}} | {:error, :indeterminable}
|
||||
"""
|
||||
def season_and_episode_from_media_filepath(media_filepath) do
|
||||
# matches s + 1 or more digits + e + 1 or more digits (case-insensitive)
|
||||
season_episode_regex = ~r/s(\d+)e(\d+)/i
|
||||
|
||||
case Regex.scan(season_episode_regex, media_filepath) do
|
||||
[[_, season, episode] | _] -> {:ok, {season, episode}}
|
||||
_ -> {:error, :indeterminable}
|
||||
end
|
||||
body
|
||||
end
|
||||
|
||||
defp generate_filepath_for(database_record, filename) do
|
||||
|
|
|
|||
|
|
@ -8,8 +8,6 @@ defmodule Pinchflat.Metadata.MetadataParser do
|
|||
and not have it, ya know?
|
||||
"""
|
||||
|
||||
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
|
||||
|
||||
@doc """
|
||||
Parses the given JSON response from yt-dlp and returns a map of
|
||||
the needful media_item attributes, along with anything needed for
|
||||
|
|
@ -26,12 +24,15 @@ defmodule Pinchflat.Metadata.MetadataParser do
|
|||
end
|
||||
|
||||
defp parse_media_metadata(metadata) do
|
||||
Map.merge(
|
||||
Map.from_struct(YtDlpMedia.response_to_struct(metadata)),
|
||||
%{
|
||||
media_filepath: metadata["filepath"]
|
||||
}
|
||||
)
|
||||
%{
|
||||
media_id: metadata["id"],
|
||||
title: metadata["title"],
|
||||
original_url: metadata["original_url"],
|
||||
description: metadata["description"],
|
||||
media_filepath: metadata["filepath"],
|
||||
livestream: !!metadata["was_live"],
|
||||
duration_seconds: metadata["duration"] && round(metadata["duration"])
|
||||
}
|
||||
end
|
||||
|
||||
defp parse_subtitle_metadata(metadata) do
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ defmodule Pinchflat.Metadata.NfoBuilder do
|
|||
|
||||
import Pinchflat.Utils.XmlUtils, only: [safe: 1]
|
||||
|
||||
alias Pinchflat.Utils.FilesystemUtils
|
||||
alias Pinchflat.Metadata.MetadataFileHelpers
|
||||
alias Pinchflat.Utils.FilesystemUtils
|
||||
|
||||
@doc """
|
||||
Builds an NFO file for a media item (read: single "episode") and
|
||||
|
|
@ -15,12 +15,12 @@ defmodule Pinchflat.Metadata.NfoBuilder do
|
|||
|
||||
Returns the filepath of the NFO file.
|
||||
"""
|
||||
def build_and_store_for_media_item(nfo_filepath, metadata) do
|
||||
nfo = build_for_media_item(nfo_filepath, metadata)
|
||||
def build_and_store_for_media_item(filepath, metadata) do
|
||||
nfo = build_for_media_item(metadata)
|
||||
|
||||
FilesystemUtils.write_p!(nfo_filepath, nfo)
|
||||
FilesystemUtils.write_p!(filepath, nfo)
|
||||
|
||||
nfo_filepath
|
||||
filepath
|
||||
end
|
||||
|
||||
@doc """
|
||||
|
|
@ -37,15 +37,10 @@ defmodule Pinchflat.Metadata.NfoBuilder do
|
|||
filepath
|
||||
end
|
||||
|
||||
defp build_for_media_item(nfo_filepath, metadata) do
|
||||
defp build_for_media_item(metadata) do
|
||||
upload_date = MetadataFileHelpers.parse_upload_date(metadata["upload_date"])
|
||||
# NOTE: the filepath here isn't the path of the media item, it's the path that
|
||||
# the NFO should be saved to. This works because the NFO's path is the same as
|
||||
# the media's path, just with a different extension. If this ever changes I'll
|
||||
# need to pass in the media item's path as well.
|
||||
{season, episode} = determine_season_and_episode_number(nfo_filepath, upload_date)
|
||||
|
||||
# Cribbed from a combination of the Kodi wiki, ytdl-nfo, and ytdl-sub.
|
||||
# WHO NEEDS A FANCY XML PARSER ANYWAY?!
|
||||
"""
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
|
||||
<episodedetails>
|
||||
|
|
@ -54,8 +49,8 @@ defmodule Pinchflat.Metadata.NfoBuilder do
|
|||
<uniqueid type="youtube" default="true">#{safe(metadata["id"])}</uniqueid>
|
||||
<plot>#{safe(metadata["description"])}</plot>
|
||||
<aired>#{safe(upload_date)}</aired>
|
||||
<season>#{safe(season)}</season>
|
||||
<episode>#{episode}</episode>
|
||||
<season>#{safe(upload_date.year)}</season>
|
||||
<episode>#{Calendar.strftime(upload_date, "%m%d")}</episode>
|
||||
<genre>YouTube</genre>
|
||||
</episodedetails>
|
||||
"""
|
||||
|
|
@ -72,11 +67,4 @@ defmodule Pinchflat.Metadata.NfoBuilder do
|
|||
</tvshow>
|
||||
"""
|
||||
end
|
||||
|
||||
defp determine_season_and_episode_number(filepath, upload_date) do
|
||||
case MetadataFileHelpers.season_and_episode_from_media_filepath(filepath) do
|
||||
{:ok, {season, episode}} -> {season, episode}
|
||||
{:error, _} -> {upload_date.year, Calendar.strftime(upload_date, "%m%d")}
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -20,49 +20,29 @@ defmodule Pinchflat.Metadata.SourceImageParser do
|
|||
def store_source_images(base_directory, source_metadata) do
|
||||
(source_metadata["thumbnails"] || [])
|
||||
|> Enum.filter(&(&1["filepath"] != nil))
|
||||
|> select_useful_images(source_metadata)
|
||||
|> select_useful_images()
|
||||
|> Enum.map(&move_image(&1, base_directory))
|
||||
|> Enum.into(%{})
|
||||
end
|
||||
|
||||
defp select_useful_images(images, source_metadata) do
|
||||
defp select_useful_images(images) do
|
||||
labelled_images =
|
||||
Enum.reduce(images, %{}, fn image_map, acc ->
|
||||
Enum.reduce(images, [], fn image_map, acc ->
|
||||
case image_map do
|
||||
%{"id" => "avatar_uncropped"} -> put_image_key(acc, :poster, image_map["filepath"])
|
||||
%{"id" => "banner_uncropped"} -> put_image_key(acc, :fanart, image_map["filepath"])
|
||||
_ -> acc
|
||||
%{"id" => "avatar_uncropped"} ->
|
||||
acc ++ [{:poster, :poster_filepath, image_map["filepath"]}]
|
||||
|
||||
%{"id" => "banner_uncropped"} ->
|
||||
acc ++ [{:fanart, :fanart_filepath, image_map["filepath"]}]
|
||||
|
||||
_ ->
|
||||
acc
|
||||
end
|
||||
end)
|
||||
|
||||
labelled_images
|
||||
|> add_fallback_poster(source_metadata)
|
||||
|> put_image_key(:banner, determine_best_banner(images))
|
||||
|> Enum.filter(fn {_key, attrs} -> attrs.current_filepath end)
|
||||
end
|
||||
|
||||
# If a poster is set, short-circuit and return the images as-is
|
||||
defp add_fallback_poster(%{poster: _} = images, _), do: images
|
||||
|
||||
# If a poster is NOT set, see if we can find a suitable image to use as a fallback
|
||||
defp add_fallback_poster(images, source_metadata) do
|
||||
case source_metadata["entries"] do
|
||||
nil -> images
|
||||
[] -> images
|
||||
[first_entry | _] -> add_poster_from_entry_thumbnail(images, first_entry)
|
||||
end
|
||||
end
|
||||
|
||||
defp add_poster_from_entry_thumbnail(images, entry) do
|
||||
thumbnail =
|
||||
(entry["thumbnails"] || [])
|
||||
|> Enum.reverse()
|
||||
|> Enum.find(& &1["filepath"])
|
||||
|
||||
case thumbnail do
|
||||
nil -> images
|
||||
_ -> put_image_key(images, :poster, thumbnail["filepath"])
|
||||
end
|
||||
|> Enum.concat([{:banner, :banner_filepath, determine_best_banner(images)}])
|
||||
|> Enum.filter(fn {_, _, tmp_filepath} -> tmp_filepath end)
|
||||
end
|
||||
|
||||
defp determine_best_banner(images) do
|
||||
|
|
@ -78,22 +58,12 @@ defmodule Pinchflat.Metadata.SourceImageParser do
|
|||
Map.get(best_candidate || %{}, "filepath")
|
||||
end
|
||||
|
||||
defp move_image({_key, attrs}, base_directory) do
|
||||
extension = Path.extname(attrs.current_filepath)
|
||||
final_filepath = Path.join([base_directory, "#{attrs.final_filename}#{extension}"])
|
||||
defp move_image({filename, source_attr_name, tmp_filepath}, base_directory) do
|
||||
extension = Path.extname(tmp_filepath)
|
||||
final_filepath = Path.join([base_directory, "#{filename}#{extension}"])
|
||||
|
||||
FilesystemUtils.cp_p!(attrs.current_filepath, final_filepath)
|
||||
FilesystemUtils.cp_p!(tmp_filepath, final_filepath)
|
||||
|
||||
{attrs.attribute_name, final_filepath}
|
||||
end
|
||||
|
||||
defp put_image_key(map, key, image) do
|
||||
attribute_atom = String.to_existing_atom("#{key}_filepath")
|
||||
|
||||
Map.put(map, key, %{
|
||||
attribute_name: attribute_atom,
|
||||
final_filename: to_string(key),
|
||||
current_filepath: image
|
||||
})
|
||||
{source_attr_name, final_filepath}
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -77,8 +77,10 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorker do
|
|||
|
||||
defp fetch_source_metadata_and_images(series_directory, source) do
|
||||
metadata_directory = MetadataFileHelpers.metadata_directory_for(source)
|
||||
tmp_output_path = "#{tmp_directory()}/#{StringUtils.random_string(16)}/source_image.%(ext)S"
|
||||
opts = [:write_all_thumbnails, convert_thumbnails: "jpg", output: tmp_output_path]
|
||||
|
||||
{:ok, metadata} = fetch_metadata_for_source(source)
|
||||
{:ok, metadata} = MediaCollection.get_source_metadata(source.original_url, opts)
|
||||
metadata_image_attrs = SourceImageParser.store_source_images(metadata_directory, metadata)
|
||||
|
||||
if source.media_profile.download_source_images && series_directory do
|
||||
|
|
@ -92,9 +94,7 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorker do
|
|||
|
||||
defp determine_series_directory(source) do
|
||||
output_path = DownloadOptionBuilder.build_output_path_for(source)
|
||||
runner_opts = [output: output_path]
|
||||
addl_opts = [use_cookies: Sources.use_cookies?(source, :metadata)]
|
||||
{:ok, %{filepath: filepath}} = MediaCollection.get_source_details(source.original_url, runner_opts, addl_opts)
|
||||
{:ok, %{filepath: filepath}} = MediaCollection.get_source_details(source.original_url, output: output_path)
|
||||
|
||||
case MetadataFileHelpers.series_directory_from_media_filepath(filepath) do
|
||||
{:ok, series_directory} -> series_directory
|
||||
|
|
@ -110,21 +110,6 @@ defmodule Pinchflat.Metadata.SourceMetadataStorageWorker do
|
|||
end
|
||||
end
|
||||
|
||||
defp fetch_metadata_for_source(source) do
|
||||
tmp_output_path = "#{tmp_directory()}/#{StringUtils.random_string(16)}/source_image.%(ext)S"
|
||||
base_opts = [convert_thumbnails: "jpg", output: tmp_output_path]
|
||||
should_use_cookies = Sources.use_cookies?(source, :metadata)
|
||||
|
||||
opts =
|
||||
if source.collection_type == :channel do
|
||||
base_opts ++ [:write_all_thumbnails, playlist_items: 0]
|
||||
else
|
||||
base_opts ++ [:write_thumbnail, playlist_items: 1]
|
||||
end
|
||||
|
||||
MediaCollection.get_source_metadata(source.original_url, opts, use_cookies: should_use_cookies)
|
||||
end
|
||||
|
||||
defp tmp_directory do
|
||||
Application.get_env(:pinchflat, :tmpfile_directory)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,40 +0,0 @@
|
|||
defmodule Pinchflat.Podcasts.OpmlFeedBuilder do
|
||||
@moduledoc """
|
||||
Methods for building an OPML feed for a list of sources.
|
||||
"""
|
||||
|
||||
import Pinchflat.Utils.XmlUtils, only: [safe: 1]
|
||||
|
||||
alias PinchflatWeb.Router.Helpers, as: Routes
|
||||
|
||||
@doc """
|
||||
Builds an OPML feed for a given list of sources.
|
||||
|
||||
Returns an XML document as a string.
|
||||
"""
|
||||
def build(url_base, sources) do
|
||||
sources_xml =
|
||||
Enum.map(
|
||||
sources,
|
||||
&"""
|
||||
<outline type="rss" text="#{safe(&1.custom_name)}" xmlUrl="#{safe(source_route(url_base, &1))}" />
|
||||
"""
|
||||
)
|
||||
|
||||
"""
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<opml version="2.0">
|
||||
<head>
|
||||
<title>All Sources</title>
|
||||
</head>
|
||||
<body>
|
||||
#{Enum.join(sources_xml, "\n")}
|
||||
</body>
|
||||
</opml>
|
||||
"""
|
||||
end
|
||||
|
||||
defp source_route(url_base, source) do
|
||||
Path.join(url_base, "#{Routes.podcast_path(PinchflatWeb.Endpoint, :rss_feed, source.uuid)}.xml")
|
||||
end
|
||||
end
|
||||
|
|
@ -5,25 +5,11 @@ defmodule Pinchflat.Podcasts.PodcastHelpers do
|
|||
"""
|
||||
|
||||
use Pinchflat.Media.MediaQuery
|
||||
use Pinchflat.Sources.SourcesQuery
|
||||
|
||||
alias Pinchflat.Repo
|
||||
alias Pinchflat.Metadata.MediaMetadata
|
||||
alias Pinchflat.Metadata.SourceMetadata
|
||||
|
||||
@doc """
|
||||
Returns a list of sources that are not marked for deletion.
|
||||
|
||||
Returns: [%Source{}]
|
||||
"""
|
||||
def opml_sources() do
|
||||
SourcesQuery.new()
|
||||
|> select([s], %{custom_name: s.custom_name, uuid: s.uuid})
|
||||
|> where([s], is_nil(s.marked_for_deletion_at))
|
||||
|> order_by(asc: :custom_name)
|
||||
|> Repo.all()
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns a list of media items that have been downloaded to disk
|
||||
and have been proven to still exist there.
|
||||
|
|
@ -42,7 +28,7 @@ defmodule Pinchflat.Podcasts.PodcastHelpers do
|
|||
|
||||
MediaQuery.new()
|
||||
|> where(^dynamic(^MediaQuery.for_source(source) and ^MediaQuery.downloaded()))
|
||||
|> order_by(desc: :uploaded_at)
|
||||
|> order_by(desc: :upload_date)
|
||||
|> Repo.maybe_limit(limit)
|
||||
|> Repo.all()
|
||||
|> Enum.filter(fn media_item -> File.exists?(media_item.media_filepath) end)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ defmodule Pinchflat.Podcasts.RssFeedBuilder do
|
|||
|
||||
import Pinchflat.Utils.XmlUtils, only: [safe: 1]
|
||||
|
||||
alias Pinchflat.Utils.DatetimeUtils
|
||||
alias Pinchflat.Podcasts.PodcastHelpers
|
||||
alias PinchflatWeb.Router.Helpers, as: Routes
|
||||
|
||||
|
|
@ -82,7 +83,7 @@ defmodule Pinchflat.Podcasts.RssFeedBuilder do
|
|||
<title>#{safe(media_item.title)}</title>
|
||||
<link>#{safe(media_item.original_url)}</link>
|
||||
<description>#{safe(media_item.description)}</description>
|
||||
<pubDate>#{Calendar.strftime(media_item.uploaded_at, @datetime_format)}</pubDate>
|
||||
<pubDate>#{generate_upload_date(media_item)}</pubDate>
|
||||
<itunes:duration>#{media_item.duration_seconds}</itunes:duration>
|
||||
<enclosure
|
||||
url="#{media_stream_path(url_base, media_item)}"
|
||||
|
|
@ -132,6 +133,12 @@ defmodule Pinchflat.Podcasts.RssFeedBuilder do
|
|||
end
|
||||
end
|
||||
|
||||
defp generate_upload_date(media_item) do
|
||||
media_item.upload_date
|
||||
|> DatetimeUtils.date_to_datetime()
|
||||
|> Calendar.strftime(@datetime_format)
|
||||
end
|
||||
|
||||
defp podcast_route(action, params) do
|
||||
Routes.podcast_path(PinchflatWeb.Endpoint, action, params)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -26,11 +26,8 @@ defmodule Pinchflat.Profiles.MediaProfile do
|
|||
sponsorblock_categories
|
||||
shorts_behaviour
|
||||
livestream_behaviour
|
||||
audio_track
|
||||
preferred_resolution
|
||||
media_container
|
||||
redownload_delay_days
|
||||
marked_for_deletion_at
|
||||
)a
|
||||
|
||||
@required_fields ~w(name output_path_template)a
|
||||
|
|
@ -55,7 +52,7 @@ defmodule Pinchflat.Profiles.MediaProfile do
|
|||
field :embed_metadata, :boolean, default: false
|
||||
|
||||
field :download_nfo, :boolean, default: false
|
||||
field :sponsorblock_behaviour, Ecto.Enum, values: [:disabled, :mark, :remove], default: :disabled
|
||||
field :sponsorblock_behaviour, Ecto.Enum, values: [:disabled, :remove], default: :disabled
|
||||
field :sponsorblock_categories, {:array, :string}, default: []
|
||||
# NOTE: these do NOT speed up indexing - the indexer still has to go
|
||||
# through the entire collection to determine if a media is a short or
|
||||
|
|
@ -66,11 +63,7 @@ defmodule Pinchflat.Profiles.MediaProfile do
|
|||
# See `build_format_clauses` in the Media context for more.
|
||||
field :shorts_behaviour, Ecto.Enum, values: ~w(include exclude only)a, default: :include
|
||||
field :livestream_behaviour, Ecto.Enum, values: ~w(include exclude only)a, default: :include
|
||||
field :audio_track, :string
|
||||
field :preferred_resolution, Ecto.Enum, values: ~w(4320p 2160p 1440p 1080p 720p 480p 360p audio)a, default: :"1080p"
|
||||
field :media_container, :string, default: nil
|
||||
|
||||
field :marked_for_deletion_at, :utc_datetime
|
||||
field :preferred_resolution, Ecto.Enum, values: ~w(4320p 2160p 1080p 720p 480p 360p audio)a, default: :"1080p"
|
||||
|
||||
has_many :sources, Source
|
||||
|
||||
|
|
|
|||
|
|
@ -1,38 +0,0 @@
|
|||
defmodule Pinchflat.Profiles.MediaProfileDeletionWorker do
|
||||
@moduledoc false
|
||||
|
||||
use Oban.Worker,
|
||||
queue: :local_data,
|
||||
tags: ["media_profiles", "local_data"]
|
||||
|
||||
require Logger
|
||||
|
||||
alias __MODULE__
|
||||
alias Pinchflat.Profiles
|
||||
|
||||
@doc """
|
||||
Starts the profile deletion worker. Does not attach it to a task like `kickoff_with_task/2`
|
||||
since deletion also cancels all tasks for the profile
|
||||
|
||||
Returns {:ok, %Oban.Job{}} | {:error, %Ecto.Changeset{}}
|
||||
"""
|
||||
def kickoff(profile, job_args \\ %{}, job_opts \\ []) do
|
||||
%{id: profile.id}
|
||||
|> Map.merge(job_args)
|
||||
|> MediaProfileDeletionWorker.new(job_opts)
|
||||
|> Oban.insert()
|
||||
end
|
||||
|
||||
@doc """
|
||||
Deletes a profile and optionally deletes its files
|
||||
|
||||
Returns :ok
|
||||
"""
|
||||
@impl Oban.Worker
|
||||
def perform(%Oban.Job{args: %{"id" => profile_id} = args}) do
|
||||
delete_files = Map.get(args, "delete_files", false)
|
||||
profile = Profiles.get_media_profile!(profile_id)
|
||||
|
||||
Profiles.delete_media_profile(profile, delete_files: delete_files)
|
||||
end
|
||||
end
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
defmodule Pinchflat.Profiles.ProfilesQuery do
|
||||
@moduledoc """
|
||||
Query helpers for the Profiles context.
|
||||
|
||||
These methods are made to be one-ish liners used
|
||||
to compose queries. Each method should strive to do
|
||||
_one_ thing. These don't need to be tested as
|
||||
they are just building blocks for other functionality
|
||||
which, itself, will be tested.
|
||||
"""
|
||||
import Ecto.Query, warn: false
|
||||
|
||||
alias Pinchflat.Profiles.MediaProfile
|
||||
|
||||
# This allows the module to be aliased and query methods to be used
|
||||
# all in one go
|
||||
# usage: use Pinchflat.Profiles.ProfilesQuery
|
||||
defmacro __using__(_opts) do
|
||||
quote do
|
||||
import Ecto.Query, warn: false
|
||||
|
||||
alias unquote(__MODULE__)
|
||||
end
|
||||
end
|
||||
|
||||
def new do
|
||||
MediaProfile
|
||||
end
|
||||
end
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
defmodule Pinchflat.PromEx do
|
||||
@moduledoc """
|
||||
Configuration for the PromEx library which provides Prometheus metrics
|
||||
"""
|
||||
|
||||
use PromEx, otp_app: :pinchflat
|
||||
|
||||
alias PromEx.Plugins
|
||||
|
||||
@impl true
|
||||
def plugins do
|
||||
[
|
||||
Plugins.Application,
|
||||
Plugins.Beam,
|
||||
{Plugins.Phoenix, router: PinchflatWeb.Router, endpoint: PinchflatWeb.Endpoint},
|
||||
Plugins.Ecto,
|
||||
Plugins.Oban,
|
||||
Plugins.PhoenixLiveView
|
||||
]
|
||||
end
|
||||
|
||||
@impl true
|
||||
def dashboard_assigns do
|
||||
[
|
||||
default_selected_interval: "30s"
|
||||
]
|
||||
end
|
||||
|
||||
@impl true
|
||||
def dashboards do
|
||||
[
|
||||
{:prom_ex, "application.json"},
|
||||
{:prom_ex, "beam.json"},
|
||||
{:prom_ex, "phoenix.json"},
|
||||
{:prom_ex, "ecto.json"},
|
||||
{:prom_ex, "oban.json"},
|
||||
{:prom_ex, "phoenix_live_view.json"}
|
||||
]
|
||||
end
|
||||
end
|
||||
|
|
@ -29,8 +29,6 @@ defmodule Pinchflat.Release do
|
|||
[
|
||||
"/config",
|
||||
"/downloads",
|
||||
"/etc/yt-dlp",
|
||||
"/etc/yt-dlp/plugins",
|
||||
Application.get_env(:pinchflat, :media_directory),
|
||||
Application.get_env(:pinchflat, :tmpfile_directory),
|
||||
Application.get_env(:pinchflat, :extras_directory),
|
||||
|
|
|
|||
|
|
@ -13,43 +13,62 @@ defmodule Pinchflat.Settings.Setting do
|
|||
:apprise_version,
|
||||
:apprise_server,
|
||||
:video_codec_preference,
|
||||
:audio_codec_preference,
|
||||
:youtube_api_key,
|
||||
:extractor_sleep_interval_seconds,
|
||||
:download_throughput_limit,
|
||||
:restrict_filenames
|
||||
:audio_codec_preference
|
||||
]
|
||||
|
||||
@required_fields [
|
||||
:onboarding,
|
||||
:pro_enabled,
|
||||
:video_codec_preference,
|
||||
:audio_codec_preference,
|
||||
:extractor_sleep_interval_seconds
|
||||
@virtual_fields [
|
||||
:video_codec_preference_string,
|
||||
:audio_codec_preference_string
|
||||
]
|
||||
|
||||
@required_fields ~w(
|
||||
onboarding
|
||||
pro_enabled
|
||||
)a
|
||||
|
||||
schema "settings" do
|
||||
field :onboarding, :boolean, default: true
|
||||
field :pro_enabled, :boolean, default: false
|
||||
field :yt_dlp_version, :string
|
||||
field :apprise_version, :string
|
||||
field :apprise_server, :string
|
||||
field :youtube_api_key, :string
|
||||
field :route_token, :string
|
||||
field :extractor_sleep_interval_seconds, :integer, default: 0
|
||||
# This is a string because it accepts values like "100K" or "4.2M"
|
||||
field :download_throughput_limit, :string
|
||||
field :restrict_filenames, :boolean, default: false
|
||||
|
||||
field :video_codec_preference, :string
|
||||
field :audio_codec_preference, :string
|
||||
field :video_codec_preference, {:array, :string}, default: []
|
||||
field :audio_codec_preference, {:array, :string}, default: []
|
||||
field :video_codec_preference_string, :string, default: nil, virtual: true
|
||||
field :audio_codec_preference_string, :string, default: nil, virtual: true
|
||||
end
|
||||
|
||||
@doc false
|
||||
def changeset(setting, attrs) do
|
||||
setting
|
||||
|> cast(attrs, @allowed_fields)
|
||||
|> cast(attrs, @virtual_fields, empty_values: [])
|
||||
|> convert_codec_preference_strings()
|
||||
|> validate_required(@required_fields)
|
||||
|> validate_number(:extractor_sleep_interval_seconds, greater_than_or_equal_to: 0)
|
||||
end
|
||||
|
||||
defp convert_codec_preference_strings(changeset) do
|
||||
fields = [
|
||||
video_codec_preference_string: :video_codec_preference,
|
||||
audio_codec_preference_string: :audio_codec_preference
|
||||
]
|
||||
|
||||
Enum.reduce(fields, changeset, fn {virtual_field, actual_field}, changeset ->
|
||||
case get_change(changeset, virtual_field) do
|
||||
nil ->
|
||||
changeset
|
||||
|
||||
value ->
|
||||
new_value =
|
||||
value
|
||||
|> String.split(">")
|
||||
|> Enum.map(&String.trim/1)
|
||||
|> Enum.reject(&(String.trim(&1) == ""))
|
||||
|> Enum.map(&String.downcase/1)
|
||||
|
||||
put_change(changeset, actual_field, new_value)
|
||||
end
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -106,7 +106,7 @@ defmodule Pinchflat.SlowIndexing.FileFollowerServer do
|
|||
{:noreply, %{state | last_activity: DateTime.utc_now()}}
|
||||
|
||||
:eof ->
|
||||
Logger.debug("Current batch of media processed. Will check again in #{@poll_interval_ms}ms")
|
||||
Logger.debug("EOF reached, waiting before trying to read new lines")
|
||||
Process.send_after(self(), :read_new_lines, @poll_interval_ms)
|
||||
|
||||
{:noreply, state}
|
||||
|
|
|
|||
|
|
@ -79,21 +79,21 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
|
|||
case {source.index_frequency_minutes, source.last_indexed_at} do
|
||||
{index_freq, _} when index_freq > 0 ->
|
||||
# If the indexing is on a schedule simply run indexing and reschedule
|
||||
perform_indexing_and_notification(source, was_forced: args["force"])
|
||||
perform_indexing_and_notification(source)
|
||||
maybe_enqueue_fast_indexing_task(source)
|
||||
reschedule_indexing(source)
|
||||
|
||||
{_, nil} ->
|
||||
# If the source has never been indexed, index it once
|
||||
# even if it's not meant to reschedule
|
||||
perform_indexing_and_notification(source, was_forced: args["force"])
|
||||
perform_indexing_and_notification(source)
|
||||
:ok
|
||||
|
||||
_ ->
|
||||
# If the source HAS been indexed and is not meant to reschedule,
|
||||
# perform a no-op (unless forced)
|
||||
if args["force"] do
|
||||
perform_indexing_and_notification(source, was_forced: true)
|
||||
perform_indexing_and_notification(source)
|
||||
end
|
||||
|
||||
:ok
|
||||
|
|
@ -103,11 +103,11 @@ defmodule Pinchflat.SlowIndexing.MediaCollectionIndexingWorker do
|
|||
Ecto.StaleEntryError -> Logger.info("#{__MODULE__} discarded: source #{source_id} stale")
|
||||
end
|
||||
|
||||
defp perform_indexing_and_notification(source, indexing_opts) do
|
||||
defp perform_indexing_and_notification(source) do
|
||||
apprise_server = Settings.get!(:apprise_server)
|
||||
|
||||
SourceNotifications.wrap_new_media_notification(apprise_server, source, fn ->
|
||||
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source, indexing_opts)
|
||||
SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source)
|
||||
end)
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -5,63 +5,41 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
|
|||
Many of these methods are made to be kickoff or be consumed by workers.
|
||||
"""
|
||||
|
||||
use Pinchflat.Media.MediaQuery
|
||||
|
||||
require Logger
|
||||
|
||||
alias Pinchflat.Repo
|
||||
alias Pinchflat.Media
|
||||
alias Pinchflat.Tasks
|
||||
alias Pinchflat.Sources
|
||||
alias Pinchflat.Sources.Source
|
||||
alias Pinchflat.Media.MediaItem
|
||||
alias Pinchflat.YtDlp.MediaCollection
|
||||
alias Pinchflat.Utils.FilesystemUtils
|
||||
alias Pinchflat.Downloading.DownloadingHelpers
|
||||
alias Pinchflat.SlowIndexing.FileFollowerServer
|
||||
alias Pinchflat.Downloading.DownloadOptionBuilder
|
||||
alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker
|
||||
|
||||
alias Pinchflat.YtDlp.Media, as: YtDlpMedia
|
||||
|
||||
@doc """
|
||||
Kills old indexing tasks and starts a new task to index the media collection.
|
||||
Starts tasks for indexing a source's media regardless of the source's indexing
|
||||
frequency. It's assumed the caller will check for indexing frequency.
|
||||
|
||||
The job is delayed based on the source's `index_frequency_minutes` setting unless
|
||||
one of the following is true:
|
||||
- The `force` option is set to true
|
||||
- The source has never been indexed before
|
||||
- The source has been indexed before, but the last indexing job was more than
|
||||
`index_frequency_minutes` ago
|
||||
|
||||
Returns {:ok, %Task{}}
|
||||
Returns {:ok, %Task{}}.
|
||||
"""
|
||||
def kickoff_indexing_task(%Source{} = source, job_args \\ %{}, job_opts \\ []) do
|
||||
job_offset_seconds = if job_args[:force], do: 0, else: calculate_job_offset_seconds(source)
|
||||
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
|
||||
Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
|
||||
|
||||
Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker", include_executing: true)
|
||||
|
||||
MediaCollectionIndexingWorker.kickoff_with_task(source, job_args, job_opts ++ [schedule_in: job_offset_seconds])
|
||||
end
|
||||
|
||||
@doc """
|
||||
A helper method to delete all indexing-related tasks for a source.
|
||||
Optionally, you can include executing tasks in the deletion process.
|
||||
|
||||
Returns :ok
|
||||
"""
|
||||
def delete_indexing_tasks(%Source{} = source, opts \\ []) do
|
||||
include_executing = Keyword.get(opts, :include_executing, false)
|
||||
|
||||
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker", include_executing: include_executing)
|
||||
Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker", include_executing: include_executing)
|
||||
MediaCollectionIndexingWorker.kickoff_with_task(source, job_args, job_opts)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Given a media source, creates (indexes) the media by creating media_items for each
|
||||
media ID in the source. Afterward, kicks off a download task for each pending media
|
||||
item belonging to the source. Returns a list of media items or changesets
|
||||
(if the media item couldn't be created).
|
||||
item belonging to the source. You can't tell me the method name isn't descriptive!
|
||||
Returns a list of media items or changesets (if the media item couldn't be created).
|
||||
|
||||
For each new media item, the method will also run a user script with the `media_indexed`
|
||||
event, if the script is present.
|
||||
|
||||
Indexing is slow and usually returns a list of all media data at once for record creation.
|
||||
To help with this, we use a file follower to watch the file that yt-dlp writes to
|
||||
|
|
@ -69,47 +47,35 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
|
|||
clarity to the user experience. This has a few things to be aware of which are documented
|
||||
below in the file watcher setup method.
|
||||
|
||||
Additionally, in the case of a repeat index we create a download archive file that
|
||||
contains some media IDs that we've indexed in the past. Note that this archive doesn't
|
||||
contain the most recent IDs but rather a subset of IDs that are offset by some amount.
|
||||
Practically, this means that we'll re-index a small handful of media that we've recently
|
||||
indexed, but this is a good thing since it'll let us pick up on any recent changes to the
|
||||
most recent media items.
|
||||
|
||||
We don't create a download archive for playlists (only channels), nor do we create one if
|
||||
the indexing was forced by the user.
|
||||
|
||||
NOTE: downloads are only enqueued if the source is set to download media. Downloads are
|
||||
also enqueued for ALL pending media items, not just the ones that were indexed in this
|
||||
job run. This should ensure that any stragglers are caught if, for some reason, they
|
||||
weren't enqueued or somehow got de-queued.
|
||||
|
||||
Available options:
|
||||
- `was_forced`: Whether the indexing was forced by the user
|
||||
Since indexing returns all media data EVERY TIME, we that that opportunity to update
|
||||
indexing metadata for media items that have already been created.
|
||||
|
||||
Returns [%MediaItem{} | %Ecto.Changeset{}]
|
||||
"""
|
||||
def index_and_enqueue_download_for_media_items(%Source{} = source, opts \\ []) do
|
||||
# The media_profile is needed to determine the quality options to _then_ determine a more
|
||||
# accurate predicted filepath
|
||||
source = Repo.preload(source, [:media_profile])
|
||||
def index_and_enqueue_download_for_media_items(%Source{} = source) do
|
||||
# See the method definition below for more info on how file watchers work
|
||||
# (important reading if you're not familiar with it)
|
||||
{:ok, media_attributes} = setup_file_watcher_and_kickoff_indexing(source, opts)
|
||||
{:ok, media_attributes} = setup_file_watcher_and_kickoff_indexing(source)
|
||||
# Reload because the source may have been updated during the (long-running) indexing process
|
||||
# and important settings like `download_media` may have changed.
|
||||
source = Repo.reload!(source)
|
||||
|
||||
result =
|
||||
Enum.map(media_attributes, fn media_attrs ->
|
||||
case Media.create_media_item_from_backend_attrs(source, media_attrs) do
|
||||
Enum.map(media_attributes, fn media_attrs_struct ->
|
||||
case DownloadingHelpers.create_media_item_and_run_script(source, media_attrs_struct) do
|
||||
{:ok, media_item} -> media_item
|
||||
{:error, changeset} -> changeset
|
||||
end
|
||||
end)
|
||||
|
||||
Sources.update_source(source, %{last_indexed_at: DateTime.utc_now()})
|
||||
DownloadingHelpers.enqueue_pending_download_tasks(source)
|
||||
# Wait 5s before enqueuing downloads to give the post-indexing user script a chance to run
|
||||
DownloadingHelpers.enqueue_pending_download_tasks(source, kickoff_delay: 5)
|
||||
|
||||
result
|
||||
end
|
||||
|
|
@ -127,20 +93,11 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
|
|||
# It attempts a graceful shutdown of the file follower after the indexing is done,
|
||||
# but the FileFollowerServer will also stop itself if it doesn't see any activity
|
||||
# for a sufficiently long time.
|
||||
defp setup_file_watcher_and_kickoff_indexing(source, opts) do
|
||||
was_forced = Keyword.get(opts, :was_forced, false)
|
||||
defp setup_file_watcher_and_kickoff_indexing(source) do
|
||||
{:ok, pid} = FileFollowerServer.start_link()
|
||||
|
||||
handler = fn filepath -> setup_file_follower_watcher(pid, filepath, source) end
|
||||
should_use_cookies = Sources.use_cookies?(source, :indexing)
|
||||
|
||||
command_opts =
|
||||
[output: DownloadOptionBuilder.build_output_path_for(source)] ++
|
||||
DownloadOptionBuilder.build_quality_options_for(source) ++
|
||||
build_download_archive_options(source, was_forced)
|
||||
|
||||
runner_opts = [file_listener_handler: handler, use_cookies: should_use_cookies]
|
||||
result = MediaCollection.get_media_attributes_for_collection(source.original_url, command_opts, runner_opts)
|
||||
result = MediaCollection.get_media_attributes_for_collection(source.original_url, file_listener_handler: handler)
|
||||
|
||||
FileFollowerServer.stop(pid)
|
||||
|
||||
|
|
@ -164,81 +121,18 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do
|
|||
end)
|
||||
end
|
||||
|
||||
defp create_media_item_and_enqueue_download(source, media_attrs) do
|
||||
defp create_media_item_and_enqueue_download(source, media_attrs_struct) do
|
||||
# Reload because the source may have been updated during the (long-running) indexing process
|
||||
# and important settings like `download_media` may have changed.
|
||||
source = Repo.reload!(source)
|
||||
|
||||
case Media.create_media_item_from_backend_attrs(source, media_attrs) do
|
||||
case DownloadingHelpers.create_media_item_and_run_script(source, media_attrs_struct) do
|
||||
{:ok, %MediaItem{} = media_item} ->
|
||||
DownloadingHelpers.kickoff_download_if_pending(media_item)
|
||||
# Wait 5s before enqueuing downloads to give the post-indexing user script a chance to run
|
||||
DownloadingHelpers.kickoff_download_if_pending(media_item, kickoff_delay: 5)
|
||||
|
||||
{:error, changeset} ->
|
||||
changeset
|
||||
end
|
||||
end
|
||||
|
||||
# Find the difference between the current time and the last time the source was indexed
|
||||
defp calculate_job_offset_seconds(%Source{last_indexed_at: nil}), do: 0
|
||||
|
||||
defp calculate_job_offset_seconds(source) do
|
||||
offset_seconds = DateTime.diff(DateTime.utc_now(), source.last_indexed_at, :second)
|
||||
index_frequency_seconds = source.index_frequency_minutes * 60
|
||||
|
||||
max(0, index_frequency_seconds - offset_seconds)
|
||||
end
|
||||
|
||||
# The download archive file works in tandem with --break-on-existing to stop
|
||||
# yt-dlp once we've hit media items we've already indexed. But we generate
|
||||
# this list with a bit of an offset so we do intentionally re-scan some media
|
||||
# items to pick up any recent changes (see `get_media_items_for_download_archive`).
|
||||
#
|
||||
# From there, we format the media IDs in the way that yt-dlp expects (ie: "<extractor> <media_id>")
|
||||
# and return the filepath to the caller.
|
||||
defp create_download_archive_file(source) do
|
||||
tmpfile = FilesystemUtils.generate_metadata_tmpfile(:txt)
|
||||
|
||||
archive_contents =
|
||||
source
|
||||
|> get_media_items_for_download_archive()
|
||||
|> Enum.map_join("\n", fn media_item -> "youtube #{media_item.media_id}" end)
|
||||
|
||||
case File.write(tmpfile, archive_contents) do
|
||||
:ok -> tmpfile
|
||||
err -> err
|
||||
end
|
||||
end
|
||||
|
||||
# Sorting by `uploaded_at` is important because we want to re-index the most recent
|
||||
# media items first but there is no guarantee of any correlation between ID and uploaded_at.
|
||||
#
|
||||
# The offset is important because we want to re-index some media items that we've
|
||||
# recently indexed to pick up on any changes. The limit is because we want this mechanism
|
||||
# to work even if, for example, the video we were using as a stopping point was deleted.
|
||||
# It's not a perfect system, but it should do well enough.
|
||||
#
|
||||
# The chosen limit and offset are arbitary, independent, and vibes-based. Feel free to
|
||||
# tweak as-needed
|
||||
defp get_media_items_for_download_archive(source) do
|
||||
MediaQuery.new()
|
||||
|> where(^MediaQuery.for_source(source))
|
||||
|> order_by(desc: :uploaded_at)
|
||||
|> limit(50)
|
||||
|> offset(20)
|
||||
|> Repo.all()
|
||||
end
|
||||
|
||||
# The download archive isn't useful for playlists (since those are ordered arbitrarily)
|
||||
# and we don't want to use it if the indexing was forced by the user. In other words,
|
||||
# only create an archive for channels that are being indexed as part of their regular
|
||||
# indexing schedule. The first indexing pass should also not create an archive.
|
||||
defp build_download_archive_options(%Source{collection_type: :playlist}, _was_forced), do: []
|
||||
defp build_download_archive_options(%Source{last_indexed_at: nil}, _was_forced), do: []
|
||||
defp build_download_archive_options(_source, true), do: []
|
||||
|
||||
defp build_download_archive_options(source, _was_forced) do
|
||||
archive_file = create_download_archive_file(source)
|
||||
|
||||
[:break_on_existing, download_archive: archive_file]
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ defmodule Pinchflat.Sources.Source do
|
|||
alias Pinchflat.Metadata.SourceMetadata
|
||||
|
||||
@allowed_fields ~w(
|
||||
enabled
|
||||
collection_name
|
||||
collection_id
|
||||
collection_type
|
||||
|
|
@ -28,7 +27,6 @@ defmodule Pinchflat.Sources.Source do
|
|||
series_directory
|
||||
index_frequency_minutes
|
||||
fast_index
|
||||
cookie_behaviour
|
||||
download_media
|
||||
last_indexed_at
|
||||
original_url
|
||||
|
|
@ -37,9 +35,6 @@ defmodule Pinchflat.Sources.Source do
|
|||
title_filter_regex
|
||||
media_profile_id
|
||||
output_path_template_override
|
||||
marked_for_deletion_at
|
||||
min_duration_seconds
|
||||
max_duration_seconds
|
||||
)a
|
||||
|
||||
# Expensive API calls are made when a source is inserted/updated so
|
||||
|
|
@ -65,7 +60,6 @@ defmodule Pinchflat.Sources.Source do
|
|||
)a
|
||||
|
||||
schema "sources" do
|
||||
field :enabled, :boolean, default: true
|
||||
# This is _not_ used as the primary key or internally in the database
|
||||
# relations. This is only used to prevent an enumeration attack on the streaming
|
||||
# and RSS feed endpoints since those _must_ be public (ie: no basic auth)
|
||||
|
|
@ -78,7 +72,6 @@ defmodule Pinchflat.Sources.Source do
|
|||
field :collection_type, Ecto.Enum, values: [:channel, :playlist]
|
||||
field :index_frequency_minutes, :integer, default: 60 * 24
|
||||
field :fast_index, :boolean, default: false
|
||||
field :cookie_behaviour, Ecto.Enum, values: [:disabled, :when_needed, :all_operations], default: :disabled
|
||||
field :download_media, :boolean, default: true
|
||||
field :last_indexed_at, :utc_datetime
|
||||
# Only download media items that were published after this date
|
||||
|
|
@ -88,17 +81,12 @@ defmodule Pinchflat.Sources.Source do
|
|||
field :title_filter_regex, :string
|
||||
field :output_path_template_override, :string
|
||||
|
||||
field :min_duration_seconds, :integer
|
||||
field :max_duration_seconds, :integer
|
||||
|
||||
field :series_directory, :string
|
||||
field :nfo_filepath, :string
|
||||
field :poster_filepath, :string
|
||||
field :fanart_filepath, :string
|
||||
field :banner_filepath, :string
|
||||
|
||||
field :marked_for_deletion_at, :utc_datetime
|
||||
|
||||
belongs_to :media_profile, MediaProfile
|
||||
|
||||
has_one :metadata, SourceMetadata, on_replace: :update
|
||||
|
|
@ -124,12 +112,9 @@ defmodule Pinchflat.Sources.Source do
|
|||
|> dynamic_default(:custom_name, fn cs -> get_field(cs, :collection_name) end)
|
||||
|> dynamic_default(:uuid, fn _ -> Ecto.UUID.generate() end)
|
||||
|> validate_required(required_fields)
|
||||
|> validate_title_regex()
|
||||
|> validate_min_and_max_durations()
|
||||
|> validate_number(:retention_period_days, greater_than_or_equal_to: 0)
|
||||
# Ensures it ends with `.{{ ext }}` or `.%(ext)s` or similar (with a little wiggle room)
|
||||
|> validate_format(:output_path_template_override, MediaProfile.ext_regex(), message: "must end with .{{ ext }}")
|
||||
|> validate_format(:original_url, youtube_channel_or_playlist_regex(), message: "must be a channel or playlist URL")
|
||||
|> cast_assoc(:metadata, with: &SourceMetadata.changeset/2, required: false)
|
||||
|> unique_constraint([:collection_id, :media_profile_id, :title_filter_regex], error_key: :original_url)
|
||||
end
|
||||
|
|
@ -143,7 +128,7 @@ defmodule Pinchflat.Sources.Source do
|
|||
@doc false
|
||||
def fast_index_frequency do
|
||||
# minutes
|
||||
10
|
||||
15
|
||||
end
|
||||
|
||||
@doc false
|
||||
|
|
@ -156,33 +141,6 @@ defmodule Pinchflat.Sources.Source do
|
|||
~w(__meta__ __struct__ metadata tasks media_items)a
|
||||
end
|
||||
|
||||
def youtube_channel_or_playlist_regex do
|
||||
# Validate that the original URL is not a video URL
|
||||
# Also matches if the string does NOT contain youtube.com or youtu.be. This preserves my tenuous support
|
||||
# for non-youtube sources.
|
||||
~r<^(?:(?!youtube\.com/(watch|shorts|embed)|youtu\.be).)*$>
|
||||
end
|
||||
|
||||
defp validate_title_regex(%{changes: %{title_filter_regex: regex}} = changeset) when is_binary(regex) do
|
||||
case Ecto.Adapters.SQL.query(Repo, "SELECT regexp_like('', ?)", [regex]) do
|
||||
{:ok, _} -> changeset
|
||||
_ -> add_error(changeset, :title_filter_regex, "is invalid")
|
||||
end
|
||||
end
|
||||
|
||||
defp validate_title_regex(changeset), do: changeset
|
||||
|
||||
defp validate_min_and_max_durations(changeset) do
|
||||
min_duration = get_change(changeset, :min_duration_seconds)
|
||||
max_duration = get_change(changeset, :max_duration_seconds)
|
||||
|
||||
case {min_duration, max_duration} do
|
||||
{min, max} when is_nil(min) or is_nil(max) -> changeset
|
||||
{min, max} when min >= max -> add_error(changeset, :max_duration_seconds, "must be greater than minumum duration")
|
||||
_ -> changeset
|
||||
end
|
||||
end
|
||||
|
||||
defimpl Jason.Encoder, for: Source do
|
||||
def encode(value, opts) do
|
||||
value
|
||||
|
|
|
|||
|
|
@ -1,38 +0,0 @@
|
|||
defmodule Pinchflat.Sources.SourceDeletionWorker do
|
||||
@moduledoc false
|
||||
|
||||
use Oban.Worker,
|
||||
queue: :local_data,
|
||||
tags: ["sources", "local_data"]
|
||||
|
||||
require Logger
|
||||
|
||||
alias __MODULE__
|
||||
alias Pinchflat.Sources
|
||||
|
||||
@doc """
|
||||
Starts the source deletion worker. Does not attach it to a task like `kickoff_with_task/2`
|
||||
since deletion also cancels all tasks for the source
|
||||
|
||||
Returns {:ok, %Task{}} | {:error, %Ecto.Changeset{}}
|
||||
"""
|
||||
def kickoff(source, job_args \\ %{}, job_opts \\ []) do
|
||||
%{id: source.id}
|
||||
|> Map.merge(job_args)
|
||||
|> SourceDeletionWorker.new(job_opts)
|
||||
|> Oban.insert()
|
||||
end
|
||||
|
||||
@doc """
|
||||
Deletes a source and optionally deletes its files
|
||||
|
||||
Returns :ok
|
||||
"""
|
||||
@impl Oban.Worker
|
||||
def perform(%Oban.Job{args: %{"id" => source_id} = args}) do
|
||||
delete_files = Map.get(args, "delete_files", false)
|
||||
source = Sources.get_source!(source_id)
|
||||
|
||||
Sources.delete_source(source, delete_files: delete_files)
|
||||
end
|
||||
end
|
||||
|
|
@ -15,8 +15,8 @@ defmodule Pinchflat.Sources do
|
|||
alias Pinchflat.Metadata.SourceMetadata
|
||||
alias Pinchflat.Utils.FilesystemUtils
|
||||
alias Pinchflat.Downloading.DownloadingHelpers
|
||||
alias Pinchflat.FastIndexing.FastIndexingWorker
|
||||
alias Pinchflat.SlowIndexing.SlowIndexingHelpers
|
||||
alias Pinchflat.FastIndexing.FastIndexingHelpers
|
||||
alias Pinchflat.Metadata.SourceMetadataStorageWorker
|
||||
|
||||
@doc """
|
||||
|
|
@ -32,19 +32,6 @@ defmodule Pinchflat.Sources do
|
|||
source.output_path_template_override || media_profile.output_path_template
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns a boolean indicating whether or not cookies should be used for a given operation.
|
||||
|
||||
Returns boolean()
|
||||
"""
|
||||
def use_cookies?(source, operation) when operation in [:indexing, :downloading, :metadata, :error_recovery] do
|
||||
case source.cookie_behaviour do
|
||||
:disabled -> false
|
||||
:all_operations -> true
|
||||
:when_needed -> operation in [:indexing, :error_recovery]
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns the list of sources. Returns [%Source{}, ...]
|
||||
"""
|
||||
|
|
@ -193,22 +180,11 @@ defmodule Pinchflat.Sources do
|
|||
end
|
||||
|
||||
defp add_source_details_to_changeset(source, changeset) do
|
||||
original_url = changeset.changes.original_url
|
||||
should_use_cookies = Ecto.Changeset.get_field(changeset, :cookie_behaviour) == :all_operations
|
||||
# Skipping sleep interval since this is UI blocking and we want to keep this as fast as possible
|
||||
addl_opts = [use_cookies: should_use_cookies, skip_sleep_interval: true]
|
||||
|
||||
case MediaCollection.get_source_details(original_url, [], addl_opts) do
|
||||
case MediaCollection.get_source_details(changeset.changes.original_url) do
|
||||
{:ok, source_details} ->
|
||||
add_source_details_by_collection_type(source, changeset, source_details)
|
||||
|
||||
err ->
|
||||
runner_error =
|
||||
case err do
|
||||
{:error, error_msg, _status_code} -> error_msg
|
||||
{:error, error_msg} -> error_msg
|
||||
end
|
||||
|
||||
{:error, runner_error, _status_code} ->
|
||||
Ecto.Changeset.add_error(
|
||||
changeset,
|
||||
:original_url,
|
||||
|
|
@ -271,40 +247,19 @@ defmodule Pinchflat.Sources do
|
|||
end
|
||||
end
|
||||
|
||||
# If the source is new (ie: not persisted), do nothing
|
||||
defp maybe_handle_media_tasks(%{data: %{__meta__: %{state: state}}}, _source) when state != :loaded do
|
||||
:ok
|
||||
end
|
||||
|
||||
# If the source is NOT new (ie: updated),
|
||||
# If the source is NOT new (ie: updated) and the download_media flag has changed,
|
||||
# enqueue or dequeue media download tasks as necessary.
|
||||
defp maybe_handle_media_tasks(changeset, source) do
|
||||
current_changes = changeset.changes
|
||||
applied_changes = Ecto.Changeset.apply_changes(changeset)
|
||||
|
||||
# We need both current_changes and applied_changes to determine
|
||||
# the course of action to take. For example, we only care if a source is supposed
|
||||
# to be `enabled` or not - we don't care if that information comes from the
|
||||
# current changes or if that's how it already was in the database.
|
||||
# Rephrased, we're essentially using it in place of `get_field/2`
|
||||
case {current_changes, applied_changes} do
|
||||
{%{download_media: true}, %{enabled: true}} ->
|
||||
case {changeset.data, changeset.changes} do
|
||||
{%{__meta__: %{state: :loaded}}, %{download_media: true}} ->
|
||||
DownloadingHelpers.enqueue_pending_download_tasks(source)
|
||||
|
||||
{%{enabled: true}, %{download_media: true}} ->
|
||||
DownloadingHelpers.enqueue_pending_download_tasks(source)
|
||||
|
||||
{%{download_media: false}, _} ->
|
||||
DownloadingHelpers.dequeue_pending_download_tasks(source)
|
||||
|
||||
{%{enabled: false}, _} ->
|
||||
{%{__meta__: %{state: :loaded}}, %{download_media: false}} ->
|
||||
DownloadingHelpers.dequeue_pending_download_tasks(source)
|
||||
|
||||
_ ->
|
||||
nil
|
||||
:ok
|
||||
end
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
defp maybe_run_indexing_task(changeset, source) do
|
||||
|
|
@ -313,10 +268,6 @@ defmodule Pinchflat.Sources do
|
|||
%{__meta__: %{state: :built}} ->
|
||||
SlowIndexingHelpers.kickoff_indexing_task(source)
|
||||
|
||||
if Ecto.Changeset.get_field(changeset, :fast_index) do
|
||||
FastIndexingHelpers.kickoff_indexing_task(source)
|
||||
end
|
||||
|
||||
# If the record has been persisted, only run indexing if the
|
||||
# indexing frequency has been changed and is now greater than 0
|
||||
%{__meta__: %{state: :loaded}} ->
|
||||
|
|
@ -342,22 +293,13 @@ defmodule Pinchflat.Sources do
|
|||
end
|
||||
|
||||
defp maybe_update_slow_indexing_task(changeset, source) do
|
||||
# See comment in `maybe_handle_media_tasks` as to why we need these
|
||||
current_changes = changeset.changes
|
||||
applied_changes = Ecto.Changeset.apply_changes(changeset)
|
||||
|
||||
case {current_changes, applied_changes} do
|
||||
{%{index_frequency_minutes: mins}, %{enabled: true}} when mins > 0 ->
|
||||
case changeset.changes do
|
||||
%{index_frequency_minutes: mins} when mins > 0 ->
|
||||
SlowIndexingHelpers.kickoff_indexing_task(source)
|
||||
|
||||
{%{enabled: true}, %{index_frequency_minutes: mins}} when mins > 0 ->
|
||||
SlowIndexingHelpers.kickoff_indexing_task(source)
|
||||
|
||||
{%{index_frequency_minutes: _}, _} ->
|
||||
SlowIndexingHelpers.delete_indexing_tasks(source, include_executing: true)
|
||||
|
||||
{%{enabled: false}, _} ->
|
||||
SlowIndexingHelpers.delete_indexing_tasks(source, include_executing: true)
|
||||
%{index_frequency_minutes: _} ->
|
||||
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
|
||||
Tasks.delete_pending_tasks_for(source, "MediaCollectionIndexingWorker")
|
||||
|
||||
_ ->
|
||||
:ok
|
||||
|
|
@ -365,25 +307,13 @@ defmodule Pinchflat.Sources do
|
|||
end
|
||||
|
||||
defp maybe_update_fast_indexing_task(changeset, source) do
|
||||
# See comment in `maybe_handle_media_tasks` as to why we need these
|
||||
current_changes = changeset.changes
|
||||
applied_changes = Ecto.Changeset.apply_changes(changeset)
|
||||
case changeset.changes do
|
||||
%{fast_index: true} ->
|
||||
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
|
||||
FastIndexingWorker.kickoff_with_task(source)
|
||||
|
||||
# This technically could be simplified since `maybe_update_slow_indexing_task`
|
||||
# has some overlap re: deleting pending tasks, but I'm keeping it separate
|
||||
# for clarity and explicitness.
|
||||
case {current_changes, applied_changes} do
|
||||
{%{fast_index: true}, %{enabled: true}} ->
|
||||
FastIndexingHelpers.kickoff_indexing_task(source)
|
||||
|
||||
{%{enabled: true}, %{fast_index: true}} ->
|
||||
FastIndexingHelpers.kickoff_indexing_task(source)
|
||||
|
||||
{%{fast_index: false}, _} ->
|
||||
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker", include_executing: true)
|
||||
|
||||
{%{enabled: false}, _} ->
|
||||
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker", include_executing: true)
|
||||
%{fast_index: false} ->
|
||||
Tasks.delete_pending_tasks_for(source, "FastIndexingWorker")
|
||||
|
||||
_ ->
|
||||
:ok
|
||||
|
|
|
|||
|
|
@ -53,6 +53,20 @@ defmodule Pinchflat.Tasks do
|
|||
)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns the list of pending tasks for a given record type and ID. Optionally allows you to specify
|
||||
which worker to include.
|
||||
|
||||
Returns [%Task{}, ...]
|
||||
"""
|
||||
def list_pending_tasks_for(record, worker_name \\ nil) do
|
||||
list_tasks_for(
|
||||
record,
|
||||
worker_name,
|
||||
[:available, :scheduled, :retryable]
|
||||
)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Gets a single task.
|
||||
|
||||
|
|
@ -113,13 +127,13 @@ defmodule Pinchflat.Tasks do
|
|||
|
||||
@doc """
|
||||
Deletes all tasks attached to a given record, cancelling any attached jobs.
|
||||
Optionally allows you to specify which worker and job states to include.
|
||||
Optionally allows you to specify which worker to include.
|
||||
|
||||
Returns :ok
|
||||
"""
|
||||
def delete_tasks_for(record, worker_name \\ nil, job_states \\ Oban.Job.states()) do
|
||||
def delete_tasks_for(record, worker_name \\ nil) do
|
||||
record
|
||||
|> list_tasks_for(worker_name, job_states)
|
||||
|> list_tasks_for(worker_name)
|
||||
|> Enum.each(&delete_task/1)
|
||||
end
|
||||
|
||||
|
|
@ -129,12 +143,10 @@ defmodule Pinchflat.Tasks do
|
|||
|
||||
Returns :ok
|
||||
"""
|
||||
def delete_pending_tasks_for(record, worker_name \\ nil, opts \\ []) do
|
||||
include_executing = Keyword.get(opts, :include_executing, false)
|
||||
base_job_states = [:available, :scheduled, :retryable]
|
||||
job_states = if include_executing, do: base_job_states ++ [:executing], else: base_job_states
|
||||
|
||||
delete_tasks_for(record, worker_name, job_states)
|
||||
def delete_pending_tasks_for(record, worker_name \\ nil) do
|
||||
record
|
||||
|> list_pending_tasks_for(worker_name)
|
||||
|> Enum.each(&delete_task/1)
|
||||
end
|
||||
|
||||
@doc """
|
||||
|
|
|
|||
17
lib/pinchflat/utils/datetime_utils.ex
Normal file
17
lib/pinchflat/utils/datetime_utils.ex
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
defmodule Pinchflat.Utils.DatetimeUtils do
|
||||
@moduledoc """
|
||||
Utility methods for working with dates and datetimes
|
||||
"""
|
||||
|
||||
@doc """
|
||||
Converts a Date to a DateTime
|
||||
|
||||
Returns %DateTime{}
|
||||
"""
|
||||
def date_to_datetime(date) do
|
||||
date
|
||||
|> Date.to_gregorian_days()
|
||||
|> Kernel.*(86_400)
|
||||
|> DateTime.from_gregorian_seconds()
|
||||
end
|
||||
end
|
||||
|
|
@ -20,24 +20,6 @@ defmodule Pinchflat.Utils.FilesystemUtils do
|
|||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Checks if two filepaths reference the same file.
|
||||
|
||||
Useful if you have a relative and absolute filepath and want to be sure they're the same file.
|
||||
Also works with symlinks.
|
||||
|
||||
Returns boolean()
|
||||
"""
|
||||
def filepaths_reference_same_file?(filepath_1, filepath_2) do
|
||||
{:ok, stat_1} = File.stat(filepath_1)
|
||||
{:ok, stat_2} = File.stat(filepath_2)
|
||||
|
||||
identifier_1 = "#{stat_1.major_device}:#{stat_1.minor_device}:#{stat_1.inode}"
|
||||
identifier_2 = "#{stat_2.major_device}:#{stat_2.minor_device}:#{stat_2.inode}"
|
||||
|
||||
identifier_1 == identifier_2
|
||||
end
|
||||
|
||||
@doc """
|
||||
Generates a temporary file and returns its path. The file is empty and has the given type.
|
||||
Generates all the directories in the path if they don't exist.
|
||||
|
|
@ -45,20 +27,8 @@ defmodule Pinchflat.Utils.FilesystemUtils do
|
|||
Returns binary()
|
||||
"""
|
||||
def generate_metadata_tmpfile(type) do
|
||||
filename = StringUtils.random_string(64)
|
||||
# This "namespacing" is more to help with development since things get
|
||||
# weird in my editor when there are thousands of files in a single directory
|
||||
first_two = String.slice(filename, 0..1)
|
||||
second_two = String.slice(filename, 2..3)
|
||||
tmpfile_directory = Application.get_env(:pinchflat, :tmpfile_directory)
|
||||
|
||||
filepath =
|
||||
Path.join([
|
||||
tmpfile_directory,
|
||||
first_two,
|
||||
second_two,
|
||||
"#{filename}.#{type}"
|
||||
])
|
||||
filepath = Path.join([tmpfile_directory, "#{StringUtils.random_string(64)}.#{type}"])
|
||||
|
||||
:ok = write_p!(filepath, "")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +0,0 @@
|
|||
defmodule Pinchflat.Utils.MapUtils do
|
||||
@moduledoc """
|
||||
Utility methods for working with maps
|
||||
"""
|
||||
|
||||
@doc """
|
||||
Converts a nested list of 2-element tuples or lists into a map.
|
||||
|
||||
Returns map()
|
||||
"""
|
||||
def from_nested_list(list) do
|
||||
Enum.reduce(list, %{}, fn
|
||||
[key, value], acc -> Map.put(acc, key, value)
|
||||
{key, value}, acc -> Map.put(acc, key, value)
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
|
@ -36,18 +36,4 @@ defmodule Pinchflat.Utils.NumberUtils do
|
|||
end
|
||||
end)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Adds jitter to a number based on a percentage. Returns 0 if the number is less than or equal to 0.
|
||||
|
||||
Returns integer()
|
||||
"""
|
||||
def add_jitter(num, jitter_percentage \\ 0.5)
|
||||
def add_jitter(num, _jitter_percentage) when num <= 0, do: 0
|
||||
|
||||
def add_jitter(num, jitter_percentage) do
|
||||
jitter = :rand.uniform(round(num * jitter_percentage))
|
||||
|
||||
round(num + jitter)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -26,22 +26,19 @@ defmodule Pinchflat.Utils.StringUtils do
|
|||
end
|
||||
|
||||
@doc """
|
||||
Wraps a string in double braces. Useful as a UI helper now that
|
||||
LiveView 1.0.0 allows `{}` for interpolation so now we can't use braces
|
||||
directly in the view.
|
||||
Truncates a string to the given length and adds `...` if the string is longer than the given length.
|
||||
Will break on a word boundary. Nothing happens if the string is shorter than the given length.
|
||||
|
||||
Returns binary()
|
||||
"""
|
||||
def double_brace(string) do
|
||||
"{{ #{string} }}"
|
||||
def truncate(string, length) do
|
||||
if String.length(string) > length do
|
||||
string
|
||||
|> String.slice(0..(length - 1))
|
||||
|> String.replace(~r/\s+\S*$/, "")
|
||||
|> Kernel.<>("...")
|
||||
else
|
||||
string
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Wraps a string in quotes if it's not already a string. Useful for working with
|
||||
error messages whose types can vary.
|
||||
|
||||
Returns binary()
|
||||
"""
|
||||
def wrap_string(message) when is_binary(message), do: message
|
||||
def wrap_string(message), do: "#{inspect(message)}"
|
||||
end
|
||||
|
|
|
|||
|
|
@ -3,11 +3,7 @@ defmodule Pinchflat.YtDlp.CommandRunner do
|
|||
Runs yt-dlp commands using the `System.cmd/3` function
|
||||
"""
|
||||
|
||||
require Logger
|
||||
|
||||
alias Pinchflat.Settings
|
||||
alias Pinchflat.Utils.CliUtils
|
||||
alias Pinchflat.Utils.NumberUtils
|
||||
alias Pinchflat.YtDlp.YtDlpCommandRunner
|
||||
alias Pinchflat.Utils.FilesystemUtils, as: FSUtils
|
||||
|
||||
|
|
@ -22,32 +18,23 @@ defmodule Pinchflat.YtDlp.CommandRunner do
|
|||
- :output_filepath - the path to save the output to. If not provided, a temporary
|
||||
file will be created and used. Useful for if you need a reference to the file
|
||||
for a file watcher.
|
||||
- :use_cookies - if true, will add a cookie file to the command options. Will not
|
||||
attach a cookie file if the user hasn't set one up.
|
||||
- :skip_sleep_interval - if true, will not add the sleep interval options to the command.
|
||||
Usually only used for commands that would be UI-blocking
|
||||
|
||||
Returns {:ok, binary()} | {:error, output, status}.
|
||||
"""
|
||||
@impl YtDlpCommandRunner
|
||||
def run(url, action_name, command_opts, output_template, addl_opts \\ []) do
|
||||
Logger.debug("Running yt-dlp command for action: #{action_name}")
|
||||
def run(url, command_opts, output_template, addl_opts \\ []) do
|
||||
# This approach lets us mock the command for testing
|
||||
command = backend_executable()
|
||||
|
||||
output_filepath = generate_output_filepath(addl_opts)
|
||||
print_to_file_opts = [{:print_to_file, output_template}, output_filepath]
|
||||
user_configured_opts = cookie_file_options(addl_opts) ++ rate_limit_options(addl_opts) ++ misc_options()
|
||||
user_configured_opts = cookie_file_options()
|
||||
# These must stay in exactly this order, hence why I'm giving it its own variable.
|
||||
all_opts = command_opts ++ print_to_file_opts ++ user_configured_opts ++ global_options()
|
||||
formatted_command_opts = [url] ++ CliUtils.parse_options(all_opts)
|
||||
|
||||
case CliUtils.wrap_cmd(backend_executable(), formatted_command_opts, stderr_to_stdout: true) do
|
||||
# yt-dlp exit codes:
|
||||
# 0 = Everything is successful
|
||||
# 100 = yt-dlp must restart for update to complete
|
||||
# 101 = Download cancelled by --max-downloads etc
|
||||
# 2 = Error in user-provided options
|
||||
# 1 = Any other error
|
||||
{_, status} when status in [0, 101] ->
|
||||
case CliUtils.wrap_cmd(command, formatted_command_opts, stderr_to_stdout: true) do
|
||||
{_, 0} ->
|
||||
# IDEA: consider deleting the file after reading it. It's in the tmp dir, so it's not
|
||||
# a huge deal, but it's still a good idea to clean up after ourselves.
|
||||
# (even on error? especially on error?)
|
||||
|
|
@ -76,24 +63,6 @@ defmodule Pinchflat.YtDlp.CommandRunner do
|
|||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Updates yt-dlp to the latest version
|
||||
|
||||
Returns {:ok, binary()} | {:error, binary()}
|
||||
"""
|
||||
@impl YtDlpCommandRunner
|
||||
def update do
|
||||
command = backend_executable()
|
||||
|
||||
case CliUtils.wrap_cmd(command, ["--update"]) do
|
||||
{output, 0} ->
|
||||
{:ok, String.trim(output)}
|
||||
|
||||
{output, _} ->
|
||||
{:error, output}
|
||||
end
|
||||
end
|
||||
|
||||
defp generate_output_filepath(addl_opts) do
|
||||
case Keyword.get(addl_opts, :output_filepath) do
|
||||
nil -> FSUtils.generate_metadata_tmpfile(:json)
|
||||
|
|
@ -109,14 +78,7 @@ defmodule Pinchflat.YtDlp.CommandRunner do
|
|||
]
|
||||
end
|
||||
|
||||
defp cookie_file_options(addl_opts) do
|
||||
case Keyword.get(addl_opts, :use_cookies) do
|
||||
true -> add_cookie_file()
|
||||
_ -> []
|
||||
end
|
||||
end
|
||||
|
||||
defp add_cookie_file do
|
||||
defp cookie_file_options do
|
||||
base_dir = Application.get_env(:pinchflat, :extras_directory)
|
||||
filename_options_map = %{cookies: "cookies.txt"}
|
||||
|
||||
|
|
@ -131,32 +93,6 @@ defmodule Pinchflat.YtDlp.CommandRunner do
|
|||
end)
|
||||
end
|
||||
|
||||
defp rate_limit_options(addl_opts) do
|
||||
throughput_limit = Settings.get!(:download_throughput_limit)
|
||||
sleep_interval_opts = sleep_interval_opts(addl_opts)
|
||||
throughput_option = if throughput_limit, do: [limit_rate: throughput_limit], else: []
|
||||
|
||||
throughput_option ++ sleep_interval_opts
|
||||
end
|
||||
|
||||
defp sleep_interval_opts(addl_opts) do
|
||||
sleep_interval = Settings.get!(:extractor_sleep_interval_seconds)
|
||||
|
||||
if sleep_interval <= 0 || Keyword.get(addl_opts, :skip_sleep_interval) do
|
||||
[]
|
||||
else
|
||||
[
|
||||
sleep_requests: NumberUtils.add_jitter(sleep_interval),
|
||||
sleep_interval: NumberUtils.add_jitter(sleep_interval),
|
||||
sleep_subtitles: NumberUtils.add_jitter(sleep_interval)
|
||||
]
|
||||
end
|
||||
end
|
||||
|
||||
defp misc_options do
|
||||
if Settings.get!(:restrict_filenames), do: [:restrict_filenames], else: []
|
||||
end
|
||||
|
||||
defp backend_executable do
|
||||
Application.get_env(:pinchflat, :yt_dlp_executable)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -10,9 +10,8 @@ defmodule Pinchflat.YtDlp.Media do
|
|||
:original_url,
|
||||
:livestream,
|
||||
:short_form_content,
|
||||
:uploaded_at,
|
||||
:duration_seconds,
|
||||
:predicted_media_filepath
|
||||
:upload_date,
|
||||
:duration_seconds
|
||||
]
|
||||
|
||||
defstruct [
|
||||
|
|
@ -22,10 +21,8 @@ defmodule Pinchflat.YtDlp.Media do
|
|||
:original_url,
|
||||
:livestream,
|
||||
:short_form_content,
|
||||
:uploaded_at,
|
||||
:duration_seconds,
|
||||
:playlist_index,
|
||||
:predicted_media_filepath
|
||||
:upload_date,
|
||||
:duration_seconds
|
||||
]
|
||||
|
||||
alias __MODULE__
|
||||
|
|
@ -39,9 +36,9 @@ defmodule Pinchflat.YtDlp.Media do
|
|||
Returns {:ok, map()} | {:error, any, ...}.
|
||||
"""
|
||||
def download(url, command_opts \\ [], addl_opts \\ []) do
|
||||
all_command_opts = [:no_simulate] ++ command_opts
|
||||
opts = [:no_simulate] ++ command_opts
|
||||
|
||||
with {:ok, output} <- backend_runner().run(url, :download, all_command_opts, "after_move:%()j", addl_opts),
|
||||
with {:ok, output} <- backend_runner().run(url, opts, "after_move:%()j", addl_opts),
|
||||
{:ok, parsed_json} <- Phoenix.json_library().decode(output) do
|
||||
{:ok, parsed_json}
|
||||
else
|
||||
|
|
@ -49,53 +46,17 @@ defmodule Pinchflat.YtDlp.Media do
|
|||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Determines if the media at the given URL is ready to be downloaded.
|
||||
Common examples of non-downloadable media are upcoming or in-progress live streams.
|
||||
|
||||
Returns {:ok, :downloadable | :ignorable} | {:error, any}
|
||||
"""
|
||||
def get_downloadable_status(url, addl_opts \\ []) do
|
||||
action = :get_downloadable_status
|
||||
command_opts = [:simulate, :skip_download]
|
||||
|
||||
case backend_runner().run(url, action, command_opts, "%(.{live_status})j", addl_opts) do
|
||||
{:ok, output} ->
|
||||
output
|
||||
|> Phoenix.json_library().decode!()
|
||||
|> parse_downloadable_status()
|
||||
|
||||
err ->
|
||||
err
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Downloads a thumbnail for a single piece of media. Usually used for
|
||||
downloading thumbnails for internal use
|
||||
|
||||
Returns {:ok, ""} | {:error, any, ...}.
|
||||
"""
|
||||
def download_thumbnail(url, command_opts \\ [], addl_opts \\ []) do
|
||||
all_command_opts = [:no_simulate, :skip_download, :write_thumbnail, convert_thumbnail: "jpg"] ++ command_opts
|
||||
|
||||
# NOTE: it doesn't seem like this command actually returns anything in `after_move` since
|
||||
# we aren't downloading the main media file
|
||||
backend_runner().run(url, :download_thumbnail, all_command_opts, "after_move:%()j", addl_opts)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns a map representing the media at the given URL.
|
||||
Optionally takes a list of additional command options to pass to yt-dlp
|
||||
or configuration-related options to pass to the runner.
|
||||
|
||||
Returns {:ok, %Media{}} | {:error, any, ...}.
|
||||
Returns {:ok, [map()]} | {:error, any, ...}.
|
||||
"""
|
||||
def get_media_attributes(url, command_opts \\ [], addl_opts \\ []) do
|
||||
all_command_opts = [:simulate, :skip_download] ++ command_opts
|
||||
def get_media_attributes(url) do
|
||||
runner = Application.get_env(:pinchflat, :yt_dlp_runner)
|
||||
command_opts = [:simulate, :skip_download]
|
||||
output_template = indexing_output_template()
|
||||
|
||||
case backend_runner().run(url, :get_media_attributes, all_command_opts, output_template, addl_opts) do
|
||||
case runner.run(url, command_opts, output_template) do
|
||||
{:ok, output} ->
|
||||
output
|
||||
|> Phoenix.json_library().decode!()
|
||||
|
|
@ -109,13 +70,9 @@ defmodule Pinchflat.YtDlp.Media do
|
|||
|
||||
@doc """
|
||||
Returns the output template for yt-dlp's indexing command.
|
||||
|
||||
NOTE: playlist_index is really only useful for playlists that will never change their order.
|
||||
NOTE: I've switched back to `original_url` (from `webpage_url`) since it's started indicating
|
||||
if something is a short via the URL again
|
||||
"""
|
||||
def indexing_output_template do
|
||||
"%(.{id,title,live_status,original_url,description,aspect_ratio,duration,upload_date,timestamp,playlist_index,filename})j"
|
||||
"%(.{id,title,was_live,webpage_url,description,aspect_ratio,duration,upload_date})j"
|
||||
end
|
||||
|
||||
@doc """
|
||||
|
|
@ -129,18 +86,16 @@ defmodule Pinchflat.YtDlp.Media do
|
|||
media_id: response["id"],
|
||||
title: response["title"],
|
||||
description: response["description"],
|
||||
original_url: response["original_url"],
|
||||
livestream: !!response["live_status"] && response["live_status"] != "not_live",
|
||||
original_url: response["webpage_url"],
|
||||
livestream: !!response["was_live"],
|
||||
duration_seconds: response["duration"] && round(response["duration"]),
|
||||
short_form_content: response["original_url"] && short_form_content?(response),
|
||||
uploaded_at: response["upload_date"] && parse_uploaded_at(response),
|
||||
playlist_index: response["playlist_index"] || 0,
|
||||
predicted_media_filepath: response["filename"]
|
||||
short_form_content: response["webpage_url"] && short_form_content?(response),
|
||||
upload_date: response["upload_date"] && MetadataFileHelpers.parse_upload_date(response["upload_date"])
|
||||
}
|
||||
end
|
||||
|
||||
defp short_form_content?(response) do
|
||||
if String.contains?(response["original_url"], "/shorts/") do
|
||||
if String.contains?(response["webpage_url"], "/shorts/") do
|
||||
true
|
||||
else
|
||||
# Sometimes shorts are returned without /shorts/ in the URL,
|
||||
|
|
@ -151,29 +106,7 @@ defmodule Pinchflat.YtDlp.Media do
|
|||
#
|
||||
# These don't fail if duration or aspect_ratio are missing
|
||||
# due to Elixir's comparison semantics
|
||||
response["duration"] <= 180 && response["aspect_ratio"] <= 0.85
|
||||
end
|
||||
end
|
||||
|
||||
defp parse_uploaded_at(%{"timestamp" => ts} = response) when is_number(ts) do
|
||||
case DateTime.from_unix(ts) do
|
||||
{:ok, datetime} -> datetime
|
||||
_ -> MetadataFileHelpers.parse_upload_date(response["upload_date"])
|
||||
end
|
||||
end
|
||||
|
||||
# This field is needed before inserting into the database, but absence
|
||||
# of this field should fail at insert-time rather than here
|
||||
defp parse_uploaded_at(%{"upload_date" => nil}), do: nil
|
||||
defp parse_uploaded_at(response), do: MetadataFileHelpers.parse_upload_date(response["upload_date"])
|
||||
|
||||
defp parse_downloadable_status(response) do
|
||||
case response["live_status"] do
|
||||
status when status in ["is_live", "is_upcoming", "post_live"] -> {:ok, :ignorable}
|
||||
status when status in ["was_live", "not_live"] -> {:ok, :downloadable}
|
||||
# This preserves my tenuous support for non-youtube sources.
|
||||
nil -> {:ok, :downloadable}
|
||||
_ -> {:error, "Unknown live status: #{response["live_status"]}"}
|
||||
response["duration"] <= 60 && response["aspect_ratio"] < 0.8
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -11,34 +11,29 @@ defmodule Pinchflat.YtDlp.MediaCollection do
|
|||
|
||||
@doc """
|
||||
Returns a list of maps representing the media in the collection.
|
||||
Optionally takes a list of additional command options to pass to yt-dlp
|
||||
or configuration-related options to pass to the runner.
|
||||
|
||||
Runner Options:
|
||||
Options:
|
||||
- :file_listener_handler - a function that will be called with the path to the
|
||||
file that will be written to when yt-dlp is done. This is useful for
|
||||
setting up a file watcher to know when the file is ready to be read.
|
||||
- :use_cookies - whether or not to use user-provided cookies when fetching the media details
|
||||
|
||||
Returns {:ok, [map()]} | {:error, any, ...}.
|
||||
"""
|
||||
def get_media_attributes_for_collection(url, command_opts \\ [], addl_opts \\ []) do
|
||||
def get_media_attributes_for_collection(url, addl_opts \\ []) do
|
||||
runner = Application.get_env(:pinchflat, :yt_dlp_runner)
|
||||
# `ignore_no_formats_error` is necessary because yt-dlp will error out if
|
||||
# the first video has not released yet (ie: is a premier). We don't care about
|
||||
# available formats since we're just getting the media details
|
||||
all_command_opts = [:simulate, :skip_download, :ignore_no_formats_error, :no_warnings] ++ command_opts
|
||||
use_cookies = Keyword.get(addl_opts, :use_cookies, false)
|
||||
command_opts = [:simulate, :skip_download, :ignore_no_formats_error, :no_warnings]
|
||||
output_template = YtDlpMedia.indexing_output_template()
|
||||
output_filepath = FilesystemUtils.generate_metadata_tmpfile(:json)
|
||||
file_listener_handler = Keyword.get(addl_opts, :file_listener_handler, false)
|
||||
runner_opts = [output_filepath: output_filepath, use_cookies: use_cookies]
|
||||
action = :get_media_attributes_for_collection
|
||||
|
||||
if file_listener_handler do
|
||||
file_listener_handler.(output_filepath)
|
||||
end
|
||||
|
||||
case backend_runner().run(url, action, all_command_opts, output_template, runner_opts) do
|
||||
case runner.run(url, command_opts, output_template, output_filepath: output_filepath) do
|
||||
{:ok, output} ->
|
||||
parsed_lines =
|
||||
output
|
||||
|
|
@ -69,7 +64,7 @@ defmodule Pinchflat.YtDlp.MediaCollection do
|
|||
|
||||
Returns {:ok, map()} | {:error, any, ...}.
|
||||
"""
|
||||
def get_source_details(source_url, command_opts \\ [], addl_opts \\ []) do
|
||||
def get_source_details(source_url, addl_opts \\ []) do
|
||||
# `ignore_no_formats_error` is necessary because yt-dlp will error out if
|
||||
# the first video has not released yet (ie: is a premier). We don't care about
|
||||
# available formats since we're just getting the source details
|
||||
|
|
@ -80,15 +75,13 @@ defmodule Pinchflat.YtDlp.MediaCollection do
|
|||
playlist_end: 1
|
||||
]
|
||||
|
||||
all_command_opts = default_opts ++ command_opts
|
||||
command_opts = default_opts ++ addl_opts
|
||||
output_template = "%(.{channel,channel_id,playlist_id,playlist_title,filename})j"
|
||||
action = :get_source_details
|
||||
|
||||
with {:ok, output} <- backend_runner().run(source_url, action, all_command_opts, output_template, addl_opts),
|
||||
with {:ok, output} <- backend_runner().run(source_url, command_opts, output_template),
|
||||
{:ok, parsed_json} <- Phoenix.json_library().decode(output) do
|
||||
{:ok, format_source_details(parsed_json)}
|
||||
else
|
||||
{:error, %Jason.DecodeError{}} -> {:error, "Error decoding JSON response"}
|
||||
err -> err
|
||||
end
|
||||
end
|
||||
|
|
@ -106,25 +99,13 @@ defmodule Pinchflat.YtDlp.MediaCollection do
|
|||
as a compressed blob for possible future use. That's why it's not getting formatted like
|
||||
`get_source_details/1`
|
||||
|
||||
! IMPORTANT ! - you'll always want to set `playlist_items: int` in `addl_opts.
|
||||
This is great if you want to also return details about the videos in the playlists,
|
||||
but it should be set in all cases to not over-fetch data.
|
||||
For channels you should usually set this to 0 since channels return all the
|
||||
metadata we need without needing to fetch the videos. On the other hand, playlists
|
||||
don't return very useful images so you can set this to 1 to get the first video's
|
||||
images, for instance.
|
||||
|
||||
Returns {:ok, map()} | {:error, any, ...}.
|
||||
"""
|
||||
def get_source_metadata(source_url, command_opts, addl_opts \\ []) do
|
||||
# This only validates that the `playlist_items` key is present. It's otherwise unused
|
||||
_playlist_items = Keyword.fetch!(command_opts, :playlist_items)
|
||||
|
||||
all_command_opts = [:skip_download] ++ command_opts
|
||||
def get_source_metadata(source_url, addl_opts \\ []) do
|
||||
opts = [playlist_items: 0] ++ addl_opts
|
||||
output_template = "playlist:%()j"
|
||||
action = :get_source_metadata
|
||||
|
||||
with {:ok, output} <- backend_runner().run(source_url, action, all_command_opts, output_template, addl_opts),
|
||||
with {:ok, output} <- backend_runner().run(source_url, opts, output_template),
|
||||
{:ok, parsed_json} <- Phoenix.json_library().decode(output) do
|
||||
{:ok, parsed_json}
|
||||
else
|
||||
|
|
|
|||
|
|
@ -1,44 +0,0 @@
|
|||
defmodule Pinchflat.YtDlp.UpdateWorker do
|
||||
@moduledoc false
|
||||
|
||||
use Oban.Worker,
|
||||
queue: :local_data,
|
||||
tags: ["local_data"]
|
||||
|
||||
require Logger
|
||||
|
||||
alias __MODULE__
|
||||
alias Pinchflat.Settings
|
||||
|
||||
@doc """
|
||||
Starts the yt-dlp update worker. Does not attach it to a task like `kickoff_with_task/2`
|
||||
|
||||
Returns {:ok, %Oban.Job{}} | {:error, %Ecto.Changeset{}}
|
||||
"""
|
||||
def kickoff do
|
||||
Oban.insert(UpdateWorker.new(%{}))
|
||||
end
|
||||
|
||||
@doc """
|
||||
Updates yt-dlp and saves the version to the settings.
|
||||
|
||||
This worker is scheduled to run via the Oban Cron plugin as well as on app boot.
|
||||
|
||||
Returns :ok
|
||||
"""
|
||||
@impl Oban.Worker
|
||||
def perform(%Oban.Job{}) do
|
||||
Logger.info("Updating yt-dlp")
|
||||
|
||||
yt_dlp_runner().update()
|
||||
|
||||
{:ok, yt_dlp_version} = yt_dlp_runner().version()
|
||||
Settings.set(yt_dlp_version: yt_dlp_version)
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
defp yt_dlp_runner do
|
||||
Application.get_env(:pinchflat, :yt_dlp_runner)
|
||||
end
|
||||
end
|
||||
|
|
@ -6,8 +6,7 @@ defmodule Pinchflat.YtDlp.YtDlpCommandRunner do
|
|||
yt-dlp command.
|
||||
"""
|
||||
|
||||
@callback run(binary(), atom(), keyword(), binary()) :: {:ok, binary()} | {:error, binary(), integer()}
|
||||
@callback run(binary(), atom(), keyword(), binary(), keyword()) :: {:ok, binary()} | {:error, binary(), integer()}
|
||||
@callback run(binary(), keyword(), binary()) :: {:ok, binary()} | {:error, binary(), integer()}
|
||||
@callback run(binary(), keyword(), binary(), keyword()) :: {:ok, binary()} | {:error, binary(), integer()}
|
||||
@callback version() :: {:ok, binary()} | {:error, binary()}
|
||||
@callback update() :: {:ok, binary()} | {:error, binary()}
|
||||
end
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ defmodule PinchflatWeb do
|
|||
layouts: [html: PinchflatWeb.Layouts]
|
||||
|
||||
import Plug.Conn
|
||||
use Gettext, backend: PinchflatWeb.Gettext
|
||||
import PinchflatWeb.Gettext
|
||||
|
||||
alias Pinchflat.Settings
|
||||
alias PinchflatWeb.Layouts
|
||||
|
|
@ -94,13 +94,12 @@ defmodule PinchflatWeb do
|
|||
# HTML escaping functionality
|
||||
import Phoenix.HTML
|
||||
# Core UI components and translation
|
||||
use Gettext, backend: PinchflatWeb.Gettext
|
||||
import PinchflatWeb.Gettext
|
||||
import PinchflatWeb.CoreComponents
|
||||
import PinchflatWeb.CustomComponents.TabComponents
|
||||
import PinchflatWeb.CustomComponents.TextComponents
|
||||
import PinchflatWeb.CustomComponents.TableComponents
|
||||
import PinchflatWeb.CustomComponents.ButtonComponents
|
||||
import Pinchflat.Utils.StringUtils, only: [double_brace: 1]
|
||||
|
||||
alias Pinchflat.Settings
|
||||
alias Pinchflat.Utils.StringUtils
|
||||
|
|
|
|||
|
|
@ -15,10 +15,10 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
Icons are provided by [heroicons](https://heroicons.com). See `icon/1` for usage.
|
||||
"""
|
||||
use Phoenix.Component, global_prefixes: ~w(x-)
|
||||
use Gettext, backend: PinchflatWeb.Gettext
|
||||
|
||||
import PinchflatWeb.Gettext
|
||||
|
||||
alias Phoenix.LiveView.JS
|
||||
alias PinchflatWeb.CustomComponents.TextComponents
|
||||
|
||||
@doc """
|
||||
Renders a modal.
|
||||
|
|
@ -81,7 +81,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
</button>
|
||||
</div>
|
||||
<div id={"#{@id}-content"}>
|
||||
{render_slot(@inner_block)}
|
||||
<%= render_slot(@inner_block) %>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -125,9 +125,9 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
]}>
|
||||
<main>
|
||||
<h5 :if={@title} class="mb-2 text-lg font-bold">
|
||||
{@title}
|
||||
<%= @title %>
|
||||
</h5>
|
||||
<p class="mt-2 text-md leading-5 opacity-80">{msg}</p>
|
||||
<p class="mt-2 text-md leading-5 opacity-80"><%= msg %></p>
|
||||
</main>
|
||||
<button
|
||||
type="button"
|
||||
|
|
@ -207,9 +207,9 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
def simple_form(assigns) do
|
||||
~H"""
|
||||
<.form :let={f} for={@for} as={@as} {@rest}>
|
||||
{render_slot(@inner_block, f)}
|
||||
<%= render_slot(@inner_block, f) %>
|
||||
<div :for={action <- @actions} class="mt-2 flex items-center justify-between gap-6">
|
||||
{render_slot(action, f)}
|
||||
<%= render_slot(action, f) %>
|
||||
</div>
|
||||
</.form>
|
||||
"""
|
||||
|
|
@ -296,11 +296,11 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
class={["rounded focus:ring-0", @inputclass]}
|
||||
{@rest}
|
||||
/>
|
||||
{@label}
|
||||
<span :if={@label_suffix} class="text-xs text-bodydark">{@label_suffix}</span>
|
||||
<%= @label %>
|
||||
<span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
|
||||
</label>
|
||||
<.help :if={@help}>{if @html_help, do: Phoenix.HTML.raw(@help), else: @help}</.help>
|
||||
<.error :for={msg <- @errors}>{msg}</.error>
|
||||
<.help :if={@help}><%= if @html_help, do: Phoenix.HTML.raw(@help), else: @help %></.help>
|
||||
<.error :for={msg <- @errors}><%= msg %></.error>
|
||||
</div>
|
||||
"""
|
||||
end
|
||||
|
|
@ -309,7 +309,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
~H"""
|
||||
<div phx-feedback-for={@name}>
|
||||
<.label for={@id}>
|
||||
{@label}<span :if={@label_suffix} class="text-xs text-bodydark">{@label_suffix}</span>
|
||||
<%= @label %><span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
|
||||
</.label>
|
||||
<section class="grid grid-cols-1 gap-2 md:grid-cols-2 max-w-prose mb-4 ml-1">
|
||||
<div :for={{option_name, option_value} <- @options} class="flex items-center">
|
||||
|
|
@ -322,12 +322,12 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
class={["rounded focus:ring-offset-0 ring-offset-0 focus:ring-0 h-5 w-5 ", @inputclass]}
|
||||
/>
|
||||
<label for={"#{@id}-#{option_value}"} class="ml-2 cursor-pointer select-none">
|
||||
{option_name}
|
||||
<%= option_name %>
|
||||
</label>
|
||||
</div>
|
||||
</section>
|
||||
<.help :if={@help}>{if @html_help, do: Phoenix.HTML.raw(@help), else: @help}</.help>
|
||||
<.error :for={msg <- @errors}>{msg}</.error>
|
||||
<.help :if={@help}><%= if @html_help, do: Phoenix.HTML.raw(@help), else: @help %></.help>
|
||||
<.error :for={msg <- @errors}><%= msg %></.error>
|
||||
</div>
|
||||
"""
|
||||
end
|
||||
|
|
@ -339,15 +339,14 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
end)
|
||||
|
||||
~H"""
|
||||
<div x-data={"{ enabled: #{@checked} }"} class="" phx-update="ignore" id={"#{@id}-wrapper"}>
|
||||
<.label :if={@label} for={@id}>
|
||||
{@label}
|
||||
<span :if={@label_suffix} class="text-xs text-bodydark">{@label_suffix}</span>
|
||||
<div x-data={"{ enabled: #{@checked}}"}>
|
||||
<.label for={@id}>
|
||||
<%= @label %>
|
||||
<span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
|
||||
</.label>
|
||||
<div class="relative flex flex-col">
|
||||
<div class="relative">
|
||||
<input type="hidden" id={@id} name={@name} x-bind:value="enabled" {@rest} />
|
||||
<%!-- This triggers a `change` event on the hidden input when the toggle is clicked --%>
|
||||
<div class="inline-block cursor-pointer" @click={"enabled = !enabled; dispatchFor('#{@id}', 'change')"}>
|
||||
<div class="inline-block cursor-pointer" @click="enabled = !enabled">
|
||||
<div x-bind:class="enabled && '!bg-primary'" class="block h-8 w-14 rounded-full bg-black"></div>
|
||||
<div
|
||||
x-bind:class="enabled && '!right-1 !translate-x-full'"
|
||||
|
|
@ -358,8 +357,8 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
>
|
||||
</div>
|
||||
</div>
|
||||
<.help :if={@help}>{if @html_help, do: Phoenix.HTML.raw(@help), else: @help}</.help>
|
||||
<.error :for={msg <- @errors}>{msg}</.error>
|
||||
<.help :if={@help}><%= if @html_help, do: Phoenix.HTML.raw(@help), else: @help %></.help>
|
||||
<.error :for={msg <- @errors}><%= msg %></.error>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
|
|
@ -369,7 +368,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
~H"""
|
||||
<div phx-feedback-for={@name}>
|
||||
<.label :if={@label} for={@id}>
|
||||
{@label}<span :if={@label_suffix} class="text-xs text-bodydark">{@label_suffix}</span>
|
||||
<%= @label %><span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
|
||||
</.label>
|
||||
<div class="flex">
|
||||
<select
|
||||
|
|
@ -384,13 +383,13 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
multiple={@multiple}
|
||||
{@rest}
|
||||
>
|
||||
<option :if={@prompt} value="">{@prompt}</option>
|
||||
{Phoenix.HTML.Form.options_for_select(@options, @value)}
|
||||
<option :if={@prompt} value=""><%= @prompt %></option>
|
||||
<%= Phoenix.HTML.Form.options_for_select(@options, @value) %>
|
||||
</select>
|
||||
{render_slot(@inner_block)}
|
||||
<%= render_slot(@inner_block) %>
|
||||
</div>
|
||||
<.help :if={@help}>{if @html_help, do: Phoenix.HTML.raw(@help), else: @help}</.help>
|
||||
<.error :for={msg <- @errors}>{msg}</.error>
|
||||
<.help :if={@help}><%= if @html_help, do: Phoenix.HTML.raw(@help), else: @help %></.help>
|
||||
<.error :for={msg <- @errors}><%= msg %></.error>
|
||||
</div>
|
||||
"""
|
||||
end
|
||||
|
|
@ -399,7 +398,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
~H"""
|
||||
<div phx-feedback-for={@name}>
|
||||
<.label for={@id}>
|
||||
{@label}<span :if={@label_suffix} class="text-xs text-bodydark">{@label_suffix}</span>
|
||||
<%= @label %><span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
|
||||
</.label>
|
||||
<textarea
|
||||
id={@id}
|
||||
|
|
@ -413,8 +412,8 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
]}
|
||||
{@rest}
|
||||
><%= Phoenix.HTML.Form.normalize_value("textarea", @value) %></textarea>
|
||||
<.help :if={@help}>{if @html_help, do: Phoenix.HTML.raw(@help), else: @help}</.help>
|
||||
<.error :for={msg <- @errors}>{msg}</.error>
|
||||
<.help :if={@help}><%= if @html_help, do: Phoenix.HTML.raw(@help), else: @help %></.help>
|
||||
<.error :for={msg <- @errors}><%= msg %></.error>
|
||||
</div>
|
||||
"""
|
||||
end
|
||||
|
|
@ -424,7 +423,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
~H"""
|
||||
<div phx-feedback-for={@name}>
|
||||
<.label for={@id}>
|
||||
{@label}<span :if={@label_suffix} class="text-xs text-bodydark">{@label_suffix}</span>
|
||||
<%= @label %><span :if={@label_suffix} class="text-xs text-bodydark"><%= @label_suffix %></span>
|
||||
</.label>
|
||||
<div class="flex items-center">
|
||||
<input
|
||||
|
|
@ -441,10 +440,10 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
]}
|
||||
{@rest}
|
||||
/>
|
||||
{render_slot(@input_append)}
|
||||
<%= render_slot(@input_append) %>
|
||||
</div>
|
||||
<.help :if={@help}>{if @html_help, do: Phoenix.HTML.raw(@help), else: @help}</.help>
|
||||
<.error :for={msg <- @errors}>{msg}</.error>
|
||||
<.help :if={@help}><%= if @html_help, do: Phoenix.HTML.raw(@help), else: @help %></.help>
|
||||
<.error :for={msg <- @errors}><%= msg %></.error>
|
||||
</div>
|
||||
"""
|
||||
end
|
||||
|
|
@ -457,7 +456,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
def help(assigns) do
|
||||
~H"""
|
||||
<p class="mt-1 text-sm leading-5">
|
||||
{render_slot(@inner_block)}
|
||||
<%= render_slot(@inner_block) %>
|
||||
</p>
|
||||
"""
|
||||
end
|
||||
|
|
@ -471,7 +470,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
def label(assigns) do
|
||||
~H"""
|
||||
<label for={@for} class="mt-5 mb-2 inline-block text-md font-medium text-black dark:text-white">
|
||||
{render_slot(@inner_block)}
|
||||
<%= render_slot(@inner_block) %>
|
||||
</label>
|
||||
"""
|
||||
end
|
||||
|
|
@ -485,7 +484,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
~H"""
|
||||
<p class="mt-1 mb-5 flex gap-3 text-md leading-6 text-rose-600 phx-no-feedback:hidden">
|
||||
<.icon name="hero-exclamation-circle-mini" class="mt-0.5 h-5 w-5 flex-none" />
|
||||
{render_slot(@inner_block)}
|
||||
<%= render_slot(@inner_block) %>
|
||||
</p>
|
||||
"""
|
||||
end
|
||||
|
|
@ -504,13 +503,13 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
<header class={[@actions != [] && "flex items-center justify-between gap-6", @class]}>
|
||||
<div>
|
||||
<h1 class="text-lg font-semibold leading-8 text-zinc-800">
|
||||
{render_slot(@inner_block)}
|
||||
<%= render_slot(@inner_block) %>
|
||||
</h1>
|
||||
<p :if={@subtitle != []} class="mt-2 text-sm leading-6 text-zinc-600">
|
||||
{render_slot(@subtitle)}
|
||||
<%= render_slot(@subtitle) %>
|
||||
</p>
|
||||
</div>
|
||||
<div class="flex-none">{render_slot(@actions)}</div>
|
||||
<div class="flex-none"><%= render_slot(@actions) %></div>
|
||||
</header>
|
||||
"""
|
||||
end
|
||||
|
|
@ -550,9 +549,9 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
<table class="w-[40rem] mt-11 sm:w-full">
|
||||
<thead class="text-sm text-left leading-6 text-zinc-500">
|
||||
<tr>
|
||||
<th :for={col <- @col} class="p-0 pb-4 pr-6 font-normal">{col[:label]}</th>
|
||||
<th :for={col <- @col} class="p-0 pb-4 pr-6 font-normal"><%= col[:label] %></th>
|
||||
<th :if={@action != []} class="relative p-0 pb-4">
|
||||
<span class="sr-only">{gettext("Actions")}</span>
|
||||
<span class="sr-only"><%= gettext("Actions") %></span>
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
|
|
@ -570,7 +569,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
<div class="block py-4 pr-6">
|
||||
<span class="absolute -inset-y-px right-0 -left-4 group-hover:bg-zinc-50 sm:rounded-l-xl" />
|
||||
<span class={["relative", i == 0 && "font-semibold text-zinc-900"]}>
|
||||
{render_slot(col, @row_item.(row))}
|
||||
<%= render_slot(col, @row_item.(row)) %>
|
||||
</span>
|
||||
</div>
|
||||
</td>
|
||||
|
|
@ -578,7 +577,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
<div class="relative whitespace-nowrap py-4 text-right text-sm font-medium">
|
||||
<span class="absolute -inset-y-px -right-4 left-0 group-hover:bg-zinc-50 sm:rounded-r-xl" />
|
||||
<span :for={action <- @action} class="relative ml-4 font-semibold leading-6 text-zinc-900 hover:text-zinc-700">
|
||||
{render_slot(action, @row_item.(row))}
|
||||
<%= render_slot(action, @row_item.(row)) %>
|
||||
</span>
|
||||
</div>
|
||||
</td>
|
||||
|
|
@ -607,8 +606,8 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
<div class="mt-2 mb-14">
|
||||
<dl class="-my-4 divide-y dark:divide-strokedark">
|
||||
<div :for={item <- @item} class="flex gap-4 py-4 text-sm leading-6 sm:gap-8">
|
||||
<dt class="w-1/4 flex-none dark:text-white">{item.title}</dt>
|
||||
<dd class="dark:text-white">{render_slot(item)}</dd>
|
||||
<dt class="w-1/4 flex-none dark:text-white"><%= item.title %></dt>
|
||||
<dd class="dark:text-white"><%= render_slot(item) %></dd>
|
||||
</div>
|
||||
</dl>
|
||||
</div>
|
||||
|
|
@ -643,13 +642,9 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
~H"""
|
||||
<ul>
|
||||
<li :for={{k, v} <- @iterable_attributes} class="mb-2 w-2/3">
|
||||
<strong>{k}:</strong>
|
||||
<strong><%= k %>:</strong>
|
||||
<code class="inline-block text-sm font-mono text-gray p-0.5 mx-0.5">
|
||||
<%= if is_binary(v) && URI.parse(v).scheme && URI.parse(v).scheme =~ "http" do %>
|
||||
<TextComponents.inline_link href={v}>{v}</TextComponents.inline_link>
|
||||
<% else %>
|
||||
{v}
|
||||
<% end %>
|
||||
<%= v %>
|
||||
</code>
|
||||
</li>
|
||||
</ul>
|
||||
|
|
@ -671,7 +666,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
<div class="mt-16">
|
||||
<.link href={@href} class="text-sm font-semibold leading-6 text-zinc-900 hover:text-zinc-700">
|
||||
<.icon name="hero-arrow-left-solid" class="h-3 w-3" />
|
||||
{render_slot(@inner_block)}
|
||||
<%= render_slot(@inner_block) %>
|
||||
</.link>
|
||||
</div>
|
||||
"""
|
||||
|
|
@ -699,7 +694,7 @@ defmodule PinchflatWeb.CoreComponents do
|
|||
attr :class, :string, default: nil
|
||||
attr :rest, :global
|
||||
|
||||
def icon(assigns) do
|
||||
def icon(%{name: "hero-" <> _} = assigns) do
|
||||
~H"""
|
||||
<span class={[@name, @class]} {@rest} />
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ defmodule PinchflatWeb.CustomComponents.ButtonComponents do
|
|||
use Phoenix.Component, global_prefixes: ~w(x-)
|
||||
|
||||
alias PinchflatWeb.CoreComponents
|
||||
alias PinchflatWeb.CustomComponents.TextComponents
|
||||
|
||||
@doc """
|
||||
Render a button
|
||||
|
|
@ -27,7 +26,7 @@ defmodule PinchflatWeb.CustomComponents.ButtonComponents do
|
|||
~H"""
|
||||
<button
|
||||
class={[
|
||||
"text-center font-medium text-white whitespace-nowrap",
|
||||
"text-center font-medium text-white",
|
||||
"#{@rounding} inline-flex items-center justify-center px-8 py-4",
|
||||
"#{@color}",
|
||||
"hover:bg-opacity-90 lg:px-8 xl:px-10",
|
||||
|
|
@ -38,7 +37,7 @@ defmodule PinchflatWeb.CustomComponents.ButtonComponents do
|
|||
disabled={@disabled}
|
||||
{@rest}
|
||||
>
|
||||
{render_slot(@inner_block)}
|
||||
<%= render_slot(@inner_block) %>
|
||||
</button>
|
||||
"""
|
||||
end
|
||||
|
|
@ -67,7 +66,7 @@ defmodule PinchflatWeb.CustomComponents.ButtonComponents do
|
|||
"font-medium text-white hover:bg-opacity-95"
|
||||
]}
|
||||
>
|
||||
{@text}
|
||||
<%= @text %>
|
||||
<CoreComponents.icon
|
||||
name="hero-chevron-down"
|
||||
class="fill-current duration-200 ease-linear mt-1"
|
||||
|
|
@ -82,7 +81,7 @@ defmodule PinchflatWeb.CustomComponents.ButtonComponents do
|
|||
<ul class="flex flex-col">
|
||||
<li :for={option <- @option}>
|
||||
<span class="flex px-5 py-2 font-medium text-bodydark2 hover:text-white cursor-pointer">
|
||||
{render_slot(option)}
|
||||
<%= render_slot(option) %>
|
||||
</span>
|
||||
</li>
|
||||
</ul>
|
||||
|
|
@ -105,7 +104,7 @@ defmodule PinchflatWeb.CustomComponents.ButtonComponents do
|
|||
|
||||
def icon_button(assigns) do
|
||||
~H"""
|
||||
<TextComponents.tooltip position="bottom" tooltip={@tooltip} tooltip_class="text-nowrap">
|
||||
<div class="group relative inline-block">
|
||||
<button
|
||||
class={[
|
||||
"flex justify-center items-center rounded-lg ",
|
||||
|
|
@ -118,7 +117,18 @@ defmodule PinchflatWeb.CustomComponents.ButtonComponents do
|
|||
>
|
||||
<CoreComponents.icon name={@icon_name} class="text-stroke" />
|
||||
</button>
|
||||
</TextComponents.tooltip>
|
||||
<div
|
||||
:if={@tooltip}
|
||||
class={[
|
||||
"hidden absolute left-1/2 top-full z-20 mt-3 -translate-x-1/2 whitespace-nowrap rounded-md",
|
||||
"px-4.5 py-1.5 text-sm font-medium opacity-0 drop-shadow-4 group-hover:opacity-100 group-hover:block bg-meta-4"
|
||||
]}
|
||||
>
|
||||
<span class="border-light absolute -top-1 left-1/2 -z-10 h-2 w-2 -translate-x-1/2 rotate-45 rounded-sm bg-meta-4">
|
||||
</span>
|
||||
<span><%= @tooltip %></span>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -34,16 +34,16 @@ defmodule PinchflatWeb.CustomComponents.TabComponents do
|
|||
x-bind:class={"openTab === '#{tab.id}' ? activeClasses : inactiveClasses"}
|
||||
class="border-b-2 py-4 w-full sm:w-fit text-sm font-medium hover:text-meta-5 md:text-base"
|
||||
>
|
||||
<span class="text-xl">{tab.title}</span>
|
||||
<span class="text-xl"><%= tab.title %></span>
|
||||
</a>
|
||||
</div>
|
||||
<div class="mx-4 my-4 lg:my-0 flex gap-5 sm:gap-10 items-center">
|
||||
{render_slot(@tab_append)}
|
||||
<%= render_slot(@tab_append) %>
|
||||
</div>
|
||||
</header>
|
||||
<div class="mt-4 min-h-60 overflow-x-auto">
|
||||
<div class="mt-4 min-h-60">
|
||||
<div :for={tab <- @tab} x-show={"openTab === '#{tab.id}'"} class="font-medium leading-relaxed">
|
||||
{render_slot(tab)}
|
||||
<%= render_slot(tab) %>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -2,8 +2,7 @@ defmodule PinchflatWeb.CustomComponents.TableComponents do
|
|||
@moduledoc false
|
||||
use Phoenix.Component
|
||||
|
||||
import PinchflatWeb.CoreComponents
|
||||
import PinchflatWeb.CustomComponents.TextComponents
|
||||
alias PinchflatWeb.CoreComponents
|
||||
|
||||
@doc """
|
||||
Renders a table component with the given rows and columns.
|
||||
|
|
@ -16,8 +15,6 @@ defmodule PinchflatWeb.CustomComponents.TableComponents do
|
|||
"""
|
||||
attr :rows, :list, required: true
|
||||
attr :table_class, :string, default: ""
|
||||
attr :sort_key, :string, default: nil
|
||||
attr :sort_direction, :string, default: nil
|
||||
|
||||
attr :row_item, :any,
|
||||
default: &Function.identity/1,
|
||||
|
|
@ -26,7 +23,6 @@ defmodule PinchflatWeb.CustomComponents.TableComponents do
|
|||
slot :col, required: true do
|
||||
attr :label, :string
|
||||
attr :class, :string
|
||||
attr :sort_key, :string
|
||||
end
|
||||
|
||||
def table(assigns) do
|
||||
|
|
@ -34,20 +30,8 @@ defmodule PinchflatWeb.CustomComponents.TableComponents do
|
|||
<table class={["w-full table-auto bg-boxdark", @table_class]}>
|
||||
<thead>
|
||||
<tr class="text-left bg-meta-4">
|
||||
<th
|
||||
:for={col <- @col}
|
||||
class={["px-4 py-4 font-medium text-white", col[:sort_key] && "cursor-pointer"]}
|
||||
phx-click={col[:sort_key] && "sort_update"}
|
||||
phx-value-sort_key={col[:sort_key]}
|
||||
>
|
||||
<div class="relative">
|
||||
{col[:label]}
|
||||
<.icon
|
||||
:if={to_string(@sort_key) == col[:sort_key]}
|
||||
name={if @sort_direction == :asc, do: "hero-chevron-up", else: "hero-chevron-down"}
|
||||
class="w-3 h-3 mt-2 ml-1 absolute"
|
||||
/>
|
||||
</div>
|
||||
<th :for={col <- @col} class="px-4 py-4 font-medium text-white xl:pl-11">
|
||||
<%= col[:label] %>
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
|
|
@ -56,11 +40,11 @@ defmodule PinchflatWeb.CustomComponents.TableComponents do
|
|||
<td
|
||||
:for={col <- @col}
|
||||
class={[
|
||||
"px-4 py-5",
|
||||
"px-4 py-5 pl-9 xl:pl-11",
|
||||
col[:class]
|
||||
]}
|
||||
>
|
||||
{render_slot(col, @row_item.(row))}
|
||||
<%= render_slot(col, @row_item.(row)) %>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
|
|
@ -85,32 +69,30 @@ defmodule PinchflatWeb.CustomComponents.TableComponents do
|
|||
<li>
|
||||
<span
|
||||
class={[
|
||||
"pagination-prev h-8 w-8 items-center justify-center rounded",
|
||||
"flex h-8 w-8 items-center justify-center rounded",
|
||||
@page_number != 1 && "cursor-pointer hover:bg-primary hover:text-white",
|
||||
@page_number <= 1 && "cursor-not-allowed"
|
||||
@page_number == 1 && "cursor-not-allowed"
|
||||
]}
|
||||
phx-click={@page_number != 1 && "page_change"}
|
||||
phx-value-direction="dec"
|
||||
>
|
||||
<.icon name="hero-chevron-left" />
|
||||
<CoreComponents.icon name="hero-chevron-left" />
|
||||
</span>
|
||||
</li>
|
||||
<li>
|
||||
<span class="mx-2">
|
||||
Page <.localized_number number={@page_number} /> of <.localized_number number={@total_pages} />
|
||||
</span>
|
||||
<span class="mx-2">Page <%= @page_number %> of <%= @total_pages %></span>
|
||||
</li>
|
||||
<li>
|
||||
<span
|
||||
class={[
|
||||
"pagination-next flex h-8 w-8 items-center justify-center rounded",
|
||||
"flex h-8 w-8 items-center justify-center rounded",
|
||||
@page_number != @total_pages && "cursor-pointer hover:bg-primary hover:text-white",
|
||||
@page_number >= @total_pages && "cursor-not-allowed"
|
||||
@page_number == @total_pages && "cursor-not-allowed"
|
||||
]}
|
||||
phx-click={@page_number != @total_pages && "page_change"}
|
||||
phx-value-direction="inc"
|
||||
>
|
||||
<.icon name="hero-chevron-right" />
|
||||
<CoreComponents.icon name="hero-chevron-right" />
|
||||
</span>
|
||||
</li>
|
||||
</ul>
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ defmodule PinchflatWeb.CustomComponents.TextComponents do
|
|||
@moduledoc false
|
||||
use Phoenix.Component
|
||||
|
||||
alias Pinchflat.Utils.NumberUtils
|
||||
alias PinchflatWeb.CoreComponents
|
||||
|
||||
@doc """
|
||||
|
|
@ -13,7 +12,7 @@ defmodule PinchflatWeb.CustomComponents.TextComponents do
|
|||
def inline_code(assigns) do
|
||||
~H"""
|
||||
<code class="inline-block text-sm font-mono text-gray bg-boxdark rounded-md p-0.5 mx-0.5 text-nowrap">
|
||||
{render_slot(@inner_block)}
|
||||
<%= render_slot(@inner_block) %>
|
||||
</code>
|
||||
"""
|
||||
end
|
||||
|
|
@ -27,7 +26,7 @@ defmodule PinchflatWeb.CustomComponents.TextComponents do
|
|||
def inline_link(assigns) do
|
||||
~H"""
|
||||
<.link href={@href} target="_blank" class="text-blue-500 hover:text-blue-300">
|
||||
{render_slot(@inner_block)}
|
||||
<%= render_slot(@inner_block) %>
|
||||
</.link>
|
||||
"""
|
||||
end
|
||||
|
|
@ -42,7 +41,7 @@ defmodule PinchflatWeb.CustomComponents.TextComponents do
|
|||
def subtle_link(assigns) do
|
||||
~H"""
|
||||
<.link href={@href} target={@target} class="underline decoration-bodydark decoration-1 hover:decoration-white">
|
||||
{render_slot(@inner_block)}
|
||||
<%= render_slot(@inner_block) %>
|
||||
</.link>
|
||||
"""
|
||||
end
|
||||
|
|
@ -63,47 +62,20 @@ defmodule PinchflatWeb.CustomComponents.TextComponents do
|
|||
end
|
||||
|
||||
@doc """
|
||||
Renders a block of text with each line broken into a separate span and links highlighted.
|
||||
Renders a block of text with each line broken into a separate span.
|
||||
"""
|
||||
attr :text, :string, required: true
|
||||
|
||||
def render_description(assigns) do
|
||||
formatted_text =
|
||||
Regex.split(~r{https?://\S+}, assigns.text, include_captures: true)
|
||||
|> Enum.map(fn
|
||||
"http" <> _ = url -> {:url, url}
|
||||
text -> Regex.split(~r{\n}, text, include_captures: true, trim: true)
|
||||
end)
|
||||
def break_on_newline(assigns) do
|
||||
broken_text =
|
||||
assigns.text
|
||||
|> String.split("\n", trim: false)
|
||||
|> Enum.intersperse(Phoenix.HTML.Tag.tag(:span, class: "inline-block mt-2"))
|
||||
|
||||
assigns = Map.put(assigns, :text, formatted_text)
|
||||
assigns = Map.put(assigns, :text, broken_text)
|
||||
|
||||
~H"""
|
||||
<span>
|
||||
<.rendered_description_line :for={line <- @text} content={line} />
|
||||
</span>
|
||||
"""
|
||||
end
|
||||
|
||||
defp rendered_description_line(%{content: {:url, url}} = assigns) do
|
||||
assigns = Map.put(assigns, :url, url)
|
||||
|
||||
~H"""
|
||||
<a href={@url} target="_blank" class="text-blue-500 hover:text-blue-300">
|
||||
{@url}
|
||||
</a>
|
||||
"""
|
||||
end
|
||||
|
||||
defp rendered_description_line(%{content: list_of_content} = assigns) do
|
||||
assigns = Map.put(assigns, :list_of_content, list_of_content)
|
||||
|
||||
~H"""
|
||||
<span
|
||||
:for={inner_content <- @list_of_content}
|
||||
class={[if(inner_content == "\n", do: "block", else: "mt-2 inline-block")]}
|
||||
>
|
||||
{inner_content}
|
||||
</span>
|
||||
<span><%= @text %></span>
|
||||
"""
|
||||
end
|
||||
|
||||
|
|
@ -119,107 +91,7 @@ defmodule PinchflatWeb.CustomComponents.TextComponents do
|
|||
assigns = Map.put(assigns, :timezone, timezone)
|
||||
|
||||
~H"""
|
||||
<time>{Calendar.strftime(Timex.Timezone.convert(@datetime, @timezone), @format)}</time>
|
||||
"""
|
||||
end
|
||||
|
||||
@doc """
|
||||
Renders a localized number using the Intl.NumberFormat API, falling back to the raw number if needed
|
||||
"""
|
||||
attr :number, :any, required: true
|
||||
|
||||
def localized_number(assigns) do
|
||||
~H"""
|
||||
<span x-data x-text={"Intl.NumberFormat().format(#{@number})"}>{@number}</span>
|
||||
"""
|
||||
end
|
||||
|
||||
@doc """
|
||||
Renders a word with a suffix if the count is not 1
|
||||
"""
|
||||
attr :word, :string, required: true
|
||||
attr :count, :integer, required: true
|
||||
attr :suffix, :string, default: "s"
|
||||
|
||||
def pluralize(assigns) do
|
||||
~H"""
|
||||
{@word}{if @count == 1, do: "", else: @suffix}
|
||||
"""
|
||||
end
|
||||
|
||||
@doc """
|
||||
Renders a human-readable byte size
|
||||
"""
|
||||
|
||||
attr :byte_size, :integer, required: true
|
||||
|
||||
def readable_filesize(assigns) do
|
||||
{num, suffix} = NumberUtils.human_byte_size(assigns.byte_size, precision: 2)
|
||||
|
||||
assigns =
|
||||
Map.merge(assigns, %{
|
||||
num: num,
|
||||
suffix: suffix
|
||||
})
|
||||
|
||||
~H"""
|
||||
<.localized_number number={@num} /> {@suffix}
|
||||
"""
|
||||
end
|
||||
|
||||
@doc """
|
||||
Renders a tooltip with the given content
|
||||
"""
|
||||
|
||||
attr :tooltip, :string, required: true
|
||||
attr :position, :string, default: ""
|
||||
attr :tooltip_class, :any, default: ""
|
||||
attr :tooltip_arrow_class, :any, default: ""
|
||||
slot :inner_block
|
||||
|
||||
def tooltip(%{position: "bottom-right"} = assigns) do
|
||||
~H"""
|
||||
<.tooltip tooltip={@tooltip} tooltip_class={@tooltip_class} tooltip_arrow_class={["-top-1", @tooltip_arrow_class]}>
|
||||
{render_slot(@inner_block)}
|
||||
</.tooltip>
|
||||
"""
|
||||
end
|
||||
|
||||
def tooltip(%{position: "bottom"} = assigns) do
|
||||
~H"""
|
||||
<.tooltip
|
||||
tooltip={@tooltip}
|
||||
tooltip_class={["left-1/2 -translate-x-1/2", @tooltip_class]}
|
||||
tooltip_arrow_class={["-top-1 left-1/2 -translate-x-1/2", @tooltip_arrow_class]}
|
||||
>
|
||||
{render_slot(@inner_block)}
|
||||
</.tooltip>
|
||||
"""
|
||||
end
|
||||
|
||||
def tooltip(assigns) do
|
||||
~H"""
|
||||
<div class="group relative inline-block cursor-pointer">
|
||||
<div>
|
||||
{render_slot(@inner_block)}
|
||||
</div>
|
||||
<div
|
||||
:if={@tooltip}
|
||||
class={[
|
||||
"hidden absolute top-full z-20 mt-3 whitespace-nowrap rounded-md",
|
||||
"p-1.5 text-sm font-medium opacity-0 drop-shadow-4 group-hover:opacity-100 group-hover:block bg-meta-4",
|
||||
"border border-form-strokedark text-wrap",
|
||||
@tooltip_class
|
||||
]}
|
||||
>
|
||||
<span class={[
|
||||
"border-t border-l border-form-strokedark absolute -z-10 h-2 w-2 rotate-45 rounded-sm bg-meta-4",
|
||||
@tooltip_arrow_class
|
||||
]}>
|
||||
</span>
|
||||
<div class="px-3">{@tooltip}</div>
|
||||
</div>
|
||||
</div>
|
||||
<time><%= Calendar.strftime(Timex.Timezone.convert(@datetime, @timezone), @format) %></time>
|
||||
"""
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -15,12 +15,11 @@ defmodule PinchflatWeb.Layouts do
|
|||
attr :text, :string, required: true
|
||||
attr :href, :any, required: true
|
||||
attr :target, :any, default: "_self"
|
||||
attr :icon_class, :string, default: ""
|
||||
|
||||
def sidebar_item(assigns) do
|
||||
~H"""
|
||||
<li class="text-bodydark1">
|
||||
<.sidebar_link icon={@icon} text={@text} href={@href} target={@target} icon_class={@icon_class} />
|
||||
<.sidebar_link icon={@icon} text={@text} href={@href} target={@target} />
|
||||
</li>
|
||||
"""
|
||||
end
|
||||
|
|
@ -62,10 +61,10 @@ defmodule PinchflatWeb.Layouts do
|
|||
x-on:click="selected = !selected"
|
||||
>
|
||||
<span class="flex items-center gap-2.5">
|
||||
<.icon name={@icon} /> {@text}
|
||||
<.icon name={@icon} /> <%= @text %>
|
||||
</span>
|
||||
<span class="text-bodydark2">
|
||||
<.icon name="hero-chevron-down" x-bind:class="{ 'rotate-180': selected }" />
|
||||
<.icon name="hero-chevron-up" x-bind:class="{ 'rotate-180': selected }" />
|
||||
</span>
|
||||
</span>
|
||||
|
||||
|
|
@ -90,7 +89,6 @@ defmodule PinchflatWeb.Layouts do
|
|||
attr :href, :any, required: true
|
||||
attr :target, :any, default: "_self"
|
||||
attr :class, :string, default: ""
|
||||
attr :icon_class, :string, default: ""
|
||||
|
||||
def sidebar_link(assigns) do
|
||||
~H"""
|
||||
|
|
@ -105,7 +103,7 @@ defmodule PinchflatWeb.Layouts do
|
|||
@class
|
||||
]}
|
||||
>
|
||||
<.icon :if={@icon} name={@icon} class={@icon_class} /> {@text}
|
||||
<.icon :if={@icon} name={@icon} /> <%= @text %>
|
||||
</.link>
|
||||
"""
|
||||
end
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
<main>
|
||||
<div class="mx-auto max-w-screen-2xl p-4 md:p-6 2xl:p-10">
|
||||
<.flash_group flash={@flash} />
|
||||
{@inner_content}
|
||||
<%= @inner_content %>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
<main>
|
||||
<div class="mx-auto max-w-screen-2xl p-4 md:p-6 2xl:p-10">
|
||||
<.flash_group flash={@flash} />
|
||||
{@inner_content}
|
||||
<%= @inner_content %>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
>
|
||||
<section>
|
||||
<div class="flex items-center justify-between gap-2 px-6 py-4">
|
||||
<a href={~p"/"} class="flex items-center">
|
||||
<a href="/" class="flex items-center">
|
||||
<img src={~p"/images/logo-2024-03-20.png"} alt="Pinchflat" class="w-auto" />
|
||||
</a>
|
||||
|
||||
|
|
@ -47,10 +47,8 @@
|
|||
text="Docs"
|
||||
target="_blank"
|
||||
href="https://github.com/kieraneglin/pinchflat/wiki"
|
||||
icon_class="scale-110"
|
||||
/>
|
||||
<.sidebar_item icon="si-github" text="Github" target="_blank" href="https://github.com/kieraneglin/pinchflat" />
|
||||
<.sidebar_item icon="si-discord" text="Discord" target="_blank" href="https://discord.gg/j7T6dCuwU4" />
|
||||
<.sidebar_item icon="hero-cog" text="Github" target="_blank" href="https://github.com/kieraneglin/pinchflat" />
|
||||
<li>
|
||||
<span
|
||||
class={[
|
||||
|
|
@ -61,7 +59,7 @@
|
|||
]}
|
||||
phx-click={show_modal("donate-modal")}
|
||||
>
|
||||
<.icon name="hero-currency-dollar" class="scale-110" /> Donate
|
||||
<.icon name="hero-currency-dollar" /> Donate
|
||||
</span>
|
||||
</li>
|
||||
<li>
|
||||
|
|
@ -69,7 +67,7 @@
|
|||
class="group relative flex items-center gap-2.5 px-4 pt-2 text-sm"
|
||||
x-on:click={"markVersionAsSeen('#{Application.spec(:pinchflat)[:vsn]}')"}
|
||||
>
|
||||
<span>Pinchflat {Application.spec(:pinchflat)[:vsn]}</span>
|
||||
<span>Pinchflat <%= Application.spec(:pinchflat)[:vsn] %></span>
|
||||
<a
|
||||
href="https://github.com/kieraneglin/pinchflat/releases"
|
||||
target="_blank"
|
||||
|
|
@ -81,7 +79,7 @@
|
|||
</a>
|
||||
</span>
|
||||
<span class="group relative flex items-center gap-2.5 px-4 pt-2 text-sm">
|
||||
yt-dlp {Settings.get!(:yt_dlp_version)}
|
||||
yt-dlp <%= Settings.get!(:yt_dlp_version) %>
|
||||
</span>
|
||||
</li>
|
||||
</ul>
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ defmodule Pinchflat.UpgradeButtonLive do
|
|||
|
||||
def render(assigns) do
|
||||
~H"""
|
||||
<form id="upgradeForm" phx-change="check_matching_text" phx-hook="supress-enter-submission">
|
||||
<form id="upgradeForm" phx-change="check_matching_text" phx-hook="supressEnterSubmission">
|
||||
<.input type="text" name="unlock-pro-textbox" value="" />
|
||||
</form>
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,6 @@
|
|||
into the text box and press the button.
|
||||
</p>
|
||||
|
||||
{live_render(@conn, Pinchflat.UpgradeButtonLive)}
|
||||
<%= live_render(@conn, Pinchflat.UpgradeButtonLive) %>
|
||||
</section>
|
||||
</.modal>
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<meta name="csrf-token" content={get_csrf_token()} />
|
||||
<.live_title>
|
||||
{assigns[:page_title] || "Pinchflat"}
|
||||
<%= assigns[:page_title] || "Pinchflat" %>
|
||||
</.live_title>
|
||||
<link phx-track-static rel="stylesheet" href={~p"/assets/app.css"} />
|
||||
<link rel="icon" type="image/x-icon" href={~p"/favicon.ico?cachebust=2024-03-20"} />
|
||||
|
|
@ -19,9 +19,8 @@
|
|||
onboarding: #{Settings.get!(:onboarding)}
|
||||
}"}
|
||||
class="dark text-bodydark bg-boxdark-2"
|
||||
data-socket-path={Path.join(Application.get_env(:pinchflat, :base_route_path), "/live")}
|
||||
>
|
||||
{@inner_content}
|
||||
<%= @inner_content %>
|
||||
|
||||
<.donate_modal conn={@conn} />
|
||||
<template x-if="!proEnabled && !onboarding">
|
||||
|
|
|
|||
|
|
@ -1,7 +1,14 @@
|
|||
defmodule PinchflatWeb.ErrorHTML do
|
||||
use PinchflatWeb, :html
|
||||
|
||||
embed_templates "error_html/*"
|
||||
# If you want to customize your error pages,
|
||||
# uncomment the embed_templates/1 call below
|
||||
# and add pages to the error directory:
|
||||
#
|
||||
# * lib/pinchflat_web/controllers/error_html/404.html.heex
|
||||
# * lib/pinchflat_web/controllers/error_html/500.html.heex
|
||||
#
|
||||
# embed_templates "error_html/*"
|
||||
|
||||
# The default is to render a plain text page based on
|
||||
# the template name. For example, "404.html" becomes
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
<section>
|
||||
<h2 class="text-title-md2 font-bold text-white">404 (not found)</h2>
|
||||
</section>
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
<section>
|
||||
<h2 class="text-title-md2 font-bold text-white">Internal Server Error</h2>
|
||||
<p class="text-body-md text-white mt-2">
|
||||
This shouldn't happen! Please make a
|
||||
<.inline_link href="https://github.com/kieraneglin/pinchflat/issues/new/choose">GitHub issue</.inline_link>
|
||||
with the following information:
|
||||
</p>
|
||||
|
||||
<ul class="list-disc ml-8 mb-8">
|
||||
<li>What you were doing when you saw this page</li>
|
||||
<li>
|
||||
Your system details and logs from
|
||||
<.inline_link href={~p"/app_info"}>app info</.inline_link>
|
||||
</li>
|
||||
<li>All the information in the textarea below (use select all + copy)</li>
|
||||
</ul>
|
||||
<textarea class="w-full min-h-96 font-mono inline-block rounded-lg" readonly>
|
||||
**Status**:
|
||||
`<%= if Map.has_key?(assigns, :status), do: @status, else: "" %>`
|
||||
|
||||
**Reason**:
|
||||
`<%= if Map.has_key?(assigns, :reason), do: inspect(@reason), else: "" %>`
|
||||
|
||||
**Stacktrace**:
|
||||
```
|
||||
<%= if Map.has_key?(assigns, :stack), do: Exception.format_stacktrace(@stack), else: "" %>
|
||||
```
|
||||
</textarea>
|
||||
</section>
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
<:option>
|
||||
<span x-data="{ copied: false }" x-on:click={~s"
|
||||
copyWithCallbacks(
|
||||
String.raw`#{Jason.Formatter.pretty_print(Phoenix.json_library().encode!(@media_item))}`,
|
||||
`#{Jason.Formatter.pretty_print(Phoenix.json_library().encode!(@media_item))}`,
|
||||
() => copied = true,
|
||||
() => copied = false
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
<div class="mb-6 flex gap-3 flex-row items-center">
|
||||
<h2 class="text-title-md2 font-bold text-black dark:text-white ml-4 truncate">
|
||||
Editing "{@media_item.title}"
|
||||
<h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">
|
||||
Editing "<%= StringUtils.truncate(@media_item.title, 35) %>"
|
||||
</h2>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
<%= if media_type(@media_item) == :video do %>
|
||||
<video controls class="max-h-128 w-full">
|
||||
<source src={~p"/media/#{@media_item.uuid}/stream?v=#{DateTime.to_unix(@media_item.updated_at)}"} type="video/mp4" />
|
||||
<video controls class="max-h-96 w-full">
|
||||
<source src={~p"/media/#{@media_item.uuid}/stream"} type="video/mp4" />
|
||||
Your browser does not support the video element.
|
||||
</video>
|
||||
<% end %>
|
||||
|
||||
<%= if media_type(@media_item) == :audio do %>
|
||||
<audio controls class="w-full">
|
||||
<source src={~p"/media/#{@media_item.uuid}/stream?v=#{DateTime.to_unix(@media_item.updated_at)}"} type="audio/mpeg" />
|
||||
<source src={~p"/media/#{@media_item.uuid}/stream"} type="audio/mpeg" />
|
||||
Your browser does not support the audio element.
|
||||
</audio>
|
||||
<% end %>
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
<div class="mb-6 flex gap-3 flex-row items-center justify-between">
|
||||
<div class="flex gap-3 items-center min-w-0">
|
||||
<div class="flex gap-3 items-center">
|
||||
<.link href={~p"/sources/#{@media_item.source_id}"}>
|
||||
<.icon name="hero-arrow-left" class="w-10 h-10 hover:dark:text-white" />
|
||||
</.link>
|
||||
<h2 class="text-title-md2 font-bold text-black dark:text-white ml-4 truncate">
|
||||
{@media_item.title}
|
||||
<h2 class="text-title-md2 font-bold text-black dark:text-white ml-4">
|
||||
<%= StringUtils.truncate(@media_item.title, 35) %>
|
||||
</h2>
|
||||
</div>
|
||||
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
</.link>
|
||||
</nav>
|
||||
</div>
|
||||
<div class="rounded-sm border py-5 pt-6 shadow-default border-strokedark bg-boxdark px-7.5">
|
||||
<div class="rounded-sm border border-stroke bg-white py-5 pt-6 shadow-default dark:border-strokedark dark:bg-boxdark px-7.5">
|
||||
<div class="max-w-full">
|
||||
<.tabbed_layout>
|
||||
<:tab_append>
|
||||
|
|
@ -24,67 +24,53 @@
|
|||
</:tab_append>
|
||||
|
||||
<:tab title="Media" id="media">
|
||||
<div class="flex flex-col gap-10 text-white">
|
||||
<section :if={@media_item.last_error} class="mt-6">
|
||||
<div class="flex items-center gap-1 mb-2">
|
||||
<.icon name="hero-exclamation-circle-solid" class="text-red-500" />
|
||||
<h3 class="font-bold text-xl">Last Error</h3>
|
||||
</div>
|
||||
<span>{@media_item.last_error}</span>
|
||||
</section>
|
||||
|
||||
<div class="flex flex-col gap-10 dark:text-white">
|
||||
<%= if media_file_exists?(@media_item) do %>
|
||||
<section class="grid grid-cols-1 xl:gap-6 mt-6">
|
||||
<section class="grid grid-cols-1 xl:grid-cols-2 xl:gap-6 mt-6">
|
||||
<div>
|
||||
<.media_preview media_item={@media_item} />
|
||||
</div>
|
||||
<aside class="mt-4">
|
||||
<h2 class="text-xl mb-2">{@media_item.title}</h2>
|
||||
<div>Uploaded: {DateTime.to_date(@media_item.uploaded_at)}</div>
|
||||
<aside class="mt-4 xl:mt-0">
|
||||
<div>Uploaded: <%= @media_item.upload_date %></div>
|
||||
<div>
|
||||
<span :if={URI.parse(@media_item.original_url).scheme =~ "http"}>
|
||||
<.subtle_link href={@media_item.original_url} target="_blank">Open Original</.subtle_link>
|
||||
<span class="mx-2">or</span>
|
||||
</span>
|
||||
<span>
|
||||
<.subtle_link
|
||||
href={~p"/media/#{@media_item.uuid}/stream?v=#{DateTime.to_unix(@media_item.updated_at)}"}
|
||||
target="_blank"
|
||||
>
|
||||
<.subtle_link href={~p"/media/#{@media_item.uuid}/stream"} target="_blank">
|
||||
Open Local Stream
|
||||
</.subtle_link>
|
||||
</span>
|
||||
</div>
|
||||
<div :if={@media_item.description} class="mt-4 text-bodydark">
|
||||
<.render_description text={@media_item.description} />
|
||||
<div class="mt-4 text-bodydark">
|
||||
<.break_on_newline text={@media_item.description} />
|
||||
</div>
|
||||
</aside>
|
||||
</section>
|
||||
<% end %>
|
||||
|
||||
<h3 class="font-bold text-xl mt-6">Raw Attributes</h3>
|
||||
<section>
|
||||
<h3 class="font-bold text-xl mb-2">Raw Attributes</h3>
|
||||
<section>
|
||||
<strong>Source:</strong>
|
||||
<.subtle_link href={~p"/sources/#{@media_item.source_id}"}>
|
||||
{@media_item.source.custom_name}
|
||||
</.subtle_link>
|
||||
<.list_items_from_map map={Map.from_struct(@media_item)} />
|
||||
</section>
|
||||
<strong>Source:</strong>
|
||||
<.subtle_link href={~p"/sources/#{@media_item.source_id}"}>
|
||||
<%= @media_item.source.custom_name %>
|
||||
</.subtle_link>
|
||||
<.list_items_from_map map={Map.from_struct(@media_item)} />
|
||||
</section>
|
||||
</div>
|
||||
</:tab>
|
||||
<:tab title="Tasks" id="tasks">
|
||||
<%= if match?([_|_], @media_item.tasks) do %>
|
||||
<.table rows={@media_item.tasks} table_class="text-white">
|
||||
<.table rows={@media_item.tasks} table_class="text-black dark:text-white">
|
||||
<:col :let={task} label="Worker">
|
||||
{task.job.worker}
|
||||
<%= task.job.worker %>
|
||||
</:col>
|
||||
<:col :let={task} label="State">
|
||||
{task.job.state}
|
||||
<%= task.job.state %>
|
||||
</:col>
|
||||
<:col :let={task} label="Scheduled At">
|
||||
<.datetime_in_zone datetime={task.job.scheduled_at} />
|
||||
<%= Calendar.strftime(task.job.scheduled_at, "%y-%m-%d %I:%M:%S %p %Z") %>
|
||||
</:col>
|
||||
</.table>
|
||||
<% else %>
|
||||
|
|
|
|||
|
|
@ -1,51 +1,24 @@
|
|||
defmodule PinchflatWeb.MediaProfiles.MediaProfileController do
|
||||
use PinchflatWeb, :controller
|
||||
use Pinchflat.Sources.SourcesQuery
|
||||
use Pinchflat.Profiles.ProfilesQuery
|
||||
|
||||
alias Pinchflat.Repo
|
||||
alias Pinchflat.Profiles
|
||||
alias Pinchflat.Sources.Source
|
||||
alias Pinchflat.Profiles.MediaProfile
|
||||
alias Pinchflat.Profiles.MediaProfileDeletionWorker
|
||||
|
||||
def index(conn, _params) do
|
||||
media_profiles_query =
|
||||
from mp in MediaProfile,
|
||||
as: :media_profile,
|
||||
where: is_nil(mp.marked_for_deletion_at),
|
||||
order_by: [asc: mp.name],
|
||||
select: map(mp, ^MediaProfile.__schema__(:fields)),
|
||||
select_merge: %{
|
||||
source_count:
|
||||
subquery(
|
||||
from s in Source,
|
||||
where: s.media_profile_id == parent_as(:media_profile).id,
|
||||
select: count(s.id)
|
||||
)
|
||||
}
|
||||
media_profiles =
|
||||
MediaProfile
|
||||
|> order_by(asc: :name)
|
||||
|> Repo.all()
|
||||
|
||||
render(conn, :index, media_profiles: Repo.all(media_profiles_query))
|
||||
render(conn, :index, media_profiles: media_profiles)
|
||||
end
|
||||
|
||||
def new(conn, params) do
|
||||
# Preload an existing media profile for faster creation
|
||||
cs_struct =
|
||||
case to_string(params["template_id"]) do
|
||||
"" -> %MediaProfile{}
|
||||
template_id -> Repo.get(MediaProfile, template_id) || %MediaProfile{}
|
||||
end
|
||||
def new(conn, _params) do
|
||||
changeset = Profiles.change_media_profile(%MediaProfile{})
|
||||
|
||||
render(conn, :new,
|
||||
layout: get_onboarding_layout(),
|
||||
changeset:
|
||||
Profiles.change_media_profile(%MediaProfile{
|
||||
cs_struct
|
||||
| id: nil,
|
||||
name: nil,
|
||||
marked_for_deletion_at: nil
|
||||
})
|
||||
)
|
||||
render(conn, :new, changeset: changeset, layout: get_onboarding_layout())
|
||||
end
|
||||
|
||||
def create(conn, %{"media_profile" => media_profile_params}) do
|
||||
|
|
@ -97,15 +70,19 @@ defmodule PinchflatWeb.MediaProfiles.MediaProfileController do
|
|||
end
|
||||
|
||||
def delete(conn, %{"id" => id} = params) do
|
||||
# This awkward comparison converts the string to a boolean
|
||||
delete_files = Map.get(params, "delete_files", "") == "true"
|
||||
delete_files = Map.get(params, "delete_files", false)
|
||||
media_profile = Profiles.get_media_profile!(id)
|
||||
{:ok, _media_profile} = Profiles.delete_media_profile(media_profile, delete_files: delete_files)
|
||||
|
||||
{:ok, _} = Profiles.update_media_profile(media_profile, %{marked_for_deletion_at: DateTime.utc_now()})
|
||||
MediaProfileDeletionWorker.kickoff(media_profile, %{delete_files: delete_files})
|
||||
flash_message =
|
||||
if delete_files do
|
||||
"Media profile, its sources, and its files deleted successfully."
|
||||
else
|
||||
"Media profile and its sources deleted successfully. Files were not deleted."
|
||||
end
|
||||
|
||||
conn
|
||||
|> put_flash(:info, "Media Profile deletion started. This may take a while to complete.")
|
||||
|> put_flash(:info, flash_message)
|
||||
|> redirect(to: ~p"/media_profiles")
|
||||
end
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue