Compare commits
443 commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8eadadc1ae | ||
|
|
151bb7b542 | ||
|
|
11c0c62a7e | ||
|
|
75f23641b3 | ||
|
|
f6360e90da | ||
|
|
02a319c350 | ||
|
|
ca1348b2c2 | ||
|
|
3f924812da | ||
|
|
b916e6a2f3 | ||
|
|
c35ec43466 | ||
|
|
a30a70fafd | ||
|
|
9aee723d1a | ||
|
|
9a8550661b | ||
|
|
458bf01599 | ||
|
|
cc8cc78b97 | ||
|
|
12cec4c042 | ||
|
|
e84d1bcde5 | ||
|
|
6a1fa78995 | ||
|
|
1b3c18e4e6 | ||
|
|
eb8498a792 | ||
|
|
8fe73ac924 | ||
|
|
b55c245fe6 | ||
|
|
f380e8ac27 | ||
|
|
bacc528ba1 | ||
|
|
6531a19c7a | ||
|
|
fc045cd74e | ||
|
|
f0e1fda0fc | ||
|
|
1722618c6d | ||
|
|
3fa7b48f4c | ||
|
|
23c64e000a | ||
|
|
f734905ba1 | ||
|
|
7140b88948 | ||
|
|
5bb90053bf | ||
|
|
846d217dca | ||
|
|
10895a7c16 | ||
|
|
7d7d242e5d | ||
|
|
b404420c76 | ||
|
|
fb9459fb5c | ||
|
|
b83f314617 | ||
|
|
7a90b008f0 | ||
|
|
3254e8a6c4 | ||
|
|
c47dccc7d7 | ||
|
|
8e071719f9 | ||
|
|
00d05a04f5 | ||
|
|
568e297d36 | ||
|
|
31d284ab0d | ||
|
|
877a86c9b0 | ||
|
|
e7552d3c62 | ||
|
|
24b79dc29e | ||
|
|
a275cbfb7a | ||
|
|
bbf7abad1b | ||
|
|
378ae39bef | ||
|
|
82cbc22577 | ||
|
|
09a1646a6a | ||
|
|
4f24ac2733 | ||
|
|
cd29239b2f | ||
|
|
1feb34621e | ||
|
|
73395b9668 | ||
|
|
dffc5c2e5d | ||
|
|
5906576a13 | ||
|
|
6734b6ab22 | ||
|
|
3f344a8cd9 | ||
|
|
d019e3aca1 | ||
|
|
f1b6fd5e31 | ||
|
|
28fe3d2b0b | ||
|
|
e7b1a239b4 | ||
|
|
d984895cba | ||
|
|
a68d9755a9 | ||
|
|
8236b95db0 | ||
|
|
78874313cc | ||
|
|
a8ce359cb7 | ||
|
|
ba6065f1b3 | ||
|
|
88553a95d2 | ||
|
|
1f1a239486 | ||
|
|
d162966cee | ||
|
|
70ad5af378 | ||
|
|
3937cb4cb8 | ||
|
|
1b39e6aa3f | ||
|
|
f111b5dda6 | ||
|
|
a6b2737a5c | ||
|
|
3ef0990ffc | ||
|
|
b26f664326 | ||
|
|
66bce2b83a | ||
|
|
7fac921afc | ||
|
|
0cf8654f47 | ||
|
|
9e845e760e | ||
|
|
ed2fbf2ae4 | ||
|
|
6fe82b2a78 | ||
|
|
a16afd1f9e | ||
|
|
5a53ffc8c2 | ||
|
|
14a32f6006 | ||
|
|
cf47032660 | ||
|
|
b6320d1a30 | ||
|
|
36f944be25 | ||
|
|
0514931140 | ||
|
|
8053bf339c | ||
|
|
c35f56bffa | ||
|
|
94e06f2d50 | ||
|
|
d925c34ce5 | ||
|
|
ae68750d9e | ||
|
|
cc2fb2ff79 | ||
|
|
fd66390648 | ||
|
|
5711963a6f | ||
|
|
49714a59ea | ||
|
|
01c101d24a | ||
|
|
a52dda6a7f | ||
|
|
efa64db04d | ||
|
|
366b5a266d | ||
|
|
9bb6c4ebd0 | ||
|
|
21e1944784 | ||
|
|
bdcd929558 | ||
|
|
91eae710b4 | ||
|
|
297709baa4 | ||
|
|
bb1074cc60 | ||
|
|
49222dd0b8 | ||
|
|
7061001f66 | ||
|
|
b23a3e91d8 | ||
|
|
7cd289e7b5 | ||
|
|
8349fa78dd | ||
|
|
0eac0ea46e | ||
|
|
d6776f9f57 | ||
|
|
57f254acaf | ||
|
|
9a3ca6ba4e | ||
|
|
f6f5561d0b | ||
|
|
7e504e03c6 | ||
|
|
afe3d0d04d | ||
|
|
55658f15da | ||
|
|
1fa3e1b599 | ||
|
|
5d488bceb4 | ||
|
|
40149d09f7 | ||
|
|
13b30a5997 | ||
|
|
3698ed1ec0 | ||
|
|
483cc61e08 | ||
|
|
71d48f704d | ||
|
|
5d8db8c5a4 | ||
|
|
c78b90826a | ||
|
|
937f3094c6 | ||
|
|
3af3a1cd24 | ||
|
|
5661ccbfd2 | ||
|
|
48f6cc5e3c | ||
|
|
1c3f255a66 | ||
|
|
d25be2c3c6 | ||
|
|
0bb77bf782 | ||
|
|
42c67a6d4d | ||
|
|
cb56ee2bee | ||
|
|
7e0726d5eb | ||
|
|
fff6448a14 | ||
|
|
ceccb4ee65 | ||
|
|
9f687c061a | ||
|
|
84fac1f80a | ||
|
|
89bedaf140 | ||
|
|
86fde7d62c | ||
|
|
305020f61e | ||
|
|
bf17ef3fab | ||
|
|
3490092e3d | ||
|
|
c5bffe51a5 | ||
|
|
44b074772b | ||
|
|
958a5cd0a4 | ||
|
|
9b1d76b4ff | ||
|
|
8803c1bd0c | ||
|
|
ecd9c25e3c | ||
|
|
ac7b47e4db | ||
|
|
2a61ac6a13 | ||
|
|
1dc727f0bb | ||
|
|
e880be9910 | ||
|
|
ddde6cba68 | ||
|
|
9f38c34900 | ||
|
|
cd13b1185e | ||
|
|
23cefd3c15 | ||
|
|
975f81f476 | ||
|
|
16ba035330 | ||
|
|
441f0e9ad1 | ||
|
|
0a0859df7a | ||
|
|
4d45cc614f | ||
|
|
887ee01cb2 | ||
|
|
8392de9207 | ||
|
|
8c2f66bea9 | ||
|
|
f98ceecd68 | ||
|
|
b3f704fbfd | ||
|
|
c33810f39f | ||
|
|
6f8a2a3f7a | ||
|
|
5a95619e22 | ||
|
|
314e587094 | ||
|
|
c55a4fa10e | ||
|
|
f7f21f44c5 | ||
|
|
530d6f181f | ||
|
|
6f14f15dd6 | ||
|
|
21a1e5ddbb | ||
|
|
278ba47519 | ||
|
|
856f14824f | ||
|
|
67197ce956 | ||
|
|
8e56273c9d | ||
|
|
5506155abe | ||
|
|
4788b54c91 | ||
|
|
ba7562c5f2 | ||
|
|
abb9d0e89e | ||
|
|
113a5ded38 | ||
|
|
62b815a3fe | ||
|
|
e141e3dcbd | ||
|
|
ef3d4cfdb4 | ||
|
|
28035f0d70 | ||
|
|
eb8f803c36 | ||
|
|
cb5ae93f76 | ||
|
|
9c3d1a5709 | ||
|
|
c7b5a93290 | ||
|
|
66f3f97460 | ||
|
|
267f11c848 | ||
|
|
d91693ba53 | ||
|
|
5012f44cfa | ||
|
|
4c16448ab3 | ||
|
|
cf03c28831 | ||
|
|
a3adf5e63c | ||
|
|
21ed97ff26 | ||
|
|
91bd895e52 | ||
|
|
c00df179ba | ||
|
|
b627f9f411 | ||
|
|
b5586d3f28 | ||
|
|
59dafff96a | ||
|
|
a83df05ce2 | ||
|
|
964b9bfdd7 | ||
|
|
220f2ad4de | ||
|
|
722b80001e | ||
|
|
3235b196b7 | ||
|
|
3a24fc24b1 | ||
|
|
9c3facfe7e | ||
|
|
217486b3c7 | ||
|
|
581735ac7d | ||
|
|
9524a40615 | ||
|
|
b6d8f07516 | ||
|
|
30dc587ebf | ||
|
|
ec5af17de7 | ||
|
|
ece4fbc792 | ||
|
|
3d67f51c29 | ||
|
|
0ff0d07c22 | ||
|
|
284acc8591 | ||
|
|
9084f1b7bb | ||
|
|
0456fa893f | ||
|
|
87750e2464 | ||
|
|
bbba2c8225 | ||
|
|
9348166bb5 | ||
|
|
b3fd6037bb | ||
|
|
591138a8a1 | ||
|
|
22adc3bc55 | ||
|
|
5ab3ec90a8 | ||
|
|
8941da8b60 | ||
|
|
f4f40c161b | ||
|
|
b440f2bcdc | ||
|
|
d30149db14 | ||
|
|
e952ea5f66 | ||
|
|
68383a16b3 | ||
|
|
b7cafb8b72 | ||
|
|
6d63016566 | ||
|
|
987cee7cb3 | ||
|
|
7c1644384d | ||
|
|
f383c82beb | ||
|
|
ce84e715c3 | ||
|
|
817de567ed | ||
|
|
efcaea314c | ||
|
|
7ef794fe74 | ||
|
|
11cc9aa789 | ||
|
|
853dbcf40f | ||
|
|
e8a0e1256b | ||
|
|
4d62df5c33 | ||
|
|
e5cd5cf08c | ||
|
|
2985514627 | ||
|
|
f7bd15d019 | ||
|
|
599f36fdf3 | ||
|
|
d90ae493dd | ||
|
|
1b456d548a | ||
|
|
bae8060c09 | ||
|
|
d3c875919f | ||
|
|
772703464b | ||
|
|
bbd081c44b | ||
|
|
261416fede | ||
|
|
6d3a59c8dc | ||
|
|
110f23f555 | ||
|
|
246c650db4 | ||
|
|
80d504f94d | ||
|
|
b316d283f7 | ||
|
|
a1b9f8a827 | ||
|
|
2b3ca12310 | ||
|
|
a9ec025706 | ||
|
|
5aa792d399 | ||
|
|
8043db0ad4 | ||
|
|
5c9b43a175 | ||
|
|
97a2023139 | ||
|
|
a8ebc9b8f7 | ||
|
|
4b7a915d39 | ||
|
|
0071358125 | ||
|
|
d722224b26 | ||
|
|
02e3b9d2e4 | ||
|
|
13d44f3ba0 | ||
|
|
9a5d187c0d | ||
|
|
419b939999 | ||
|
|
e24e9bfe1e | ||
|
|
35957b426d | ||
|
|
638269d870 | ||
|
|
21163239da | ||
|
|
240e386871 | ||
|
|
964675443b | ||
|
|
58ceca86b1 | ||
|
|
1b23b7da78 | ||
|
|
4287385af6 | ||
|
|
b344303f75 | ||
|
|
d49eb967a4 | ||
|
|
248e2c32f2 | ||
|
|
ffaa36105b | ||
|
|
e5a4dda117 | ||
|
|
6d72b7ca11 | ||
|
|
57c5148207 | ||
|
|
5856eefd7c | ||
|
|
3a8c803a54 | ||
|
|
8e00670491 | ||
|
|
5b6ef295f8 | ||
|
|
80dd2ff21c | ||
|
|
8f6f992670 | ||
|
|
f93c211de3 | ||
|
|
5f1ad517c2 | ||
|
|
a4d6e78bce | ||
|
|
b7ae32220b | ||
|
|
df09111831 | ||
|
|
c9900d91b0 | ||
|
|
b2dbd6dec5 | ||
|
|
f7d1b725d7 | ||
|
|
b2adc63a6a | ||
|
|
b8de46d2ad | ||
|
|
3359e3c111 | ||
|
|
621693ecc8 | ||
|
|
b567eb4808 | ||
|
|
e5724c6faa | ||
|
|
1423e50ccd | ||
|
|
3f439e4277 | ||
|
|
2f3e4b128b | ||
|
|
be54fccc63 | ||
|
|
df8f6a2acd | ||
|
|
8bbdd712f9 | ||
|
|
c5e30530c7 | ||
|
|
9c29414e6d | ||
|
|
6fcebe7c4b | ||
|
|
595ea2bfbd | ||
|
|
e0b9245ba3 | ||
|
|
b937343768 | ||
|
|
81546cff69 | ||
|
|
e94ea39187 | ||
|
|
8441ece6f0 | ||
|
|
6eed3d7775 | ||
|
|
f725235f44 | ||
|
|
c5ff2567a8 | ||
|
|
38d48bae37 | ||
|
|
d0c8aa9338 | ||
|
|
df889c0c6e | ||
|
|
0246842000 | ||
|
|
7081dad94e | ||
|
|
769dc62b87 | ||
|
|
d6d0dc537c | ||
|
|
87f414c8c7 | ||
|
|
288d44155d | ||
|
|
b50f7c12f4 | ||
|
|
f06728b0a7 | ||
|
|
aff4ecccc6 | ||
|
|
e2a8d3339a | ||
|
|
53391372c3 | ||
|
|
0598f5fbda | ||
|
|
c565fec59d | ||
|
|
6e526d7745 | ||
|
|
efa0632883 | ||
|
|
59c88fa0db | ||
|
|
b9b3b9eed3 | ||
|
|
8002f93220 | ||
|
|
986059626f | ||
|
|
fcd4af0efd | ||
|
|
80f1051e8a | ||
|
|
878e467f69 | ||
|
|
5e8143dd1d | ||
|
|
2306a5740c | ||
|
|
e1e0e29432 | ||
|
|
c450fa1f9a | ||
|
|
20f1214b68 | ||
|
|
81f2127e71 | ||
|
|
519685fc32 | ||
|
|
5a2113bd03 | ||
|
|
50632f6392 | ||
|
|
9b607b3d1b | ||
|
|
a483219746 | ||
|
|
fc68a6e741 | ||
|
|
50042607d7 | ||
|
|
8b01ada68a | ||
|
|
daecf9dfae | ||
|
|
a62e59dd3a | ||
|
|
9d2468a50f | ||
|
|
68a65a8e3d | ||
|
|
924d4ce8a6 | ||
|
|
798721d724 | ||
|
|
0e58973d43 | ||
|
|
776afdd1fd | ||
|
|
2901e4ec4d | ||
|
|
d35bb695ae | ||
|
|
941ebd42d9 | ||
|
|
0afd37351d | ||
|
|
aef80eb52d | ||
|
|
65ff84aa6a | ||
|
|
80679b02ba | ||
|
|
c17138bf7d | ||
|
|
093e14bd5d | ||
|
|
13e03a624c | ||
|
|
9eecd6d2c4 | ||
|
|
be141d0073 | ||
|
|
68582af374 | ||
|
|
12789b6be8 | ||
|
|
fd8d305651 | ||
|
|
9ae8c243c9 | ||
|
|
405e7097e3 | ||
|
|
a5ff4a1dd8 | ||
|
|
b5d0b76fe6 | ||
|
|
da3f89314b | ||
|
|
6f653f190a | ||
|
|
9974845470 | ||
|
|
fba47e9665 | ||
|
|
9d3cf14cdd | ||
|
|
e982b19bc5 | ||
|
|
7c8f06f427 | ||
|
|
41f6558c2d | ||
|
|
576199bb2f | ||
|
|
5b0b64bf3b | ||
|
|
0d86d84307 | ||
|
|
061079f0f6 | ||
|
|
bc334df719 | ||
|
|
7ab62674ed | ||
|
|
71f126f9d2 | ||
|
|
04fb19c80c | ||
|
|
d7ab68c067 | ||
|
|
6665ecaa12 | ||
|
|
462fa50423 | ||
|
|
401a9be0ec | ||
|
|
402a1fec50 | ||
|
|
dbe35fc023 | ||
|
|
a57951c038 | ||
|
|
c84231dd4f | ||
|
|
9330d66fcd | ||
|
|
e3d3f453da | ||
|
|
f5c0f3ca31 | ||
|
|
ac4bee75ef | ||
|
|
3202197740 |
163 changed files with 50368 additions and 164 deletions
5
.gitignore
vendored
5
.gitignore
vendored
|
|
@ -16,6 +16,11 @@ database/music_library.db
|
||||||
database/music_library.db-shm
|
database/music_library.db-shm
|
||||||
database/music_library.db-wal
|
database/music_library.db-wal
|
||||||
database/music_library.db.backup_*
|
database/music_library.db.backup_*
|
||||||
|
database/video_library.db
|
||||||
|
database/video_library.db-shm
|
||||||
|
database/video_library.db-wal
|
||||||
|
database/video_library.db.backup_*
|
||||||
|
database/*.histbak
|
||||||
database/api_call_history.json
|
database/api_call_history.json
|
||||||
storage/image_cache/
|
storage/image_cache/
|
||||||
logs/*.log
|
logs/*.log
|
||||||
|
|
|
||||||
BIN
Issue/1.png
Normal file
BIN
Issue/1.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 563 KiB |
73
api/video/__init__.py
Normal file
73
api/video/__init__.py
Normal file
|
|
@ -0,0 +1,73 @@
|
||||||
|
"""SoulSync — VIDEO side API package (isolated).
|
||||||
|
|
||||||
|
A SEPARATE Flask blueprint from the music API (api_v1). It reads only
|
||||||
|
database/video_library.db via VideoDatabase and imports nothing from the music
|
||||||
|
API or music database layer. Registered in web_server.py with a single additive
|
||||||
|
line at url_prefix '/api/video', so music routing is untouched.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import threading
|
||||||
|
|
||||||
|
from flask import Blueprint
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api")
|
||||||
|
|
||||||
|
# Lazily-created, process-wide VideoDatabase handle. VideoDatabase itself guards
|
||||||
|
# schema init once-per-path, so this just avoids re-opening the wrapper.
|
||||||
|
_video_db = None
|
||||||
|
_video_db_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def get_video_db():
|
||||||
|
"""Return the shared VideoDatabase instance (created on first use)."""
|
||||||
|
global _video_db
|
||||||
|
if _video_db is None:
|
||||||
|
with _video_db_lock:
|
||||||
|
if _video_db is None:
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
_video_db = VideoDatabase()
|
||||||
|
return _video_db
|
||||||
|
|
||||||
|
|
||||||
|
def create_video_blueprint() -> Blueprint:
|
||||||
|
"""Build the isolated /api/video blueprint with all video sub-routes."""
|
||||||
|
bp = Blueprint("video_api", __name__)
|
||||||
|
|
||||||
|
from .dashboard import register_routes as reg_dashboard
|
||||||
|
from .scan import register_routes as reg_scan
|
||||||
|
from .library import register_routes as reg_library
|
||||||
|
from .libraries import register_routes as reg_libraries
|
||||||
|
from .poster import register_routes as reg_poster
|
||||||
|
from .enrichment import register_routes as reg_enrichment
|
||||||
|
from .detail import register_routes as reg_detail
|
||||||
|
from .search import register_routes as reg_search
|
||||||
|
from .discover import register_routes as reg_discover
|
||||||
|
from .calendar import register_routes as reg_calendar
|
||||||
|
from .watchlist import register_routes as reg_watchlist
|
||||||
|
from .wishlist import register_routes as reg_wishlist
|
||||||
|
from .youtube import register_routes as reg_youtube
|
||||||
|
from .downloads import register_routes as reg_downloads
|
||||||
|
from .manual_import import register_routes as reg_manual_import
|
||||||
|
from .automations import register_routes as reg_automations
|
||||||
|
reg_dashboard(bp)
|
||||||
|
reg_scan(bp)
|
||||||
|
reg_library(bp)
|
||||||
|
reg_libraries(bp)
|
||||||
|
reg_poster(bp)
|
||||||
|
reg_enrichment(bp)
|
||||||
|
reg_detail(bp)
|
||||||
|
reg_search(bp)
|
||||||
|
reg_discover(bp)
|
||||||
|
reg_calendar(bp)
|
||||||
|
reg_watchlist(bp)
|
||||||
|
reg_wishlist(bp)
|
||||||
|
reg_youtube(bp)
|
||||||
|
reg_downloads(bp)
|
||||||
|
reg_manual_import(bp)
|
||||||
|
reg_automations(bp)
|
||||||
|
|
||||||
|
return bp
|
||||||
28
api/video/automations.py
Normal file
28
api/video/automations.py
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
"""Video automation builder support (isolated /api/video routes).
|
||||||
|
|
||||||
|
The automation ENGINE is app-wide (shared with the music side), but the
|
||||||
|
builder block palette is scoped: the video builder must only ever offer
|
||||||
|
video + generic blocks, never music-only ones. This endpoint returns that
|
||||||
|
scoped slice via ``core.automation.blocks.blocks_for_scope('video')``.
|
||||||
|
|
||||||
|
ISOLATION: imports only the shared automation block definitions (which
|
||||||
|
themselves import nothing music-specific) — no music API / music DB.
|
||||||
|
Reading/running/toggling video automations goes through the shared
|
||||||
|
``/api/automations`` endpoints (owned_by='video' rows); only the scoped
|
||||||
|
block palette lives here.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from flask import jsonify
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.automations")
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
@bp.route("/automations/blocks", methods=["GET"])
|
||||||
|
def video_automation_blocks():
|
||||||
|
from core.automation.blocks import blocks_for_scope
|
||||||
|
return jsonify(blocks_for_scope("video"))
|
||||||
79
api/video/calendar.py
Normal file
79
api/video/calendar.py
Normal file
|
|
@ -0,0 +1,79 @@
|
||||||
|
"""Video Calendar — upcoming TV episodes for OWNED shows.
|
||||||
|
|
||||||
|
GET /api/video/calendar?days=N → episodes airing from today through today+N-1,
|
||||||
|
grouped client-side into the agenda view. Isolated: reads only video_library.db
|
||||||
|
via VideoDatabase, writes nothing, never touches the music side.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import date, datetime, timedelta
|
||||||
|
|
||||||
|
from flask import jsonify, request
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video.calendar")
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
@bp.route("/calendar", methods=["GET"])
|
||||||
|
def video_calendar():
|
||||||
|
from . import get_video_db
|
||||||
|
try:
|
||||||
|
days = request.args.get("days", default=7, type=int) or 7
|
||||||
|
days = max(1, min(days, 31)) # one week (or a few) per view
|
||||||
|
today = date.today()
|
||||||
|
# Optional ?start=YYYY-MM-DD for week navigation; default = today.
|
||||||
|
start = today
|
||||||
|
start_s = request.args.get("start")
|
||||||
|
if start_s:
|
||||||
|
try:
|
||||||
|
start = datetime.strptime(start_s, "%Y-%m-%d").date()
|
||||||
|
except ValueError:
|
||||||
|
start = today
|
||||||
|
if abs((start - today).days) > 400: # sane bound around today
|
||||||
|
start = today
|
||||||
|
end = start + timedelta(days=days - 1)
|
||||||
|
db = get_video_db()
|
||||||
|
|
||||||
|
# One-time backfill: existing shows matched TVDB before air time was
|
||||||
|
# captured, so re-queue them once (background, only those missing it).
|
||||||
|
try:
|
||||||
|
if (db.get_setting("airs_time_backfill") or "") != "1":
|
||||||
|
n = db.requeue_shows_for_airtime()
|
||||||
|
db.set_setting("airs_time_backfill", "1")
|
||||||
|
if n:
|
||||||
|
logger.info("calendar: queued %d shows for TVDB air-time backfill", n)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("airs_time backfill queue failed")
|
||||||
|
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
# scope: 'watchlist' (default — shows you follow/track) or 'all' (every
|
||||||
|
# airing show in the library). The toggle on the page sends ?scope=.
|
||||||
|
scope = (request.args.get("scope") or "watchlist").lower()
|
||||||
|
eps = db.calendar_upcoming(start.isoformat(), end.isoformat(),
|
||||||
|
server_source=resolve_video_server(),
|
||||||
|
watchlist_only=(scope != "all"))
|
||||||
|
|
||||||
|
# Per-date counts drive the day-strip dots without a second query.
|
||||||
|
counts: dict[str, int] = {}
|
||||||
|
owned = 0
|
||||||
|
for e in eps:
|
||||||
|
counts[e["air_date"]] = counts.get(e["air_date"], 0) + 1
|
||||||
|
if e.get("has_file"):
|
||||||
|
owned += 1
|
||||||
|
return jsonify({
|
||||||
|
"today": today.isoformat(), # real today (for the highlight)
|
||||||
|
"start": start.isoformat(), # window start (may be a future week)
|
||||||
|
"end": end.isoformat(),
|
||||||
|
"days": days,
|
||||||
|
"scope": "all" if scope == "all" else "watchlist",
|
||||||
|
"counts_by_date": counts,
|
||||||
|
"total": len(eps),
|
||||||
|
"owned": owned,
|
||||||
|
"episodes": eps,
|
||||||
|
})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video calendar failed")
|
||||||
|
return jsonify({"error": "calendar failed"}), 500
|
||||||
28
api/video/dashboard.py
Normal file
28
api/video/dashboard.py
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
"""Video dashboard endpoint — live counts from video.db.
|
||||||
|
|
||||||
|
GET /api/video/dashboard -> {library:{...}, downloads:{...}, watchlist, wishlist}
|
||||||
|
With an empty database every value is a real 0.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from flask import jsonify
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.dashboard")
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
@bp.route("/dashboard", methods=["GET"])
|
||||||
|
def video_dashboard():
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
try:
|
||||||
|
server = resolve_video_server() # the VIDEO server, not music's active
|
||||||
|
stats = get_video_db().dashboard_stats(server_source=server)
|
||||||
|
stats["server"] = server
|
||||||
|
return jsonify(stats)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to build video dashboard stats")
|
||||||
|
return jsonify({"error": "Failed to load video dashboard stats"}), 500
|
||||||
135
api/video/detail.py
Normal file
135
api/video/detail.py
Normal file
|
|
@ -0,0 +1,135 @@
|
||||||
|
"""Video detail payloads (drill-in pages).
|
||||||
|
|
||||||
|
GET /api/video/detail/show/<id> → show + seasons→episodes tree (owned roll-ups)
|
||||||
|
GET /api/video/detail/movie/<id> → movie + owned/file info
|
||||||
|
|
||||||
|
Reads only video.db; isolated from the music API.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from flask import jsonify, request
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.detail")
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
@bp.route("/monitor", methods=["POST"])
|
||||||
|
def video_set_monitor():
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
kind, item_id = body.get("kind"), body.get("id")
|
||||||
|
if kind not in ("movie", "show") or not isinstance(item_id, int):
|
||||||
|
return jsonify({"error": "bad request"}), 400
|
||||||
|
ok = get_video_db().set_monitored(kind, item_id, bool(body.get("monitored")))
|
||||||
|
if not ok:
|
||||||
|
return jsonify({"error": "not found"}), 404
|
||||||
|
return jsonify({"success": True, "monitored": bool(body.get("monitored"))})
|
||||||
|
|
||||||
|
@bp.route("/detail/show/<int:show_id>", methods=["GET"])
|
||||||
|
def video_show_detail(show_id):
|
||||||
|
from . import get_video_db
|
||||||
|
data = get_video_db().show_detail(show_id)
|
||||||
|
if not data:
|
||||||
|
return jsonify({"error": "not found"}), 404
|
||||||
|
return jsonify(data)
|
||||||
|
|
||||||
|
@bp.route("/detail/movie/<int:movie_id>", methods=["GET"])
|
||||||
|
def video_movie_detail(movie_id):
|
||||||
|
from . import get_video_db
|
||||||
|
data = get_video_db().movie_detail(movie_id)
|
||||||
|
if not data:
|
||||||
|
return jsonify({"error": "not found"}), 404
|
||||||
|
return jsonify(data)
|
||||||
|
|
||||||
|
@bp.route("/detail/show/<int:show_id>/refresh-art", methods=["POST"])
|
||||||
|
def video_show_refresh_art(show_id):
|
||||||
|
"""Lazy on-view backfill: pull missing season posters / episode art from
|
||||||
|
TMDB and cache them. Best-effort — never errors the page."""
|
||||||
|
try:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
res = get_video_enrichment_engine().refresh_show_art(show_id)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("refresh-art failed for show %s", show_id)
|
||||||
|
res = {"ok": False, "reason": "error"}
|
||||||
|
return jsonify(res)
|
||||||
|
|
||||||
|
@bp.route("/detail/movie/<int:movie_id>/refresh-art", methods=["POST"])
|
||||||
|
def video_movie_refresh_art(movie_id):
|
||||||
|
"""Lazy on-view backfill for a movie (cast / genres / backdrop / ratings)."""
|
||||||
|
try:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
res = get_video_enrichment_engine().refresh_movie_art(movie_id)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("refresh-art failed for movie %s", movie_id)
|
||||||
|
res = {"ok": False, "reason": "error"}
|
||||||
|
return jsonify(res)
|
||||||
|
|
||||||
|
@bp.route("/tmdb/<kind>/<int:tmdb_id>", methods=["GET"])
|
||||||
|
def video_tmdb_detail(kind, tmdb_id):
|
||||||
|
"""Full detail for a TMDB title not in the library (the search → detail
|
||||||
|
view). May return {redirect:{source,kind,id}} if it's actually owned."""
|
||||||
|
if kind not in ("movie", "show"):
|
||||||
|
return jsonify({"error": "bad kind"}), 400
|
||||||
|
try:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
d = get_video_enrichment_engine().tmdb_detail(kind, tmdb_id)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("tmdb detail failed for %s %s", kind, tmdb_id)
|
||||||
|
d = None
|
||||||
|
if not d:
|
||||||
|
return jsonify({"error": "not found"}), 404
|
||||||
|
return jsonify(d)
|
||||||
|
|
||||||
|
@bp.route("/tmdb/show/<int:tv_id>/season/<int:season_number>", methods=["GET"])
|
||||||
|
def video_tmdb_season(tv_id, season_number):
|
||||||
|
"""Lazy per-season episodes for a TMDB (un-owned) show detail."""
|
||||||
|
try:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
d = get_video_enrichment_engine().tmdb_season(tv_id, season_number)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("tmdb season failed for %s S%s", tv_id, season_number)
|
||||||
|
d = None
|
||||||
|
if not d:
|
||||||
|
return jsonify({"error": "not found"}), 404
|
||||||
|
return jsonify(d)
|
||||||
|
|
||||||
|
@bp.route("/episode/<int:tmdb_id>/<int:season>/<int:episode>", methods=["GET"])
|
||||||
|
def video_episode_extra(tmdb_id, season, episode):
|
||||||
|
"""Episode expand: guest stars + bigger still (by the SHOW's tmdb id)."""
|
||||||
|
try:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
d = get_video_enrichment_engine().episode_extra(tmdb_id, season, episode)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("episode extra failed for %s S%sE%s", tmdb_id, season, episode)
|
||||||
|
d = None
|
||||||
|
if not d:
|
||||||
|
return jsonify({"error": "not found"}), 404
|
||||||
|
return jsonify(d)
|
||||||
|
|
||||||
|
@bp.route("/person/<int:tmdb_id>", methods=["GET"])
|
||||||
|
def video_person_detail(tmdb_id):
|
||||||
|
"""In-app person page: bio + filmography (each credit annotated owned/not)."""
|
||||||
|
try:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
d = get_video_enrichment_engine().person_detail(tmdb_id)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("person detail failed for %s", tmdb_id)
|
||||||
|
d = None
|
||||||
|
if not d:
|
||||||
|
return jsonify({"error": "not found"}), 404
|
||||||
|
return jsonify(d)
|
||||||
|
|
||||||
|
@bp.route("/detail/<kind>/<int:item_id>/extras", methods=["GET"])
|
||||||
|
def video_detail_extras(kind, item_id):
|
||||||
|
"""Live TMDB extras (trailer / where-to-watch / similar) for the detail page."""
|
||||||
|
if kind not in ("movie", "show"):
|
||||||
|
return jsonify({}), 400
|
||||||
|
try:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
return jsonify(get_video_enrichment_engine().item_extras(kind, item_id))
|
||||||
|
except Exception:
|
||||||
|
logger.exception("extras failed for %s %s", kind, item_id)
|
||||||
|
return jsonify({})
|
||||||
386
api/video/discover.py
Normal file
386
api/video/discover.py
Normal file
|
|
@ -0,0 +1,386 @@
|
||||||
|
"""Video discover API — browse TMDB (movies/TV the user doesn't own yet).
|
||||||
|
|
||||||
|
GET /api/video/discover/hero → trending titles w/ backdrops (slideshow)
|
||||||
|
GET /api/video/discover/genres → {movie:[{id,name}], show:[{id,name}]}
|
||||||
|
GET /api/video/discover/list?... → one shelf/grid of items, e.g.
|
||||||
|
?key=trending → trending movies + shows
|
||||||
|
?key=<curated>&page= → a canned list (popular_movies, top_shows…)
|
||||||
|
?kind=movie|show&genre=&year=&decade=&sort=&page= → a filtered browse
|
||||||
|
|
||||||
|
Items are annotated with ``library_id`` when already owned (so the card links to
|
||||||
|
the owned detail, not the TMDB preview). Reads only the enrichment engine +
|
||||||
|
video.db; isolated from the music API.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import concurrent.futures
|
||||||
|
|
||||||
|
from flask import jsonify, request
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.discover")
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
@bp.route("/discover/hero", methods=["GET"])
|
||||||
|
def video_discover_hero():
|
||||||
|
"""A few trending titles that have a backdrop — drives the hero slideshow."""
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
try:
|
||||||
|
items = [x for x in (get_video_enrichment_engine().trending() or [])
|
||||||
|
if x.get("backdrop")][:6]
|
||||||
|
except Exception:
|
||||||
|
logger.exception("discover hero failed")
|
||||||
|
items = []
|
||||||
|
return jsonify({"items": items})
|
||||||
|
|
||||||
|
@bp.route("/discover/taste", methods=["GET"])
|
||||||
|
def video_discover_taste():
|
||||||
|
"""The user's most-owned genres (movies + shows) → personalized rails."""
|
||||||
|
from . import get_video_db
|
||||||
|
try:
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
srv = resolve_video_server()
|
||||||
|
except Exception:
|
||||||
|
srv = None
|
||||||
|
db = get_video_db()
|
||||||
|
try:
|
||||||
|
return jsonify({"movie": db.top_owned_genres("movie", srv, 6),
|
||||||
|
"show": db.top_owned_genres("show", srv, 6)})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("discover taste failed")
|
||||||
|
return jsonify({"movie": [], "show": []})
|
||||||
|
|
||||||
|
@bp.route("/discover/morelike", methods=["GET"])
|
||||||
|
def video_discover_morelike():
|
||||||
|
"""'More like <owned title>' rails — TMDB recommendations seeded from a few
|
||||||
|
random titles you already own (interleaved movie/show, max 3 rails)."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
try:
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
srv = resolve_video_server()
|
||||||
|
except Exception:
|
||||||
|
srv = None
|
||||||
|
db = get_video_db()
|
||||||
|
eng = get_video_enrichment_engine()
|
||||||
|
try:
|
||||||
|
seeds = db.random_owned_titles(2, srv)
|
||||||
|
movies = [s for s in seeds if s["kind"] == "movie"]
|
||||||
|
shows = [s for s in seeds if s["kind"] == "show"]
|
||||||
|
ordered = []
|
||||||
|
while (movies or shows) and len(ordered) < 3:
|
||||||
|
if movies:
|
||||||
|
ordered.append(movies.pop(0))
|
||||||
|
if shows and len(ordered) < 3:
|
||||||
|
ordered.append(shows.pop(0))
|
||||||
|
rails = []
|
||||||
|
for s in ordered:
|
||||||
|
items = [it for it in eng.recommendations(s["kind"], s["tmdb_id"])
|
||||||
|
if it.get("tmdb_id") != s["tmdb_id"]]
|
||||||
|
if len(items) >= 4:
|
||||||
|
rails.append({"title": "More like " + s["title"], "items": items[:30]})
|
||||||
|
return jsonify({"rails": rails})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("discover morelike failed")
|
||||||
|
return jsonify({"rails": []})
|
||||||
|
|
||||||
|
@bp.route("/discover/foryou", methods=["GET"])
|
||||||
|
def video_discover_foryou():
|
||||||
|
"""A single 'Recommended for you' wall blended from many owned titles — a title
|
||||||
|
recommended by more of your library ranks higher (consensus)."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
from core.video.discovery_recs import blend_recommendations
|
||||||
|
try:
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
srv = resolve_video_server()
|
||||||
|
except Exception:
|
||||||
|
srv = None
|
||||||
|
db = get_video_db()
|
||||||
|
eng = get_video_enrichment_engine()
|
||||||
|
try:
|
||||||
|
seeds = db.random_owned_titles(6, srv) # up to 6 movies + 6 shows
|
||||||
|
seed_ids = [s["tmdb_id"] for s in seeds if s.get("tmdb_id")]
|
||||||
|
rec_lists = [eng.recommendations(s["kind"], s["tmdb_id"])
|
||||||
|
for s in seeds if s.get("tmdb_id")]
|
||||||
|
items = blend_recommendations(rec_lists, exclude_ids=seed_ids, limit=40)
|
||||||
|
return jsonify({"items": items})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("discover foryou failed")
|
||||||
|
return jsonify({"items": []})
|
||||||
|
|
||||||
|
@bp.route("/discover/gaps", methods=["GET"])
|
||||||
|
def video_discover_gaps():
|
||||||
|
"""'What am I missing?' rails — franchises you've started but not finished, and
|
||||||
|
more from the directors/creators you own the most. Powered by the gap engine."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
from core.video.discovery_gaps import collection_gaps, filmography_gaps
|
||||||
|
try:
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
srv = resolve_video_server()
|
||||||
|
except Exception:
|
||||||
|
srv = None
|
||||||
|
db = get_video_db()
|
||||||
|
eng = get_video_enrichment_engine()
|
||||||
|
try:
|
||||||
|
# Lazy collection-id backfill: movies matched before the collection column
|
||||||
|
# exists have no franchise id. Fill a small batch each load (self-healing);
|
||||||
|
# isolated so a backfill hiccup never breaks the gap rails.
|
||||||
|
try:
|
||||||
|
for mv in db.movies_missing_collection(srv, limit=20):
|
||||||
|
coll = eng.movie_collection(mv["tmdb_id"])
|
||||||
|
if coll is not None:
|
||||||
|
db.set_movie_collection(mv["id"], coll.get("id"), coll.get("name"))
|
||||||
|
except Exception:
|
||||||
|
logger.exception("collection-id backfill batch failed")
|
||||||
|
|
||||||
|
owned = db.owned_movie_tmdb_ids(srv)
|
||||||
|
ignored = db.ignored_keys()
|
||||||
|
rails = []
|
||||||
|
# Complete your collections — top franchises you've started, missing entries.
|
||||||
|
for coll in db.owned_movie_collections(srv, limit=8):
|
||||||
|
missing = collection_gaps(owned, eng.collection(coll["collection_id"]))
|
||||||
|
if missing:
|
||||||
|
name = (coll.get("name") or "Collection").strip()
|
||||||
|
rails.append({"title": "Complete the " + name, "kind": "collection",
|
||||||
|
"items": missing[:30]})
|
||||||
|
# More from the people you own the most (directors / creators).
|
||||||
|
for person in db.top_owned_people(min_titles=2, limit=6, server_source=srv):
|
||||||
|
p = eng.person_detail(person["tmdb_id"])
|
||||||
|
if not p:
|
||||||
|
continue
|
||||||
|
missing = filmography_gaps(owned, p.get("credits") or [],
|
||||||
|
kinds=("movie",), min_vote_count=50, limit=30)
|
||||||
|
if ignored:
|
||||||
|
missing = [m for m in missing
|
||||||
|
if f"{m.get('kind')}:{m.get('tmdb_id')}" not in ignored]
|
||||||
|
if len(missing) >= 3:
|
||||||
|
rails.append({"title": "More from " + person["name"], "kind": "person",
|
||||||
|
"items": missing})
|
||||||
|
return jsonify({"rails": rails})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("discover gaps failed")
|
||||||
|
return jsonify({"rails": []})
|
||||||
|
|
||||||
|
@bp.route("/discover/trailer", methods=["GET"])
|
||||||
|
def video_discover_trailer():
|
||||||
|
"""Best YouTube trailer {key,name} for a tmdb title (hero 'Trailer' button)."""
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
kind = request.args.get("kind", "movie")
|
||||||
|
try:
|
||||||
|
tmdb_id = int(request.args.get("tmdb_id"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return jsonify({"trailer": None})
|
||||||
|
try:
|
||||||
|
tr = get_video_enrichment_engine().trailer(kind, tmdb_id)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("discover trailer failed")
|
||||||
|
tr = None
|
||||||
|
return jsonify({"trailer": tr or None})
|
||||||
|
|
||||||
|
@bp.route("/discover/ignore", methods=["GET", "POST"])
|
||||||
|
def video_discover_ignore():
|
||||||
|
"""The Discover 'Not interested' list. GET -> {items}. POST
|
||||||
|
{action:'add'|'remove', kind, tmdb_id, title?, year?, poster?}."""
|
||||||
|
from . import get_video_db
|
||||||
|
db = get_video_db()
|
||||||
|
try:
|
||||||
|
if request.method == "GET":
|
||||||
|
return jsonify({"items": db.list_ignored()})
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
kind, tmdb_id = body.get("kind"), body.get("tmdb_id")
|
||||||
|
if body.get("action") == "remove":
|
||||||
|
db.remove_ignored(kind, tmdb_id)
|
||||||
|
return jsonify({"success": True})
|
||||||
|
ok = db.add_ignored(kind, tmdb_id, body.get("title"), body.get("year"), body.get("poster"))
|
||||||
|
return jsonify({"success": ok})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("discover ignore failed")
|
||||||
|
return jsonify({"success": False, "items": []})
|
||||||
|
|
||||||
|
@bp.route("/discover/languages", methods=["GET", "POST"])
|
||||||
|
def video_discover_languages():
|
||||||
|
"""Get/set the preferred original-languages for general rails (ISO-639-1 codes).
|
||||||
|
POST {languages: ['en','ko']} (or 'en,ko'); GET returns the current list."""
|
||||||
|
from . import get_video_db
|
||||||
|
db = get_video_db()
|
||||||
|
try:
|
||||||
|
if request.method == "POST":
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
langs = body.get("languages")
|
||||||
|
if isinstance(langs, list):
|
||||||
|
val = ",".join(str(c).strip().lower() for c in langs if str(c).strip())
|
||||||
|
else:
|
||||||
|
val = ",".join(c.strip().lower() for c in str(langs or "").split(",") if c.strip())
|
||||||
|
db.set_setting("discover_languages", val or "en")
|
||||||
|
return jsonify({"success": True,
|
||||||
|
"languages": [c for c in (val or "en").split(",") if c]})
|
||||||
|
raw = db.get_setting("discover_languages", "en") or "en"
|
||||||
|
return jsonify({"languages": [c.strip() for c in raw.split(",") if c.strip()]})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("discover languages get/set failed")
|
||||||
|
return jsonify({"languages": ["en"]})
|
||||||
|
|
||||||
|
@bp.route("/discover/providers-pref", methods=["GET", "POST"])
|
||||||
|
def video_discover_providers_pref():
|
||||||
|
"""The user's subscribed streaming services (TMDB provider ids) — drives the
|
||||||
|
'On your streaming services' rail. POST {providers:[8,9]} (or '8,9'); GET returns them."""
|
||||||
|
from . import get_video_db
|
||||||
|
db = get_video_db()
|
||||||
|
try:
|
||||||
|
if request.method == "POST":
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
p = body.get("providers")
|
||||||
|
if isinstance(p, list):
|
||||||
|
val = ",".join(str(c).strip() for c in p if str(c).strip())
|
||||||
|
else:
|
||||||
|
val = ",".join(c.strip() for c in str(p or "").split(",") if c.strip())
|
||||||
|
db.set_setting("discover_providers", val)
|
||||||
|
return jsonify({"success": True, "providers": [c for c in val.split(",") if c]})
|
||||||
|
raw = db.get_setting("discover_providers", "") or ""
|
||||||
|
return jsonify({"providers": [c.strip() for c in raw.split(",") if c.strip()]})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("discover providers-pref failed")
|
||||||
|
return jsonify({"providers": []})
|
||||||
|
|
||||||
|
@bp.route("/discover/genres", methods=["GET"])
|
||||||
|
def video_discover_genres():
|
||||||
|
"""Genre id→name maps for both kinds (powers the genre rails + filter)."""
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
eng = get_video_enrichment_engine()
|
||||||
|
try:
|
||||||
|
return jsonify({"movie": eng.genre_list("movie"), "show": eng.genre_list("show")})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("discover genres failed")
|
||||||
|
return jsonify({"movie": [], "show": []})
|
||||||
|
|
||||||
|
@bp.route("/discover/list", methods=["GET"])
|
||||||
|
def video_discover_list():
|
||||||
|
"""One shelf (rail) or one page of a filtered browse — see module docstring.
|
||||||
|
|
||||||
|
``pages`` (1–3, default 1) fetches that many consecutive TMDB pages and
|
||||||
|
concatenates them (deduped) in one response — so a rail can show ~40 items
|
||||||
|
and still look full after 'Hide owned' drops the ones you have."""
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
eng = get_video_enrichment_engine()
|
||||||
|
try:
|
||||||
|
page = max(1, int(request.args.get("page", 1) or 1))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
page = 1
|
||||||
|
try:
|
||||||
|
pages = min(3, max(1, int(request.args.get("pages", 1) or 1)))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pages = 1
|
||||||
|
key = (request.args.get("key") or "").strip()
|
||||||
|
kind = request.args.get("kind", "movie")
|
||||||
|
genre = request.args.get("genre") or None
|
||||||
|
year = request.args.get("year") or None
|
||||||
|
decade = request.args.get("decade") or None
|
||||||
|
providers = request.args.get("providers") or None
|
||||||
|
if providers and "," in providers:
|
||||||
|
providers = providers.replace(",", "|") # TMDB with_watch_providers OR-join
|
||||||
|
sort = request.args.get("sort") or "popularity.desc"
|
||||||
|
# Netflix-class discover extensions (all optional). Comma -> pipe = TMDB OR-join.
|
||||||
|
keywords = (request.args.get("keywords") or "").replace(",", "|") or None
|
||||||
|
companies = (request.args.get("companies") or "").replace(",", "|") or None
|
||||||
|
networks = (request.args.get("networks") or "").replace(",", "|") or None
|
||||||
|
cast = request.args.get("cast") or None
|
||||||
|
crew = request.args.get("crew") or None
|
||||||
|
min_runtime = request.args.get("min_runtime") or None
|
||||||
|
max_runtime = request.args.get("max_runtime") or None
|
||||||
|
certification = request.args.get("certification") or None
|
||||||
|
release_window = request.args.get("release_window") or None
|
||||||
|
try:
|
||||||
|
vote_count_min = int(request.args["vote_count_min"]) if request.args.get("vote_count_min") else None
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
vote_count_min = None
|
||||||
|
lang = (request.args.get("lang") or "").strip().lower() or None # explicit (foreign rail / browse)
|
||||||
|
# The Browse-all grid sends `lang=any` to opt OUT of the rail language preference
|
||||||
|
# entirely (ad-hoc search shows every language); a real code (en/ko/…) filters to it.
|
||||||
|
any_lang = lang in ("any", "all")
|
||||||
|
if any_lang:
|
||||||
|
lang = None
|
||||||
|
hide_owned = (request.args.get("hide_owned") or "") in ("1", "true", "yes")
|
||||||
|
# Preferred original-languages (multi) for GENERAL/curated rails — so the feeds
|
||||||
|
# aren't flooded with foreign titles (e.g. Bollywood in Popular/Trending). A rail
|
||||||
|
# with an explicit `lang` (a dedicated foreign rail) or `lang=any` (browse) bypasses
|
||||||
|
# this. Default 'en'.
|
||||||
|
prefer_langs = None
|
||||||
|
if not lang and not any_lang:
|
||||||
|
try:
|
||||||
|
from . import get_video_db
|
||||||
|
raw = get_video_db().get_setting("discover_languages", "en") or "en"
|
||||||
|
prefer_langs = {c.strip().lower() for c in raw.split(",") if c.strip()} or None
|
||||||
|
except Exception:
|
||||||
|
prefer_langs = {"en"}
|
||||||
|
|
||||||
|
def fetch(p):
|
||||||
|
if key == "trending":
|
||||||
|
return eng.trending()
|
||||||
|
if key == "trending_today":
|
||||||
|
return eng.trending(window="day") # the ranked 'Top 10 today' chart (mixed)
|
||||||
|
if key == "trending_movies_today":
|
||||||
|
return eng.trending(window="day", kind="movie") # split 'Top 10 Movies Today'
|
||||||
|
if key == "trending_tv_today":
|
||||||
|
return eng.trending(window="day", kind="show") # split 'Top 10 TV Shows Today'
|
||||||
|
if key:
|
||||||
|
return eng.discover_curated(key, page=p)
|
||||||
|
return eng.discover_filter(
|
||||||
|
kind, genre=genre, year=year, decade=decade, providers=providers,
|
||||||
|
sort_by=sort, page=p, language=lang, keywords=keywords, companies=companies,
|
||||||
|
networks=networks, cast=cast, crew=crew, min_runtime=min_runtime,
|
||||||
|
max_runtime=max_runtime, certification=certification,
|
||||||
|
vote_count_min=vote_count_min, release_window=release_window)
|
||||||
|
|
||||||
|
try:
|
||||||
|
items, seen = [], set()
|
||||||
|
|
||||||
|
def consume(batch):
|
||||||
|
"""Filter a page into `items` (dedup + hide-owned + language). Returns False
|
||||||
|
once TMDB hands back an empty page (it ran out — stop paging)."""
|
||||||
|
if not batch:
|
||||||
|
return False
|
||||||
|
for it in batch:
|
||||||
|
dk = (it.get("kind"), it.get("tmdb_id"))
|
||||||
|
if dk in seen:
|
||||||
|
continue
|
||||||
|
seen.add(dk)
|
||||||
|
if hide_owned and it.get("library_id") is not None:
|
||||||
|
continue
|
||||||
|
if prefer_langs:
|
||||||
|
ol = (it.get("original_language") or "").lower()
|
||||||
|
if ol and ol not in prefer_langs:
|
||||||
|
continue # known foreign language not in your preference
|
||||||
|
items.append(it)
|
||||||
|
return True
|
||||||
|
|
||||||
|
# When filtering (hide-owned or language), page DEEPER and drop items server-side
|
||||||
|
# so a heavy library's rail still fills to ~target instead of showing 4 cards. We
|
||||||
|
# fetch the extra pages in concurrent WAVES (TTLCache + the TMDB client are
|
||||||
|
# thread-safe) so digging 20 pages deep costs ~4 page-latencies, not 20.
|
||||||
|
need_fill = hide_owned or bool(prefer_langs)
|
||||||
|
target = 20 if need_fill else 0
|
||||||
|
if key in ("trending", "trending_today", "trending_movies_today", "trending_tv_today"):
|
||||||
|
consume(fetch(page)) # a single fixed chart; never paged
|
||||||
|
elif not need_fill:
|
||||||
|
for offset in range(pages): # no filtering: respect requested page count
|
||||||
|
if not consume(fetch(page + offset)):
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
MAX_PAGES, WAVE = 20, 5
|
||||||
|
offset, ran_out = 0, False
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(max_workers=WAVE) as ex:
|
||||||
|
while offset < MAX_PAGES and len(items) < target and not ran_out:
|
||||||
|
batches = list(ex.map(fetch, [page + offset + i for i in range(WAVE)]))
|
||||||
|
for batch in batches:
|
||||||
|
if not consume(batch):
|
||||||
|
ran_out = True # an empty page = TMDB has no more
|
||||||
|
offset += WAVE
|
||||||
|
return jsonify({"items": items, "page": page})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("discover list failed (key=%s)", key)
|
||||||
|
return jsonify({"items": [], "page": page})
|
||||||
514
api/video/downloads.py
Normal file
514
api/video/downloads.py
Normal file
|
|
@ -0,0 +1,514 @@
|
||||||
|
"""Video-side download SETTINGS (isolated).
|
||||||
|
|
||||||
|
Persists the video download configuration in video.db's ``video_settings`` KV
|
||||||
|
table — fully separate from the music ``soulseek.*`` paths so the two libraries
|
||||||
|
never share a folder or collide. The actual download fulfillment engine (wishlist
|
||||||
|
→ search → grab) is a later roadmap phase; these endpoints just store/serve the
|
||||||
|
config the Settings → Downloads tab edits.
|
||||||
|
|
||||||
|
Folders:
|
||||||
|
- INPUT (download) folder is SHARED with the music side — it's the same
|
||||||
|
``config_manager`` key the music Download Settings use (``soulseek.download_path``),
|
||||||
|
so changing it on either side changes both (one physical download dir, simpler
|
||||||
|
Docker mounts). We only READ/WRITE that shared key; no music code is touched.
|
||||||
|
- OUTPUT (library) folders are video-specific and live in video.db, one per type:
|
||||||
|
``movies_path`` / ``tv_path`` / ``youtube_path``.
|
||||||
|
The engine routes a finished download to the library path matching its type. (Legacy
|
||||||
|
single video ``transfer_path`` is migrated into ``movies_path`` on first read.)
|
||||||
|
|
||||||
|
Connection settings that are genuinely SHARED with music (the slskd instance, the
|
||||||
|
torrent/usenet clients, Prowlarr indexers) are NOT stored here — those live in the
|
||||||
|
music config_manager and are surfaced on the shared Indexers tab + shared slskd
|
||||||
|
block (a deliberate shared boundary, since they're one physical resource).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from flask import jsonify, request
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.downloads")
|
||||||
|
|
||||||
|
# Video-specific OUTPUT library folders (video.db). The INPUT folder is the shared
|
||||||
|
# music key below — not in this list.
|
||||||
|
_PATH_KEYS = ("movies_path", "tv_path", "youtube_path")
|
||||||
|
# The shared input/download dir — the SAME config key the music side reads/writes.
|
||||||
|
_SHARED_DOWNLOAD_KEY = "soulseek.download_path"
|
||||||
|
|
||||||
|
# slskd CONNECTION settings genuinely SHARED with music — one slskd instance serves
|
||||||
|
# both sides, so these live in the app-wide config_manager (soulseek.*), NOT video.db.
|
||||||
|
# Deliberately excludes the music download/transfer PATHS and source mode/quality —
|
||||||
|
# those are video-specific (stored in video.db). Maps the video field name -> the
|
||||||
|
# shared config key + default. (config_manager is shared app config, not music code.)
|
||||||
|
_SLSKD_KEYS = {
|
||||||
|
"slskd_url": ("soulseek.slskd_url", "http://localhost:5030"),
|
||||||
|
"api_key": ("soulseek.api_key", ""),
|
||||||
|
"search_timeout": ("soulseek.search_timeout", 60),
|
||||||
|
"search_timeout_buffer": ("soulseek.search_timeout_buffer", 15),
|
||||||
|
"search_min_delay_seconds": ("soulseek.search_min_delay_seconds", 0),
|
||||||
|
"min_peer_upload_speed": ("soulseek.min_peer_upload_speed", 0),
|
||||||
|
"max_peer_queue": ("soulseek.max_peer_queue", 0),
|
||||||
|
"download_timeout": ("soulseek.download_timeout", 600), # seconds (UI shows minutes)
|
||||||
|
"auto_clear_searches": ("soulseek.auto_clear_searches", True),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _evaluate_hits(raw, profile, scope, want_season, want_episode) -> list:
|
||||||
|
"""Parse → evaluate → rank a list of raw indexer hits against the quality profile.
|
||||||
|
Shared by the mock search and the live slskd start/poll endpoints."""
|
||||||
|
from core.video.quality_eval import evaluate_release
|
||||||
|
from core.video.release_parse import parse_release
|
||||||
|
results = []
|
||||||
|
for hit in raw:
|
||||||
|
parsed = parse_release(hit.get("title"))
|
||||||
|
size_gb = round((hit.get("size_bytes") or 0) / (1024 ** 3), 1)
|
||||||
|
verdict = evaluate_release(parsed, profile, scope=scope, want_season=want_season,
|
||||||
|
want_episode=want_episode, size_gb=size_gb)
|
||||||
|
# Availability = how downloadable the source is (slskd: free slot/queue/speed score
|
||||||
|
# from group_video_files; torrents/mock: seeders/peers). Ranks within a quality tier
|
||||||
|
# so we grab a free-slot/empty-queue release over one stuck behind a huge queue.
|
||||||
|
avail = hit.get("availability")
|
||||||
|
if avail is None:
|
||||||
|
avail = hit.get("seeders") if hit.get("seeders") is not None else (hit.get("peers") or 0)
|
||||||
|
results.append({
|
||||||
|
"title": hit.get("title"), "size_gb": size_gb, "size_bytes": hit.get("size_bytes") or 0,
|
||||||
|
"seeders": hit.get("seeders"), "peers": hit.get("peers"),
|
||||||
|
"username": hit.get("username"), "slots": hit.get("slots"),
|
||||||
|
"queue": hit.get("queue"), "speed": hit.get("speed"),
|
||||||
|
"filename": hit.get("filename"), "_avail": avail,
|
||||||
|
"quality_label": verdict["quality_label"], "accepted": verdict["accepted"],
|
||||||
|
"rejected": verdict["rejected"], "score": verdict["score"],
|
||||||
|
"resolution": parsed.get("resolution"), "source": parsed.get("source"),
|
||||||
|
"codec": parsed.get("codec"), "hdr": parsed.get("hdr"),
|
||||||
|
"audio": parsed.get("audio"), "group": parsed.get("group"),
|
||||||
|
"repack": parsed.get("repack") or parsed.get("proper"),
|
||||||
|
})
|
||||||
|
# accepted first, then quality-profile score, then availability, then bigger file.
|
||||||
|
results.sort(key=lambda r: (r["accepted"], r["score"], r["_avail"], r["size_bytes"]), reverse=True)
|
||||||
|
for r in results:
|
||||||
|
r.pop("_avail", None)
|
||||||
|
return results[:40]
|
||||||
|
|
||||||
|
|
||||||
|
def _search_ints(body):
|
||||||
|
def _int(v):
|
||||||
|
try:
|
||||||
|
return int(v)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
return _int(body.get("season")), _int(body.get("episode")), _int(body.get("season_end"))
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
@bp.route("/downloads/config", methods=["GET"])
|
||||||
|
def video_downloads_config():
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.download_config import load as load_source
|
||||||
|
from config.settings import config_manager
|
||||||
|
db = get_video_db()
|
||||||
|
out = {k: db.get_setting(k) or "" for k in _PATH_KEYS}
|
||||||
|
if not out["movies_path"]: # migrate the legacy single transfer folder → Movies
|
||||||
|
out["movies_path"] = db.get_setting("transfer_path") or ""
|
||||||
|
out["download_path"] = config_manager.get(_SHARED_DOWNLOAD_KEY, "") or "" # shared w/ music
|
||||||
|
out.update(load_source(db)) # download_mode + hybrid_order
|
||||||
|
return jsonify(out)
|
||||||
|
|
||||||
|
@bp.route("/downloads/config", methods=["POST"])
|
||||||
|
def video_downloads_config_save():
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.download_config import save as save_source
|
||||||
|
db = get_video_db()
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
for key in _PATH_KEYS:
|
||||||
|
if key in body:
|
||||||
|
db.set_setting(key, (str(body.get(key) or "")).strip())
|
||||||
|
if "download_path" in body: # SHARED with music — write the same config key
|
||||||
|
from config.settings import config_manager
|
||||||
|
config_manager.set(_SHARED_DOWNLOAD_KEY, (str(body.get("download_path") or "")).strip())
|
||||||
|
save_source(db, body) # download_mode + hybrid_order (validated)
|
||||||
|
return jsonify({"status": "saved"})
|
||||||
|
|
||||||
|
@bp.route("/downloads/history", methods=["GET"])
|
||||||
|
def video_downloads_history():
|
||||||
|
"""Paged permanent history of grabs (movies + episodes). ?kind=movie|show,
|
||||||
|
?search=, ?outcome=, ?page=, ?limit=. Always returns counts for the tabs."""
|
||||||
|
from . import get_video_db
|
||||||
|
try:
|
||||||
|
db = get_video_db()
|
||||||
|
kind = request.args.get("kind")
|
||||||
|
res = db.query_download_history(
|
||||||
|
kind=kind if kind in ("movie", "show") else None,
|
||||||
|
search=request.args.get("search", ""),
|
||||||
|
outcome=request.args.get("outcome") or None,
|
||||||
|
page=request.args.get("page", 1), limit=request.args.get("limit", 40))
|
||||||
|
return jsonify({"success": True, "counts": db.download_history_counts(), **res})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to list video download history")
|
||||||
|
return jsonify({"success": False, "error": "Failed to load history"}), 500
|
||||||
|
|
||||||
|
@bp.route("/downloads/history/<int:history_id>", methods=["GET"])
|
||||||
|
def video_downloads_history_detail(history_id):
|
||||||
|
from . import get_video_db
|
||||||
|
d = get_video_db().download_history_detail(history_id)
|
||||||
|
if not d:
|
||||||
|
return jsonify({"success": False, "error": "not found"}), 404
|
||||||
|
return jsonify({"success": True, "item": d})
|
||||||
|
|
||||||
|
@bp.route("/downloads/history/<int:history_id>", methods=["DELETE"])
|
||||||
|
def video_downloads_history_delete(history_id):
|
||||||
|
"""Forget one grab — the 'Re-download' action. Removing its history row lets the
|
||||||
|
scans re-add + re-grab it (useful after you've deleted the file)."""
|
||||||
|
from . import get_video_db
|
||||||
|
return jsonify({"success": get_video_db().delete_download_history(history_id)})
|
||||||
|
|
||||||
|
@bp.route("/downloads/history/clear", methods=["POST"])
|
||||||
|
def video_downloads_history_clear():
|
||||||
|
"""Clear the permanent history (all, or one kind via {kind})."""
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
n = get_video_db().clear_download_history(kind=body.get("kind"))
|
||||||
|
return jsonify({"success": True, "removed": n})
|
||||||
|
|
||||||
|
@bp.route("/downloads/meta/<kind>/<int:tmdb_id>", methods=["GET"])
|
||||||
|
def video_download_meta(kind, tmdb_id):
|
||||||
|
"""Lazy TMDB detail for a download's expand drawer (logo, cast w/ photos, trailer,
|
||||||
|
where-to-watch, rating/runtime/genres…), keyed by the grabbed title's TMDB id.
|
||||||
|
Best-effort — the drawer still shows the download facts without it."""
|
||||||
|
if kind not in ("movie", "show"):
|
||||||
|
return jsonify({}), 400
|
||||||
|
try:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
d = get_video_enrichment_engine().tmdb_full_detail(kind, tmdb_id) or {}
|
||||||
|
if not d:
|
||||||
|
return jsonify({})
|
||||||
|
extras = d.get("_extras") or {}
|
||||||
|
tr = extras.get("trailer") or {}
|
||||||
|
director = next((c.get("name") for c in (d.get("crew") or [])
|
||||||
|
if (c.get("job") or "").lower() in ("director", "creator")), None)
|
||||||
|
# episode-specific detail (still + that episode's own title/overview/air date) when a
|
||||||
|
# specific episode is downloading — more relevant than the show synopsis.
|
||||||
|
episode = None
|
||||||
|
sn, en = request.args.get("season"), request.args.get("episode")
|
||||||
|
if kind == "show" and sn and en:
|
||||||
|
try:
|
||||||
|
season = get_video_enrichment_engine().tmdb_season(tmdb_id, int(sn)) or {}
|
||||||
|
ep = next((e for e in (season.get("episodes") or [])
|
||||||
|
if str(e.get("episode_number")) == str(int(en))), None)
|
||||||
|
if ep:
|
||||||
|
episode = {"season": int(sn), "episode": int(en), "title": ep.get("title"),
|
||||||
|
"overview": ep.get("overview"), "air_date": ep.get("air_date"),
|
||||||
|
"still_url": ep.get("still_url")}
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
pass
|
||||||
|
return jsonify({
|
||||||
|
"title": d.get("title"), "overview": d.get("overview"), "tagline": d.get("tagline"),
|
||||||
|
"backdrop_url": d.get("backdrop_url"), "logo": d.get("logo"),
|
||||||
|
"genres": d.get("genres") or [], "rating": d.get("rating"),
|
||||||
|
"runtime_minutes": d.get("runtime_minutes"), "year": d.get("year"),
|
||||||
|
"network": d.get("network"), "studio": d.get("studio"),
|
||||||
|
"status": d.get("status"), "director": director, "episode": episode,
|
||||||
|
"cast": [{"name": c.get("name"), "character": c.get("character"), "photo": c.get("photo")}
|
||||||
|
for c in (d.get("cast") or [])[:10]],
|
||||||
|
"trailer_url": ("https://www.youtube.com/watch?v=" + tr["key"]) if tr.get("key") else None,
|
||||||
|
"providers": (extras.get("providers") or [])[:6],
|
||||||
|
"providers_link": extras.get("providers_link"),
|
||||||
|
})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("download meta failed for %s %s", kind, tmdb_id)
|
||||||
|
return jsonify({})
|
||||||
|
|
||||||
|
@bp.route("/downloads/yt-meta/<video_id>", methods=["GET"])
|
||||||
|
def video_download_yt_meta(video_id):
|
||||||
|
"""Cached extra detail for a YouTube download's drawer (duration / views / thumbnail)."""
|
||||||
|
from . import get_video_db
|
||||||
|
try:
|
||||||
|
return jsonify(get_video_db().youtube_video_detail(video_id) or {})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("yt meta failed for %s", video_id)
|
||||||
|
return jsonify({})
|
||||||
|
|
||||||
|
@bp.route("/downloads/quality", methods=["GET"])
|
||||||
|
def video_quality_profile():
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.quality_profile import load
|
||||||
|
return jsonify(load(get_video_db()))
|
||||||
|
|
||||||
|
@bp.route("/downloads/quality", methods=["POST"])
|
||||||
|
def video_quality_profile_save():
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.quality_profile import save
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
return jsonify(save(get_video_db(), body))
|
||||||
|
|
||||||
|
@bp.route("/organization", methods=["GET"])
|
||||||
|
def video_organization():
|
||||||
|
"""The library-organisation settings: naming templates + post-process toggles."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.organization import load
|
||||||
|
return jsonify(load(get_video_db()))
|
||||||
|
|
||||||
|
@bp.route("/organization", methods=["POST"])
|
||||||
|
def video_organization_save():
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.organization import save
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
return jsonify(save(get_video_db(), body))
|
||||||
|
|
||||||
|
@bp.route("/downloads/evaluate", methods=["POST"])
|
||||||
|
def video_quality_evaluate():
|
||||||
|
"""Judge a video file the user already owns against their quality profile —
|
||||||
|
powers the Download modal's 'In your library · … (below your target)' line.
|
||||||
|
Body: {"file": {resolution, video_codec, …}}."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.quality_eval import evaluate_owned
|
||||||
|
from core.video.quality_profile import load
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
profile = load(get_video_db())
|
||||||
|
return jsonify(evaluate_owned(body.get("file"), profile))
|
||||||
|
|
||||||
|
@bp.route("/downloads/search", methods=["POST"])
|
||||||
|
def video_downloads_search():
|
||||||
|
"""Search a scope (movie / episode / season / series) and return candidates
|
||||||
|
ranked + filtered against the stored quality profile. The indexer is mocked
|
||||||
|
for now (core.video.mock_search) — the parse→evaluate→rank pipeline is real,
|
||||||
|
so swapping in slskd/Prowlarr later needs no change here.
|
||||||
|
Body: {scope, title, year?, season?, episode?, season_end?}."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.mock_search import mock_search
|
||||||
|
from core.video.quality_profile import load as load_profile
|
||||||
|
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
scope = str(body.get("scope") or "movie").lower()
|
||||||
|
title = body.get("title") or ""
|
||||||
|
source = str(body.get("source") or "").lower()
|
||||||
|
want_season, want_episode, season_end = _search_ints(body)
|
||||||
|
profile = load_profile(get_video_db())
|
||||||
|
live = False
|
||||||
|
if source == "soulseek":
|
||||||
|
from core.video.slskd_search import build_query, slskd_search
|
||||||
|
sres = slskd_search(build_query(scope, title, year=body.get("year"),
|
||||||
|
season=want_season, episode=want_episode))
|
||||||
|
if not sres.get("configured"):
|
||||||
|
return jsonify({"scope": scope, "results": [], "error": "slskd isn't configured — set its URL on Settings → Downloads."})
|
||||||
|
if sres.get("error"):
|
||||||
|
return jsonify({"scope": scope, "results": [], "error": "slskd: " + str(sres["error"])})
|
||||||
|
raw, live = sres["hits"], True
|
||||||
|
else:
|
||||||
|
raw = mock_search(scope, title, year=body.get("year"), season=want_season,
|
||||||
|
episode=want_episode, season_end=season_end, source=source)
|
||||||
|
return jsonify({"scope": scope, "live": live,
|
||||||
|
"results": _evaluate_hits(raw, profile, scope, want_season, want_episode)})
|
||||||
|
|
||||||
|
@bp.route("/downloads/search/start", methods=["POST"])
|
||||||
|
def video_downloads_search_start():
|
||||||
|
"""Begin a search. For mock sources the results come back immediately; for
|
||||||
|
Soulseek it returns a slskd search id to poll (results trickle in over ~30s,
|
||||||
|
like the music side) — fixes 'no results' from waiting too briefly."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.mock_search import mock_search
|
||||||
|
from core.video.quality_profile import load as load_profile
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
scope = str(body.get("scope") or "movie").lower()
|
||||||
|
title = body.get("title") or ""
|
||||||
|
source = str(body.get("source") or "").lower()
|
||||||
|
want_season, want_episode, season_end = _search_ints(body)
|
||||||
|
|
||||||
|
if source == "soulseek":
|
||||||
|
from core.video.slskd_search import build_query, search_timeout_ms, start_search
|
||||||
|
res = start_search(build_query(scope, title, year=body.get("year"),
|
||||||
|
season=want_season, episode=want_episode))
|
||||||
|
if not res.get("configured"):
|
||||||
|
return jsonify({"error": "slskd isn't configured — set its URL on Settings → Downloads."})
|
||||||
|
if res.get("error"):
|
||||||
|
return jsonify({"error": "slskd: " + str(res["error"])})
|
||||||
|
# how long the client should keep polling (slskd keeps searching this long).
|
||||||
|
return jsonify({"id": res["id"], "live": True, "complete": False,
|
||||||
|
"poll_ms": search_timeout_ms() + 8000})
|
||||||
|
# mock sources resolve in one shot
|
||||||
|
profile = load_profile(get_video_db())
|
||||||
|
raw = mock_search(scope, title, year=body.get("year"), season=want_season,
|
||||||
|
episode=want_episode, season_end=season_end, source=source)
|
||||||
|
return jsonify({"id": None, "live": False, "complete": True,
|
||||||
|
"results": _evaluate_hits(raw, profile, scope, want_season, want_episode)})
|
||||||
|
|
||||||
|
@bp.route("/downloads/search/poll", methods=["GET"])
|
||||||
|
def video_downloads_search_poll():
|
||||||
|
"""Current ranked results for an in-flight slskd search. Query: id, scope,
|
||||||
|
title, season?, episode?. The client polls until it stops growing or times out."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.quality_profile import load as load_profile
|
||||||
|
from core.video.slskd_search import poll_search
|
||||||
|
sid = request.args.get("id")
|
||||||
|
scope = str(request.args.get("scope") or "movie").lower()
|
||||||
|
want_season, want_episode, _ = _search_ints(request.args)
|
||||||
|
if not sid:
|
||||||
|
return jsonify({"results": [], "live": True, "total_files": 0})
|
||||||
|
profile = load_profile(get_video_db())
|
||||||
|
polled = poll_search(sid)
|
||||||
|
return jsonify({"live": True, "total_files": polled["total_files"],
|
||||||
|
"results": _evaluate_hits(polled["hits"], profile, scope, want_season, want_episode)})
|
||||||
|
|
||||||
|
@bp.route("/downloads/grab", methods=["POST"])
|
||||||
|
def video_downloads_grab():
|
||||||
|
"""Start a real download of a chosen release and track it. v1: Soulseek only.
|
||||||
|
Body: {kind, title, release_title, source, username, filename, size_bytes,
|
||||||
|
quality_label}."""
|
||||||
|
from . import get_video_db
|
||||||
|
from config.settings import config_manager
|
||||||
|
from core.video.download_monitor import ensure_started
|
||||||
|
from core.video.download_pipeline import target_dir_for
|
||||||
|
from core.video.slskd_download import start_download
|
||||||
|
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
source = str(body.get("source") or "soulseek").lower()
|
||||||
|
if source != "soulseek":
|
||||||
|
return jsonify({"ok": False, "error": "Only Soulseek grabs are wired up so far."}), 400
|
||||||
|
username, filename = body.get("username"), body.get("filename")
|
||||||
|
if not username or not filename:
|
||||||
|
return jsonify({"ok": False, "error": "Missing the release's source info."}), 400
|
||||||
|
|
||||||
|
db = get_video_db()
|
||||||
|
paths = {k: db.get_setting(k) or "" for k in ("movies_path", "tv_path", "youtube_path")}
|
||||||
|
if not paths["movies_path"]:
|
||||||
|
paths["movies_path"] = db.get_setting("transfer_path") or ""
|
||||||
|
target = target_dir_for(body.get("kind"), paths)
|
||||||
|
if not target:
|
||||||
|
return jsonify({"ok": False, "error": "Set the library folder for this type on Settings → Downloads."}), 400
|
||||||
|
|
||||||
|
started = start_download(username, filename, body.get("size_bytes") or 0)
|
||||||
|
if not started.get("ok"):
|
||||||
|
return jsonify({"ok": False, "error": started.get("error") or "slskd refused the download."}), 502
|
||||||
|
|
||||||
|
import json as _json
|
||||||
|
from core.video.slskd_search import build_query
|
||||||
|
# The OTHER accepted results become the retry pool; the search context drives
|
||||||
|
# the alternate-query requery when the pool runs dry.
|
||||||
|
ctx = body.get("search_ctx") if isinstance(body.get("search_ctx"), dict) else {}
|
||||||
|
candidates = [c for c in (body.get("candidates") or []) if isinstance(c, dict) and c.get("filename") != filename]
|
||||||
|
first_query = build_query(ctx.get("scope") or body.get("kind") or "movie", ctx.get("title") or body.get("title"),
|
||||||
|
year=ctx.get("year"), season=ctx.get("season"), episode=ctx.get("episode"))
|
||||||
|
dl_id = db.add_video_download({
|
||||||
|
"kind": str(body.get("kind") or "movie"), "title": body.get("title"),
|
||||||
|
"release_title": body.get("release_title") or body.get("filename"),
|
||||||
|
"source": "soulseek", "username": username, "filename": filename,
|
||||||
|
"size_bytes": int(body.get("size_bytes") or 0), "quality_label": body.get("quality_label"),
|
||||||
|
"target_dir": target, "status": "downloading",
|
||||||
|
"media_id": (str(body.get("media_id")) if body.get("media_id") is not None else None),
|
||||||
|
"media_source": body.get("media_source"), "year": body.get("year"),
|
||||||
|
"poster_url": body.get("poster_url"),
|
||||||
|
"candidates": _json.dumps(candidates), "search_ctx": _json.dumps(ctx),
|
||||||
|
"tried_queries": _json.dumps([first_query] if first_query else []),
|
||||||
|
"tried_files": _json.dumps([filename]), "attempts": 0,
|
||||||
|
})
|
||||||
|
ensure_started(get_video_db)
|
||||||
|
return jsonify({"ok": True, "id": dl_id})
|
||||||
|
|
||||||
|
@bp.route("/downloads/active", methods=["GET"])
|
||||||
|
def video_downloads_active():
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.download_monitor import ensure_started
|
||||||
|
db = get_video_db()
|
||||||
|
ensure_started(get_video_db) # also (re)start the monitor when the page is open
|
||||||
|
return jsonify({"downloads": db.list_video_downloads()})
|
||||||
|
|
||||||
|
@bp.route("/downloads/status", methods=["GET"])
|
||||||
|
def video_downloads_status():
|
||||||
|
"""Lightweight live-tracking lookup — used by the Download modal's result
|
||||||
|
card (by ``id``) and a movie/show detail page (by ``media_id`` +
|
||||||
|
``media_source``, returning that title's most relevant download so the page
|
||||||
|
can show live progress). Returns ``{"download": {...}|null}``."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.download_monitor import ensure_started
|
||||||
|
db = get_video_db()
|
||||||
|
ensure_started(get_video_db)
|
||||||
|
dl_id = request.args.get("id")
|
||||||
|
if dl_id:
|
||||||
|
try:
|
||||||
|
return jsonify({"download": db.get_video_download(int(dl_id))})
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return jsonify({"download": None})
|
||||||
|
media_id = request.args.get("media_id")
|
||||||
|
if media_id:
|
||||||
|
media_source = request.args.get("media_source")
|
||||||
|
match = [r for r in db.list_video_downloads()
|
||||||
|
if str(r.get("media_id")) == str(media_id)
|
||||||
|
and (not media_source or r.get("media_source") == media_source)]
|
||||||
|
# list_video_downloads orders active-first then newest — so an active
|
||||||
|
# download (or else the most recent) for this title is simply match[0].
|
||||||
|
return jsonify({"download": match[0] if match else None})
|
||||||
|
return jsonify({"download": None})
|
||||||
|
|
||||||
|
@bp.route("/downloads/cancel", methods=["POST"])
|
||||||
|
def video_downloads_cancel():
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.slskd_download import cancel_download
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
db = get_video_db()
|
||||||
|
dl = db.get_video_download(body.get("id"))
|
||||||
|
if not dl:
|
||||||
|
return jsonify({"ok": False, "error": "Download not found."}), 404
|
||||||
|
if dl["status"] in ("completed", "failed", "cancelled"):
|
||||||
|
return jsonify({"ok": True, "already": True})
|
||||||
|
cancel_download(dl.get("username"), dl.get("filename")) # best-effort; mark regardless
|
||||||
|
import time
|
||||||
|
db.update_video_download(dl["id"], status="cancelled", error="Cancelled",
|
||||||
|
completed_at=time.strftime("%Y-%m-%d %H:%M:%S"))
|
||||||
|
return jsonify({"ok": True})
|
||||||
|
|
||||||
|
@bp.route("/downloads/retry", methods=["POST"])
|
||||||
|
def video_downloads_retry():
|
||||||
|
"""Re-grab the SAME release (basic retry). Auto-retry + alternate-query retry
|
||||||
|
come in a later phase."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.download_monitor import ensure_started
|
||||||
|
from core.video.slskd_download import start_download
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
db = get_video_db()
|
||||||
|
dl = db.get_video_download(body.get("id"))
|
||||||
|
if not dl:
|
||||||
|
return jsonify({"ok": False, "error": "Download not found."}), 404
|
||||||
|
if not dl.get("username") or not dl.get("filename"):
|
||||||
|
return jsonify({"ok": False, "error": "Nothing to retry from."}), 400
|
||||||
|
started = start_download(dl["username"], dl["filename"], dl.get("size_bytes") or 0)
|
||||||
|
if not started.get("ok"):
|
||||||
|
return jsonify({"ok": False, "error": started.get("error") or "slskd refused the download."}), 502
|
||||||
|
db.update_video_download(dl["id"], status="downloading", progress=0, error=None,
|
||||||
|
dest_path=None, completed_at=None)
|
||||||
|
ensure_started(get_video_db)
|
||||||
|
return jsonify({"ok": True})
|
||||||
|
|
||||||
|
@bp.route("/downloads/clear", methods=["POST"])
|
||||||
|
def video_downloads_clear():
|
||||||
|
from . import get_video_db
|
||||||
|
return jsonify({"cleared": get_video_db().clear_finished_video_downloads()})
|
||||||
|
|
||||||
|
@bp.route("/downloads/youtube-quality", methods=["GET"])
|
||||||
|
def video_youtube_quality():
|
||||||
|
# Separate, smaller profile — YouTube is yt-dlp, not scene/p2p releases.
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.youtube_quality import load
|
||||||
|
return jsonify(load(get_video_db()))
|
||||||
|
|
||||||
|
@bp.route("/downloads/youtube-quality", methods=["POST"])
|
||||||
|
def video_youtube_quality_save():
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.youtube_quality import save
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
return jsonify(save(get_video_db(), body))
|
||||||
|
|
||||||
|
@bp.route("/downloads/slskd", methods=["GET"])
|
||||||
|
def video_slskd_config():
|
||||||
|
# SHARED with music — same slskd instance. Reads the app-wide config_manager.
|
||||||
|
from config.settings import config_manager
|
||||||
|
return jsonify({k: config_manager.get(cfg, default)
|
||||||
|
for k, (cfg, default) in _SLSKD_KEYS.items()})
|
||||||
|
|
||||||
|
@bp.route("/downloads/slskd", methods=["POST"])
|
||||||
|
def video_slskd_config_save():
|
||||||
|
from config.settings import config_manager
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
for k, (cfg, _default) in _SLSKD_KEYS.items():
|
||||||
|
if k in body:
|
||||||
|
config_manager.set(cfg, body.get(k))
|
||||||
|
return jsonify({"status": "saved", "shared": True})
|
||||||
219
api/video/enrichment.py
Normal file
219
api/video/enrichment.py
Normal file
|
|
@ -0,0 +1,219 @@
|
||||||
|
"""Video enrichment API — mirrors the music enrichment endpoints so the shared
|
||||||
|
Manage-Workers modal can drive video workers by pointing at /api/video/...
|
||||||
|
|
||||||
|
GET /api/video/enrichment/services
|
||||||
|
GET /api/video/enrichment/<service>/status
|
||||||
|
POST /api/video/enrichment/<service>/pause | /resume
|
||||||
|
GET /api/video/enrichment/<service>/breakdown
|
||||||
|
GET /api/video/enrichment/<service>/unmatched?kind=&status=&q=&limit=&offset=
|
||||||
|
POST /api/video/enrichment/<service>/retry {kind, scope, item_id}
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from flask import jsonify, request
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.enrichment")
|
||||||
|
|
||||||
|
|
||||||
|
def _int(val, default):
|
||||||
|
try:
|
||||||
|
return int(val)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
def engine():
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
return get_video_enrichment_engine()
|
||||||
|
|
||||||
|
def _yt_enricher():
|
||||||
|
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||||
|
return get_youtube_date_enricher()
|
||||||
|
|
||||||
|
@bp.route("/enrichment/services", methods=["GET"])
|
||||||
|
def video_enrichment_services():
|
||||||
|
try:
|
||||||
|
return jsonify({"services": engine().services()})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video enrichment services failed")
|
||||||
|
return jsonify({"services": []})
|
||||||
|
|
||||||
|
@bp.route("/enrichment/config", methods=["GET"])
|
||||||
|
def video_enrichment_config():
|
||||||
|
from . import get_video_db
|
||||||
|
db = get_video_db()
|
||||||
|
return jsonify({
|
||||||
|
"tmdb_api_key": db.get_setting("tmdb_api_key") or "",
|
||||||
|
"tvdb_api_key": db.get_setting("tvdb_api_key") or "",
|
||||||
|
"omdb_api_key": db.get_setting("omdb_api_key") or "",
|
||||||
|
# Backfill-worker keys (free, optional) + no-key toggles.
|
||||||
|
"fanart_api_key": db.get_setting("fanart_api_key") or "",
|
||||||
|
"opensubtitles_api_key": db.get_setting("opensubtitles_api_key") or "",
|
||||||
|
"trakt_api_key": db.get_setting("trakt_api_key") or "",
|
||||||
|
"ryd_enabled": (db.get_setting("ryd_enabled") or "1") == "1",
|
||||||
|
"sponsorblock_enabled": (db.get_setting("sponsorblock_enabled") or "1") == "1",
|
||||||
|
"dearrow_enabled": (db.get_setting("dearrow_enabled") or "1") == "1",
|
||||||
|
"tvmaze_enabled": (db.get_setting("tvmaze_enabled") or "1") == "1",
|
||||||
|
"anilist_enabled": (db.get_setting("anilist_enabled") or "0") == "1",
|
||||||
|
"wikidata_enabled": (db.get_setting("wikidata_enabled") or "1") == "1",
|
||||||
|
"billboard_autoplay": (db.get_setting("billboard_autoplay") or "1") == "1",
|
||||||
|
"watch_region": (db.get_setting("watch_region") or "US").upper(),
|
||||||
|
})
|
||||||
|
|
||||||
|
@bp.route("/prefs", methods=["GET"])
|
||||||
|
def video_prefs():
|
||||||
|
# Lightweight UI prefs for the detail page (no API keys).
|
||||||
|
from . import get_video_db
|
||||||
|
db = get_video_db()
|
||||||
|
return jsonify({
|
||||||
|
"billboard_autoplay": (db.get_setting("billboard_autoplay") or "1") == "1",
|
||||||
|
"watch_region": (db.get_setting("watch_region") or "US").upper(),
|
||||||
|
})
|
||||||
|
|
||||||
|
@bp.route("/enrichment/config", methods=["POST"])
|
||||||
|
def video_enrichment_config_save():
|
||||||
|
from . import get_video_db
|
||||||
|
db = get_video_db()
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
keys_changed = False
|
||||||
|
|
||||||
|
def put_key(field):
|
||||||
|
nonlocal keys_changed
|
||||||
|
if field in body:
|
||||||
|
val = body.get(field) or ""
|
||||||
|
if val != (db.get_setting(field) or ""):
|
||||||
|
keys_changed = True
|
||||||
|
db.set_setting(field, val)
|
||||||
|
|
||||||
|
put_key("tmdb_api_key")
|
||||||
|
put_key("tvdb_api_key")
|
||||||
|
put_key("fanart_api_key")
|
||||||
|
put_key("opensubtitles_api_key")
|
||||||
|
put_key("trakt_api_key")
|
||||||
|
# No-key worker on/off toggles (read live by the worker — no rebuild needed).
|
||||||
|
for flag in ("ryd_enabled", "sponsorblock_enabled", "dearrow_enabled",
|
||||||
|
"tvmaze_enabled", "anilist_enabled", "wikidata_enabled"):
|
||||||
|
if flag in body:
|
||||||
|
db.set_setting(flag, "1" if body.get(flag) else "0")
|
||||||
|
if "billboard_autoplay" in body:
|
||||||
|
db.set_setting("billboard_autoplay", "1" if body.get("billboard_autoplay") else "0")
|
||||||
|
if "watch_region" in body:
|
||||||
|
region = (body.get("watch_region") or "US").strip().upper()[:2] or "US"
|
||||||
|
db.set_setting("watch_region", region)
|
||||||
|
if "omdb_api_key" in body:
|
||||||
|
new_key = body.get("omdb_api_key") or ""
|
||||||
|
changed = new_key != (db.get_setting("omdb_api_key") or "")
|
||||||
|
db.set_setting("omdb_api_key", new_key)
|
||||||
|
keys_changed = keys_changed or changed
|
||||||
|
# A new/changed OMDb key → re-try every title that still has no rating
|
||||||
|
# (covers items wrongly marked 'synced' during a prior bad-key run).
|
||||||
|
if new_key and changed:
|
||||||
|
try:
|
||||||
|
for kind in ("movie", "show"):
|
||||||
|
db.enrichment_retry("omdb", kind, scope="failed")
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video enrichment: omdb re-try reset failed")
|
||||||
|
# Only rebuild the workers when an API KEY actually changed — a prefs-only
|
||||||
|
# save (autoplay/region) must not churn the running enrichment engine
|
||||||
|
# (that restart was re-logging the OMDb limit warning on every save).
|
||||||
|
if keys_changed:
|
||||||
|
try:
|
||||||
|
from core.video.enrichment.engine import rebuild_video_enrichment_engine
|
||||||
|
rebuild_video_enrichment_engine()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video enrichment: engine rebuild after key change failed")
|
||||||
|
return jsonify({"status": "saved"})
|
||||||
|
|
||||||
|
@bp.route("/enrichment/<service>/status", methods=["GET"])
|
||||||
|
def video_enrichment_status(service):
|
||||||
|
if service == "youtube": # the standalone date enricher (not an engine worker)
|
||||||
|
return jsonify(_yt_enricher().stats())
|
||||||
|
w = engine().worker(service)
|
||||||
|
if not w:
|
||||||
|
return jsonify({"error": "unknown service"}), 404
|
||||||
|
return jsonify(w.get_stats())
|
||||||
|
|
||||||
|
@bp.route("/enrichment/<service>/pause", methods=["POST"])
|
||||||
|
def video_enrichment_pause(service):
|
||||||
|
if service == "youtube":
|
||||||
|
_yt_enricher().pause()
|
||||||
|
return jsonify({"status": "paused"})
|
||||||
|
w = engine().worker(service)
|
||||||
|
if not w:
|
||||||
|
return jsonify({"error": "unknown service"}), 404
|
||||||
|
w.pause()
|
||||||
|
return jsonify({"status": "paused"})
|
||||||
|
|
||||||
|
@bp.route("/enrichment/<service>/resume", methods=["POST"])
|
||||||
|
def video_enrichment_resume(service):
|
||||||
|
if service == "youtube":
|
||||||
|
_yt_enricher().resume()
|
||||||
|
return jsonify({"status": "running"})
|
||||||
|
w = engine().worker(service)
|
||||||
|
if not w:
|
||||||
|
return jsonify({"error": "unknown service"}), 404
|
||||||
|
w.resume()
|
||||||
|
return jsonify({"status": "running"})
|
||||||
|
|
||||||
|
@bp.route("/enrichment/priority", methods=["GET", "POST"])
|
||||||
|
def video_enrichment_priority():
|
||||||
|
from . import get_video_db
|
||||||
|
db = get_video_db()
|
||||||
|
if request.method == "POST":
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
kind = body.get("priority") or ""
|
||||||
|
if kind not in ("", "movie", "show"):
|
||||||
|
return jsonify({"error": "bad priority"}), 400
|
||||||
|
db.set_setting("enrichment_priority", kind)
|
||||||
|
return jsonify({"success": True, "priority": kind})
|
||||||
|
return jsonify({"priority": db.get_setting("enrichment_priority") or ""})
|
||||||
|
|
||||||
|
@bp.route("/enrichment/<service>/test", methods=["POST"])
|
||||||
|
def video_enrichment_test(service):
|
||||||
|
w = engine().worker(service)
|
||||||
|
if not w:
|
||||||
|
return jsonify({"success": False, "error": "unknown service"}), 404
|
||||||
|
try:
|
||||||
|
ok, msg = w.client.test()
|
||||||
|
return jsonify({"success": bool(ok), "message": msg, "error": None if ok else msg})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video enrichment test failed for %s", service)
|
||||||
|
return jsonify({"success": False, "error": "Test failed"})
|
||||||
|
|
||||||
|
@bp.route("/enrichment/<service>/breakdown", methods=["GET"])
|
||||||
|
def video_enrichment_breakdown(service):
|
||||||
|
from . import get_video_db
|
||||||
|
return jsonify({"service": service, "breakdown": get_video_db().enrichment_breakdown(service)})
|
||||||
|
|
||||||
|
@bp.route("/enrichment/<service>/unmatched", methods=["GET"])
|
||||||
|
def video_enrichment_unmatched(service):
|
||||||
|
from . import get_video_db
|
||||||
|
kind = request.args.get("kind", "movie")
|
||||||
|
res = get_video_db().enrichment_unmatched(
|
||||||
|
service, kind,
|
||||||
|
status=request.args.get("status", "not_found"),
|
||||||
|
search=request.args.get("q") or None,
|
||||||
|
limit=_int(request.args.get("limit"), 50),
|
||||||
|
offset=_int(request.args.get("offset"), 0))
|
||||||
|
res.update({"service": service, "kind": kind})
|
||||||
|
return jsonify(res)
|
||||||
|
|
||||||
|
@bp.route("/enrichment/retry-all-failed", methods=["POST"])
|
||||||
|
def video_enrichment_retry_all_failed():
|
||||||
|
"""Global re-queue: reset every failed/not_found item across ALL workers and
|
||||||
|
kinds (one-click recovery after an outage). Returns the total re-queued."""
|
||||||
|
from . import get_video_db
|
||||||
|
return jsonify({"success": True, "reset": get_video_db().retry_all_failed()})
|
||||||
|
|
||||||
|
@bp.route("/enrichment/<service>/retry", methods=["POST"])
|
||||||
|
def video_enrichment_retry(service):
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
n = get_video_db().enrichment_retry(
|
||||||
|
service, body.get("kind", "movie"),
|
||||||
|
scope=body.get("scope", "failed"), item_id=body.get("item_id"))
|
||||||
|
return jsonify({"success": True, "reset": n})
|
||||||
182
api/video/libraries.py
Normal file
182
api/video/libraries.py
Normal file
|
|
@ -0,0 +1,182 @@
|
||||||
|
"""Video library mapping endpoints.
|
||||||
|
|
||||||
|
GET /api/video/libraries -> discover the active server's Movies/TV libraries
|
||||||
|
+ the user's current selection.
|
||||||
|
POST /api/video/libraries -> save {movies, tv} (library titles) for the active
|
||||||
|
server. The scanner then reads only those.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from flask import jsonify, request
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.libraries")
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
@bp.route("/libraries", methods=["GET"])
|
||||||
|
def video_libraries():
|
||||||
|
from . import get_video_db
|
||||||
|
try:
|
||||||
|
from core.video.sources import list_video_libraries, resolve_video_server
|
||||||
|
libs = list_video_libraries() or {"server": None, "movies": [], "tv": []}
|
||||||
|
server = libs.get("server") or resolve_video_server()
|
||||||
|
libs["selected"] = (get_video_db().get_library_selection(server)
|
||||||
|
if server else {"movies": None, "tv": None})
|
||||||
|
return jsonify(libs)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to list video libraries")
|
||||||
|
return jsonify({"error": "Failed to list video libraries"}), 500
|
||||||
|
|
||||||
|
@bp.route("/server", methods=["GET"])
|
||||||
|
def video_server_status():
|
||||||
|
"""Which server the video side uses + which of Plex/Jellyfin are configured
|
||||||
|
(so the UI can show a picker, or a 'connect a server' message)."""
|
||||||
|
try:
|
||||||
|
from core.video.sources import (resolve_video_server,
|
||||||
|
video_plex_config, video_jellyfin_config)
|
||||||
|
plex = bool(video_plex_config().get("base_url"))
|
||||||
|
jelly = bool(video_jellyfin_config().get("base_url"))
|
||||||
|
return jsonify({"server": resolve_video_server(), "plex": plex, "jellyfin": jelly})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video server status failed")
|
||||||
|
return jsonify({"server": None, "plex": False, "jellyfin": False})
|
||||||
|
|
||||||
|
@bp.route("/server", methods=["POST"])
|
||||||
|
def video_server_set():
|
||||||
|
"""Set the explicit video-side server pick (only meaningful when both Plex
|
||||||
|
and Jellyfin are configured)."""
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
choice = body.get("server")
|
||||||
|
if choice not in ("plex", "jellyfin"):
|
||||||
|
return jsonify({"error": "bad server"}), 400
|
||||||
|
get_video_db().set_setting("video_server", choice)
|
||||||
|
return jsonify({"status": "saved", "server": choice})
|
||||||
|
|
||||||
|
@bp.route("/server-config", methods=["GET"])
|
||||||
|
def video_server_config_get():
|
||||||
|
"""The video side's OWN server connection — its stored creds when set, else
|
||||||
|
the values INHERITED (read-only) from music. 'inherited' flags tell the UI a
|
||||||
|
field is a placeholder it can override; tokens/keys are returned masked."""
|
||||||
|
try:
|
||||||
|
from core.video.sources import video_plex_config, video_jellyfin_config
|
||||||
|
p, j = video_plex_config(), video_jellyfin_config()
|
||||||
|
|
||||||
|
def mask(v):
|
||||||
|
v = v or ""
|
||||||
|
return ("•" * 12) if v else ""
|
||||||
|
return jsonify({
|
||||||
|
"plex": {"base_url": p.get("base_url") or "", "token": mask(p.get("token")),
|
||||||
|
"has_token": bool(p.get("token")), "inherited": p.get("source") == "music"},
|
||||||
|
"jellyfin": {"base_url": j.get("base_url") or "", "api_key": mask(j.get("api_key")),
|
||||||
|
"has_key": bool(j.get("api_key")), "inherited": j.get("source") == "music"},
|
||||||
|
})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video server-config get failed")
|
||||||
|
return jsonify({"plex": {}, "jellyfin": {}})
|
||||||
|
|
||||||
|
@bp.route("/server-config", methods=["POST"])
|
||||||
|
def video_server_config_set():
|
||||||
|
"""Save the video side's OWN Plex/Jellyfin creds to video.db (NEVER the music
|
||||||
|
config). An empty/blank field clears that override → video falls back to
|
||||||
|
inheriting music's value. A masked token (all •) is left untouched."""
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
db = get_video_db()
|
||||||
|
|
||||||
|
def is_mask(v):
|
||||||
|
return bool(v) and set(str(v)) == {"•"}
|
||||||
|
|
||||||
|
def put(key, val):
|
||||||
|
if is_mask(val):
|
||||||
|
return # unchanged masked secret — keep what's stored
|
||||||
|
db.set_setting(key, (val or "").strip())
|
||||||
|
|
||||||
|
plex = body.get("plex") or {}
|
||||||
|
jelly = body.get("jellyfin") or {}
|
||||||
|
if "base_url" in plex:
|
||||||
|
put("video_plex_url", plex.get("base_url"))
|
||||||
|
if "token" in plex:
|
||||||
|
put("video_plex_token", plex.get("token"))
|
||||||
|
if "base_url" in jelly:
|
||||||
|
put("video_jellyfin_url", jelly.get("base_url"))
|
||||||
|
if "api_key" in jelly:
|
||||||
|
put("video_jellyfin_key", jelly.get("api_key"))
|
||||||
|
return jsonify({"status": "saved"})
|
||||||
|
|
||||||
|
@bp.route("/server-config/test", methods=["POST"])
|
||||||
|
def video_server_config_test():
|
||||||
|
"""Test the video side's effective connection for one server, using its OWN
|
||||||
|
stored/inherited creds (so it verifies exactly what the video scan will use)."""
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
which = body.get("server")
|
||||||
|
if which not in ("plex", "jellyfin"):
|
||||||
|
return jsonify({"success": False, "error": "bad server"}), 400
|
||||||
|
try:
|
||||||
|
if which == "plex":
|
||||||
|
from core.video.sources import video_plex_config, PLEX_SCAN_TIMEOUT
|
||||||
|
cfg = video_plex_config()
|
||||||
|
if not cfg.get("base_url") or not cfg.get("token"):
|
||||||
|
return jsonify({"success": False, "error": "Plex URL/token not set"})
|
||||||
|
from plexapi.server import PlexServer
|
||||||
|
srv = PlexServer(cfg["base_url"], cfg["token"], timeout=PLEX_SCAN_TIMEOUT)
|
||||||
|
return jsonify({"success": True, "message": "Connected to " + (srv.friendlyName or "Plex")})
|
||||||
|
from core.video.sources import video_jellyfin_config, video_jellyfin_test
|
||||||
|
ok, message = video_jellyfin_test(video_jellyfin_config())
|
||||||
|
if ok:
|
||||||
|
return jsonify({"success": True, "message": message})
|
||||||
|
return jsonify({"success": False, "error": message})
|
||||||
|
except Exception as e:
|
||||||
|
return jsonify({"success": False, "error": str(e) or "connection failed"})
|
||||||
|
|
||||||
|
@bp.route("/jellyfin/users", methods=["GET"])
|
||||||
|
def video_jellyfin_users():
|
||||||
|
"""List the Jellyfin server's users so the video side can pick one (its
|
||||||
|
libraries are scoped to that user) — mirrors the music user picker. Uses
|
||||||
|
video's own effective Jellyfin creds."""
|
||||||
|
from . import get_video_db
|
||||||
|
try:
|
||||||
|
from core.video.sources import video_jellyfin_config
|
||||||
|
cfg = video_jellyfin_config()
|
||||||
|
base = (cfg.get("base_url") or "").rstrip("/")
|
||||||
|
key = cfg.get("api_key") or ""
|
||||||
|
if not base or not key:
|
||||||
|
return jsonify({"success": False, "users": []})
|
||||||
|
import requests
|
||||||
|
r = requests.get(base + "/Users", headers={"X-Emby-Token": key}, timeout=8)
|
||||||
|
if r.status_code != 200:
|
||||||
|
return jsonify({"success": False, "users": [], "error": "HTTP %d" % r.status_code})
|
||||||
|
users = r.json() or []
|
||||||
|
out = [{"id": u.get("Id"), "name": u.get("Name"),
|
||||||
|
"admin": bool((u.get("Policy") or {}).get("IsAdministrator"))}
|
||||||
|
for u in users if u.get("Id")]
|
||||||
|
selected = get_video_db().get_setting("video_jellyfin_user") or ""
|
||||||
|
return jsonify({"success": True, "users": out, "selected": selected})
|
||||||
|
except Exception as e:
|
||||||
|
return jsonify({"success": False, "users": [], "error": str(e)})
|
||||||
|
|
||||||
|
@bp.route("/jellyfin/user", methods=["POST"])
|
||||||
|
def video_jellyfin_user_set():
|
||||||
|
"""Persist the chosen Jellyfin user (its Id) for the video side."""
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
get_video_db().set_setting("video_jellyfin_user", (body.get("user") or "").strip())
|
||||||
|
return jsonify({"status": "saved"})
|
||||||
|
|
||||||
|
@bp.route("/libraries", methods=["POST"])
|
||||||
|
def save_video_libraries():
|
||||||
|
from . import get_video_db
|
||||||
|
try:
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
server = resolve_video_server()
|
||||||
|
if not server:
|
||||||
|
return jsonify({"error": "no video server"}), 400
|
||||||
|
get_video_db().set_library_selection(server, body.get("movies"), body.get("tv"))
|
||||||
|
return jsonify({"status": "saved", "server": server})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to save video library selection")
|
||||||
|
return jsonify({"error": "Failed to save selection"}), 500
|
||||||
34
api/video/library.py
Normal file
34
api/video/library.py
Normal file
|
|
@ -0,0 +1,34 @@
|
||||||
|
"""Video library listing endpoint.
|
||||||
|
|
||||||
|
GET /api/video/library -> {"movies": [...], "shows": [...]}
|
||||||
|
Reads what the last scan mirrored from the media server into video.db.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from flask import jsonify, request
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.library")
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
@bp.route("/library", methods=["GET"])
|
||||||
|
def video_library():
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
try:
|
||||||
|
return jsonify(get_video_db().query_library(
|
||||||
|
request.args.get("kind", "movies"),
|
||||||
|
search=request.args.get("search") or None,
|
||||||
|
letter=request.args.get("letter") or None,
|
||||||
|
sort=request.args.get("sort", "title"),
|
||||||
|
status=request.args.get("status", "all"),
|
||||||
|
page=request.args.get("page", 1),
|
||||||
|
limit=request.args.get("limit", 75),
|
||||||
|
server_source=resolve_video_server(),
|
||||||
|
))
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to query video library")
|
||||||
|
return jsonify({"error": "Failed to load video library"}), 500
|
||||||
160
api/video/manual_import.py
Normal file
160
api/video/manual_import.py
Normal file
|
|
@ -0,0 +1,160 @@
|
||||||
|
"""Manual / failed-import resolution API (isolated).
|
||||||
|
|
||||||
|
When a finished download can't be auto-placed it's parked as ``import_failed`` with
|
||||||
|
the file left on disk (``dest_path`` points at it). The Import page surfaces these and
|
||||||
|
lets the user place them by hand:
|
||||||
|
|
||||||
|
GET /api/video/import/failed → the queue of unplaced downloads
|
||||||
|
POST /api/video/import/<id>/place → force-import to the user's chosen identity
|
||||||
|
POST /api/video/import/<id>/dismiss → drop the row (optionally delete the file)
|
||||||
|
|
||||||
|
The identity picker on the page reuses the existing /api/video/search (TMDB, with a
|
||||||
|
``library_id`` annotation for owned titles) — no new search endpoint needed here.
|
||||||
|
Reads only the video engine + video.db; nothing from the music side.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
from flask import jsonify, request
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.manual_import")
|
||||||
|
|
||||||
|
_KIND_FOR_SCOPE = {"movie": "movie", "episode": "show"}
|
||||||
|
|
||||||
|
|
||||||
|
def _ctx(row):
|
||||||
|
try:
|
||||||
|
c = json.loads(row.get("search_ctx") or "{}")
|
||||||
|
return c if isinstance(c, dict) else {}
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def _failed_view(row):
|
||||||
|
"""The render-ready shape for one unplaced download."""
|
||||||
|
c = _ctx(row)
|
||||||
|
return {
|
||||||
|
"id": row.get("id"),
|
||||||
|
"title": row.get("title"),
|
||||||
|
"kind": row.get("kind"),
|
||||||
|
"year": row.get("year"),
|
||||||
|
"reason": row.get("error"),
|
||||||
|
"file": row.get("dest_path"), # where the file is sitting, unplaced
|
||||||
|
"release_title": row.get("release_title"),
|
||||||
|
"poster_url": row.get("poster_url"),
|
||||||
|
"scope": c.get("scope"),
|
||||||
|
"season": c.get("season"),
|
||||||
|
"episode": c.get("episode"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
@bp.route("/import/failed", methods=["GET"])
|
||||||
|
def video_import_failed():
|
||||||
|
from . import get_video_db
|
||||||
|
rows = get_video_db().get_import_failed_video_downloads()
|
||||||
|
return jsonify({"success": True, "items": [_failed_view(r) for r in rows]})
|
||||||
|
|
||||||
|
@bp.route("/import/<int:dl_id>/place", methods=["POST"])
|
||||||
|
def video_import_place(dl_id):
|
||||||
|
"""Force-import an unplaced file to the user's chosen identity. Body:
|
||||||
|
{scope, title, year, season, episode, episode_title, media_id}."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video import organization
|
||||||
|
from core.video.download_pipeline import target_dir_for
|
||||||
|
from core.video.importer import real_fs, run_import
|
||||||
|
from core.video.mediainfo import probe
|
||||||
|
|
||||||
|
db = get_video_db()
|
||||||
|
row = db.get_video_download(dl_id)
|
||||||
|
if not row or row.get("status") != "import_failed":
|
||||||
|
return jsonify({"success": False, "error": "Not an unplaced import."}), 404
|
||||||
|
src = row.get("dest_path")
|
||||||
|
if not src or not os.path.exists(src):
|
||||||
|
return jsonify({"success": False, "error": "The file is no longer on disk."}), 410
|
||||||
|
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
scope = str(body.get("scope") or "").lower()
|
||||||
|
if scope not in ("movie", "episode"):
|
||||||
|
return jsonify({"success": False, "error": "Choose a movie or an episode."}), 400
|
||||||
|
|
||||||
|
paths = {k: db.get_setting(k) or "" for k in ("movies_path", "tv_path", "youtube_path")}
|
||||||
|
if not paths["movies_path"]:
|
||||||
|
paths["movies_path"] = db.get_setting("transfer_path") or ""
|
||||||
|
override = {
|
||||||
|
"scope": scope,
|
||||||
|
"title": body.get("title"),
|
||||||
|
"year": body.get("year"),
|
||||||
|
"season": body.get("season"),
|
||||||
|
"episode": body.get("episode"),
|
||||||
|
"episode_title": body.get("episode_title"),
|
||||||
|
"media_id": body.get("media_id"),
|
||||||
|
"target_dir": target_dir_for(_KIND_FOR_SCOPE[scope], paths),
|
||||||
|
}
|
||||||
|
settings = organization.load(db)
|
||||||
|
prober = probe if settings.get("verify_with_ffprobe", True) else None
|
||||||
|
patch = run_import(row, src, fs=real_fs(), prober=prober, settings=settings,
|
||||||
|
force=True, override=override)
|
||||||
|
try:
|
||||||
|
db.update_video_download(dl_id, **patch)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("manual place: failed to persist import %s", dl_id)
|
||||||
|
return jsonify({"success": False, "error": "Couldn't save the result."}), 500
|
||||||
|
ok = patch.get("status") == "completed"
|
||||||
|
if ok:
|
||||||
|
# Write NFO + artwork sidecars for the chosen identity (best-effort), then
|
||||||
|
# refresh the server + DB the same way the auto-download path does
|
||||||
|
# (batch-complete → scan chain), so the manually-placed title shows up
|
||||||
|
# without waiting for a scheduled scan.
|
||||||
|
sidecar_dl = {"kind": _KIND_FOR_SCOPE[scope], "media_source": "tmdb",
|
||||||
|
"media_id": override.get("media_id"),
|
||||||
|
"poster_url": row.get("poster_url"),
|
||||||
|
"search_ctx": json.dumps({"scope": scope, "season": override.get("season"),
|
||||||
|
"episode": override.get("episode")})}
|
||||||
|
if settings.get("save_artwork") or settings.get("write_nfo"):
|
||||||
|
try:
|
||||||
|
from core.video.download_monitor import write_sidecars
|
||||||
|
from core.video.importer import real_fs
|
||||||
|
write_sidecars(db, sidecar_dl, patch["dest_path"], settings, real_fs())
|
||||||
|
except Exception:
|
||||||
|
logger.exception("manual place: sidecar write failed for %s", dl_id)
|
||||||
|
if settings.get("download_subtitles"):
|
||||||
|
try:
|
||||||
|
from core.video.download_monitor import write_subtitles_for
|
||||||
|
from core.video.importer import real_fs
|
||||||
|
write_subtitles_for(db, sidecar_dl, patch["dest_path"], settings, real_fs())
|
||||||
|
except Exception:
|
||||||
|
logger.exception("manual place: subtitle fetch failed for %s", dl_id)
|
||||||
|
try:
|
||||||
|
from core.video.download_events import notify_batch_complete
|
||||||
|
notify_batch_complete({"completed": 1, "manual": True})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("manual place: batch-complete notify failed for %s", dl_id)
|
||||||
|
return jsonify({"success": ok, "status": patch.get("status"),
|
||||||
|
"dest_path": patch.get("dest_path"), "error": patch.get("error")})
|
||||||
|
|
||||||
|
@bp.route("/import/<int:dl_id>/dismiss", methods=["POST"])
|
||||||
|
def video_import_dismiss(dl_id):
|
||||||
|
"""Drop a failed-import row. Body {delete_file: bool} optionally removes the
|
||||||
|
unplaced file from disk too."""
|
||||||
|
from . import get_video_db
|
||||||
|
db = get_video_db()
|
||||||
|
row = db.get_video_download(dl_id)
|
||||||
|
if not row or row.get("status") != "import_failed":
|
||||||
|
return jsonify({"success": False, "error": "Not an unplaced import."}), 404
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
if body.get("delete_file"):
|
||||||
|
src = row.get("dest_path")
|
||||||
|
if src and os.path.exists(src):
|
||||||
|
try:
|
||||||
|
os.remove(src)
|
||||||
|
except OSError:
|
||||||
|
logger.warning("dismiss: could not delete %s", src)
|
||||||
|
db.update_video_download(dl_id, status="cancelled",
|
||||||
|
error="Dismissed from manual import")
|
||||||
|
return jsonify({"success": True})
|
||||||
152
api/video/poster.py
Normal file
152
api/video/poster.py
Normal file
|
|
@ -0,0 +1,152 @@
|
||||||
|
"""Video poster proxy.
|
||||||
|
|
||||||
|
GET /api/video/poster/<kind>/<id> streams a movie/show poster from the media
|
||||||
|
server, server-side (so the Plex token / Jellyfin key never reaches the
|
||||||
|
browser). Falls back to 404 so the frontend shows its placeholder.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from flask import Response, abort, request
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.poster")
|
||||||
|
|
||||||
|
|
||||||
|
def _req_width():
|
||||||
|
"""Optional ?w= thumbnail width (clamped) so the calendar/library don't pull
|
||||||
|
full-size art into tiny cells. None = original."""
|
||||||
|
try:
|
||||||
|
w = request.args.get("w", type=int)
|
||||||
|
except Exception:
|
||||||
|
w = None
|
||||||
|
if not w:
|
||||||
|
return None
|
||||||
|
return max(48, min(1600, w))
|
||||||
|
|
||||||
|
|
||||||
|
def _tmdb_resize(url, w, backdrop):
|
||||||
|
"""Rewrite a TMDB image URL's size segment (/t/p/<size>/...) to a bucket near
|
||||||
|
``w`` so episode stills load small. Best-effort — unchanged if it doesn't match."""
|
||||||
|
import re
|
||||||
|
buckets = [300, 780, 1280] if backdrop else [185, 342, 500, 780]
|
||||||
|
pick = next((b for b in buckets if w <= b), buckets[-1])
|
||||||
|
return re.sub(r"/t/p/[^/]+/", "/t/p/w%d/" % pick, url, count=1)
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
def _stream_art(kind, item_id, art):
|
||||||
|
from . import get_video_db
|
||||||
|
ref = get_video_db().get_art_ref(kind, item_id, art)
|
||||||
|
if not ref or not ref.get("poster_url"):
|
||||||
|
abort(404)
|
||||||
|
try:
|
||||||
|
import requests
|
||||||
|
w = _req_width()
|
||||||
|
backdrop = art == "backdrop"
|
||||||
|
path = ref["poster_url"]
|
||||||
|
# Enrichment can store a full external URL (e.g. a TMDB season poster) —
|
||||||
|
# stream it directly; otherwise it's a server path needing the token.
|
||||||
|
if path.startswith("http://") or path.startswith("https://"):
|
||||||
|
if w and "image.tmdb.org" in path:
|
||||||
|
path = _tmdb_resize(path, w, backdrop)
|
||||||
|
upstream = requests.get(path, timeout=15, stream=True)
|
||||||
|
if upstream.status_code != 200:
|
||||||
|
abort(404)
|
||||||
|
resp = Response(upstream.iter_content(8192),
|
||||||
|
content_type=upstream.headers.get("Content-Type", "image/jpeg"))
|
||||||
|
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||||||
|
return resp
|
||||||
|
# Art is served from the VIDEO side's effective connection (its own
|
||||||
|
# creds, or inherited from music) — that's where the item was scanned.
|
||||||
|
from core.video.sources import video_plex_config, video_jellyfin_config
|
||||||
|
source = ref.get("server_source")
|
||||||
|
fallback = None
|
||||||
|
if source == "plex":
|
||||||
|
cfg = video_plex_config()
|
||||||
|
base, token = cfg.get("base_url"), cfg.get("token")
|
||||||
|
if not base or not token:
|
||||||
|
abort(404)
|
||||||
|
base = base.rstrip("/")
|
||||||
|
params = {"X-Plex-Token": token}
|
||||||
|
if w:
|
||||||
|
# Plex photo transcoder → a right-sized JPEG instead of full art;
|
||||||
|
# fall back to the original if a server has transcoding disabled.
|
||||||
|
from urllib.parse import quote
|
||||||
|
h = int(w * (9 / 16 if backdrop else 3 / 2))
|
||||||
|
url = (base + "/photo/:/transcode?width=%d&height=%d&minSize=1&upscale=1&url=%s"
|
||||||
|
% (w, h, quote(ref["poster_url"], safe="")))
|
||||||
|
fallback = base + ref["poster_url"]
|
||||||
|
else:
|
||||||
|
url = base + ref["poster_url"]
|
||||||
|
elif source == "jellyfin":
|
||||||
|
cfg = video_jellyfin_config()
|
||||||
|
base, key = cfg.get("base_url"), cfg.get("api_key")
|
||||||
|
if not base:
|
||||||
|
abort(404)
|
||||||
|
image = "Backdrop" if backdrop else "Primary"
|
||||||
|
url = base.rstrip("/") + f"/Items/{ref['server_id']}/Images/{image}"
|
||||||
|
params = {"api_key": key} if key else {}
|
||||||
|
if w:
|
||||||
|
params["maxWidth"] = w
|
||||||
|
params["quality"] = 90
|
||||||
|
else:
|
||||||
|
abort(404)
|
||||||
|
|
||||||
|
upstream = requests.get(url, params=params, timeout=15, stream=True)
|
||||||
|
if upstream.status_code != 200 and fallback:
|
||||||
|
upstream = requests.get(fallback, params=params, timeout=15, stream=True)
|
||||||
|
if upstream.status_code != 200:
|
||||||
|
abort(404)
|
||||||
|
ctype = upstream.headers.get("Content-Type", "image/jpeg")
|
||||||
|
resp = Response(upstream.iter_content(8192), content_type=ctype)
|
||||||
|
resp.headers["Cache-Control"] = "public, max-age=86400"
|
||||||
|
return resp
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video %s proxy failed for %s/%s", art, kind, item_id)
|
||||||
|
abort(404)
|
||||||
|
|
||||||
|
@bp.route("/poster/<kind>/<int:item_id>", methods=["GET"])
|
||||||
|
def video_poster(kind, item_id):
|
||||||
|
return _stream_art(kind, item_id, "poster")
|
||||||
|
|
||||||
|
@bp.route("/backdrop/<kind>/<int:item_id>", methods=["GET"])
|
||||||
|
def video_backdrop(kind, item_id):
|
||||||
|
return _stream_art(kind, item_id, "backdrop")
|
||||||
|
|
||||||
|
@bp.route("/img", methods=["GET"])
|
||||||
|
def video_img_proxy():
|
||||||
|
"""Same-origin image proxy (SSRF-safe allowlist). TMDB for accent-sampling,
|
||||||
|
and YouTube CDN (avatars/banners/thumbnails) so channel art loads reliably
|
||||||
|
regardless of hotlink/CORS policy."""
|
||||||
|
from flask import request
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
url = request.args.get("u", "")
|
||||||
|
if not url.startswith("https://"):
|
||||||
|
abort(404)
|
||||||
|
host = (urlparse(url).hostname or "").lower()
|
||||||
|
# image.tmdb.org + any YouTube image host (yt3/yt4.ggpht, googleusercontent, *.ytimg)
|
||||||
|
ok = host == "image.tmdb.org" or any(
|
||||||
|
host == s or host.endswith("." + s) for s in ("ytimg.com", "ggpht.com", "googleusercontent.com"))
|
||||||
|
if not ok:
|
||||||
|
abort(404)
|
||||||
|
try:
|
||||||
|
import requests
|
||||||
|
# A browser UA — Google's image CDN (yt3/googleusercontent) 403s some
|
||||||
|
# avatars when fetched with no User-Agent, which blanked search results.
|
||||||
|
upstream = requests.get(url, timeout=15, stream=True, headers={
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||||
|
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||||
|
"Accept": "image/avif,image/webp,image/apng,image/*,*/*;q=0.8",
|
||||||
|
})
|
||||||
|
if upstream.status_code != 200:
|
||||||
|
abort(404)
|
||||||
|
resp = Response(upstream.iter_content(8192),
|
||||||
|
content_type=upstream.headers.get("Content-Type", "image/jpeg"))
|
||||||
|
resp.headers["Cache-Control"] = "public, max-age=604800"
|
||||||
|
resp.headers["Access-Control-Allow-Origin"] = "*"
|
||||||
|
return resp
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video image proxy failed for %s", url)
|
||||||
|
abort(404)
|
||||||
61
api/video/scan.py
Normal file
61
api/video/scan.py
Normal file
|
|
@ -0,0 +1,61 @@
|
||||||
|
"""Video library scan endpoints.
|
||||||
|
|
||||||
|
POST /api/video/scan/request -> start a background scan of the active server
|
||||||
|
GET /api/video/scan/status -> current scan progress/state
|
||||||
|
|
||||||
|
The scan READS the media server (source of truth) into video.db. Triggering the
|
||||||
|
server's own rescan (post-download) is wired separately into the download flow.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from flask import jsonify, request
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.scan")
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
@bp.route("/scan/request", methods=["POST"])
|
||||||
|
def video_scan_request():
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.scanner import get_video_scanner
|
||||||
|
from core.video.sources import get_active_video_source
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
mode = body.get("mode", "full")
|
||||||
|
# Which library to scan — movies and TV are independent libraries, so the
|
||||||
|
# UI can target one or both. The scanner normalizes/validates it.
|
||||||
|
media_type = body.get("media_type", "all")
|
||||||
|
scanner = get_video_scanner(get_video_db())
|
||||||
|
return jsonify(scanner.request_scan(get_active_video_source, mode, media_type))
|
||||||
|
|
||||||
|
@bp.route("/scan/status", methods=["GET"])
|
||||||
|
def video_scan_status():
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.scanner import get_video_scanner
|
||||||
|
return jsonify(get_video_scanner(get_video_db()).get_status())
|
||||||
|
|
||||||
|
@bp.route("/scan/stop", methods=["POST"])
|
||||||
|
def video_scan_stop():
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.scanner import get_video_scanner
|
||||||
|
return jsonify(get_video_scanner(get_video_db()).cancel())
|
||||||
|
|
||||||
|
# ── Server-side scan (distinct from the SoulSync-reads-server scan above) ──
|
||||||
|
# This tells the media server (Plex/Jellyfin) to rescan its OWN folders so
|
||||||
|
# newly-downloaded files get indexed — the manual twin of the post-download
|
||||||
|
# 'Scan Video Server' automation. media_type scopes it to Movies / TV / both.
|
||||||
|
@bp.route("/scan/server", methods=["POST"])
|
||||||
|
def video_scan_server():
|
||||||
|
from core.video.sources import refresh_video_server_sections
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
media_type = body.get("media_type", "all")
|
||||||
|
return jsonify(refresh_video_server_sections(media_type))
|
||||||
|
|
||||||
|
@bp.route("/scan/server/status", methods=["GET"])
|
||||||
|
def video_scan_server_status():
|
||||||
|
# {scanning: true|false|null} — null when the adapter can't report state.
|
||||||
|
from core.video.sources import video_server_scan_in_progress
|
||||||
|
media_type = request.args.get("media_type", "all")
|
||||||
|
return jsonify({"scanning": video_server_scan_in_progress(media_type)})
|
||||||
43
api/video/search.py
Normal file
43
api/video/search.py
Normal file
|
|
@ -0,0 +1,43 @@
|
||||||
|
"""Video search API (in-app, isolated).
|
||||||
|
|
||||||
|
GET /api/video/search?q=... → TMDB multi-search (movies / shows / people),
|
||||||
|
movie/show results annotated with library_id
|
||||||
|
if already owned.
|
||||||
|
|
||||||
|
Everything resolves back into SoulSync — results link to the library detail
|
||||||
|
(owned) or the TMDB-backed detail (not owned); people open the in-app person
|
||||||
|
page. Reads only the video engine + video.db.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from flask import jsonify, request
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.search")
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
@bp.route("/search", methods=["GET"])
|
||||||
|
def video_search():
|
||||||
|
q = (request.args.get("q") or "").strip()
|
||||||
|
if not q:
|
||||||
|
return jsonify({"results": [], "query": ""})
|
||||||
|
try:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
results = get_video_enrichment_engine().search(q)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video search failed for %r", q)
|
||||||
|
results = []
|
||||||
|
return jsonify({"results": results, "query": q})
|
||||||
|
|
||||||
|
@bp.route("/trending", methods=["GET"])
|
||||||
|
def video_trending():
|
||||||
|
try:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
results = get_video_enrichment_engine().trending()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video trending failed")
|
||||||
|
results = []
|
||||||
|
return jsonify({"results": results})
|
||||||
122
api/video/watchlist.py
Normal file
122
api/video/watchlist.py
Normal file
|
|
@ -0,0 +1,122 @@
|
||||||
|
"""Video watchlist API — the user's curated follow-list of shows + people.
|
||||||
|
|
||||||
|
Mirrors the music watchlist's add/remove/list/check shape. v1 just manages
|
||||||
|
membership (so cards can toggle + the Watchlist page can render); the
|
||||||
|
monitoring/discovery engine that turns a follow into new downloads is a later
|
||||||
|
phase. Reads/writes only video_library.db via the shared VideoDatabase.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from flask import jsonify, request
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.watchlist")
|
||||||
|
|
||||||
|
_KINDS = ("show", "person")
|
||||||
|
|
||||||
|
|
||||||
|
def _server():
|
||||||
|
"""Active video server_source (scopes the airing-show default). None on error."""
|
||||||
|
try:
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
return resolve_video_server()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
@bp.route("/watchlist", methods=["GET"])
|
||||||
|
def video_watchlist_list():
|
||||||
|
"""All watchlist entries grouped by kind (for the tabbed page).
|
||||||
|
|
||||||
|
Shows include actively-airing library shows by default, scoped to the
|
||||||
|
active video server so Plex/Jellyfin libraries don't commingle."""
|
||||||
|
from . import get_video_db
|
||||||
|
try:
|
||||||
|
db = get_video_db()
|
||||||
|
server = _server()
|
||||||
|
kind = request.args.get("kind")
|
||||||
|
counts = db.watchlist_counts(server_source=server)
|
||||||
|
if kind in _KINDS:
|
||||||
|
# Paged + searchable, like the library page.
|
||||||
|
res = db.query_watchlist(
|
||||||
|
kind, search=request.args.get("search", ""),
|
||||||
|
sort=request.args.get("sort", "default"),
|
||||||
|
page=request.args.get("page", 1), limit=request.args.get("limit", 60),
|
||||||
|
server_source=server)
|
||||||
|
return jsonify({"success": True, "kind": kind, "counts": counts, **res})
|
||||||
|
# No kind → grouped (counts + first-glance lists).
|
||||||
|
rows = db.list_watchlist(server_source=server)
|
||||||
|
shows = [r for r in rows if r.get("kind") == "show"]
|
||||||
|
people = [r for r in rows if r.get("kind") == "person"]
|
||||||
|
return jsonify({"success": True, "shows": shows, "people": people, "counts": counts})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to list video watchlist")
|
||||||
|
return jsonify({"success": False, "error": "Failed to load watchlist"}), 500
|
||||||
|
|
||||||
|
@bp.route("/watchlist/counts", methods=["GET"])
|
||||||
|
def video_watchlist_counts():
|
||||||
|
from . import get_video_db
|
||||||
|
try:
|
||||||
|
return jsonify({"success": True, **get_video_db().watchlist_counts(server_source=_server())})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to count video watchlist")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
||||||
|
@bp.route("/watchlist/add", methods=["POST"])
|
||||||
|
def video_watchlist_add():
|
||||||
|
"""Add a show/person. Body: {kind, tmdb_id, title, poster_url?, library_id?}."""
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
kind = body.get("kind")
|
||||||
|
tmdb_id = body.get("tmdb_id")
|
||||||
|
title = (body.get("title") or "").strip()
|
||||||
|
if kind not in _KINDS or not tmdb_id or not title:
|
||||||
|
return jsonify({"success": False, "error": "kind, tmdb_id and title are required"}), 400
|
||||||
|
try:
|
||||||
|
ok = get_video_db().add_to_watchlist(
|
||||||
|
kind, int(tmdb_id), title,
|
||||||
|
poster_url=body.get("poster_url") or None,
|
||||||
|
library_id=body.get("library_id") or None)
|
||||||
|
if not ok:
|
||||||
|
return jsonify({"success": False, "error": "Could not add to watchlist"}), 400
|
||||||
|
return jsonify({"success": True, "watched": True})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to add to video watchlist")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
||||||
|
@bp.route("/watchlist/remove", methods=["POST"])
|
||||||
|
def video_watchlist_remove():
|
||||||
|
"""Remove a show/person. Body: {kind, tmdb_id}."""
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
kind = body.get("kind")
|
||||||
|
tmdb_id = body.get("tmdb_id")
|
||||||
|
if kind not in _KINDS or not tmdb_id:
|
||||||
|
return jsonify({"success": False, "error": "kind and tmdb_id are required"}), 400
|
||||||
|
try:
|
||||||
|
removed = get_video_db().remove_from_watchlist(kind, int(tmdb_id))
|
||||||
|
return jsonify({"success": True, "watched": False, "removed": removed})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to remove from video watchlist")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
||||||
|
@bp.route("/watchlist/check", methods=["POST"])
|
||||||
|
def video_watchlist_check():
|
||||||
|
"""Hydrate cards. Body: {kind, tmdb_ids: [...]} → {results: {id: true}}.
|
||||||
|
Only watched ids appear in results (absent = not watched)."""
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
kind = body.get("kind")
|
||||||
|
ids = body.get("tmdb_ids") or []
|
||||||
|
if kind not in _KINDS:
|
||||||
|
return jsonify({"success": False, "error": "kind is required"}), 400
|
||||||
|
try:
|
||||||
|
state = get_video_db().watchlist_state(kind, ids, server_source=_server())
|
||||||
|
# JSON object keys must be strings.
|
||||||
|
return jsonify({"success": True, "results": {str(k): True for k in state}})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to check video watchlist")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
185
api/video/wishlist.py
Normal file
185
api/video/wishlist.py
Normal file
|
|
@ -0,0 +1,185 @@
|
||||||
|
"""Video wishlist API — the curated 'get this' list (movies + episodes).
|
||||||
|
|
||||||
|
Atomic units are movies and episodes; adding a whole show or a season just hands
|
||||||
|
us the explicit episodes to expand into rows. v1 manages membership + the tabbed
|
||||||
|
Movies/TV page; the search/download engine that fulfils a wish is a later phase.
|
||||||
|
Reads/writes only video_library.db via the shared VideoDatabase.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from flask import jsonify, request
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.wishlist")
|
||||||
|
|
||||||
|
_KINDS = ("movie", "show")
|
||||||
|
_SCOPES = ("movie", "show", "season", "episode")
|
||||||
|
|
||||||
|
|
||||||
|
def _server():
|
||||||
|
"""Active video server_source (stored on rows, informational). None on error."""
|
||||||
|
try:
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
return resolve_video_server()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
@bp.route("/wishlist", methods=["GET"])
|
||||||
|
def video_wishlist_list():
|
||||||
|
"""Paged slice for a tab (kind='movie'|'show'), or counts-only with no kind.
|
||||||
|
Shows are grouped show→season→episode with wanted/done roll-ups."""
|
||||||
|
from . import get_video_db
|
||||||
|
try:
|
||||||
|
db = get_video_db()
|
||||||
|
counts = db.wishlist_counts()
|
||||||
|
kind = request.args.get("kind")
|
||||||
|
if kind in _KINDS:
|
||||||
|
res = db.query_wishlist(
|
||||||
|
kind, search=request.args.get("search", ""), sort=request.args.get("sort", "added"),
|
||||||
|
page=request.args.get("page", 1), limit=request.args.get("limit", 60))
|
||||||
|
return jsonify({"success": True, "kind": kind, "counts": counts, **res})
|
||||||
|
return jsonify({"success": True, "counts": counts})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to list video wishlist")
|
||||||
|
return jsonify({"success": False, "error": "Failed to load wishlist"}), 500
|
||||||
|
|
||||||
|
@bp.route("/wishlist/counts", methods=["GET"])
|
||||||
|
def video_wishlist_counts():
|
||||||
|
from . import get_video_db
|
||||||
|
try:
|
||||||
|
db = get_video_db()
|
||||||
|
counts = db.wishlist_counts() # {movie, show, episode, total(movie+ep)}
|
||||||
|
yt = db.youtube_wishlist_counts() # {channel, video} (its own table-shape)
|
||||||
|
counts["video"] = yt.get("video", 0)
|
||||||
|
counts["channel"] = yt.get("channel", 0)
|
||||||
|
# The header/sidebar badge is the WHOLE wishlist — movies + episodes + YouTube
|
||||||
|
# videos — so fold the YouTube count into the total (it was movies+episodes only,
|
||||||
|
# which is why a YouTube-only wishlist showed no badge).
|
||||||
|
counts["total"] = counts.get("total", 0) + counts["video"]
|
||||||
|
return jsonify({"success": True, **counts})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to count video wishlist")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
||||||
|
@bp.route("/wishlist/add", methods=["POST"])
|
||||||
|
def video_wishlist_add():
|
||||||
|
"""Add a movie or a set of a show's episodes. Body is one of:
|
||||||
|
{"movie": {tmdb_id, title, year?, poster_url?, library_id?}}
|
||||||
|
{"show": {tmdb_id, title, poster_url?, library_id?},
|
||||||
|
"episodes": [{season_number, episode_number, title?, air_date?}, …]}"""
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
srv = _server()
|
||||||
|
db = get_video_db()
|
||||||
|
try:
|
||||||
|
movie = body.get("movie")
|
||||||
|
if movie and movie.get("tmdb_id") and (movie.get("title") or "").strip():
|
||||||
|
ok = db.add_movie_to_wishlist(
|
||||||
|
int(movie["tmdb_id"]), movie["title"].strip(), year=movie.get("year"),
|
||||||
|
poster_url=movie.get("poster_url") or None,
|
||||||
|
library_id=movie.get("library_id") or None, server_source=srv)
|
||||||
|
return jsonify({"success": ok, "added": 1 if ok else 0, "counts": db.wishlist_counts()})
|
||||||
|
|
||||||
|
show = body.get("show")
|
||||||
|
episodes = body.get("episodes") or []
|
||||||
|
if show and show.get("tmdb_id") and (show.get("title") or "").strip() and episodes:
|
||||||
|
n = db.add_episodes_to_wishlist(
|
||||||
|
int(show["tmdb_id"]), show["title"].strip(), episodes,
|
||||||
|
poster_url=show.get("poster_url") or None,
|
||||||
|
library_id=show.get("library_id") or None, server_source=srv)
|
||||||
|
return jsonify({"success": n > 0, "added": n, "counts": db.wishlist_counts()})
|
||||||
|
|
||||||
|
return jsonify({"success": False, "error": "movie or show+episodes required"}), 400
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to add to video wishlist")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
||||||
|
@bp.route("/wishlist/remove", methods=["POST"])
|
||||||
|
def video_wishlist_remove():
|
||||||
|
"""Remove at any granularity. Body: {scope, tmdb_id, season_number?, episode_number?}
|
||||||
|
where scope ∈ movie|show|season|episode."""
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
scope, tmdb_id = body.get("scope"), body.get("tmdb_id")
|
||||||
|
if scope not in _SCOPES or not tmdb_id:
|
||||||
|
return jsonify({"success": False, "error": "scope and tmdb_id are required"}), 400
|
||||||
|
try:
|
||||||
|
db = get_video_db()
|
||||||
|
removed = db.remove_from_wishlist(
|
||||||
|
scope, tmdb_id=int(tmdb_id),
|
||||||
|
season_number=body.get("season_number"), episode_number=body.get("episode_number"))
|
||||||
|
return jsonify({"success": True, "removed": removed, "counts": db.wishlist_counts()})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to remove from video wishlist")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
||||||
|
@bp.route("/wishlist/clear", methods=["POST"])
|
||||||
|
def video_wishlist_clear():
|
||||||
|
"""Empty an entire wishlist tab. Body: {kind} where kind ∈ movie|show|youtube."""
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
kind = body.get("kind")
|
||||||
|
if kind not in ("movie", "show", "youtube"):
|
||||||
|
return jsonify({"success": False, "error": "kind must be movie|show|youtube"}), 400
|
||||||
|
try:
|
||||||
|
db = get_video_db()
|
||||||
|
removed = db.clear_wishlist(kind)
|
||||||
|
return jsonify({"success": True, "removed": removed,
|
||||||
|
"counts": db.wishlist_counts(), "youtube_counts": db.youtube_wishlist_counts()})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to clear video wishlist")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
||||||
|
@bp.route("/wishlist/backfill-art", methods=["POST"])
|
||||||
|
def video_wishlist_backfill_art():
|
||||||
|
"""Fill episode stills + season posters for rows that predate art-capture.
|
||||||
|
One tmdb_season call per (show, season); best-effort. Returns rows filled."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
db = get_video_db()
|
||||||
|
eng = get_video_enrichment_engine()
|
||||||
|
updated = 0
|
||||||
|
try:
|
||||||
|
for grp in db.wishlist_art_backfill_targets():
|
||||||
|
try:
|
||||||
|
se = eng.tmdb_season(grp["tmdb_id"], grp["season_number"]) or {}
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
if se.get("poster_url"):
|
||||||
|
updated += db.set_wishlist_season_poster(grp["tmdb_id"], grp["season_number"], se["poster_url"])
|
||||||
|
for ep in (se.get("episodes") or []):
|
||||||
|
en = ep.get("episode_number")
|
||||||
|
if en is None:
|
||||||
|
continue
|
||||||
|
if ep.get("still_url") and db.set_wishlist_still(grp["tmdb_id"], grp["season_number"], en, ep["still_url"]):
|
||||||
|
updated += 1
|
||||||
|
if ep.get("overview"):
|
||||||
|
db.set_wishlist_episode_overview(grp["tmdb_id"], grp["season_number"], en, ep["overview"])
|
||||||
|
return jsonify({"success": True, "updated": updated})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("wishlist art backfill failed")
|
||||||
|
return jsonify({"success": False, "updated": updated})
|
||||||
|
|
||||||
|
@bp.route("/wishlist/check", methods=["POST"])
|
||||||
|
def video_wishlist_check():
|
||||||
|
"""Hydrate cards/modal. Body: {movie_ids: [...], show_tmdb_id?} →
|
||||||
|
{movies: [ids already wished], episodes: ['S_E' already wished]}."""
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
try:
|
||||||
|
db = get_video_db()
|
||||||
|
st = db.wishlist_state(
|
||||||
|
movie_ids=body.get("movie_ids") or [], show_tmdb_id=body.get("show_tmdb_id"))
|
||||||
|
out = {"success": True, "movies": sorted(st["movies"]), "episodes": sorted(st["episodes"])}
|
||||||
|
shows = body.get("shows") # multi-show membership for the calendar button
|
||||||
|
if shows:
|
||||||
|
keys = db.wishlist_keys_for_shows(shows)
|
||||||
|
out["by_show"] = {str(tid): sorted(ks) for tid, ks in keys.items()}
|
||||||
|
return jsonify(out)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Failed to check video wishlist")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
494
api/video/youtube.py
Normal file
494
api/video/youtube.py
Normal file
|
|
@ -0,0 +1,494 @@
|
||||||
|
"""Video YouTube API — follow a channel as a "show", its uploads flow to the
|
||||||
|
wishlist (visual-first: no downloading yet).
|
||||||
|
|
||||||
|
GET /youtube/resolve?url=... → preview a pasted channel (meta + recent
|
||||||
|
uploads) + whether it's already followed.
|
||||||
|
POST /youtube/follow → {url} (or a pre-resolved channel) → follow +
|
||||||
|
wish its videos. Returns the channel + counts.
|
||||||
|
POST /youtube/unfollow → {youtube_id} → un-follow (keeps wished videos).
|
||||||
|
GET /youtube/channels → followed channels (for the watchlist page).
|
||||||
|
GET /youtube/wishlist → wished videos grouped by channel (+ counts).
|
||||||
|
POST /youtube/wishlist/remove → {scope: channel|video, source_id}.
|
||||||
|
|
||||||
|
yt-dlp only; reads/writes only video_library.db.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from flask import jsonify, request
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_api.youtube")
|
||||||
|
|
||||||
|
# Cap how many recent uploads a follow pulls into the wishlist. Channels can have
|
||||||
|
# thousands of videos; flat listing is cheap but we don't want to wish them all.
|
||||||
|
_FOLLOW_LIMIT = 30
|
||||||
|
_RESOLVE_LIMIT = 24
|
||||||
|
|
||||||
|
|
||||||
|
def _server():
|
||||||
|
try:
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
return resolve_video_server()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def register_routes(bp):
|
||||||
|
@bp.route("/youtube/video/<video_id>/segments", methods=["GET"])
|
||||||
|
def video_youtube_segments(video_id):
|
||||||
|
"""SponsorBlock crowd segments stored for a video (sponsor/intro/outro/…),
|
||||||
|
so the player can offer skips. [] until the SponsorBlock worker enriches it."""
|
||||||
|
from . import get_video_db
|
||||||
|
return jsonify({"video_id": video_id,
|
||||||
|
"segments": get_video_db().youtube_video_segments(video_id)})
|
||||||
|
|
||||||
|
@bp.route("/youtube/resolve", methods=["GET"])
|
||||||
|
def video_youtube_resolve():
|
||||||
|
"""Preview a pasted channel URL without committing. Returns the channel +
|
||||||
|
recent uploads, plus ``following`` so the button can hydrate."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video import youtube as yt
|
||||||
|
url = (request.args.get("url") or "").strip()
|
||||||
|
if not url:
|
||||||
|
return jsonify({"success": False, "error": "url is required"}), 400
|
||||||
|
try:
|
||||||
|
limit = int(request.args.get("limit") or _RESOLVE_LIMIT)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
limit = _RESOLVE_LIMIT
|
||||||
|
try:
|
||||||
|
# A playlist link → resolve as a playlist (curator-ordered, partial set).
|
||||||
|
if yt.parse_playlist_id(url):
|
||||||
|
pl = yt.resolve_playlist(url, limit=max(1, min(50, limit)))
|
||||||
|
if not pl:
|
||||||
|
return jsonify({"success": False, "error": "Could not read that playlist"}), 404
|
||||||
|
following = bool(get_video_db().playlist_watch_state([pl["playlist_id"]]))
|
||||||
|
return jsonify({"success": True, "playlist": pl, "following": following})
|
||||||
|
channel = yt.resolve_channel(url, limit=max(1, min(50, limit)))
|
||||||
|
if not channel:
|
||||||
|
return jsonify({"success": False,
|
||||||
|
"error": "Not a YouTube channel or playlist link (paste a channel "
|
||||||
|
"URL like youtube.com/@handle, or a playlist link)"}), 404
|
||||||
|
following = bool(get_video_db().channel_watch_state([channel["youtube_id"]]))
|
||||||
|
return jsonify({"success": True, "channel": channel, "following": following})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube resolve failed for %r", url)
|
||||||
|
return jsonify({"success": False, "error": "Could not read that link"}), 500
|
||||||
|
|
||||||
|
@bp.route("/youtube/follow", methods=["POST"])
|
||||||
|
def video_youtube_follow():
|
||||||
|
"""Follow a channel + wish its recent uploads. Body: {url} (re-resolved) or
|
||||||
|
{channel: {...}} already resolved (avoids a second yt-dlp call)."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video import youtube as yt
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
db = get_video_db()
|
||||||
|
try:
|
||||||
|
channel = body.get("channel")
|
||||||
|
if not channel:
|
||||||
|
url = (body.get("url") or "").strip()
|
||||||
|
if not url:
|
||||||
|
return jsonify({"success": False, "error": "url or channel required"}), 400
|
||||||
|
channel = yt.resolve_channel(url, limit=_FOLLOW_LIMIT)
|
||||||
|
if not channel or not channel.get("youtube_id"):
|
||||||
|
return jsonify({"success": False, "error": "Could not resolve channel"}), 404
|
||||||
|
|
||||||
|
followed = db.add_channel_to_watchlist(channel)
|
||||||
|
added = db.add_videos_to_wishlist(channel, channel.get("videos") or [], server_source=_server())
|
||||||
|
if followed: # followed channels get their full upload-date catalog in the background
|
||||||
|
try:
|
||||||
|
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||||
|
get_youtube_date_enricher().enqueue(channel.get("youtube_id"), channel.get("title"))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return jsonify({"success": followed, "following": followed, "added_videos": added,
|
||||||
|
"channel": {k: channel.get(k) for k in ("youtube_id", "title", "avatar_url")},
|
||||||
|
"counts": db.youtube_wishlist_counts()})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube follow failed")
|
||||||
|
return jsonify({"success": False, "error": "Failed to follow channel"}), 500
|
||||||
|
|
||||||
|
@bp.route("/youtube/unfollow", methods=["POST"])
|
||||||
|
def video_youtube_unfollow():
|
||||||
|
"""Un-follow a channel. Body: {youtube_id}. Wished videos are left in place."""
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
cid = (body.get("youtube_id") or "").strip()
|
||||||
|
if not cid:
|
||||||
|
return jsonify({"success": False, "error": "youtube_id is required"}), 400
|
||||||
|
try:
|
||||||
|
get_video_db().remove_channel_from_watchlist(cid)
|
||||||
|
return jsonify({"success": True, "following": False})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube unfollow failed")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
||||||
|
@bp.route("/youtube/playlist/follow", methods=["POST"])
|
||||||
|
def video_youtube_playlist_follow():
|
||||||
|
"""Follow a YouTube playlist. Body: {playlist:{...}} (already resolved) or {url}."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video import youtube as yt
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
db = get_video_db()
|
||||||
|
try:
|
||||||
|
playlist = body.get("playlist")
|
||||||
|
if not playlist:
|
||||||
|
url = (body.get("url") or "").strip()
|
||||||
|
playlist = yt.resolve_playlist(url) if url else None
|
||||||
|
if not playlist or not playlist.get("playlist_id"):
|
||||||
|
return jsonify({"success": False, "error": "Could not resolve playlist"}), 404
|
||||||
|
ok = db.add_playlist_to_watchlist(playlist)
|
||||||
|
if ok and playlist.get("videos"): # remember the count straight away
|
||||||
|
try:
|
||||||
|
db.cache_channel_videos(playlist["playlist_id"], playlist["videos"])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return jsonify({"success": ok, "following": ok,
|
||||||
|
"playlist": {k: playlist.get(k) for k in ("playlist_id", "title", "thumbnail_url")}})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube playlist follow failed")
|
||||||
|
return jsonify({"success": False, "error": "Failed to follow playlist"}), 500
|
||||||
|
|
||||||
|
@bp.route("/youtube/playlist/unfollow", methods=["POST"])
|
||||||
|
def video_youtube_playlist_unfollow():
|
||||||
|
from . import get_video_db
|
||||||
|
pid = ((request.get_json(silent=True) or {}).get("playlist_id") or "").strip()
|
||||||
|
if not pid:
|
||||||
|
return jsonify({"success": False, "error": "playlist_id is required"}), 400
|
||||||
|
try:
|
||||||
|
get_video_db().remove_playlist_from_watchlist(pid)
|
||||||
|
return jsonify({"success": True, "following": False})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube playlist unfollow failed")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
||||||
|
@bp.route("/youtube/channels", methods=["GET"])
|
||||||
|
def video_youtube_channels():
|
||||||
|
"""Followed channels (newest first) for the watchlist page. Also sweeps:
|
||||||
|
any followed channel not date-enriched recently gets queued for the
|
||||||
|
background enricher (so existing follows get picked up, not just new ones)."""
|
||||||
|
from . import get_video_db
|
||||||
|
try:
|
||||||
|
db = get_video_db()
|
||||||
|
channels = db.list_watchlist_channels()
|
||||||
|
playlists = db.list_watchlist_playlists()
|
||||||
|
try:
|
||||||
|
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||||
|
enr = get_youtube_date_enricher()
|
||||||
|
for c in channels:
|
||||||
|
if not db.channel_dates_enriched_recently(c["youtube_id"]):
|
||||||
|
enr.enqueue(c["youtube_id"], c.get("title"))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return jsonify({"success": True, "channels": channels, "playlists": playlists,
|
||||||
|
"counts": db.youtube_wishlist_counts()})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube channels list failed")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
||||||
|
@bp.route("/youtube/channel/<channel_id>/settings", methods=["GET"])
|
||||||
|
def video_youtube_channel_settings(channel_id):
|
||||||
|
"""Per-channel overrides (custom show-name + quality), plus the global quality
|
||||||
|
default so the modal can show 'using default' until the user overrides."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.youtube_quality import default_profile, load as load_quality
|
||||||
|
db = get_video_db()
|
||||||
|
return jsonify({"success": True, "settings": db.get_channel_settings(channel_id) or {},
|
||||||
|
"default_quality": load_quality(db) or default_profile()})
|
||||||
|
|
||||||
|
@bp.route("/youtube/channel/<channel_id>/settings", methods=["POST"])
|
||||||
|
def video_youtube_channel_settings_save(channel_id):
|
||||||
|
"""Save per-channel overrides. A blank custom_name / absent quality clears that
|
||||||
|
override (falls back to the channel title / global quality). Body:
|
||||||
|
{custom_name?, quality?{...}}."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.youtube_quality import normalize as normalize_quality
|
||||||
|
db = get_video_db()
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
out = {}
|
||||||
|
name = str(body.get("custom_name") or "").strip()
|
||||||
|
if name:
|
||||||
|
out["custom_name"] = name
|
||||||
|
if body.get("quality"): # falsy/absent → no override (use the global default)
|
||||||
|
out["quality"] = normalize_quality(body.get("quality"))
|
||||||
|
keep = str(body.get("retention") or "").strip()
|
||||||
|
if keep and keep != "all": # blank/'all' → no policy (keep everything)
|
||||||
|
out["retention"] = keep
|
||||||
|
db.set_channel_settings(channel_id, out)
|
||||||
|
return jsonify({"success": True, "settings": out})
|
||||||
|
|
||||||
|
@bp.route("/youtube/wishlist", methods=["GET"])
|
||||||
|
def video_youtube_wishlist():
|
||||||
|
"""Wished videos grouped by channel (channel = group, videos = feed)."""
|
||||||
|
from . import get_video_db
|
||||||
|
try:
|
||||||
|
db = get_video_db()
|
||||||
|
res = db.query_youtube_wishlist(
|
||||||
|
search=request.args.get("search", ""), sort=request.args.get("sort", "added"),
|
||||||
|
page=request.args.get("page", 1), limit=request.args.get("limit", 60))
|
||||||
|
return jsonify({"success": True, "counts": db.youtube_wishlist_counts(), **res})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube wishlist list failed")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
||||||
|
@bp.route("/youtube/channel/<channel_id>", methods=["GET"])
|
||||||
|
def video_youtube_channel_detail(channel_id):
|
||||||
|
"""Full channel detail for the in-app channel page: meta + a deeper page of
|
||||||
|
uploads, the follow state, and per-video wished flags. Resolves live."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video import youtube as yt
|
||||||
|
try:
|
||||||
|
limit = int(request.args.get("limit") or 60)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
limit = 60
|
||||||
|
try:
|
||||||
|
db = get_video_db()
|
||||||
|
cid = str(channel_id).strip()
|
||||||
|
following = bool(db.channel_watch_state([cid]))
|
||||||
|
# Opening a channel page → (re)remember it in the background (followed or
|
||||||
|
# not — you're looking at it). The enricher caches list + meta + dates.
|
||||||
|
try:
|
||||||
|
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||||
|
get_youtube_date_enricher().enqueue(cid)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# CACHE-FIRST: a remembered channel renders instantly (no yt-dlp). The
|
||||||
|
# page's background re-stream + the enricher keep it fresh.
|
||||||
|
meta = db.get_channel_meta(cid)
|
||||||
|
cached_vids = db.get_channel_videos(cid)
|
||||||
|
from_cache = bool(meta and cached_vids)
|
||||||
|
if from_cache:
|
||||||
|
channel = {"youtube_id": cid, "title": meta.get("title"), "handle": meta.get("handle"),
|
||||||
|
"description": meta.get("description"), "avatar_url": meta.get("avatar_url"),
|
||||||
|
"banner_url": meta.get("banner_url"), "subscriber_count": meta.get("subscriber_count"),
|
||||||
|
"view_count": meta.get("view_count"), "tags": meta.get("tags") or [],
|
||||||
|
"videos": cached_vids}
|
||||||
|
else:
|
||||||
|
# MISS → fetch live (yt-dlp header + recent uploads) and remember it.
|
||||||
|
channel = yt.resolve_channel("https://www.youtube.com/channel/" + cid,
|
||||||
|
limit=max(1, min(90, limit)))
|
||||||
|
if not channel or not channel.get("youtube_id"):
|
||||||
|
return jsonify({"success": False, "error": "Channel not found"}), 404
|
||||||
|
cid = channel["youtube_id"]
|
||||||
|
db.cache_channel_meta(cid, channel)
|
||||||
|
db.cache_channel_videos(cid, channel.get("videos") or [])
|
||||||
|
if channel.get("avatar_url"):
|
||||||
|
try:
|
||||||
|
db.set_wishlist_channel_poster(cid, channel["avatar_url"])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
vids = channel.get("videos") or []
|
||||||
|
ids = [v.get("youtube_id") for v in vids]
|
||||||
|
wished = db.youtube_video_wish_state(ids)
|
||||||
|
# Dates → year-seasons. Cached list already carries them; only pull a
|
||||||
|
# fresh RSS (recent ~15) on a live MISS so the cache-hit stays instant.
|
||||||
|
dates = db.get_video_dates(ids)
|
||||||
|
if not from_cache:
|
||||||
|
try:
|
||||||
|
dates.update(yt.channel_recent_dates(cid) or {})
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
for v in vids:
|
||||||
|
v["wished"] = v.get("youtube_id") in wished
|
||||||
|
if not v.get("published_at") and dates.get(v.get("youtube_id")):
|
||||||
|
v["published_at"] = dates[v["youtube_id"]]
|
||||||
|
try:
|
||||||
|
db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v.get("published_at")}
|
||||||
|
for v in vids if v.get("published_at")])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return jsonify({"success": True, "kind": "channel", "source": "youtube",
|
||||||
|
"channel": channel, "following": following, "from_cache": from_cache})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube channel detail failed for %r", channel_id)
|
||||||
|
return jsonify({"success": False, "error": "Could not load channel"}), 500
|
||||||
|
|
||||||
|
@bp.route("/youtube/channel/<channel_id>/videos", methods=["POST"])
|
||||||
|
def video_youtube_channel_videos(channel_id):
|
||||||
|
"""ONE InnerTube page of a channel's videos — the channel page streams the
|
||||||
|
WHOLE catalog by re-POSTing with the continuation token from each response
|
||||||
|
(each page fetched once; no yt-dlp re-scan). POST (not GET) keeps the giant
|
||||||
|
continuation token out of the URL/access logs; the frontend paces the calls,
|
||||||
|
so each is fast and never trips the slow-request warning. Returns the videos
|
||||||
|
(dates refined from cache) + next ``continuation`` (null = no more)."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video import youtube as yt
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
cont = (body.get("continuation") or "").strip() or None
|
||||||
|
try:
|
||||||
|
db = get_video_db()
|
||||||
|
page = yt.innertube_channel_videos_page(channel_id, continuation=cont)
|
||||||
|
videos, token = page.get("videos") or [], page.get("continuation")
|
||||||
|
ids = [v.get("youtube_id") for v in videos if v.get("youtube_id")]
|
||||||
|
cached = db.get_video_dates(ids)
|
||||||
|
wished = db.youtube_video_wish_state(ids)
|
||||||
|
for v in videos:
|
||||||
|
vid = v.get("youtube_id")
|
||||||
|
if cached.get(vid): # a cached (possibly exact) date wins
|
||||||
|
v["published_at"] = cached[vid]
|
||||||
|
v["wished"] = vid in wished
|
||||||
|
try:
|
||||||
|
db.cache_channel_videos(channel_id, videos) # remember the list
|
||||||
|
db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v.get("published_at")}
|
||||||
|
for v in videos if v.get("youtube_id") and v.get("published_at")])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return jsonify({"success": True, "videos": videos, "continuation": token})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube channel videos batch failed for %r", channel_id)
|
||||||
|
return jsonify({"success": False, "videos": [], "continuation": None}), 200
|
||||||
|
|
||||||
|
@bp.route("/youtube/video/<video_id>", methods=["GET"])
|
||||||
|
def video_youtube_video_detail(video_id):
|
||||||
|
"""Full metadata for one video (description, views, likes, duration, tags) —
|
||||||
|
fetched lazily when a video is selected. Persists the description onto its
|
||||||
|
wishlist row so re-opening is instant."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video import youtube as yt
|
||||||
|
try:
|
||||||
|
v = yt.video_detail(video_id)
|
||||||
|
if not v or not v.get("youtube_id"):
|
||||||
|
return jsonify({"success": False, "error": "Video not found"}), 404
|
||||||
|
db = get_video_db()
|
||||||
|
# DeArrow crowd-sourced better title (if enriched) — surfaced as an
|
||||||
|
# alternative on the detail page.
|
||||||
|
try:
|
||||||
|
v["dearrow_title"] = db.youtube_video_dearrow_title(v["youtube_id"])
|
||||||
|
except Exception:
|
||||||
|
v["dearrow_title"] = None
|
||||||
|
if v.get("description"):
|
||||||
|
try:
|
||||||
|
db.set_wishlist_video_overview(v["youtube_id"], v["description"])
|
||||||
|
except Exception:
|
||||||
|
pass # persistence is best-effort; the detail still returns
|
||||||
|
if v.get("published_at"): # learned a real date → cache it for year-seasons
|
||||||
|
try:
|
||||||
|
db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v["published_at"]}])
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return jsonify({"success": True, "video": v})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube video detail failed for %r", video_id)
|
||||||
|
return jsonify({"success": False, "error": "Could not load video"}), 500
|
||||||
|
|
||||||
|
@bp.route("/youtube/search", methods=["GET"])
|
||||||
|
def video_youtube_search():
|
||||||
|
"""Channel search results for the search page (alongside TMDB), each with a
|
||||||
|
followed flag so the card can hydrate. Best-effort."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video import youtube as yt
|
||||||
|
q = (request.args.get("q") or "").strip()
|
||||||
|
if not q:
|
||||||
|
return jsonify({"success": True, "channels": []})
|
||||||
|
try:
|
||||||
|
chans = yt.search_channels(q)
|
||||||
|
following = get_video_db().channel_watch_state([c["youtube_id"] for c in chans])
|
||||||
|
for c in chans:
|
||||||
|
c["following"] = c["youtube_id"] in following
|
||||||
|
return jsonify({"success": True, "channels": chans})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube search failed for %r", q)
|
||||||
|
return jsonify({"success": False, "channels": []})
|
||||||
|
|
||||||
|
@bp.route("/youtube/playlists/<channel_id>", methods=["GET"])
|
||||||
|
def video_youtube_playlists(channel_id):
|
||||||
|
"""The channel's playlists (collapsible sections on the channel page), each
|
||||||
|
flagged ``following`` so its Add-to-watchlist button hydrates."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video import youtube as yt
|
||||||
|
try:
|
||||||
|
pls = yt.channel_playlists(channel_id)
|
||||||
|
followed = get_video_db().playlist_watch_state([p.get("playlist_id") for p in pls])
|
||||||
|
for p in pls:
|
||||||
|
p["following"] = p.get("playlist_id") in followed
|
||||||
|
return jsonify({"success": True, "playlists": pls})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube playlists failed for %r", channel_id)
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
||||||
|
@bp.route("/youtube/playlist/<playlist_id>", methods=["GET"])
|
||||||
|
def video_youtube_playlist(playlist_id):
|
||||||
|
"""A playlist's videos + metadata + follow state. Serves BOTH the channel-page
|
||||||
|
playlist expansion (reads ``videos``) and the standalone playlist detail view
|
||||||
|
(reads ``playlist`` + ``following``). Curator-ordered — a partial set, NOT
|
||||||
|
grouped by year. Per-video wished + cached-date flags hydrate the toggles."""
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video import youtube as yt
|
||||||
|
try:
|
||||||
|
limit = int(request.args.get("limit") or 200)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
limit = 200
|
||||||
|
try:
|
||||||
|
db = get_video_db()
|
||||||
|
# Pull the WHOLE playlist. With cookies configured (youtube.cookies_browser,
|
||||||
|
# like ytdl-sub) yt-dlp pages the full list; anonymous, YouTube throttles
|
||||||
|
# us to ~100, so the InnerTube catalog (~200 + the true header count) is the
|
||||||
|
# better source. Use whichever got more, with the true total.
|
||||||
|
pl = yt.resolve_playlist("https://www.youtube.com/playlist?list=" + playlist_id, limit=5000)
|
||||||
|
if not pl:
|
||||||
|
return jsonify({"success": False, "error": "Playlist not found"}), 404
|
||||||
|
try:
|
||||||
|
cat = yt.innertube_playlist_catalog(playlist_id)
|
||||||
|
if len(cat.get("videos") or []) > len(pl.get("videos") or []):
|
||||||
|
pl["videos"] = cat["videos"]
|
||||||
|
total = max(pl.get("video_count") or 0, cat.get("total") or 0)
|
||||||
|
if total:
|
||||||
|
pl["video_count"] = total
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
vids = pl.get("videos") or []
|
||||||
|
ids = [v.get("youtube_id") for v in vids]
|
||||||
|
wished = db.youtube_video_wish_state(ids)
|
||||||
|
dates = db.get_video_dates(ids)
|
||||||
|
for v in vids:
|
||||||
|
v["wished"] = v.get("youtube_id") in wished
|
||||||
|
if not v.get("published_at") and dates.get(v.get("youtube_id")):
|
||||||
|
v["published_at"] = dates[v["youtube_id"]]
|
||||||
|
try:
|
||||||
|
db.cache_channel_videos(playlist_id, vids) # remember the count for the watchlist card
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return jsonify({"success": True, "videos": vids, "playlist": pl,
|
||||||
|
"following": bool(db.playlist_watch_state([pl["playlist_id"]])),
|
||||||
|
"kind": "playlist", "source": "youtube"})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube playlist failed for %r", playlist_id)
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
||||||
|
@bp.route("/youtube/wishlist/add", methods=["POST"])
|
||||||
|
def video_youtube_wishlist_add():
|
||||||
|
"""Wish specific videos (per-video add from the channel page). Body:
|
||||||
|
{channel: {youtube_id, title, avatar_url?}, videos: [{youtube_id, title, …}]}."""
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
channel = body.get("channel") or {}
|
||||||
|
videos = body.get("videos") or []
|
||||||
|
if not channel.get("youtube_id") or not videos:
|
||||||
|
return jsonify({"success": False, "error": "channel and videos required"}), 400
|
||||||
|
try:
|
||||||
|
db = get_video_db()
|
||||||
|
n = db.add_videos_to_wishlist(channel, videos, server_source=_server())
|
||||||
|
return jsonify({"success": n > 0, "added": n, "counts": db.youtube_wishlist_counts()})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube wishlist add failed")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
||||||
|
@bp.route("/youtube/wishlist/remove", methods=["POST"])
|
||||||
|
def video_youtube_wishlist_remove():
|
||||||
|
"""Remove wished videos. Body: {scope: 'channel'|'video', source_id}."""
|
||||||
|
from . import get_video_db
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
scope = body.get("scope")
|
||||||
|
source_id = (body.get("source_id") or "").strip()
|
||||||
|
if scope not in ("channel", "video") or not source_id:
|
||||||
|
return jsonify({"success": False, "error": "scope and source_id are required"}), 400
|
||||||
|
try:
|
||||||
|
db = get_video_db()
|
||||||
|
removed = db.remove_youtube_from_wishlist(scope, source_id)
|
||||||
|
return jsonify({"success": True, "removed": removed, "counts": db.youtube_wishlist_counts()})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube wishlist remove failed")
|
||||||
|
return jsonify({"success": False, "error": "Failed"}), 500
|
||||||
|
|
@ -10,30 +10,37 @@ Three top-level lists:
|
||||||
- `ACTIONS` — DO blocks: process_wishlist, scan_library, etc.
|
- `ACTIONS` — DO blocks: process_wishlist, scan_library, etc.
|
||||||
- `NOTIFICATIONS` — THEN blocks: discord/pushbullet/telegram/webhook,
|
- `NOTIFICATIONS` — THEN blocks: discord/pushbullet/telegram/webhook,
|
||||||
plus fire_signal and run_script then-actions.
|
plus fire_signal and run_script then-actions.
|
||||||
|
|
||||||
|
Each block carries an optional ``scope`` tag so the SAME definitions can
|
||||||
|
feed both the music and the (isolated) video automation builders:
|
||||||
|
- ``"both"`` — generic; shown on both sides (schedule, notifications, …).
|
||||||
|
- ``"video"`` — video-only; shown only on the video builder.
|
||||||
|
- absent — treated as music-only (the default); never shown on video.
|
||||||
|
Use :func:`blocks_for_scope` to get the filtered lists for one side.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
TRIGGERS: list[dict] = [
|
TRIGGERS: list[dict] = [
|
||||||
{"type": "schedule", "label": "Schedule", "icon": "clock", "description": "Run on a timer interval", "available": True,
|
{"type": "schedule", "label": "Schedule", "icon": "clock", "scope": "both", "description": "Run on a timer interval", "available": True,
|
||||||
"config_fields": [
|
"config_fields": [
|
||||||
{"key": "interval", "type": "number", "label": "Every", "default": 6, "min": 1},
|
{"key": "interval", "type": "number", "label": "Every", "default": 6, "min": 1},
|
||||||
{"key": "unit", "type": "select", "label": "Unit",
|
{"key": "unit", "type": "select", "label": "Unit",
|
||||||
"options": [{"value": "minutes", "label": "Minutes"}, {"value": "hours", "label": "Hours"}, {"value": "days", "label": "Days"}],
|
"options": [{"value": "minutes", "label": "Minutes"}, {"value": "hours", "label": "Hours"}, {"value": "days", "label": "Days"}],
|
||||||
"default": "hours"}
|
"default": "hours"}
|
||||||
]},
|
]},
|
||||||
{"type": "daily_time", "label": "Daily Time", "icon": "clock", "description": "Run every day at a specific time", "available": True,
|
{"type": "daily_time", "label": "Daily Time", "icon": "clock", "scope": "both", "description": "Run every day at a specific time", "available": True,
|
||||||
"config_fields": [
|
"config_fields": [
|
||||||
{"key": "time", "type": "time", "label": "At", "default": "03:00"}
|
{"key": "time", "type": "time", "label": "At", "default": "03:00"}
|
||||||
]},
|
]},
|
||||||
{"type": "weekly_time", "label": "Weekly Schedule", "icon": "calendar", "description": "Run on specific days of the week at a set time", "available": True,
|
{"type": "weekly_time", "label": "Weekly Schedule", "icon": "calendar", "scope": "both", "description": "Run on specific days of the week at a set time", "available": True,
|
||||||
"config_fields": [
|
"config_fields": [
|
||||||
{"key": "time", "type": "time", "label": "At", "default": "03:00"},
|
{"key": "time", "type": "time", "label": "At", "default": "03:00"},
|
||||||
{"key": "days", "type": "multi_select", "label": "Days",
|
{"key": "days", "type": "multi_select", "label": "Days",
|
||||||
"options": [{"value": "mon", "label": "Mon"}, {"value": "tue", "label": "Tue"}, {"value": "wed", "label": "Wed"},
|
"options": [{"value": "mon", "label": "Mon"}, {"value": "tue", "label": "Tue"}, {"value": "wed", "label": "Wed"},
|
||||||
{"value": "thu", "label": "Thu"}, {"value": "fri", "label": "Fri"}, {"value": "sat", "label": "Sat"}, {"value": "sun", "label": "Sun"}]}
|
{"value": "thu", "label": "Thu"}, {"value": "fri", "label": "Fri"}, {"value": "sat", "label": "Sat"}, {"value": "sun", "label": "Sun"}]}
|
||||||
]},
|
]},
|
||||||
{"type": "app_started", "label": "App Started", "icon": "power", "description": "When SoulSync starts up", "available": True},
|
{"type": "app_started", "label": "App Started", "icon": "power", "scope": "both", "description": "When SoulSync starts up", "available": True},
|
||||||
{"type": "track_downloaded", "label": "Track Downloaded", "icon": "download", "description": "When a track finishes downloading", "available": True,
|
{"type": "track_downloaded", "label": "Track Downloaded", "icon": "download", "description": "When a track finishes downloading", "available": True,
|
||||||
"has_conditions": True,
|
"has_conditions": True,
|
||||||
"condition_fields": ["artist", "title", "album", "quality"],
|
"condition_fields": ["artist", "title", "album", "quality"],
|
||||||
|
|
@ -106,16 +113,24 @@ TRIGGERS: list[dict] = [
|
||||||
"description": "When duplicate cleaner finishes", "available": True,
|
"description": "When duplicate cleaner finishes", "available": True,
|
||||||
"variables": ["files_scanned", "duplicates_found", "space_freed"]},
|
"variables": ["files_scanned", "duplicates_found", "space_freed"]},
|
||||||
# Signal trigger
|
# Signal trigger
|
||||||
{"type": "signal_received", "label": "Signal Received", "icon": "zap",
|
{"type": "signal_received", "label": "Signal Received", "icon": "zap", "scope": "both",
|
||||||
"description": "When another automation fires a named signal", "available": True,
|
"description": "When another automation fires a named signal", "available": True,
|
||||||
"config_fields": [
|
"config_fields": [
|
||||||
{"key": "signal_name", "type": "signal_input", "label": "Signal Name"}
|
{"key": "signal_name", "type": "signal_input", "label": "Signal Name"}
|
||||||
],
|
],
|
||||||
"variables": ["signal_name"]},
|
"variables": ["signal_name"]},
|
||||||
# Webhook trigger
|
# Webhook trigger
|
||||||
{"type": "webhook_received", "label": "Webhook Received", "icon": "globe",
|
{"type": "webhook_received", "label": "Webhook Received", "icon": "globe", "scope": "both",
|
||||||
"description": "When an external API request is received (POST /api/v1/request)", "available": True,
|
"description": "When an external API request is received (POST /api/v1/request)", "available": True,
|
||||||
"variables": ["query", "request_id", "source"]},
|
"variables": ["query", "request_id", "source"]},
|
||||||
|
|
||||||
|
# ── Video side (scope='video') — the post-download scan chain triggers ──
|
||||||
|
{"type": "video_batch_complete", "label": "Video Download Batch Done", "icon": "check-circle", "scope": "video",
|
||||||
|
"description": "When a batch of video downloads finishes", "available": True,
|
||||||
|
"variables": ["completed"]},
|
||||||
|
{"type": "video_library_scan_completed", "label": "Video Library Scan Done", "icon": "hard-drive", "scope": "video",
|
||||||
|
"description": "When the media server finishes rescanning your video sections", "available": True,
|
||||||
|
"variables": ["server"]},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -155,7 +170,7 @@ ACTIONS: list[dict] = [
|
||||||
{"key": "refresh_first", "type": "checkbox", "label": "Refresh playlists before sync (regenerate snapshots)", "default": False},
|
{"key": "refresh_first", "type": "checkbox", "label": "Refresh playlists before sync (regenerate snapshots)", "default": False},
|
||||||
{"key": "skip_wishlist", "type": "checkbox", "label": "Skip wishlist processing", "default": False},
|
{"key": "skip_wishlist", "type": "checkbox", "label": "Skip wishlist processing", "default": False},
|
||||||
]},
|
]},
|
||||||
{"type": "notify_only", "label": "Notify Only", "icon": "bell", "description": "No action — just send notification", "available": True},
|
{"type": "notify_only", "label": "Notify Only", "icon": "bell", "scope": "both", "description": "No action — just send notification", "available": True},
|
||||||
# Phase 3 actions
|
# Phase 3 actions
|
||||||
{"type": "start_database_update", "label": "Update Database", "icon": "database",
|
{"type": "start_database_update", "label": "Update Database", "icon": "database",
|
||||||
"description": "Trigger library database refresh", "available": True,
|
"description": "Trigger library database refresh", "available": True,
|
||||||
|
|
@ -184,7 +199,7 @@ ACTIONS: list[dict] = [
|
||||||
"description": "Clear quarantine, download queue, import folder, and search history in one sweep", "available": True},
|
"description": "Clear quarantine, download queue, import folder, and search history in one sweep", "available": True},
|
||||||
{"type": "deep_scan_library", "label": "Deep Scan Library", "icon": "search",
|
{"type": "deep_scan_library", "label": "Deep Scan Library", "icon": "search",
|
||||||
"description": "Full library comparison without losing enrichment data", "available": True},
|
"description": "Full library comparison without losing enrichment data", "available": True},
|
||||||
{"type": "run_script", "label": "Run Script", "icon": "terminal",
|
{"type": "run_script", "label": "Run Script", "icon": "terminal", "scope": "both",
|
||||||
"description": "Execute a script from the scripts folder", "available": True},
|
"description": "Execute a script from the scripts folder", "available": True},
|
||||||
{"type": "search_and_download", "label": "Search & Download", "icon": "download",
|
{"type": "search_and_download", "label": "Search & Download", "icon": "download",
|
||||||
"description": "Search for a track and download the best match", "available": True,
|
"description": "Search for a track and download the best match", "available": True,
|
||||||
|
|
@ -192,28 +207,154 @@ ACTIONS: list[dict] = [
|
||||||
{"key": "query", "type": "text", "label": "Search Query",
|
{"key": "query", "type": "text", "label": "Search Query",
|
||||||
"placeholder": "Artist - Track (leave empty to use trigger's query)"}
|
"placeholder": "Artist - Track (leave empty to use trigger's query)"}
|
||||||
]},
|
]},
|
||||||
|
|
||||||
|
# ── Video side (isolated app, shared engine) ──────────────────────────
|
||||||
|
# Tagged scope='video' so they appear ONLY on the video automation
|
||||||
|
# builder, never the music one. Their handlers bridge into core.video.
|
||||||
|
{"type": "video_scan_library", "label": "Scan Video Library", "icon": "refresh", "scope": "video",
|
||||||
|
"description": "Tell the media server to rescan your selected movie/TV sections, then read what it found into SoulSync", "available": True,
|
||||||
|
"config_fields": [
|
||||||
|
{"key": "mode", "type": "select", "label": "Mode",
|
||||||
|
"options": [{"value": "full", "label": "Full (add + refresh)"},
|
||||||
|
{"value": "incremental", "label": "Incremental (recent only)"},
|
||||||
|
{"value": "deep", "label": "Deep (also remove missing)"}],
|
||||||
|
"default": "full"},
|
||||||
|
{"key": "media_type", "type": "select", "label": "Library",
|
||||||
|
"options": [{"value": "all", "label": "Movies + TV"},
|
||||||
|
{"value": "movie", "label": "Movies only"},
|
||||||
|
{"value": "show", "label": "TV only"}],
|
||||||
|
"default": "all"}
|
||||||
|
]},
|
||||||
|
# Post-download chain actions (two stages, like music's scan_library +
|
||||||
|
# start_database_update). Stage 1 nudges the server; stage 2 reads it in.
|
||||||
|
{"type": "video_scan_server", "label": "Scan Video Server", "icon": "refresh", "scope": "video",
|
||||||
|
"description": "Get the server to index new downloads (skips the scan if it already has them), waits until it finishes, then fires 'Video Library Scan Done'", "available": True,
|
||||||
|
"config_fields": [
|
||||||
|
{"key": "media_type", "type": "select", "label": "Library",
|
||||||
|
"options": [{"value": "all", "label": "Movies + TV"},
|
||||||
|
{"value": "movie", "label": "Movies only"},
|
||||||
|
{"value": "show", "label": "TV only"}],
|
||||||
|
"default": "all"},
|
||||||
|
{"key": "skip_if_present", "type": "checkbox", "label": "Skip the scan if the server already has the download", "default": True},
|
||||||
|
{"key": "probe_grace_minutes", "type": "number", "label": "Give the server's auto-scan this long to ingest first (min)", "default": 2, "min": 0},
|
||||||
|
{"key": "max_wait_minutes", "type": "number", "label": "Max wait for scan (min)", "default": 60, "min": 1},
|
||||||
|
{"key": "debounce_seconds", "type": "number", "label": "Fallback wait if status unknown (sec)", "default": 120, "min": 10}
|
||||||
|
]},
|
||||||
|
{"type": "video_update_database", "label": "Update Video Database", "icon": "database", "scope": "video",
|
||||||
|
"description": "Read newly-indexed media from the server into SoulSync (incremental)", "available": True,
|
||||||
|
"config_fields": [
|
||||||
|
{"key": "mode", "type": "select", "label": "Mode",
|
||||||
|
"options": [{"value": "incremental", "label": "Incremental (recent only)"},
|
||||||
|
{"value": "full", "label": "Full (add + refresh)"}],
|
||||||
|
"default": "incremental"},
|
||||||
|
{"key": "media_type", "type": "select", "label": "Library",
|
||||||
|
"options": [{"value": "all", "label": "Movies + TV"},
|
||||||
|
{"value": "movie", "label": "Movies only"},
|
||||||
|
{"value": "show", "label": "TV only"}],
|
||||||
|
"default": "all"}
|
||||||
|
]},
|
||||||
|
{"type": "video_update_database_hourly", "label": "Update Video Database (Hourly)", "icon": "database", "scope": "video",
|
||||||
|
"description": "The same incremental server-read on an hourly schedule, so manual library additions (which Plex auto-scans) appear within the hour instead of waiting for the weekly deep scan. Pair with a 1-hour Schedule trigger.", "available": True},
|
||||||
|
# Per-library deep-scan presets (the system 'Auto-Deep Scan TV/Movie Library' run
|
||||||
|
# these). Scope + deep mode are baked in by the registration wrapper, so no config
|
||||||
|
# fields — drag one in and it just deep-scans that library.
|
||||||
|
{"type": "video_deep_scan_tv", "label": "Deep Scan TV Library", "icon": "search", "scope": "video",
|
||||||
|
"description": "Full reconcile of the TV library: re-read every show from the server and drop ones it no longer has (a read, NOT a Plex disk-scan; never touches movies)", "available": True},
|
||||||
|
{"type": "video_deep_scan_movies", "label": "Deep Scan Movie Library", "icon": "search", "scope": "video",
|
||||||
|
"description": "Full reconcile of the Movie library: re-read every movie from the server and drop ones it no longer has (a read, NOT a Plex disk-scan; never touches TV)", "available": True},
|
||||||
|
{"type": "video_refresh_airing_schedules", "label": "Refresh Airing TV Schedules", "icon": "calendar", "scope": "video",
|
||||||
|
"description": "Re-pull the latest TMDB episode schedules (air dates, stills) for the still-airing shows on your watchlist, so the calendar the airing automation reads is current — newly-announced or rescheduled episodes get picked up instead of being missed. Skips ended/canceled shows. Pair with a daily Schedule a couple hours before the airing run.", "available": True},
|
||||||
|
{"type": "video_clean_youtube_episodes", "label": "Clean Old YouTube Episodes", "icon": "trash", "scope": "video",
|
||||||
|
"description": "Delete downloaded YouTube channel episodes that fall outside each channel's keep window (set per channel via its cog → Keep: e.g. last 30 episodes / last 3–6 months). Removes the video + its sidecars but keeps the history record so it's never re-downloaded. No-op for channels left on 'keep everything' (the default). Playlists are excluded. Pair with a daily Schedule.", "available": True},
|
||||||
|
{"type": "video_add_airing_episodes", "label": "Wishlist Today's Airings", "icon": "calendar", "scope": "video",
|
||||||
|
"description": "Sonarr-style: add every episode airing today (for shows you follow) to the wishlist, skipping ones you already own. Also tidies the watchlist by dropping shows that have ended/been canceled.", "available": True,
|
||||||
|
"config_fields": [
|
||||||
|
{"key": "prune_ended", "type": "checkbox", "label": "Also remove ended/canceled shows from the watchlist", "default": True}
|
||||||
|
]},
|
||||||
|
# ── Watchlist → Wishlist pipeline ────────────────────────────────────────
|
||||||
|
# Stage 1 — scans that FILL the wishlist from what you follow.
|
||||||
|
{"type": "video_scan_watchlist_people", "label": "Scan Watchlist People", "icon": "users", "scope": "video",
|
||||||
|
"description": "For everyone you follow on the watchlist, wishlist every movie they acted in or directed that you don't already own — the whole back catalog plus anything upcoming (kept as 'monitored' until it's released). First run backlogs everything; later runs are fast.", "available": True},
|
||||||
|
{"type": "video_scan_watchlist_channels", "label": "Scan Watchlist Channels", "icon": "youtube", "scope": "video",
|
||||||
|
"description": "For every YouTube channel you follow, wishlist its new long-form uploads (Shorts excluded). Forward-looking from when you followed, plus a safety net that always keeps the last N videos. Pair with a 6-hourly Schedule trigger — channels post at all hours.", "available": True,
|
||||||
|
"config_fields": [
|
||||||
|
{"key": "backfill_count", "type": "number", "label": "Always keep the last N videos from each channel", "default": 10, "min": 0}
|
||||||
|
]},
|
||||||
|
{"type": "video_scan_watchlist_playlists", "label": "Scan Watchlist Playlists", "icon": "list", "scope": "video",
|
||||||
|
"description": "For every YouTube playlist you follow, wishlist its videos you don't have yet (and anything later added to it) — mirrors the whole list. Each playlist becomes its own show in the library (playlist-as-show). Shorts excluded. Pair with a schedule.", "available": True},
|
||||||
|
# Stage 2 — processors that DRAIN the wishlist by downloading.
|
||||||
|
{"type": "video_process_movie_wishlist", "label": "Process Movie Wishlist", "icon": "download", "scope": "video",
|
||||||
|
"description": "Auto-grab your wished, released MOVIES from Soulseek: searches each, picks the best release per your quality profile, and downloads it (the monitor finishes + organises it). Skips unreleased ('monitored') ones and any already downloading. Needs the Movie library folder + slskd set. Pair with an hourly schedule.", "available": True,
|
||||||
|
"config_fields": [
|
||||||
|
{"key": "max_concurrent", "type": "number", "label": "Max simultaneous searches", "default": 3, "min": 1}
|
||||||
|
]},
|
||||||
|
{"type": "video_process_episode_wishlist", "label": "Process Episode Wishlist", "icon": "download", "scope": "video",
|
||||||
|
"description": "Auto-grab your wished EPISODES from Soulseek: searches each, picks the best release per your quality profile, and downloads it. Fed by the 'Wishlist Today's Airings' scan. Needs the TV library folder + slskd set. Pair with an hourly schedule.", "available": True,
|
||||||
|
"config_fields": [
|
||||||
|
{"key": "max_concurrent", "type": "number", "label": "Max simultaneous searches", "default": 3, "min": 1}
|
||||||
|
]},
|
||||||
|
{"type": "video_process_youtube_wishlist", "label": "Process YouTube Wishlist", "icon": "download", "scope": "video",
|
||||||
|
"description": "Download wished YouTube videos (yt-dlp), organised as a Plex 'TV by date' show (channel/year/date). Queues the WHOLE wishlist — the setting only limits how many download at the same time; each finished one starts the next, so it all drains. A completed download leaves the wishlist. Needs the YouTube library folder set on Settings → Downloads.", "available": True,
|
||||||
|
"config_fields": [
|
||||||
|
{"key": "max_concurrent", "type": "number", "label": "Max simultaneous downloads", "default": 3, "min": 1}
|
||||||
|
]},
|
||||||
|
# Video twins of the music maintenance actions. Distinct action_type (the
|
||||||
|
# system seeder keys on action_type, so a shared key would collide with the
|
||||||
|
# music row) but the SAME shared handler — the cleanup operates on the common
|
||||||
|
# download/search state, so behaviour stays identical. scope='video' keeps
|
||||||
|
# them on the video builder only; the music blocks above are untouched.
|
||||||
|
{"type": "video_clean_search_history", "label": "Clean Search History", "icon": "trash-2", "scope": "video",
|
||||||
|
"description": "Remove old searches from Soulseek", "available": True},
|
||||||
|
{"type": "video_clean_completed_downloads", "label": "Clean Completed Downloads", "icon": "check-square", "scope": "video",
|
||||||
|
"description": "Clear completed downloads and empty directories", "available": True},
|
||||||
|
{"type": "video_full_cleanup", "label": "Full Cleanup", "icon": "trash", "scope": "video",
|
||||||
|
"description": "Clear quarantine, download queue, import folder, and search history in one sweep", "available": True},
|
||||||
|
# Custom (NOT a shared handler): backs up video_library.db, not the music DB.
|
||||||
|
{"type": "video_backup_database", "label": "Backup Database", "icon": "save", "scope": "video",
|
||||||
|
"description": "Create a timestamped backup of the video library database", "available": True},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
NOTIFICATIONS: list[dict] = [
|
NOTIFICATIONS: list[dict] = [
|
||||||
{"type": "discord_webhook", "label": "Discord Webhook", "icon": "message", "description": "Send a Discord notification", "available": True,
|
{"type": "discord_webhook", "label": "Discord Webhook", "icon": "message", "scope": "both", "description": "Send a Discord notification", "available": True,
|
||||||
"variables": ["time", "name", "run_count", "status"]},
|
"variables": ["time", "name", "run_count", "status"]},
|
||||||
{"type": "pushbullet", "label": "Pushbullet", "icon": "push", "description": "Push notification to phone/desktop", "available": True,
|
{"type": "pushbullet", "label": "Pushbullet", "icon": "push", "scope": "both", "description": "Push notification to phone/desktop", "available": True,
|
||||||
"variables": ["time", "name", "run_count", "status"]},
|
"variables": ["time", "name", "run_count", "status"]},
|
||||||
{"type": "telegram", "label": "Telegram", "icon": "message", "description": "Send a Telegram message", "available": True,
|
{"type": "telegram", "label": "Telegram", "icon": "message", "scope": "both", "description": "Send a Telegram message", "available": True,
|
||||||
"variables": ["time", "name", "run_count", "status"]},
|
"variables": ["time", "name", "run_count", "status"]},
|
||||||
{"type": "webhook", "label": "Webhook (POST)", "icon": "globe", "description": "Send a POST request to any URL", "available": True,
|
{"type": "webhook", "label": "Webhook (POST)", "icon": "globe", "scope": "both", "description": "Send a POST request to any URL", "available": True,
|
||||||
"variables": ["time", "name", "run_count", "status"]},
|
"variables": ["time", "name", "run_count", "status"]},
|
||||||
# Signal fire action
|
# Signal fire action
|
||||||
{"type": "fire_signal", "label": "Fire Signal", "icon": "zap",
|
{"type": "fire_signal", "label": "Fire Signal", "icon": "zap", "scope": "both",
|
||||||
"description": "Fire a signal that other automations can listen for", "available": True,
|
"description": "Fire a signal that other automations can listen for", "available": True,
|
||||||
"config_fields": [
|
"config_fields": [
|
||||||
{"key": "signal_name", "type": "signal_input", "label": "Signal Name"}
|
{"key": "signal_name", "type": "signal_input", "label": "Signal Name"}
|
||||||
]},
|
]},
|
||||||
# Run script then-action
|
# Run script then-action
|
||||||
{"type": "run_script", "label": "Run Script", "icon": "terminal",
|
{"type": "run_script", "label": "Run Script", "icon": "terminal", "scope": "both",
|
||||||
"description": "Execute a script after the action completes", "available": True,
|
"description": "Execute a script after the action completes", "available": True,
|
||||||
"config_fields": [
|
"config_fields": [
|
||||||
{"key": "script_name", "type": "script_select", "label": "Script"}
|
{"key": "script_name", "type": "script_select", "label": "Script"}
|
||||||
]},
|
]},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _in_scope(block: dict, scope: str) -> bool:
|
||||||
|
"""A block belongs to ``scope`` if it's generic (``"both"``) or tagged for
|
||||||
|
that side. Untagged blocks default to ``"music"`` (the original behaviour),
|
||||||
|
so the video builder never picks up music-only blocks by accident."""
|
||||||
|
s = block.get("scope", "music")
|
||||||
|
return s == "both" or s == scope
|
||||||
|
|
||||||
|
|
||||||
|
def blocks_for_scope(scope: str = "music") -> dict:
|
||||||
|
"""Return the trigger/action/notification lists filtered to one side.
|
||||||
|
|
||||||
|
``scope="music"`` reproduces the pre-scope behaviour (everything except
|
||||||
|
video-only blocks); ``scope="video"`` returns generic + video-only blocks.
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"triggers": [b for b in TRIGGERS if _in_scope(b, scope)],
|
||||||
|
"actions": [b for b in ACTIONS if _in_scope(b, scope)],
|
||||||
|
"notifications": [b for b in NOTIFICATIONS if _in_scope(b, scope)],
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -105,11 +105,10 @@ def auto_update_discovery_pool(config: Dict[str, Any], deps: AutomationDeps) ->
|
||||||
_MAX_BACKUPS = 5
|
_MAX_BACKUPS = 5
|
||||||
|
|
||||||
|
|
||||||
def auto_backup_database(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]:
|
def _backup_db_at(db_path: str, deps: AutomationDeps, automation_id) -> Dict[str, Any]:
|
||||||
"""Create a hot SQLite backup, then prune old backups so only the
|
"""Create a hot SQLite backup of ``db_path``, then prune old backups so only
|
||||||
newest ``_MAX_BACKUPS`` remain."""
|
the newest ``_MAX_BACKUPS`` remain. Shared by the music and video backup
|
||||||
automation_id = config.get('_automation_id')
|
actions — only the DB file differs (they can't share one backup)."""
|
||||||
db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db')
|
|
||||||
if not os.path.exists(db_path):
|
if not os.path.exists(db_path):
|
||||||
return {'status': 'error', 'reason': 'Database file not found'}
|
return {'status': 'error', 'reason': 'Database file not found'}
|
||||||
|
|
||||||
|
|
@ -146,6 +145,20 @@ def auto_backup_database(config: Dict[str, Any], deps: AutomationDeps) -> Dict[s
|
||||||
return {'status': 'completed', 'backup_path': backup_path, 'size_mb': str(size_mb)}
|
return {'status': 'completed', 'backup_path': backup_path, 'size_mb': str(size_mb)}
|
||||||
|
|
||||||
|
|
||||||
|
def auto_backup_database(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]:
|
||||||
|
"""Hot SQLite backup of the MUSIC database (``DATABASE_PATH``)."""
|
||||||
|
db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db')
|
||||||
|
return _backup_db_at(db_path, deps, config.get('_automation_id'))
|
||||||
|
|
||||||
|
|
||||||
|
def auto_backup_video_database(config: Dict[str, Any], deps: AutomationDeps) -> Dict[str, Any]:
|
||||||
|
"""Hot SQLite backup of the VIDEO database (``VIDEO_DATABASE_PATH`` /
|
||||||
|
video_library.db). Same logic as the music backup but a different DB file —
|
||||||
|
the two can't share one backup, so this is a video-specific action."""
|
||||||
|
db_path = os.environ.get('VIDEO_DATABASE_PATH', 'database/video_library.db')
|
||||||
|
return _backup_db_at(db_path, deps, config.get('_automation_id'))
|
||||||
|
|
||||||
|
|
||||||
# ─── refresh_beatport_cache ──────────────────────────────────────────
|
# ─── refresh_beatport_cache ──────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,7 @@ from core.automation.handlers.maintenance import (
|
||||||
auto_cleanup_wishlist,
|
auto_cleanup_wishlist,
|
||||||
auto_update_discovery_pool,
|
auto_update_discovery_pool,
|
||||||
auto_backup_database,
|
auto_backup_database,
|
||||||
|
auto_backup_video_database,
|
||||||
auto_refresh_beatport_cache,
|
auto_refresh_beatport_cache,
|
||||||
)
|
)
|
||||||
from core.automation.handlers.download_cleanup import (
|
from core.automation.handlers.download_cleanup import (
|
||||||
|
|
@ -35,6 +36,17 @@ from core.automation.handlers.download_cleanup import (
|
||||||
)
|
)
|
||||||
from core.automation.handlers.run_script import auto_run_script
|
from core.automation.handlers.run_script import auto_run_script
|
||||||
from core.automation.handlers.search_and_download import auto_search_and_download
|
from core.automation.handlers.search_and_download import auto_search_and_download
|
||||||
|
from core.automation.handlers.video_auto_wishlist_airing import auto_video_add_airing_episodes
|
||||||
|
from core.automation.handlers.video_refresh_airing_schedules import auto_video_refresh_airing_schedules
|
||||||
|
from core.automation.handlers.video_clean_youtube import auto_video_clean_youtube_episodes
|
||||||
|
from core.automation.handlers.video_scan_watchlist_people import auto_video_scan_watchlist_people
|
||||||
|
from core.automation.handlers.video_scan_watchlist_channels import auto_video_scan_watchlist_channels
|
||||||
|
from core.automation.handlers.video_process_youtube_wishlist import auto_video_process_youtube_wishlist
|
||||||
|
from core.automation.handlers.video_scan_watchlist_playlists import auto_video_scan_watchlist_playlists
|
||||||
|
from core.automation.handlers.video_process_wishlist import auto_video_process_wishlist, is_running
|
||||||
|
from core.automation.handlers.video_scan_library import (
|
||||||
|
auto_video_scan_library, auto_video_scan_server, auto_video_update_database,
|
||||||
|
)
|
||||||
from core.automation.handlers.progress_callbacks import (
|
from core.automation.handlers.progress_callbacks import (
|
||||||
progress_init,
|
progress_init,
|
||||||
progress_finish,
|
progress_finish,
|
||||||
|
|
@ -150,6 +162,25 @@ def register_all(deps: AutomationDeps) -> None:
|
||||||
'clean_search_history',
|
'clean_search_history',
|
||||||
lambda config: auto_clean_search_history(config, deps),
|
lambda config: auto_clean_search_history(config, deps),
|
||||||
)
|
)
|
||||||
|
# Video twin — same handler, distinct action_type so the system seeder
|
||||||
|
# (keyed on action_type) creates a separate video-owned row.
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_clean_search_history',
|
||||||
|
lambda config: auto_clean_search_history(config, deps),
|
||||||
|
)
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_clean_completed_downloads',
|
||||||
|
lambda config: auto_clean_completed_downloads(config, deps),
|
||||||
|
)
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_full_cleanup',
|
||||||
|
lambda config: auto_full_cleanup(config, deps),
|
||||||
|
)
|
||||||
|
# Video DB backup — its OWN handler (video_library.db, not the music DB).
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_backup_database',
|
||||||
|
lambda config: auto_backup_video_database(config, deps),
|
||||||
|
)
|
||||||
engine.register_action_handler(
|
engine.register_action_handler(
|
||||||
'clean_completed_downloads',
|
'clean_completed_downloads',
|
||||||
lambda config: auto_clean_completed_downloads(config, deps),
|
lambda config: auto_clean_completed_downloads(config, deps),
|
||||||
|
|
@ -167,6 +198,96 @@ def register_all(deps: AutomationDeps) -> None:
|
||||||
lambda config: auto_search_and_download(config, deps),
|
lambda config: auto_search_and_download(config, deps),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Video side (isolated app, shared engine). The video twins are tagged
|
||||||
|
# owned_by='video' on their automation rows so they never surface on the
|
||||||
|
# music automations page; the handlers bridge into core.video.
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_scan_library',
|
||||||
|
lambda config: auto_video_scan_library(config, deps),
|
||||||
|
)
|
||||||
|
# Per-library deep scans (the video twin of music's 'Auto-Deep Scan Library',
|
||||||
|
# split because Movies and TV are independent libraries). A deep scan READS the
|
||||||
|
# server's current state into video.db + prunes what's gone (a full reconcile) —
|
||||||
|
# it does NOT tell Plex to rescan its disk, so it runs through the read-only
|
||||||
|
# update-database handler in 'deep' mode, not the nudge+read scan-library one.
|
||||||
|
# Distinct action types so the seeder (which keys on action_type) sees two
|
||||||
|
# separate automations; both reuse the one handler, scoped via media_type.
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_deep_scan_movies',
|
||||||
|
lambda config: auto_video_update_database({**config, 'media_type': 'movie', 'mode': config.get('mode') or 'deep'}, deps),
|
||||||
|
)
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_deep_scan_tv',
|
||||||
|
lambda config: auto_video_update_database({**config, 'media_type': 'show', 'mode': config.get('mode') or 'deep'}, deps),
|
||||||
|
)
|
||||||
|
# Post-download chain: scan the server, then (on the scan-done event) update the DB.
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_scan_server',
|
||||||
|
lambda config: auto_video_scan_server(config, deps),
|
||||||
|
)
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_update_database',
|
||||||
|
lambda config: auto_video_update_database(config, deps),
|
||||||
|
)
|
||||||
|
# Same incremental update, but on an hourly SCHEDULE (not just after a SoulSync scan) —
|
||||||
|
# so manual library additions (which Plex auto-scans) show up within the hour instead of
|
||||||
|
# waiting for the weekly deep scan. A distinct action_type because the seeder keys on it.
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_update_database_hourly',
|
||||||
|
lambda config: auto_video_update_database({'mode': 'incremental', **config}, deps),
|
||||||
|
)
|
||||||
|
# Sonarr-style: wishlist every episode airing today (for followed shows).
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_add_airing_episodes',
|
||||||
|
lambda config: auto_video_add_airing_episodes(config, deps),
|
||||||
|
)
|
||||||
|
# Keep the calendar honest: re-pull TMDB episode schedules for still-airing watchlist
|
||||||
|
# shows (the airing automation above reads the LOCAL calendar, so it needs this fresh).
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_refresh_airing_schedules',
|
||||||
|
lambda config: auto_video_refresh_airing_schedules(config, deps),
|
||||||
|
)
|
||||||
|
# YouTube retention: delete channel episodes outside each channel's keep window (opt-in
|
||||||
|
# per channel; default keeps everything). The history row stays so it's not re-downloaded.
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_clean_youtube_episodes',
|
||||||
|
lambda config: auto_video_clean_youtube_episodes(config, deps),
|
||||||
|
)
|
||||||
|
# ── Watchlist → Wishlist pipeline ─────────────────────────────────────────
|
||||||
|
# Stage 1 — SCANS that fill the wishlist from what you follow.
|
||||||
|
# People: wishlist every un-owned movie followed actors/directors made (catalog + upcoming).
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_scan_watchlist_people',
|
||||||
|
lambda config: auto_video_scan_watchlist_people(config, deps),
|
||||||
|
)
|
||||||
|
# Channels: new long-form uploads from followed YouTube channels (forward + last-N net).
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_scan_watchlist_channels',
|
||||||
|
lambda config: auto_video_scan_watchlist_channels(config, deps),
|
||||||
|
)
|
||||||
|
# Playlists: mirror followed YouTube playlists (whole list + new additions; playlist-as-show).
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_scan_watchlist_playlists',
|
||||||
|
lambda config: auto_video_scan_watchlist_playlists(config, deps),
|
||||||
|
)
|
||||||
|
# Stage 2 — PROCESSORS that drain the wishlist by downloading. Movie/episode go through
|
||||||
|
# slskd (search → pick best → grab); the guard skips an hourly tick while a drain is still
|
||||||
|
# working. YouTube goes through yt-dlp (queue all, a few concurrent).
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_process_movie_wishlist',
|
||||||
|
lambda config: auto_video_process_wishlist(config, deps, media_type='movie'),
|
||||||
|
lambda: is_running('movie'),
|
||||||
|
)
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_process_episode_wishlist',
|
||||||
|
lambda config: auto_video_process_wishlist(config, deps, media_type='episode'),
|
||||||
|
lambda: is_running('episode'),
|
||||||
|
)
|
||||||
|
engine.register_action_handler(
|
||||||
|
'video_process_youtube_wishlist',
|
||||||
|
lambda config: auto_video_process_youtube_wishlist(config, deps),
|
||||||
|
)
|
||||||
|
|
||||||
# Progress + history callbacks: the engine invokes these around
|
# Progress + history callbacks: the engine invokes these around
|
||||||
# each handler run. Lift the closures from
|
# each handler run. Lift the closures from
|
||||||
# `web_server._register_automation_handlers` into thin lambdas
|
# `web_server._register_automation_handlers` into thin lambdas
|
||||||
|
|
|
||||||
216
core/automation/handlers/video_auto_wishlist_airing.py
Normal file
216
core/automation/handlers/video_auto_wishlist_airing.py
Normal file
|
|
@ -0,0 +1,216 @@
|
||||||
|
"""Automation handler: ``video_add_airing_episodes`` action.
|
||||||
|
|
||||||
|
Sonarr-style "monitor airings": add every episode airing TODAY — for the TV shows you
|
||||||
|
follow on the video watchlist — to the video WISHLIST, skipping ones you already own.
|
||||||
|
Runs on a daily schedule so the day's airings queue up to be grabbed automatically.
|
||||||
|
|
||||||
|
Like the other video handlers it lives on the SHARED automation side (so it may import
|
||||||
|
``core.video`` / ``api.video`` — the isolation contract only forbids the reverse) and
|
||||||
|
owns its own progress reporting (``_manages_own_progress``). The calendar read + wishlist
|
||||||
|
write are injected seams, so the handler is a pure function: tests pass fakes and never
|
||||||
|
touch a DB or a media server; production lazily binds the real calls.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import date
|
||||||
|
from typing import Any, Callable, Dict, List, Optional
|
||||||
|
|
||||||
|
from core.automation.deps import AutomationDeps
|
||||||
|
|
||||||
|
|
||||||
|
def _default_fetch_airing(today: str) -> List[Dict[str, Any]]:
|
||||||
|
"""Production wiring: the calendar's episodes airing on ``today`` for followed shows."""
|
||||||
|
from api.video import get_video_db
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
return get_video_db().calendar_upcoming(
|
||||||
|
today, today, server_source=resolve_video_server(), watchlist_only=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _default_add_episodes(show_tmdb_id: Any, show_title: Any, episodes: List[Dict[str, Any]],
|
||||||
|
library_id: Any = None, poster_url: Any = None) -> int:
|
||||||
|
from api.video import get_video_db
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
return get_video_db().add_episodes_to_wishlist(
|
||||||
|
show_tmdb_id, show_title, episodes, poster_url=poster_url, library_id=library_id,
|
||||||
|
server_source=resolve_video_server())
|
||||||
|
|
||||||
|
|
||||||
|
def _show_poster_url(library_id: Any) -> Optional[str]:
|
||||||
|
"""The SAME poster a manual add stores for a library show — the show poster proxy
|
||||||
|
path the wishlist orb renders directly. Mirrors the get-modal's
|
||||||
|
pUrl = '/api/video/poster/show/<library_id>'. Without it the orb falls back to the
|
||||||
|
show's initials, reading as 'not matched'."""
|
||||||
|
return ('/api/video/poster/show/%s' % library_id) if library_id is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def _default_season_meta(tmdb_id: Any, season_number: Any):
|
||||||
|
"""The SAME TMDB season fetch the show modal uses for a manual add — so auto-added
|
||||||
|
episodes carry identical stills / overviews / season posters, not the patchy values
|
||||||
|
the local DB happens to hold."""
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
return get_video_enrichment_engine().tmdb_season(tmdb_id, season_number)
|
||||||
|
|
||||||
|
|
||||||
|
# ── watchlist hygiene: drop shows that have ended/been canceled ───────────────
|
||||||
|
_TERMINAL = ('ended', 'canceled', 'cancelled', 'completed')
|
||||||
|
|
||||||
|
|
||||||
|
def _is_terminal_status(status) -> bool:
|
||||||
|
"""A show that won't air again — pointless to keep on the (watch-for-new) list."""
|
||||||
|
return str(status or '').strip().lower() in _TERMINAL
|
||||||
|
|
||||||
|
|
||||||
|
def _default_fetch_follows() -> List[Dict[str, Any]]:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().followed_shows()
|
||||||
|
|
||||||
|
|
||||||
|
def _default_show_status(tmdb_id: Any) -> Optional[str]:
|
||||||
|
"""TMDB status for a follow with no local status (a tmdb-only follow). Cached by
|
||||||
|
the engine; returns None on any hiccup so we never prune on uncertainty."""
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
d = get_video_enrichment_engine().tmdb_full_detail('show', tmdb_id) or {}
|
||||||
|
return d.get('status')
|
||||||
|
|
||||||
|
|
||||||
|
def _default_remove_show(tmdb_id: Any) -> None:
|
||||||
|
from api.video import get_video_db
|
||||||
|
get_video_db().remove_from_watchlist('show', tmdb_id)
|
||||||
|
|
||||||
|
|
||||||
|
def prune_ended_show_follows(deps, automation_id=None, *, fetch_follows=None,
|
||||||
|
show_status=None, remove_show=None) -> int:
|
||||||
|
"""Remove explicitly-followed shows that are no longer airing.
|
||||||
|
|
||||||
|
Auto-airing LIBRARY shows are already excluded by status, so this targets explicit
|
||||||
|
eye-button follows (which persist regardless). For follows with no local status (a
|
||||||
|
tmdb-only follow), look the status up on TMDB. Only prunes on a DEFINITIVE terminal
|
||||||
|
status — unknown status is left alone. Pure: all I/O injected. Returns count removed."""
|
||||||
|
fetch_follows = fetch_follows or _default_fetch_follows
|
||||||
|
show_status = show_status or _default_show_status
|
||||||
|
remove_show = remove_show or _default_remove_show
|
||||||
|
removed = 0
|
||||||
|
for f in (fetch_follows() or []):
|
||||||
|
tid = f.get('tmdb_id')
|
||||||
|
if not tid:
|
||||||
|
continue
|
||||||
|
status = f.get('status')
|
||||||
|
if not status: # tmdb-only follow — ask TMDB
|
||||||
|
try:
|
||||||
|
status = show_status(tid)
|
||||||
|
except Exception: # noqa: BLE001 - never prune on a lookup failure
|
||||||
|
status = None
|
||||||
|
if status and _is_terminal_status(status):
|
||||||
|
try:
|
||||||
|
remove_show(tid)
|
||||||
|
removed += 1
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id, log_line="Removed ended show '%s' from the watchlist"
|
||||||
|
% (f.get('title') or tid), log_type='info')
|
||||||
|
except Exception: # noqa: BLE001, S110 - a progress-log failure must not abort pruning
|
||||||
|
pass
|
||||||
|
return removed
|
||||||
|
|
||||||
|
|
||||||
|
def _season_lookup(season_meta, tmdb_id, season_number, cache):
|
||||||
|
"""(season_poster_url, {episode_number: tmdb_episode}) for a show+season, fetched
|
||||||
|
once and cached. A TMDB hiccup degrades to empty (DB values fill in)."""
|
||||||
|
key = (tmdb_id, season_number)
|
||||||
|
if key not in cache:
|
||||||
|
try:
|
||||||
|
sm = season_meta(tmdb_id, season_number) or {}
|
||||||
|
except Exception: # noqa: BLE001 - never let a metadata fetch break the run
|
||||||
|
sm = {}
|
||||||
|
emap = {e.get('episode_number'): e for e in (sm.get('episodes') or []) if isinstance(e, dict)}
|
||||||
|
cache[key] = (sm.get('poster_url'), emap)
|
||||||
|
return cache[key]
|
||||||
|
|
||||||
|
|
||||||
|
def auto_video_add_airing_episodes(
|
||||||
|
config: Dict[str, Any],
|
||||||
|
deps: AutomationDeps,
|
||||||
|
*,
|
||||||
|
fetch_airing: Optional[Callable[[str], List[Dict[str, Any]]]] = None,
|
||||||
|
add_episodes: Optional[Callable[[Any, Any, List[Dict[str, Any]]], int]] = None,
|
||||||
|
today_fn: Optional[Callable[[], str]] = None,
|
||||||
|
season_meta: Optional[Callable[[Any, Any], Any]] = None,
|
||||||
|
prune_follows: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||||
|
show_status: Optional[Callable[[Any], Any]] = None,
|
||||||
|
remove_show: Optional[Callable[[Any], None]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Add today's airing (unowned, followed-show) episodes to the video wishlist, and
|
||||||
|
first tidy the watchlist by dropping shows that have ended / been canceled.
|
||||||
|
|
||||||
|
Returns ``{'status': 'completed', 'episodes_added': int, 'shows': int,
|
||||||
|
'shows_pruned': int, ...}``."""
|
||||||
|
fetch_airing = fetch_airing or _default_fetch_airing
|
||||||
|
add_episodes = add_episodes or _default_add_episodes
|
||||||
|
season_meta = season_meta or _default_season_meta
|
||||||
|
today_fn = today_fn or (lambda: date.today().isoformat())
|
||||||
|
automation_id = config.get('_automation_id')
|
||||||
|
prune_ended = config.get('prune_ended', True)
|
||||||
|
try:
|
||||||
|
today = today_fn()
|
||||||
|
# Watchlist hygiene first: a followed show that has since ended/been canceled
|
||||||
|
# won't air again, so drop it (ended LIBRARY shows are already auto-excluded;
|
||||||
|
# this catches explicit eye-button follows).
|
||||||
|
pruned = 0
|
||||||
|
if prune_ended:
|
||||||
|
deps.update_progress(automation_id, phase='Tidying the watchlist…', progress=10,
|
||||||
|
log_line='Removing shows that have ended or been canceled', log_type='info')
|
||||||
|
pruned = prune_ended_show_follows(deps, automation_id, fetch_follows=prune_follows,
|
||||||
|
show_status=show_status, remove_show=remove_show)
|
||||||
|
deps.update_progress(automation_id, phase="Checking today's airings…", progress=25,
|
||||||
|
log_line='Reading the calendar for episodes airing today', log_type='info')
|
||||||
|
rows = fetch_airing(today) or []
|
||||||
|
|
||||||
|
# Group what to wish for by show: airing today, NOT already owned, with a
|
||||||
|
# real season/episode. add_episodes_to_wishlist is idempotent, so re-runs
|
||||||
|
# never duplicate.
|
||||||
|
by_show: Dict[tuple, Dict[str, Any]] = {}
|
||||||
|
season_cache: Dict[tuple, tuple] = {}
|
||||||
|
for r in rows:
|
||||||
|
if r.get('has_file'):
|
||||||
|
continue
|
||||||
|
tid = r.get('show_tmdb_id')
|
||||||
|
sn, en = r.get('season_number'), r.get('episode_number')
|
||||||
|
if not tid or sn is None or en is None:
|
||||||
|
continue
|
||||||
|
# Pull the SAME TMDB metadata a manual add gets (still + overview + season
|
||||||
|
# poster); fall back to the calendar/DB values if TMDB is unavailable.
|
||||||
|
poster, emap = _season_lookup(season_meta, tid, sn, season_cache)
|
||||||
|
tm = emap.get(en) or {}
|
||||||
|
# library_id (the show's library row id, given as show_id) is REQUIRED — the
|
||||||
|
# wishlist resolves a show's synopsis + cast from /detail/show/<library_id>;
|
||||||
|
# without it the show shows as un-matched with no synopsis/actors.
|
||||||
|
grp = by_show.setdefault((tid, r.get('show_title')),
|
||||||
|
{'library_id': r.get('show_id'), 'eps': []})
|
||||||
|
grp['eps'].append({
|
||||||
|
'season_number': sn,
|
||||||
|
'episode_number': en,
|
||||||
|
'title': r.get('title') or tm.get('title'),
|
||||||
|
'air_date': r.get('air_date') or tm.get('air_date'),
|
||||||
|
'overview': tm.get('overview') or r.get('overview'),
|
||||||
|
'still_url': tm.get('still_url') or r.get('still_url'),
|
||||||
|
'season_poster_url': poster,
|
||||||
|
})
|
||||||
|
|
||||||
|
added = 0
|
||||||
|
for (tid, title), grp in by_show.items():
|
||||||
|
added += int(add_episodes(tid, title, grp['eps'], grp['library_id'],
|
||||||
|
_show_poster_url(grp['library_id'])) or 0)
|
||||||
|
shows = len(by_show)
|
||||||
|
|
||||||
|
done = ('Added %d airing episode(s) across %d show(s) to the wishlist'
|
||||||
|
% (added, shows)) if added else 'No new airing episodes to wishlist today'
|
||||||
|
if pruned:
|
||||||
|
done += ' · pruned %d ended show(s)' % pruned
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line=done, log_type='success')
|
||||||
|
return {'status': 'completed', 'episodes_added': added, 'shows': shows,
|
||||||
|
'shows_pruned': pruned, '_manages_own_progress': True}
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||||
|
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||||
130
core/automation/handlers/video_clean_youtube.py
Normal file
130
core/automation/handlers/video_clean_youtube.py
Normal file
|
|
@ -0,0 +1,130 @@
|
||||||
|
"""Automation handler: ``video_clean_youtube_episodes`` action.
|
||||||
|
|
||||||
|
YouTube retention: for each followed channel that has a per-channel retention policy (set in
|
||||||
|
the channel's cog modal — default 'keep everything'), delete the episodes that fall outside
|
||||||
|
the keep window (video file + its ``-thumb``/``.nfo`` sidecars). The history row is KEPT and
|
||||||
|
flagged pruned, so the scan's download-dedup still excludes it — a cleaned episode is never
|
||||||
|
re-downloaded. Playlists are mirror-the-whole-thing, so they're out of scope (channels only).
|
||||||
|
|
||||||
|
Runs daily. Pure — channel list, per-channel policy, episode list, file deletion + the prune
|
||||||
|
mark are all injected seams; tests drive it with fakes and never touch a disk or DB.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from datetime import date
|
||||||
|
from typing import Any, Callable, Dict, List, Optional
|
||||||
|
|
||||||
|
from core.automation.deps import AutomationDeps
|
||||||
|
from core.video.retention import episodes_to_prune, parse_retention
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _default_fetch_channels() -> List[Any]:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().youtube_channels_with_downloads()
|
||||||
|
|
||||||
|
|
||||||
|
def _default_channel_retention(channel_id: Any) -> Any:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return (get_video_db().get_channel_settings(channel_id) or {}).get("retention")
|
||||||
|
|
||||||
|
|
||||||
|
def _default_fetch_episodes(channel_id: Any) -> List[Dict[str, Any]]:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().youtube_channel_episodes(channel_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _default_mark_pruned(history_id: Any, when: str) -> bool:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().mark_download_pruned(history_id, when)
|
||||||
|
|
||||||
|
|
||||||
|
def _silent_remove(path: str) -> None:
|
||||||
|
try:
|
||||||
|
os.remove(path)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _default_delete_files(ep: Dict[str, Any]):
|
||||||
|
"""Delete the episode's video + its sidecars. Returns (ok, freed_bytes). Only ever touches
|
||||||
|
the exact recorded ``dest_path`` and its same-stem sidecars — never walks a folder. A file
|
||||||
|
already gone counts as success (mark it pruned); a real delete error does NOT (so it retries)."""
|
||||||
|
path = ep.get("dest_path")
|
||||||
|
if not path:
|
||||||
|
return False, 0
|
||||||
|
size = 0
|
||||||
|
try:
|
||||||
|
if os.path.exists(path):
|
||||||
|
size = os.path.getsize(path)
|
||||||
|
os.remove(path)
|
||||||
|
except OSError:
|
||||||
|
logger.exception("retention: could not delete %s", path)
|
||||||
|
return False, 0
|
||||||
|
stem = os.path.splitext(path)[0]
|
||||||
|
for sc in (stem + ".nfo", stem + "-thumb.jpg", stem + "-thumb.jpeg",
|
||||||
|
stem + "-thumb.png", stem + "-thumb.webp"):
|
||||||
|
if os.path.exists(sc):
|
||||||
|
_silent_remove(sc)
|
||||||
|
return True, size
|
||||||
|
|
||||||
|
|
||||||
|
def _fmt_gb(b: int) -> str:
|
||||||
|
gb = (b or 0) / (1024 ** 3)
|
||||||
|
return ("%.1f GB" % gb) if gb >= 0.1 else "%d MB" % round((b or 0) / (1024 ** 2))
|
||||||
|
|
||||||
|
|
||||||
|
def auto_video_clean_youtube_episodes(
|
||||||
|
config: Dict[str, Any],
|
||||||
|
deps: AutomationDeps,
|
||||||
|
*,
|
||||||
|
fetch_channels: Optional[Callable[[], List[Any]]] = None,
|
||||||
|
channel_retention: Optional[Callable[[Any], Any]] = None,
|
||||||
|
fetch_episodes: Optional[Callable[[Any], List[Dict[str, Any]]]] = None,
|
||||||
|
delete_files: Optional[Callable[[Dict[str, Any]], Any]] = None,
|
||||||
|
mark_pruned: Optional[Callable[[Any, str], Any]] = None,
|
||||||
|
today_fn: Optional[Callable[[], str]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Delete out-of-window episodes for channels with a retention policy. Returns
|
||||||
|
``{'status': 'completed', 'deleted': int, 'channels': int, 'freed_bytes': int, ...}``."""
|
||||||
|
fetch_channels = fetch_channels or _default_fetch_channels
|
||||||
|
channel_retention = channel_retention or _default_channel_retention
|
||||||
|
fetch_episodes = fetch_episodes or _default_fetch_episodes
|
||||||
|
delete_files = delete_files or _default_delete_files
|
||||||
|
mark_pruned = mark_pruned or _default_mark_pruned
|
||||||
|
today_fn = today_fn or (lambda: date.today().isoformat())
|
||||||
|
automation_id = config.get("_automation_id")
|
||||||
|
try:
|
||||||
|
today = today_fn()
|
||||||
|
deps.update_progress(automation_id, phase="Checking retention…", progress=10,
|
||||||
|
log_line="Looking for channels with a retention policy", log_type="info")
|
||||||
|
deleted = freed = checked = 0
|
||||||
|
for cid in (fetch_channels() or []):
|
||||||
|
policy = channel_retention(cid)
|
||||||
|
if not parse_retention(policy): # 'keep everything' / unset → skip
|
||||||
|
continue
|
||||||
|
checked += 1
|
||||||
|
prune = episodes_to_prune(fetch_episodes(cid) or [], policy, today=today)
|
||||||
|
for ep in prune:
|
||||||
|
ok, size = delete_files(ep)
|
||||||
|
if not ok:
|
||||||
|
continue
|
||||||
|
mark_pruned(ep.get("id"), today) # keep the row → scan won't re-download it
|
||||||
|
deleted += 1
|
||||||
|
freed += size or 0
|
||||||
|
deps.update_progress(automation_id, log_type="info",
|
||||||
|
log_line="Removed '%s'" % (ep.get("title") or ep.get("media_id") or "episode"))
|
||||||
|
|
||||||
|
msg = ("Removed %d old episode(s) · freed %s" % (deleted, _fmt_gb(freed))) if deleted \
|
||||||
|
else "Nothing to clean — every channel within its retention window"
|
||||||
|
deps.update_progress(automation_id, status="finished", progress=100, phase="Complete",
|
||||||
|
log_line=msg, log_type="success")
|
||||||
|
return {"status": "completed", "deleted": deleted, "channels": checked,
|
||||||
|
"freed_bytes": freed, "_manages_own_progress": True}
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
deps.update_progress(automation_id, status="error", phase="Error", log_line=str(e), log_type="error")
|
||||||
|
return {"status": "error", "error": str(e), "_manages_own_progress": True}
|
||||||
293
core/automation/handlers/video_process_wishlist.py
Normal file
293
core/automation/handlers/video_process_wishlist.py
Normal file
|
|
@ -0,0 +1,293 @@
|
||||||
|
"""Automation handlers: ``video_process_movie_wishlist`` / ``video_process_episode_wishlist``.
|
||||||
|
|
||||||
|
The Soulseek counterpart of the YouTube wishlist drain — the piece that finally makes the
|
||||||
|
people/airing scans pay off. For wished, RELEASED movies (and aired episodes) it searches
|
||||||
|
Soulseek, picks the best release per the quality profile, and enqueues the download; the
|
||||||
|
existing ``download_monitor`` finishes + organises + archives it, exactly like a manual grab.
|
||||||
|
|
||||||
|
Shape mirrors the YouTube drain (Boulder: "same standard"): it processes the WHOLE eligible
|
||||||
|
wishlist (no total cap), but the slow part — each item needs a ~20s blocking Soulseek search
|
||||||
|
— runs only a FEW at a time (``max_concurrent``). A ``guard`` keeps the next hourly tick from
|
||||||
|
overlapping a run that's still working, so it can't pile up.
|
||||||
|
|
||||||
|
Movies are gated on ``status='wanted'`` (released; skips 'monitored'/unreleased). Episodes
|
||||||
|
are all-wished (the airing scan only adds aired ones). Items already downloading are skipped
|
||||||
|
so re-runs never double-grab. The pick is the top ACCEPTED release — the ranker already
|
||||||
|
encodes the quality profile's accept/reject/score, so no extra rules here.
|
||||||
|
|
||||||
|
Shared automation side (may import ``core.video`` / ``api.video``); owns its own progress.
|
||||||
|
The search + enqueue are injected seams, so selection/pick/record are pure + unit-tested.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import threading
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||||
|
|
||||||
|
from core.automation.deps import AutomationDeps
|
||||||
|
|
||||||
|
|
||||||
|
# ── pure helpers ──────────────────────────────────────────────────────────────
|
||||||
|
def pick_best(candidates: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
|
||||||
|
"""The best ACCEPTED release from a ranked candidate list. ``_evaluate_hits`` already
|
||||||
|
sorts best-first (accepted, score, availability), so the first accepted is the pick."""
|
||||||
|
for c in candidates or []:
|
||||||
|
if c.get("accepted"):
|
||||||
|
return c
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def item_key(item: Dict[str, Any], media_type: str) -> tuple:
|
||||||
|
"""Stable identity for de-duping a wished item against active downloads."""
|
||||||
|
if media_type == "movie":
|
||||||
|
return ("movie", str(item.get("tmdb_id")))
|
||||||
|
return ("episode", str(item.get("show_tmdb_id")),
|
||||||
|
int(item.get("season_number") or 0), int(item.get("episode_number") or 0))
|
||||||
|
|
||||||
|
|
||||||
|
def active_download_keys(active: Iterable[Dict[str, Any]]) -> set:
|
||||||
|
"""Identity keys for the movie/episode downloads already in flight, so we don't
|
||||||
|
re-grab them. Episodes read season/episode out of the row's ``search_ctx``."""
|
||||||
|
keys = set()
|
||||||
|
for d in active or []:
|
||||||
|
kind = str(d.get("kind") or "").lower()
|
||||||
|
if kind == "movie":
|
||||||
|
keys.add(("movie", str(d.get("media_id"))))
|
||||||
|
elif kind == "episode":
|
||||||
|
ctx = d.get("search_ctx")
|
||||||
|
if isinstance(ctx, str):
|
||||||
|
try:
|
||||||
|
ctx = json.loads(ctx)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
ctx = {}
|
||||||
|
ctx = ctx if isinstance(ctx, dict) else {}
|
||||||
|
keys.add(("episode", str(d.get("media_id")),
|
||||||
|
int(ctx.get("season") or 0), int(ctx.get("episode") or 0)))
|
||||||
|
return keys
|
||||||
|
|
||||||
|
|
||||||
|
def search_context(item: Dict[str, Any], media_type: str) -> Dict[str, Any]:
|
||||||
|
"""The ``search_ctx`` the download row carries (drives the monitor's requery)."""
|
||||||
|
if media_type == "movie":
|
||||||
|
return {"scope": "movie", "title": item.get("title"), "year": item.get("year")}
|
||||||
|
return {"scope": "episode", "title": item.get("show_title"),
|
||||||
|
"season": item.get("season_number"), "episode": item.get("episode_number"),
|
||||||
|
"year": (str(item.get("air_date") or "")[:4] or None)}
|
||||||
|
|
||||||
|
|
||||||
|
def build_download_record(item: Dict[str, Any], best: Dict[str, Any], candidates: List[Dict[str, Any]],
|
||||||
|
*, media_type: str, target_dir: str, query: Any) -> Dict[str, Any]:
|
||||||
|
"""The ``add_video_download`` row for a chosen release — identical shape to a manual
|
||||||
|
grab, so the monitor finishes it the same way (other accepted hits become the retry
|
||||||
|
pool)."""
|
||||||
|
ctx = search_context(item, media_type)
|
||||||
|
# stash the chosen source's peer stats so the drawer can show its availability snapshot
|
||||||
|
# (free slot / queue depth / speed at grab time). Retry ignores the extra key.
|
||||||
|
peer = {k: best.get(k) for k in ("slots", "queue", "speed", "availability") if best.get(k) is not None}
|
||||||
|
if peer:
|
||||||
|
ctx = {**ctx, "peer": peer}
|
||||||
|
rest = [c for c in (candidates or []) if c.get("filename") != best.get("filename")]
|
||||||
|
media_id = str(item.get("tmdb_id") if media_type == "movie" else item.get("show_tmdb_id"))
|
||||||
|
return {
|
||||||
|
"kind": media_type, "title": ctx["title"],
|
||||||
|
"release_title": best.get("title") or best.get("filename"),
|
||||||
|
"source": "soulseek", "username": best.get("username"), "filename": best.get("filename"),
|
||||||
|
"size_bytes": int(best.get("size_bytes") or 0), "quality_label": best.get("quality_label"),
|
||||||
|
"target_dir": target_dir, "status": "downloading",
|
||||||
|
"media_id": media_id, "media_source": "tmdb", "year": ctx.get("year"),
|
||||||
|
"poster_url": item.get("poster_url"),
|
||||||
|
"candidates": json.dumps(rest), "search_ctx": json.dumps(ctx),
|
||||||
|
"tried_queries": json.dumps([query] if query else []),
|
||||||
|
"tried_files": json.dumps([best.get("filename")]), "attempts": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ── production seams ──────────────────────────────────────────────────────────
|
||||||
|
def _default_fetch_items(media_type: str) -> List[Dict[str, Any]]:
|
||||||
|
from api.video import get_video_db
|
||||||
|
db = get_video_db()
|
||||||
|
return db.movie_wishlist_to_download() if media_type == "movie" else db.episode_wishlist_to_download()
|
||||||
|
|
||||||
|
|
||||||
|
def _default_active_keys(media_type: str) -> set:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return active_download_keys(get_video_db().get_active_video_downloads())
|
||||||
|
|
||||||
|
|
||||||
|
def _default_target_dir(media_type: str) -> str:
|
||||||
|
from api.video import get_video_db
|
||||||
|
db = get_video_db()
|
||||||
|
if media_type == "movie":
|
||||||
|
return db.get_setting("movies_path") or db.get_setting("transfer_path") or ""
|
||||||
|
return db.get_setting("tv_path") or ""
|
||||||
|
|
||||||
|
|
||||||
|
def _default_search(item: Dict[str, Any], media_type: str):
|
||||||
|
"""A bounded blocking Soulseek search → ranked candidates (same path the retry worker +
|
||||||
|
manual search use). Returns [] for a real empty result, or **None** if the search never
|
||||||
|
ran (slskd not configured / errored / rate-limited) so the caller can say so."""
|
||||||
|
from api.video.downloads import _evaluate_hits
|
||||||
|
from core.video.download_monitor import _search_for_retry
|
||||||
|
from core.video.quality_profile import load as load_profile
|
||||||
|
from core.video.slskd_search import build_query
|
||||||
|
from api.video import get_video_db
|
||||||
|
ctx = search_context(item, media_type)
|
||||||
|
query = build_query(ctx["scope"], ctx["title"], year=ctx.get("year"),
|
||||||
|
season=ctx.get("season"), episode=ctx.get("episode"))
|
||||||
|
res = _search_for_retry(query) or {}
|
||||||
|
if res.get("started") is False:
|
||||||
|
return None, res.get("error") # slskd didn't accept the search — pass the reason
|
||||||
|
profile = load_profile(get_video_db())
|
||||||
|
return _evaluate_hits(res.get("hits") or [], profile, ctx["scope"],
|
||||||
|
ctx.get("season"), ctx.get("episode")), None
|
||||||
|
|
||||||
|
|
||||||
|
def _default_enqueue(item: Dict[str, Any], best: Dict[str, Any], candidates: List[Dict[str, Any]],
|
||||||
|
media_type: str, target_dir: str) -> bool:
|
||||||
|
"""Start the slskd transfer + write the download row (exactly like the manual flow),
|
||||||
|
then ensure the monitor is running. Returns True if slskd accepted it."""
|
||||||
|
from api.video import get_video_db
|
||||||
|
from core.video.download_monitor import ensure_started
|
||||||
|
from core.video.slskd_download import start_download
|
||||||
|
from core.video.slskd_search import build_query
|
||||||
|
started = start_download(best.get("username"), best.get("filename"), best.get("size_bytes") or 0)
|
||||||
|
if not started.get("ok"):
|
||||||
|
return False
|
||||||
|
ctx = search_context(item, media_type)
|
||||||
|
query = build_query(ctx["scope"], ctx["title"], year=ctx.get("year"),
|
||||||
|
season=ctx.get("season"), episode=ctx.get("episode"))
|
||||||
|
get_video_db().add_video_download(
|
||||||
|
build_download_record(item, best, candidates, media_type=media_type,
|
||||||
|
target_dir=target_dir, query=query))
|
||||||
|
ensure_started(get_video_db)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
# ── guard: keep an in-progress drain from overlapping the next tick ───────────
|
||||||
|
_running: Dict[str, bool] = {"movie": False, "episode": False}
|
||||||
|
|
||||||
|
|
||||||
|
def is_running(media_type: str) -> bool:
|
||||||
|
return bool(_running.get(media_type))
|
||||||
|
|
||||||
|
|
||||||
|
def auto_video_process_wishlist(
|
||||||
|
config: Dict[str, Any],
|
||||||
|
deps: AutomationDeps,
|
||||||
|
*,
|
||||||
|
media_type: str = "movie",
|
||||||
|
fetch_items: Optional[Callable[[str], List[Dict[str, Any]]]] = None,
|
||||||
|
active_keys: Optional[Callable[[str], Iterable]] = None,
|
||||||
|
target_dir: Optional[Callable[[str], str]] = None,
|
||||||
|
search: Optional[Callable[[Dict[str, Any], str], List[Dict[str, Any]]]] = None,
|
||||||
|
enqueue: Optional[Callable[..., bool]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Auto-grab the wished movies (or episodes): search Soulseek, pick the best release,
|
||||||
|
enqueue. Processes the whole eligible wishlist, a few searches at a time.
|
||||||
|
|
||||||
|
Returns ``{'status': 'completed', 'searched': int, 'grabbed': int, ...}``."""
|
||||||
|
fetch_items = fetch_items or _default_fetch_items
|
||||||
|
active_keys = active_keys or _default_active_keys
|
||||||
|
target_dir = target_dir or _default_target_dir
|
||||||
|
search = search or _default_search
|
||||||
|
enqueue = enqueue or _default_enqueue
|
||||||
|
automation_id = config.get('_automation_id')
|
||||||
|
concurrency = max(1, int(config.get('max_concurrent', 3) or 3))
|
||||||
|
label = 'movie' if media_type == 'movie' else 'episode'
|
||||||
|
|
||||||
|
_running[media_type] = True
|
||||||
|
try:
|
||||||
|
root = target_dir(media_type)
|
||||||
|
if not root:
|
||||||
|
where = 'Movie' if media_type == 'movie' else 'TV'
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line='%s library folder not set — skipping (Settings → Downloads)' % where,
|
||||||
|
log_type='info')
|
||||||
|
return {'status': 'completed', 'searched': 0, 'grabbed': 0,
|
||||||
|
'skipped': 'no_folder', '_manages_own_progress': True}
|
||||||
|
|
||||||
|
deps.update_progress(automation_id, phase='Checking the wishlist…', progress=5,
|
||||||
|
log_line='Looking for wished %ss to grab' % label, log_type='info')
|
||||||
|
items = fetch_items(media_type) or []
|
||||||
|
active = set(active_keys(media_type) or set())
|
||||||
|
todo = [it for it in items if item_key(it, media_type) not in active]
|
||||||
|
if not todo:
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line='Nothing new to grab (%d already in flight)' % len(active),
|
||||||
|
log_type='info')
|
||||||
|
return {'status': 'completed', 'searched': 0, 'grabbed': 0, '_manages_own_progress': True}
|
||||||
|
|
||||||
|
grabbed = [0]
|
||||||
|
searched = [0]
|
||||||
|
noresults = [0] # search came back empty (the source had nothing)
|
||||||
|
rejected = [0] # source had hits, but none passed the quality profile
|
||||||
|
notrun = [0] # the search never ran (slskd didn't accept it)
|
||||||
|
total = len(todo)
|
||||||
|
lock = threading.Lock()
|
||||||
|
|
||||||
|
def _one(it):
|
||||||
|
found = search(it, media_type)
|
||||||
|
# the seam returns (candidates, error); tolerate a bare list/None too (test fakes)
|
||||||
|
cands, err = found if isinstance(found, tuple) else (found, None)
|
||||||
|
didnt_run = cands is None # slskd not configured / errored / rate-limited
|
||||||
|
cands = cands or []
|
||||||
|
best = pick_best(cands)
|
||||||
|
ok = bool(best) and bool(enqueue(it, best, cands, media_type, root))
|
||||||
|
name = it.get('title') or it.get('show_title') or '?'
|
||||||
|
if media_type == 'episode':
|
||||||
|
name = "%s S%02dE%02d" % (name, int(it.get('season_number') or 0),
|
||||||
|
int(it.get('episode_number') or 0))
|
||||||
|
# tell apart: grabbed / search-didn't-run / source-empty / hits-but-all-rejected.
|
||||||
|
if ok:
|
||||||
|
msg, lt = "Grabbed '%s'" % name, 'success'
|
||||||
|
elif didnt_run:
|
||||||
|
msg = ("Search didn't run for '%s' — slskd error: %s" % (name, err)) if err \
|
||||||
|
else ("Search didn't run for '%s' — slskd not responding?" % name)
|
||||||
|
lt = 'warning'
|
||||||
|
elif not cands:
|
||||||
|
msg, lt = "No search results for '%s'" % name, 'info'
|
||||||
|
else:
|
||||||
|
why = (cands[0].get('rejected') or 'none met your quality profile')
|
||||||
|
msg, lt = "%d result(s) for '%s', none accepted — %s" % (len(cands), name, why), 'info'
|
||||||
|
with lock:
|
||||||
|
searched[0] += 1
|
||||||
|
if ok:
|
||||||
|
grabbed[0] += 1
|
||||||
|
elif didnt_run:
|
||||||
|
notrun[0] += 1
|
||||||
|
elif not cands:
|
||||||
|
noresults[0] += 1
|
||||||
|
else:
|
||||||
|
rejected[0] += 1
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id, phase='Searching + grabbing…',
|
||||||
|
progress=10 + int(85 * searched[0] / max(total, 1)),
|
||||||
|
log_line=msg, log_type=lt)
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=concurrency) as ex:
|
||||||
|
list(ex.map(_one, todo))
|
||||||
|
|
||||||
|
# Headline with the WHY breakdown: it's the difference between "the source has
|
||||||
|
# nothing" (noresults) and "it has stuff but your quality profile rejects it" (rejected).
|
||||||
|
tail = []
|
||||||
|
if notrun[0]:
|
||||||
|
tail.append("%d search(es) didn't run (slskd?)" % notrun[0])
|
||||||
|
if noresults[0]:
|
||||||
|
tail.append('%d had no results' % noresults[0])
|
||||||
|
if rejected[0]:
|
||||||
|
tail.append('%d rejected on quality' % rejected[0])
|
||||||
|
breakdown = (' · ' + ', '.join(tail)) if tail else ''
|
||||||
|
done = ('Grabbed %d %s(s) of %d searched%s' % (grabbed[0], label, searched[0], breakdown)) if grabbed[0] \
|
||||||
|
else ('Searched %d %s(s), grabbed 0%s' % (searched[0], label, breakdown))
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line=done, log_type='success' if grabbed[0] else 'info')
|
||||||
|
return {'status': 'completed', 'searched': searched[0], 'grabbed': grabbed[0],
|
||||||
|
'noresults': noresults[0], 'rejected': rejected[0], 'notrun': notrun[0],
|
||||||
|
'_manages_own_progress': True}
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||||
|
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||||
|
finally:
|
||||||
|
_running[media_type] = False
|
||||||
200
core/automation/handlers/video_process_youtube_wishlist.py
Normal file
200
core/automation/handlers/video_process_youtube_wishlist.py
Normal file
|
|
@ -0,0 +1,200 @@
|
||||||
|
"""Automation handler: ``video_process_youtube_wishlist`` action.
|
||||||
|
|
||||||
|
The drain side of the YouTube fulfillment lane. The watchlist-channels scan keeps the
|
||||||
|
wishlist fed; THIS queues wished videos for download and keeps a few flowing at a time.
|
||||||
|
|
||||||
|
There is NO cap on how much of the wishlist gets processed — it queues the WHOLE thing.
|
||||||
|
The only limit is how many download SIMULTANEOUSLY (``max_concurrent``, default 3): every
|
||||||
|
wished video becomes a ``queued`` row in the shared ``video_downloads`` table, the handler
|
||||||
|
starts up to the limit, and each finished download starts the next (one-out-one-in, in the
|
||||||
|
worker) so the entire queue drains in a controlled stream. This mirrors the music side's
|
||||||
|
download worker — a concurrency cap plus a small inter-download delay (handled in the
|
||||||
|
worker) to avoid yt-dlp 429s — but stays on the isolated video side.
|
||||||
|
|
||||||
|
The worker (``core.video.youtube_download``) downloads → organises (channel/year/date) →
|
||||||
|
archives to history → removes the video from the wishlist, so a completed grab leaves the
|
||||||
|
wishlist and won't be re-queued; videos already queued or downloading are skipped so re-runs
|
||||||
|
never double-grab.
|
||||||
|
|
||||||
|
Shared automation side (may import ``core.video`` / ``api.video``); owns its own progress.
|
||||||
|
All I/O is injected as seams, so selection + the pump are pure unit-testable functions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||||
|
|
||||||
|
from core.automation.deps import AutomationDeps
|
||||||
|
|
||||||
|
|
||||||
|
def videos_to_enqueue(wanted: List[Dict[str, Any]], already_ids: Iterable) -> List[Dict[str, Any]]:
|
||||||
|
"""Wished videos not already queued or downloading. NO cap — the whole backlog is
|
||||||
|
queued; concurrency is bounded at start time, not here. Pure."""
|
||||||
|
already = {str(x) for x in (already_ids or ()) if x}
|
||||||
|
out: List[Dict[str, Any]] = []
|
||||||
|
for v in wanted or []:
|
||||||
|
vid = v.get("video_id")
|
||||||
|
if vid and str(vid) not in already:
|
||||||
|
out.append(v)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def slots_free(running: int, max_concurrent: int) -> int:
|
||||||
|
"""How many new downloads may start now given how many are already fetching. Pure."""
|
||||||
|
return max(0, int(max_concurrent) - max(0, int(running)))
|
||||||
|
|
||||||
|
|
||||||
|
def enqueue_ctx(video: Dict[str, Any], channel_settings: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
"""The download row's ``search_ctx``, applying per-channel overrides: a custom show-name
|
||||||
|
(the ``$channel`` folder token) and/or a quality override. Pure."""
|
||||||
|
cs = channel_settings if isinstance(channel_settings, dict) else {}
|
||||||
|
ctx = {
|
||||||
|
"channel": cs.get("custom_name") or video.get("channel_title"),
|
||||||
|
"channel_id": video.get("channel_id"), # so the drawer can open the in-app channel page
|
||||||
|
"video_title": video.get("video_title"),
|
||||||
|
"published_at": video.get("published_at"),
|
||||||
|
}
|
||||||
|
if cs.get("quality"):
|
||||||
|
ctx["quality"] = cs["quality"]
|
||||||
|
return ctx
|
||||||
|
|
||||||
|
|
||||||
|
# ── production seams ──────────────────────────────────────────────────────────
|
||||||
|
def _default_youtube_root() -> str:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().get_setting("youtube_path") or ""
|
||||||
|
|
||||||
|
|
||||||
|
def _default_fetch_wanted() -> List[Dict[str, Any]]:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().youtube_wishlist_to_download()
|
||||||
|
|
||||||
|
|
||||||
|
def _default_active_ids() -> List[Any]:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return [d.get("media_id") for d in get_video_db().get_active_video_downloads()
|
||||||
|
if d.get("source") == "youtube" and d.get("media_id")]
|
||||||
|
|
||||||
|
|
||||||
|
def _default_running_count() -> int:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().count_active_youtube_downloads()
|
||||||
|
|
||||||
|
|
||||||
|
def _default_enqueue(video: Dict[str, Any], root: str) -> Any:
|
||||||
|
"""Create a QUEUED download row (no thread spawned here — the pump starts it). Returns
|
||||||
|
the row id."""
|
||||||
|
import json
|
||||||
|
from api.video import get_video_db
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
db = get_video_db()
|
||||||
|
ctx = enqueue_ctx(video, db.get_channel_settings(video.get("channel_id")))
|
||||||
|
ctx["server_source"] = resolve_video_server()
|
||||||
|
return db.add_video_download({
|
||||||
|
"kind": "youtube", "source": "youtube", "media_source": "youtube",
|
||||||
|
"title": video.get("video_title") or video.get("channel_title"),
|
||||||
|
"media_id": video.get("video_id"), "target_dir": root, "status": "queued",
|
||||||
|
"year": video.get("published_at"), "poster_url": video.get("thumbnail_url"),
|
||||||
|
"search_ctx": json.dumps(ctx),
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def _default_start_next() -> Any:
|
||||||
|
"""Claim + start the next queued YouTube download (or None). The worker chains the rest."""
|
||||||
|
from api.video import get_video_db
|
||||||
|
from core.video.youtube_download import start_next_queued
|
||||||
|
return start_next_queued(get_video_db)
|
||||||
|
|
||||||
|
|
||||||
|
def _default_reap() -> int:
|
||||||
|
"""Recover downloads orphaned by a restart (stuck 'downloading', no live worker)."""
|
||||||
|
from api.video import get_video_db
|
||||||
|
from core.video.youtube_download import requeue_orphaned_youtube
|
||||||
|
return requeue_orphaned_youtube(get_video_db)
|
||||||
|
|
||||||
|
|
||||||
|
def auto_video_process_youtube_wishlist(
|
||||||
|
config: Dict[str, Any],
|
||||||
|
deps: AutomationDeps,
|
||||||
|
*,
|
||||||
|
youtube_root: Optional[Callable[[], str]] = None,
|
||||||
|
fetch_wanted: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||||
|
active_ids: Optional[Callable[[], Iterable]] = None,
|
||||||
|
running_count: Optional[Callable[[], int]] = None,
|
||||||
|
enqueue: Optional[Callable[[Dict[str, Any], str], Any]] = None,
|
||||||
|
start_next: Optional[Callable[[], Any]] = None,
|
||||||
|
reap: Optional[Callable[[], int]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Queue the whole YouTube wishlist for download and start up to ``max_concurrent`` now.
|
||||||
|
|
||||||
|
Returns ``{'status': 'completed', 'queued': int, 'started': int, 'running': int, ...}``."""
|
||||||
|
youtube_root = youtube_root or _default_youtube_root
|
||||||
|
fetch_wanted = fetch_wanted or _default_fetch_wanted
|
||||||
|
active_ids = active_ids or _default_active_ids
|
||||||
|
running_count = running_count or _default_running_count
|
||||||
|
enqueue = enqueue or _default_enqueue
|
||||||
|
start_next = start_next or _default_start_next
|
||||||
|
reap = reap or _default_reap
|
||||||
|
automation_id = config.get('_automation_id')
|
||||||
|
max_concurrent = max(1, int(config.get('max_concurrent', 3) or 3))
|
||||||
|
|
||||||
|
try:
|
||||||
|
root = youtube_root()
|
||||||
|
if not root:
|
||||||
|
# Always-on automation: a missing folder isn't a failure, it's "not set up for
|
||||||
|
# YouTube" — skip quietly so non-YouTube users don't see a recurring error.
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line='YouTube library folder not set — skipping (Settings → Downloads)',
|
||||||
|
log_type='info')
|
||||||
|
return {'status': 'completed', 'queued': 0, 'started': 0, 'running': 0,
|
||||||
|
'skipped': 'no_youtube_folder', '_manages_own_progress': True}
|
||||||
|
|
||||||
|
# Recover any downloads orphaned by a restart (stuck 'downloading', no worker) so
|
||||||
|
# they don't wedge the concurrency count — they go back to 'queued' and re-run.
|
||||||
|
recovered = int(reap() or 0)
|
||||||
|
if recovered:
|
||||||
|
deps.update_progress(automation_id, log_type='info',
|
||||||
|
log_line='Recovered %d stalled download(s) from a restart' % recovered)
|
||||||
|
|
||||||
|
deps.update_progress(automation_id, phase='Checking the YouTube wishlist…', progress=15,
|
||||||
|
log_line='Queueing new videos for download', log_type='info')
|
||||||
|
wanted = fetch_wanted() or []
|
||||||
|
already = list(active_ids() or [])
|
||||||
|
new = videos_to_enqueue(wanted, already)
|
||||||
|
|
||||||
|
queued = 0
|
||||||
|
for v in new:
|
||||||
|
try:
|
||||||
|
if enqueue(v, root) is not None:
|
||||||
|
queued += 1
|
||||||
|
except Exception: # noqa: BLE001 - one bad enqueue shouldn't stop the rest
|
||||||
|
deps.update_progress(automation_id, log_type='warning',
|
||||||
|
log_line="Couldn't queue '%s'" % (v.get('video_title') or v.get('video_id')))
|
||||||
|
|
||||||
|
# Fill the concurrency slots now; each finished download starts the next, so the
|
||||||
|
# whole queue drains on its own from here.
|
||||||
|
deps.update_progress(automation_id, phase='Starting downloads…', progress=70,
|
||||||
|
log_line='Queued %d new video(s)' % queued, log_type='info')
|
||||||
|
started = 0
|
||||||
|
for _ in range(slots_free(running_count() or 0, max_concurrent)):
|
||||||
|
if start_next() is None:
|
||||||
|
break
|
||||||
|
started += 1
|
||||||
|
|
||||||
|
running = (running_count() or 0)
|
||||||
|
if queued or started:
|
||||||
|
done = 'Queued %d new · %d downloading now (the rest drain automatically)' % (queued, running)
|
||||||
|
log_type = 'success'
|
||||||
|
elif running:
|
||||||
|
done = '%d already downloading; nothing new to queue' % running
|
||||||
|
log_type = 'info'
|
||||||
|
else:
|
||||||
|
done = 'No wished YouTube videos to download'
|
||||||
|
log_type = 'info'
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line=done, log_type=log_type)
|
||||||
|
return {'status': 'completed', 'queued': queued, 'started': started, 'running': running,
|
||||||
|
'_manages_own_progress': True}
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||||
|
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||||
90
core/automation/handlers/video_refresh_airing_schedules.py
Normal file
90
core/automation/handlers/video_refresh_airing_schedules.py
Normal file
|
|
@ -0,0 +1,90 @@
|
||||||
|
"""Automation handler: ``video_refresh_airing_schedules`` action.
|
||||||
|
|
||||||
|
Keeps the TV calendar honest. The "Wishlist Episodes Airing Today" automation reads the
|
||||||
|
LOCAL ``episodes`` table — which is only refreshed from TMDB on initial enrichment, a manual
|
||||||
|
re-match, or lazily when you open a show's page. So a newly-announced or rescheduled episode
|
||||||
|
can sit unknown indefinitely, and the airing automation misses it.
|
||||||
|
|
||||||
|
This runs daily (a couple hours before the airing run) and re-pulls each still-airing
|
||||||
|
watchlist show's season/episode schedule from TMDB (air dates, stills, overviews) so the
|
||||||
|
calendar is current when the airing automation reads it.
|
||||||
|
|
||||||
|
Scope: the EFFECTIVE watchlist's continuing LIBRARY shows (follows ∪ airing library shows,
|
||||||
|
skipping ended/canceled — they'll never air again, and skipping tmdb-only follows, which have
|
||||||
|
no episodes table to refresh). Like the other video handlers it owns its progress reporting
|
||||||
|
(``_manages_own_progress``); the show fetch + per-show refresh are injected seams, so the
|
||||||
|
handler is a pure function tests drive with fakes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Callable, Dict, List, Optional
|
||||||
|
|
||||||
|
from core.automation.deps import AutomationDeps
|
||||||
|
|
||||||
|
|
||||||
|
def _default_fetch_shows() -> List[Dict[str, Any]]:
|
||||||
|
"""Production wiring: the still-airing watchlist shows that live in the library."""
|
||||||
|
from api.video import get_video_db
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
return get_video_db().watchlist_continuing_shows(resolve_video_server())
|
||||||
|
|
||||||
|
|
||||||
|
def _default_refresh_show(library_id: Any) -> Dict[str, Any]:
|
||||||
|
"""Re-pull a library show's TMDB season/episode schedule (the lazy on-view backfill,
|
||||||
|
invoked deliberately). Re-matches + cascades episodes, so air dates/stills refresh.
|
||||||
|
``with_ratings=False`` — we only need schedules, and the per-show OMDb ratings call
|
||||||
|
would burn the daily quota across a whole watchlist."""
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
return get_video_enrichment_engine().refresh_show_art(library_id, with_ratings=False) or {}
|
||||||
|
|
||||||
|
|
||||||
|
def auto_video_refresh_airing_schedules(
|
||||||
|
config: Dict[str, Any],
|
||||||
|
deps: AutomationDeps,
|
||||||
|
*,
|
||||||
|
fetch_shows: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||||
|
refresh_show: Optional[Callable[[Any], Dict[str, Any]]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Refresh the TMDB episode schedule for every still-airing watchlist library show, so
|
||||||
|
the airing automation's calendar read is current.
|
||||||
|
|
||||||
|
Returns ``{'status': 'completed', 'refreshed': int, 'failed': int, 'shows': int, ...}``."""
|
||||||
|
fetch_shows = fetch_shows or _default_fetch_shows
|
||||||
|
refresh_show = refresh_show or _default_refresh_show
|
||||||
|
automation_id = config.get('_automation_id')
|
||||||
|
try:
|
||||||
|
deps.update_progress(automation_id, phase='Finding shows to refresh…', progress=8,
|
||||||
|
log_line='Reading your watchlist for still-airing shows', log_type='info')
|
||||||
|
shows = fetch_shows() or []
|
||||||
|
total = len(shows)
|
||||||
|
if not total:
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line='No airing shows on your watchlist to refresh', log_type='success')
|
||||||
|
return {'status': 'completed', 'refreshed': 0, 'failed': 0, 'shows': 0,
|
||||||
|
'_manages_own_progress': True}
|
||||||
|
|
||||||
|
refreshed = failed = 0
|
||||||
|
for i, s in enumerate(shows):
|
||||||
|
title = s.get('title') or ('show %s' % s.get('library_id'))
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id, phase='Refreshing TV schedules…', progress=10 + int(i / total * 85),
|
||||||
|
log_line="Pulling the latest episodes for '%s' (%d/%d)" % (title, i + 1, total),
|
||||||
|
log_type='info')
|
||||||
|
try:
|
||||||
|
ok = bool((refresh_show(s.get('library_id')) or {}).get('ok'))
|
||||||
|
except Exception: # noqa: BLE001 - one show failing must not stop the rest
|
||||||
|
ok = False
|
||||||
|
if ok:
|
||||||
|
refreshed += 1
|
||||||
|
else:
|
||||||
|
failed += 1
|
||||||
|
|
||||||
|
done = 'Refreshed %d show schedule(s)' % refreshed + (' · %d failed' % failed if failed else '')
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line=done, log_type='success')
|
||||||
|
return {'status': 'completed', 'refreshed': refreshed, 'failed': failed, 'shows': total,
|
||||||
|
'_manages_own_progress': True}
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||||
|
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||||
424
core/automation/handlers/video_scan_library.py
Normal file
424
core/automation/handlers/video_scan_library.py
Normal file
|
|
@ -0,0 +1,424 @@
|
||||||
|
"""Automation handler: ``video_scan_library`` action.
|
||||||
|
|
||||||
|
The VIDEO twin of music's ``scan_library``. It does two things, in order:
|
||||||
|
|
||||||
|
1. Tells the active media server (Plex/Jellyfin) to rescan ONLY the
|
||||||
|
user's selected VIDEO sections (the movie + TV libraries chosen in
|
||||||
|
Settings) — never the music library.
|
||||||
|
2. Reads the server's current state into ``video.db`` so freshly-added
|
||||||
|
media shows up as owned on the video side.
|
||||||
|
|
||||||
|
Both run through injected seams (``server_refresh`` / ``run_video_scan``)
|
||||||
|
so the handler stays a pure function: production lazily binds the real
|
||||||
|
``core.video`` functions; tests pass fakes and never spin up Flask, a DB,
|
||||||
|
or a media-server client.
|
||||||
|
|
||||||
|
ISOLATION NOTE: this handler lives on the SHARED automation side, so it
|
||||||
|
is allowed to import ``core.video`` — the isolation contract only forbids
|
||||||
|
``core/video`` and ``api/video`` from importing the MUSIC side, not the
|
||||||
|
other way round. Video core stays import-clean; the bridge lives here.
|
||||||
|
|
||||||
|
Like ``scan_library``, the handler owns its own progress reporting
|
||||||
|
(``_manages_own_progress: True``) so the engine doesn't stomp the live
|
||||||
|
phase string with a generic 'completed' label.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
from typing import Any, Callable, Dict, Optional
|
||||||
|
|
||||||
|
from core.automation.deps import AutomationDeps
|
||||||
|
|
||||||
|
|
||||||
|
def _default_server_refresh(media_type: str = "all") -> Dict[str, Any]:
|
||||||
|
"""Production wiring: nudge the media server to rescan its video sections.
|
||||||
|
``media_type`` scopes it to the Movie or TV section; 'all' nudges both."""
|
||||||
|
from core.video.sources import refresh_video_server_sections
|
||||||
|
return refresh_video_server_sections(media_type)
|
||||||
|
|
||||||
|
|
||||||
|
def _default_run_video_scan(mode: str, media_type: str = "all") -> Dict[str, Any]:
|
||||||
|
"""Production wiring: read the server into video.db (blocking). ``media_type``
|
||||||
|
scopes it to one library ('movie' / 'show'); 'all' does both."""
|
||||||
|
from api.video import get_video_db
|
||||||
|
from core.video.scanner import get_video_scanner
|
||||||
|
from core.video.sources import get_active_video_source
|
||||||
|
return get_video_scanner(get_video_db()).scan_sync(get_active_video_source, mode, media_type)
|
||||||
|
|
||||||
|
|
||||||
|
def auto_video_scan_library(
|
||||||
|
config: Dict[str, Any],
|
||||||
|
deps: AutomationDeps,
|
||||||
|
*,
|
||||||
|
server_refresh: Optional[Callable[[], Dict[str, Any]]] = None,
|
||||||
|
run_video_scan: Optional[Callable[..., Dict[str, Any]]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Trigger a server-side video rescan, then mirror the result into video.db.
|
||||||
|
|
||||||
|
``config['media_type']`` scopes the scan to one library — 'movie' or 'show'
|
||||||
|
(TV); 'all' (default) does both. Movies and TV are independent libraries, so
|
||||||
|
the Movie scan never touches TV and vice-versa.
|
||||||
|
|
||||||
|
Returns one of:
|
||||||
|
- ``{'status': 'completed', '_manages_own_progress': True, 'movies': .., 'shows': .., 'episodes': ..}``
|
||||||
|
- ``{'status': 'skipped', 'reason': '...'}`` (another scan was already running)
|
||||||
|
- ``{'status': 'error', 'error': '...', '_manages_own_progress': True}``
|
||||||
|
"""
|
||||||
|
server_refresh = server_refresh or _default_server_refresh
|
||||||
|
run_video_scan = run_video_scan or _default_run_video_scan
|
||||||
|
|
||||||
|
automation_id = config.get('_automation_id')
|
||||||
|
# 'full' is the safe default — upsert everything, never prune. 'deep' prunes
|
||||||
|
# what the server no longer has; only use it when the config asks explicitly.
|
||||||
|
mode = config.get('mode') or 'full'
|
||||||
|
media_type = config.get('media_type') or 'all'
|
||||||
|
lib_label = {'movie': 'Movie', 'show': 'TV'}.get(media_type, 'video')
|
||||||
|
|
||||||
|
try:
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id,
|
||||||
|
phase=f'Asking media server to rescan {lib_label} sections...',
|
||||||
|
progress=10,
|
||||||
|
log_line=f'Triggering server-side {lib_label} scan',
|
||||||
|
log_type='info',
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 1 — best-effort server nudge, scoped to the same library we're about
|
||||||
|
# to read. A server that can't be triggered (none configured, or an adapter
|
||||||
|
# without refresh support) is surfaced as a warning, NOT a hard failure: the
|
||||||
|
# read below still mirrors whatever the server currently reports.
|
||||||
|
refresh = server_refresh(media_type) or {}
|
||||||
|
if not refresh.get('ok'):
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id,
|
||||||
|
log_line='Server scan trigger unavailable: ' + str(refresh.get('error') or 'unknown'),
|
||||||
|
log_type='warning',
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
sections = refresh.get('sections')
|
||||||
|
detail = str(sections) + ' video section(s)' if sections else 'selected video section(s)'
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id,
|
||||||
|
progress=30,
|
||||||
|
log_line='Server rescanning ' + detail,
|
||||||
|
log_type='success',
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 2 — read the server into video.db (blocking; mirrors music's
|
||||||
|
# scan handler blocking until the scan resolves).
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id,
|
||||||
|
phase=f'Reading {lib_label} library into SoulSync...',
|
||||||
|
progress=45,
|
||||||
|
)
|
||||||
|
result = run_video_scan(mode, media_type) or {}
|
||||||
|
state = result.get('state')
|
||||||
|
# Singleton scanner already busy with another scan (e.g. the Movie + TV
|
||||||
|
# deep scans firing close together) — skip cleanly, don't error.
|
||||||
|
if state == 'in_progress':
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id, status='finished', phase='Skipped',
|
||||||
|
log_line='Another video scan is already running — skipping this run',
|
||||||
|
log_type='info',
|
||||||
|
)
|
||||||
|
return {'status': 'skipped', 'reason': 'a video scan is already running',
|
||||||
|
'_manages_own_progress': True}
|
||||||
|
if state == 'error':
|
||||||
|
err = result.get('error') or 'Video library scan failed'
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id,
|
||||||
|
status='error',
|
||||||
|
phase='Error',
|
||||||
|
log_line=err,
|
||||||
|
log_type='error',
|
||||||
|
)
|
||||||
|
return {'status': 'error', 'error': err, '_manages_own_progress': True}
|
||||||
|
|
||||||
|
movies = int(result.get('movies', 0) or 0)
|
||||||
|
shows = int(result.get('shows', 0) or 0)
|
||||||
|
episodes = int(result.get('episodes', 0) or 0)
|
||||||
|
# Summary names only the scanned library so a TV scan doesn't read "0 movies".
|
||||||
|
if media_type == 'movie':
|
||||||
|
summary = f'Movie library scanned: {movies} movies'
|
||||||
|
elif media_type == 'show':
|
||||||
|
summary = f'TV library scanned: {shows} shows, {episodes} episodes'
|
||||||
|
else:
|
||||||
|
summary = f'Video library scanned: {movies} movies, {shows} shows, {episodes} episodes'
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id,
|
||||||
|
status='finished',
|
||||||
|
progress=100,
|
||||||
|
phase='Complete',
|
||||||
|
log_line=summary,
|
||||||
|
log_type='success',
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
'status': 'completed',
|
||||||
|
'_manages_own_progress': True,
|
||||||
|
'movies': movies,
|
||||||
|
'shows': shows,
|
||||||
|
'episodes': episodes,
|
||||||
|
}
|
||||||
|
|
||||||
|
except Exception as e: # noqa: BLE001 — automation handlers must never raise into the engine
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id,
|
||||||
|
status='error',
|
||||||
|
phase='Error',
|
||||||
|
log_line=str(e),
|
||||||
|
log_type='error',
|
||||||
|
)
|
||||||
|
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||||
|
|
||||||
|
|
||||||
|
# ── post-download chain (video twin of music's scan_library → start_database_update) ──
|
||||||
|
# Split into two stages so the calendar/library reflect a download promptly, mirroring
|
||||||
|
# the music side: tell the server to rescan, WAIT until its scan queue is actually idle
|
||||||
|
# (a big library can take 10-20 min — a fixed wait would read too early), then fire
|
||||||
|
# 'video_library_scan_completed'; stage 2 listens for that and reads the fresh state.
|
||||||
|
|
||||||
|
def _default_scan_status(media_type: str = "all"):
|
||||||
|
"""Production wiring: True/False if the active server is/ isn't scanning, or None
|
||||||
|
when it can't report (so the caller falls back to a fixed wait)."""
|
||||||
|
from core.video.sources import video_server_scan_in_progress
|
||||||
|
return video_server_scan_in_progress(media_type)
|
||||||
|
|
||||||
|
|
||||||
|
def _default_latest_completed(media_type: str):
|
||||||
|
"""The newest completed grab of a type — the probe target for the smart skip."""
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().latest_completed_download(media_type)
|
||||||
|
|
||||||
|
|
||||||
|
def _default_server_has_item(media_type: str, item) -> bool:
|
||||||
|
"""Does the active server already have this specific grab indexed?"""
|
||||||
|
from core.video.sources import video_server_has_item
|
||||||
|
return video_server_has_item(media_type, item)
|
||||||
|
|
||||||
|
|
||||||
|
def wait_for_server_scan(scan_status, sleep, *, grace_seconds: int = 15,
|
||||||
|
interval_seconds: int = 10, cap_seconds: int = 3600,
|
||||||
|
fallback_seconds: int = 120) -> int:
|
||||||
|
"""Block until the media server's scan queue goes idle, then return ~seconds waited.
|
||||||
|
|
||||||
|
``scan_status()`` returns True (scanning), False (idle) or None (can't tell). After
|
||||||
|
a short grace — so the just-triggered scan has time to register as running — poll
|
||||||
|
every ``interval_seconds`` until idle or ``cap_seconds`` (a generous backstop for a
|
||||||
|
huge library). If the server can't report its state, fall back to a fixed
|
||||||
|
``fallback_seconds`` wait — exactly the old behaviour. ``sleep`` is injected and
|
||||||
|
elapsed is counted from the sleeps, so tests never actually block."""
|
||||||
|
grace = max(0, grace_seconds)
|
||||||
|
if grace:
|
||||||
|
sleep(grace)
|
||||||
|
waited = grace
|
||||||
|
try:
|
||||||
|
status = scan_status()
|
||||||
|
except Exception: # noqa: BLE001 - status is best-effort
|
||||||
|
status = None
|
||||||
|
if status is None:
|
||||||
|
extra = max(0, fallback_seconds - waited)
|
||||||
|
if extra:
|
||||||
|
sleep(extra)
|
||||||
|
return waited + extra
|
||||||
|
while status is True and waited < cap_seconds:
|
||||||
|
sleep(interval_seconds)
|
||||||
|
waited += interval_seconds
|
||||||
|
try:
|
||||||
|
status = scan_status()
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
status = None
|
||||||
|
if status is None: # lost the ability to tell — stop waiting, don't hang
|
||||||
|
break
|
||||||
|
return waited
|
||||||
|
|
||||||
|
|
||||||
|
def probe_present_libraries(candidates, has_item, sleep, *, grace_seconds: int = 120,
|
||||||
|
interval_seconds: int = 15):
|
||||||
|
"""Which libraries does the server ALREADY have the newest grab for?
|
||||||
|
|
||||||
|
``candidates`` = [(scope, item), …]. A freshly-dropped file takes ~1-2 min to show
|
||||||
|
up even with the server's auto-scan ON, so we POLL: check each candidate, and give
|
||||||
|
the server up to ``grace_seconds`` (re-checking every ``interval_seconds``) to
|
||||||
|
ingest before giving up on it. Returns the set of scopes confirmed present (→ skip
|
||||||
|
their crawl); anything still missing when grace expires stays out (→ gets crawled).
|
||||||
|
A probe that errors is treated as 'not present' (conservative). ``sleep`` injected."""
|
||||||
|
pending = list(candidates or [])
|
||||||
|
present = set()
|
||||||
|
waited = 0
|
||||||
|
while True:
|
||||||
|
still = []
|
||||||
|
for scope, item in pending:
|
||||||
|
try:
|
||||||
|
if has_item(scope, item):
|
||||||
|
present.add(scope)
|
||||||
|
continue
|
||||||
|
except Exception: # noqa: BLE001, S110 - uncertainty → keep probing, then scan
|
||||||
|
pass
|
||||||
|
still.append((scope, item))
|
||||||
|
pending = still
|
||||||
|
if not pending or waited >= grace_seconds:
|
||||||
|
return present
|
||||||
|
sleep(interval_seconds)
|
||||||
|
waited += interval_seconds
|
||||||
|
|
||||||
|
|
||||||
|
_LIB = {'movie': 'Movie', 'show': 'TV'}
|
||||||
|
|
||||||
|
|
||||||
|
def auto_video_scan_server(
|
||||||
|
config: Dict[str, Any],
|
||||||
|
deps: AutomationDeps,
|
||||||
|
*,
|
||||||
|
server_refresh: Optional[Callable[..., Dict[str, Any]]] = None,
|
||||||
|
sleep: Optional[Callable[[float], None]] = None,
|
||||||
|
emit: Optional[Callable[[str, Dict[str, Any]], None]] = None,
|
||||||
|
scan_status: Optional[Callable[..., Any]] = None,
|
||||||
|
latest_completed: Optional[Callable[[str], Any]] = None,
|
||||||
|
server_has_item: Optional[Callable[[str, Any], bool]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Stage 1: get the server to index our new downloads, then fire
|
||||||
|
'video_library_scan_completed' so the DB-update twin reads the fresh state.
|
||||||
|
|
||||||
|
SMART SKIP (``skip_if_present``, default on): scanning is expensive, and many
|
||||||
|
servers auto-ingest new files. So per library, we first probe whether the server
|
||||||
|
already has the NEWEST grab of that type (from download history) — if it does, it
|
||||||
|
already picked everything up, and we skip that library's crawl. Only libraries the
|
||||||
|
server is missing get the (expensive) rescan + poll-until-idle. We always emit so
|
||||||
|
stage 2 still READS the new items into video.db. (Video twin of music's scan_library.)"""
|
||||||
|
server_refresh = server_refresh or _default_server_refresh
|
||||||
|
sleep = sleep or time.sleep
|
||||||
|
emit = emit or deps.engine.emit
|
||||||
|
scan_status = scan_status or _default_scan_status
|
||||||
|
latest_completed = latest_completed or _default_latest_completed
|
||||||
|
server_has_item = server_has_item or _default_server_has_item
|
||||||
|
automation_id = config.get('_automation_id')
|
||||||
|
media_type = config.get('media_type') or 'all'
|
||||||
|
skip_if_present = config.get('skip_if_present', True)
|
||||||
|
try:
|
||||||
|
fallback = int(config.get('debounce_seconds') or 120)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
fallback = 120
|
||||||
|
try:
|
||||||
|
cap = int(config.get('max_wait_minutes') or 60) * 60
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
cap = 3600
|
||||||
|
try:
|
||||||
|
_pg = config.get('probe_grace_minutes') # 0 is valid (probe once, no wait)
|
||||||
|
grace = max(0, int(float(_pg if _pg is not None else 2) * 60))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
grace = 120
|
||||||
|
try:
|
||||||
|
# Which libraries actually need the expensive crawl? Skip any the server already
|
||||||
|
# has the newest grab for (it auto-ingested → nothing to find). Fresh drops take
|
||||||
|
# ~1-2 min to appear even with auto-scan ON, so POLL the probe over a grace
|
||||||
|
# window instead of checking once immediately (which would always miss).
|
||||||
|
scopes = ['movie', 'show'] if media_type == 'all' else \
|
||||||
|
([media_type] if media_type in ('movie', 'show') else ['movie', 'show'])
|
||||||
|
present = set()
|
||||||
|
if skip_if_present:
|
||||||
|
candidates = []
|
||||||
|
for sc in scopes:
|
||||||
|
try:
|
||||||
|
latest = latest_completed(sc)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
latest = None
|
||||||
|
if latest:
|
||||||
|
candidates.append((sc, latest))
|
||||||
|
if candidates:
|
||||||
|
deps.update_progress(automation_id, progress=15,
|
||||||
|
phase='Checking whether the server already has the new downloads…',
|
||||||
|
log_line='Probing the server (giving its auto-scan time to ingest)…',
|
||||||
|
log_type='info')
|
||||||
|
present = probe_present_libraries(candidates, server_has_item, sleep,
|
||||||
|
grace_seconds=grace)
|
||||||
|
for sc in present:
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id, log_line=f'{_LIB[sc]} library: server already has the newest grab — skipping its scan',
|
||||||
|
log_type='info')
|
||||||
|
to_scan = [sc for sc in scopes if sc not in present]
|
||||||
|
|
||||||
|
waited, server_name = 0, ''
|
||||||
|
if to_scan:
|
||||||
|
scan_scope = 'all' if set(to_scan) == {'movie', 'show'} else to_scan[0]
|
||||||
|
lib_label = _LIB.get(scan_scope, 'video')
|
||||||
|
deps.update_progress(automation_id, phase=f'Asking media server to rescan {lib_label}…', progress=20,
|
||||||
|
log_line=f'Triggering server-side {lib_label} scan', log_type='info')
|
||||||
|
refresh = server_refresh(scan_scope) or {}
|
||||||
|
if not refresh.get('ok'):
|
||||||
|
deps.update_progress(automation_id,
|
||||||
|
log_line='Server scan trigger unavailable: ' + str(refresh.get('error') or 'unknown'),
|
||||||
|
log_type='warning')
|
||||||
|
server_name = refresh.get('server') or ''
|
||||||
|
deps.update_progress(automation_id, phase='Waiting for the server to finish indexing…', progress=55)
|
||||||
|
waited = wait_for_server_scan(lambda: scan_status(scan_scope), sleep,
|
||||||
|
fallback_seconds=fallback, cap_seconds=cap)
|
||||||
|
else:
|
||||||
|
deps.update_progress(automation_id, progress=55,
|
||||||
|
log_line='Server already has all the new downloads — no scan needed',
|
||||||
|
log_type='success')
|
||||||
|
|
||||||
|
# Always emit with the ORIGINAL scope so stage 2 reads everything we grabbed.
|
||||||
|
emit('video_library_scan_completed', {'server': server_name, 'media_type': media_type})
|
||||||
|
done = (f'Server scan finished (~{waited}s) — updating the database' if to_scan
|
||||||
|
else 'Skipped server scan — updating the database')
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line=done, log_type='success')
|
||||||
|
return {'status': 'completed', '_manages_own_progress': True,
|
||||||
|
'scanned': to_scan, 'skipped': [s for s in scopes if s not in to_scan]}
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||||
|
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||||
|
|
||||||
|
|
||||||
|
def auto_video_update_database(
|
||||||
|
config: Dict[str, Any],
|
||||||
|
deps: AutomationDeps,
|
||||||
|
*,
|
||||||
|
run_video_scan: Optional[Callable[..., Dict[str, Any]]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Stage 2: read the (now-rescanned) server into video.db — INCREMENTAL by default
|
||||||
|
(newest-first, stop after N consecutive known). Video twin of start_database_update.
|
||||||
|
|
||||||
|
``media_type`` scopes it to the Movie or TV library ('all' does both). When fired by
|
||||||
|
the post-download chain it inherits the scope of the scan that ran (carried on the
|
||||||
|
event), so a TV-only rescan updates only TV."""
|
||||||
|
run_video_scan = run_video_scan or _default_run_video_scan
|
||||||
|
automation_id = config.get('_automation_id')
|
||||||
|
mode = config.get('mode') or 'incremental'
|
||||||
|
media_type = (config.get('media_type')
|
||||||
|
or (config.get('_event_data') or {}).get('media_type')
|
||||||
|
or 'all')
|
||||||
|
lib_label = {'movie': 'Movie', 'show': 'TV'}.get(media_type, 'video')
|
||||||
|
# 'deep'/'full' re-read the whole library; 'incremental' only grabs new items.
|
||||||
|
phase = (f'Re-reading the {lib_label} library from the server…' if mode in ('deep', 'full')
|
||||||
|
else f'Reading new {lib_label} media into SoulSync…')
|
||||||
|
try:
|
||||||
|
deps.update_progress(automation_id, phase=phase, progress=40)
|
||||||
|
result = run_video_scan(mode, media_type) or {}
|
||||||
|
if result.get('state') == 'in_progress':
|
||||||
|
deps.update_progress(automation_id, status='finished', phase='Skipped',
|
||||||
|
log_line='Another video scan is already running — skipping this run',
|
||||||
|
log_type='info')
|
||||||
|
return {'status': 'skipped', 'reason': 'a video scan is already running',
|
||||||
|
'_manages_own_progress': True}
|
||||||
|
if result.get('state') == 'error':
|
||||||
|
err = result.get('error') or 'Video database update failed'
|
||||||
|
deps.update_progress(automation_id, status='error', phase='Error', log_line=err, log_type='error')
|
||||||
|
return {'status': 'error', 'error': err, '_manages_own_progress': True}
|
||||||
|
movies = int(result.get('movies', 0) or 0)
|
||||||
|
shows = int(result.get('shows', 0) or 0)
|
||||||
|
episodes = int(result.get('episodes', 0) or 0)
|
||||||
|
if media_type == 'movie':
|
||||||
|
summary = f'Video database updated: {movies} movies'
|
||||||
|
elif media_type == 'show':
|
||||||
|
summary = f'Video database updated: {shows} shows, {episodes} episodes'
|
||||||
|
else:
|
||||||
|
summary = f'Video database updated: {movies} movies, {shows} shows, {episodes} episodes'
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line=summary, log_type='success')
|
||||||
|
return {'status': 'completed', '_manages_own_progress': True,
|
||||||
|
'movies': movies, 'shows': shows, 'episodes': episodes}
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||||
|
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||||
235
core/automation/handlers/video_scan_watchlist_channels.py
Normal file
235
core/automation/handlers/video_scan_watchlist_channels.py
Normal file
|
|
@ -0,0 +1,235 @@
|
||||||
|
"""Automation handler: ``video_scan_watchlist_channels`` action.
|
||||||
|
|
||||||
|
The "what's new from the channels I follow" scan — the piece that lets SoulSync replace
|
||||||
|
ytdl-sub-style YouTube auto-downloaders. For every YouTube channel on the video watchlist,
|
||||||
|
look at its recent uploads and wishlist the new long-form videos the user doesn't already
|
||||||
|
have, so the (future) fulfillment engine can grab them.
|
||||||
|
|
||||||
|
This runs on a SHORT schedule (channels publish at all hours — pair it with a 6-hourly
|
||||||
|
Schedule trigger; 3h is fine too, the scan is cheap). It's forward-looking and dup-proof:
|
||||||
|
|
||||||
|
* **Baseline = follow time.** What the user had before following isn't our concern — only
|
||||||
|
uploads published on/after they followed the channel (the watchlist row's ``date_added``)
|
||||||
|
are "new" and get wishlisted, forever forward.
|
||||||
|
* **Last-N safety net.** A per-channel default (10) reaches a little BEFORE the baseline so
|
||||||
|
the user is always kept current on the most recent videos even right after following / if
|
||||||
|
a scan was missed. Global setting now; per-channel override (the hover settings modal,
|
||||||
|
like watchlist-artist settings) comes later.
|
||||||
|
* **Long-form only.** Shorts are excluded (the channel's Videos tab + a duration floor);
|
||||||
|
livestreams/premieres that haven't aired are skipped (future-dated).
|
||||||
|
* **Never duplicates.** Each candidate is diffed against what's already wishlisted, what's
|
||||||
|
already been downloaded (the permanent download-history ledger — empty for YouTube until
|
||||||
|
the fulfillment engine lands, wired here so it's correct the day it does), and what the
|
||||||
|
user has dismissed.
|
||||||
|
|
||||||
|
Like the other video handlers it lives on the SHARED automation side (may import
|
||||||
|
``core.video`` / ``api.video`` — the isolation contract only forbids the reverse) and owns
|
||||||
|
its own progress (``_manages_own_progress``). All I/O is injected as seams, so the selection
|
||||||
|
logic is a pure, unit-testable function; production lazily binds the real calls.
|
||||||
|
|
||||||
|
NOTE (follow-up): "remove from the YouTube wishlist" currently just deletes the row, so the
|
||||||
|
last-N net could re-add a video the user deliberately removed. The scan already respects a
|
||||||
|
``dismissed_ids`` seam — wiring the wishlist-remove to record a dismissal (once the wishlist
|
||||||
|
is a real download queue) closes that loop. Tracked, out of scope for this scan-only phase.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import date
|
||||||
|
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||||
|
|
||||||
|
from core.automation.deps import AutomationDeps
|
||||||
|
|
||||||
|
|
||||||
|
# ── pure selection ────────────────────────────────────────────────────────────
|
||||||
|
def is_short(video: Dict[str, Any], min_seconds: int) -> bool:
|
||||||
|
"""A YouTube Short — a known duration under the floor. Unknown duration is NOT
|
||||||
|
assumed short (the Videos-tab listing already excludes Shorts; this is a backstop)."""
|
||||||
|
d = video.get("duration_seconds")
|
||||||
|
return isinstance(d, (int, float)) and 0 < d < min_seconds
|
||||||
|
|
||||||
|
|
||||||
|
def long_form_uploads(uploads: Iterable, min_seconds: int) -> List[Dict[str, Any]]:
|
||||||
|
"""The channel's real videos, newest-first order preserved: drop Shorts + entries
|
||||||
|
with no id."""
|
||||||
|
return [v for v in (uploads or [])
|
||||||
|
if isinstance(v, dict) and v.get("youtube_id") and not is_short(v, min_seconds)]
|
||||||
|
|
||||||
|
|
||||||
|
def _day(value: Any) -> Optional[str]:
|
||||||
|
s = str(value or "").strip()
|
||||||
|
return s[:10] or None
|
||||||
|
|
||||||
|
|
||||||
|
def select_channel_video_gaps(
|
||||||
|
uploads: List[Dict[str, Any]],
|
||||||
|
*,
|
||||||
|
baseline_date: Optional[str],
|
||||||
|
backfill_count: int,
|
||||||
|
wishlisted_ids: Iterable = (),
|
||||||
|
dismissed_ids: Iterable = (),
|
||||||
|
downloaded_ids: Iterable = (),
|
||||||
|
today: str,
|
||||||
|
min_seconds: int = 60,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""The pure core: which of a channel's recent uploads to wishlist now.
|
||||||
|
|
||||||
|
``uploads`` is the channel's recent videos newest-first (rich shape: youtube_id, title,
|
||||||
|
published_at, duration_seconds, thumbnail_url, …). Keeps a long-form upload if it's not
|
||||||
|
already wishlisted / downloaded / dismissed AND either it's within the newest
|
||||||
|
``backfill_count`` (the safety net) or it was published on/after ``baseline_date`` (the
|
||||||
|
forward-looking part). Future-dated (unaired premiere/stream) is skipped. No I/O."""
|
||||||
|
longs = long_form_uploads(uploads, min_seconds)
|
||||||
|
n = max(0, int(backfill_count or 0))
|
||||||
|
net_ids = {v["youtube_id"] for v in longs[:n]}
|
||||||
|
excluded = set(wishlisted_ids or ()) | set(dismissed_ids or ()) | set(downloaded_ids or ())
|
||||||
|
|
||||||
|
out: List[Dict[str, Any]] = []
|
||||||
|
for v in longs:
|
||||||
|
vid = v["youtube_id"]
|
||||||
|
if vid in excluded:
|
||||||
|
continue
|
||||||
|
d = _day(v.get("published_at"))
|
||||||
|
if d and today and d > today:
|
||||||
|
continue # scheduled / unaired — not out yet
|
||||||
|
if vid in net_ids or (d and baseline_date and d >= baseline_date):
|
||||||
|
out.append(v)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# ── production seams ──────────────────────────────────────────────────────────
|
||||||
|
def _default_fetch_channels() -> List[Dict[str, Any]]:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().list_watchlist_channels()
|
||||||
|
|
||||||
|
|
||||||
|
def _default_fetch_uploads(channel_id: Any, limit: int) -> List[Dict[str, Any]]:
|
||||||
|
"""A channel's recent uploads (Videos tab → long-form, with durations) with upload
|
||||||
|
dates merged on from the cache + a cheap RSS pull — the same composition the channel
|
||||||
|
detail endpoint uses. Caches any dates it learns so later scans get them free."""
|
||||||
|
from api.video import get_video_db
|
||||||
|
from core.video import youtube as yt
|
||||||
|
db = get_video_db()
|
||||||
|
cid = str(channel_id)
|
||||||
|
channel = yt.resolve_channel("https://www.youtube.com/channel/" + cid,
|
||||||
|
limit=max(1, min(90, int(limit)))) or {}
|
||||||
|
vids = channel.get("videos") or []
|
||||||
|
ids = [v.get("youtube_id") for v in vids if v.get("youtube_id")]
|
||||||
|
dates = db.get_video_dates(ids) or {}
|
||||||
|
try:
|
||||||
|
dates.update(yt.channel_recent_dates(cid) or {})
|
||||||
|
except Exception: # noqa: BLE001, S110 - RSS is best-effort; cached dates still apply
|
||||||
|
pass
|
||||||
|
for v in vids:
|
||||||
|
if not v.get("published_at") and dates.get(v.get("youtube_id")):
|
||||||
|
v["published_at"] = dates[v["youtube_id"]]
|
||||||
|
try:
|
||||||
|
db.cache_video_dates([{"youtube_id": v["youtube_id"], "published_at": v.get("published_at")}
|
||||||
|
for v in vids if v.get("published_at")])
|
||||||
|
except Exception: # noqa: BLE001, S110 - caching is opportunistic
|
||||||
|
pass
|
||||||
|
return vids
|
||||||
|
|
||||||
|
|
||||||
|
def _default_wishlisted_ids(channel_id: Any) -> List[Any]:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().wishlisted_video_ids_for_channel(channel_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _default_dismissed_ids(channel_id: Any) -> List[Any]:
|
||||||
|
# No YouTube "dismissed" store yet (video_ignored is movie/show-level). Returns empty;
|
||||||
|
# wiring wishlist-remove → dismiss is the tracked follow-up (see module docstring).
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _default_downloaded_ids(channel_id: Any) -> List[Any]:
|
||||||
|
# Already-downloaded YouTube videos (global; a video id is unique). A completed download
|
||||||
|
# leaves the wishlist, so without this the scan would re-add + re-download it.
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().downloaded_youtube_video_ids()
|
||||||
|
|
||||||
|
|
||||||
|
def _default_add_videos(channel: Dict[str, Any], videos: List[Dict[str, Any]]) -> int:
|
||||||
|
from api.video import get_video_db
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
return get_video_db().add_videos_to_wishlist(channel, videos, server_source=resolve_video_server())
|
||||||
|
|
||||||
|
|
||||||
|
def auto_video_scan_watchlist_channels(
|
||||||
|
config: Dict[str, Any],
|
||||||
|
deps: AutomationDeps,
|
||||||
|
*,
|
||||||
|
fetch_channels: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||||
|
fetch_uploads: Optional[Callable[[Any, int], List[Dict[str, Any]]]] = None,
|
||||||
|
wishlisted_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||||
|
dismissed_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||||
|
downloaded_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||||
|
add_videos: Optional[Callable[[Dict[str, Any], List[Dict[str, Any]]], int]] = None,
|
||||||
|
today_fn: Optional[Callable[[], str]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Scan every followed YouTube channel and wishlist its new long-form uploads.
|
||||||
|
|
||||||
|
Returns ``{'status': 'completed', 'channels': int, 'videos_added': int, ...}``."""
|
||||||
|
fetch_channels = fetch_channels or _default_fetch_channels
|
||||||
|
fetch_uploads = fetch_uploads or _default_fetch_uploads
|
||||||
|
wishlisted_ids = wishlisted_ids or _default_wishlisted_ids
|
||||||
|
dismissed_ids = dismissed_ids or _default_dismissed_ids
|
||||||
|
downloaded_ids = downloaded_ids or _default_downloaded_ids
|
||||||
|
add_videos = add_videos or _default_add_videos
|
||||||
|
today_fn = today_fn or (lambda: date.today().isoformat())
|
||||||
|
automation_id = config.get('_automation_id')
|
||||||
|
backfill = max(0, int(config.get('backfill_count', 10) or 0))
|
||||||
|
min_seconds = max(0, int(config.get('min_seconds', 60) or 0))
|
||||||
|
limit = max(backfill, 30)
|
||||||
|
|
||||||
|
try:
|
||||||
|
today = today_fn()
|
||||||
|
deps.update_progress(automation_id, phase='Reading your watchlist…', progress=5,
|
||||||
|
log_line='Loading the channels you follow', log_type='info')
|
||||||
|
channels = fetch_channels() or []
|
||||||
|
if not channels:
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line='No channels on the watchlist to scan', log_type='info')
|
||||||
|
return {'status': 'completed', 'channels': 0, 'videos_added': 0,
|
||||||
|
'_manages_own_progress': True}
|
||||||
|
|
||||||
|
added = 0
|
||||||
|
total = len(channels)
|
||||||
|
for i, ch in enumerate(channels):
|
||||||
|
cid = ch.get('youtube_id')
|
||||||
|
ctitle = ch.get('title') or cid
|
||||||
|
deps.update_progress(automation_id, phase='Scanning channels…',
|
||||||
|
progress=10 + int(80 * i / max(total, 1)),
|
||||||
|
log_line="Checking %s for new videos" % ctitle, log_type='info')
|
||||||
|
if not cid:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
uploads = fetch_uploads(cid, limit) or []
|
||||||
|
except Exception: # noqa: BLE001 - one flaky channel shouldn't abort the scan
|
||||||
|
deps.update_progress(automation_id, log_line="Couldn't reach %s — skipping" % ctitle,
|
||||||
|
log_type='warning')
|
||||||
|
continue
|
||||||
|
|
||||||
|
baseline = _day(ch.get('date_added')) or today
|
||||||
|
gaps = select_channel_video_gaps(
|
||||||
|
uploads, baseline_date=baseline, backfill_count=backfill,
|
||||||
|
wishlisted_ids=wishlisted_ids(cid), dismissed_ids=dismissed_ids(cid),
|
||||||
|
downloaded_ids=downloaded_ids(cid), today=today, min_seconds=min_seconds)
|
||||||
|
if gaps:
|
||||||
|
n = int(add_videos({'youtube_id': cid, 'title': ctitle,
|
||||||
|
'avatar_url': ch.get('poster_url')}, gaps) or 0)
|
||||||
|
added += n
|
||||||
|
if n:
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id, log_type='success',
|
||||||
|
log_line="Wishlisted %d new video(s) from %s" % (n, ctitle))
|
||||||
|
|
||||||
|
done = ('Wishlisted %d new video(s) across %d channel(s)' % (added, total)) if added \
|
||||||
|
else ('Channels are up to date — nothing new across %d channel(s)' % total)
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line=done, log_type='success')
|
||||||
|
return {'status': 'completed', 'channels': total, 'videos_added': added,
|
||||||
|
'_manages_own_progress': True}
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||||
|
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||||
296
core/automation/handlers/video_scan_watchlist_people.py
Normal file
296
core/automation/handlers/video_scan_watchlist_people.py
Normal file
|
|
@ -0,0 +1,296 @@
|
||||||
|
"""Automation handler: ``video_scan_watchlist_people`` action.
|
||||||
|
|
||||||
|
The video "watchlist" follows ongoing *things* — shows, channels, and people. For shows
|
||||||
|
the ``video_add_airing_episodes`` automation already keeps the wishlist fed. This handler
|
||||||
|
does the equivalent for the PEOPLE you follow:
|
||||||
|
|
||||||
|
For every person on the watchlist, look up their filmography and wishlist every MOVIE the
|
||||||
|
user doesn't already own — the whole back catalog they acted in or directed, plus anything
|
||||||
|
upcoming. The first run is the meaty one (it backlogs everything); later runs are fast
|
||||||
|
because they skip movies already on the wishlist and only promote ones that have since been
|
||||||
|
released.
|
||||||
|
|
||||||
|
Design decisions (Boulder):
|
||||||
|
* MOVIES ONLY — no TV episodes for a person (shows are handled by their own automation).
|
||||||
|
* Both ACTOR (Acting credits, minus "playing themselves") and DIRECTOR credits count.
|
||||||
|
* UNRELEASED movies are added as ``status='monitored'`` so the wishlist/download engine
|
||||||
|
leaves them alone until they're out; a later scan PROMOTES them to ``'wanted'``.
|
||||||
|
* Best-in-class UX: grab as much data as possible at add time (backdrop, overview, genres,
|
||||||
|
runtime, rating, top cast, director, release date, + provenance "because you follow X")
|
||||||
|
and stash it as a rich ``detail_json`` blob on the wishlist row, so the UI renders a full
|
||||||
|
card later without re-fetching.
|
||||||
|
|
||||||
|
Like the other video handlers this lives on the SHARED automation side (so it may import
|
||||||
|
``core.video`` / ``api.video`` — the isolation contract only forbids the reverse) and owns
|
||||||
|
its own progress (``_manages_own_progress``). All I/O is injected as seams, so the logic is
|
||||||
|
a pure function in tests (no DB, no TMDB); production lazily binds the real calls.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import date
|
||||||
|
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||||
|
|
||||||
|
from core.automation.deps import AutomationDeps
|
||||||
|
from core.video.discovery_gaps import filmography_gaps
|
||||||
|
|
||||||
|
|
||||||
|
# ── pure credit classification ────────────────────────────────────────────────
|
||||||
|
# An actor "playing themselves" in a documentary / talk show / award broadcast isn't a
|
||||||
|
# dramatic role — drop those so the wishlist stays films they actually acted in.
|
||||||
|
_SELF_MARKERS = ('self', 'himself', 'herself', 'themselves', 'archive footage',
|
||||||
|
'archival footage')
|
||||||
|
|
||||||
|
|
||||||
|
def is_self_credit(role) -> bool:
|
||||||
|
"""True for a 'plays themselves' / archive-footage credit (role text only)."""
|
||||||
|
r = str(role or '').strip().lower()
|
||||||
|
return bool(r) and any(m in r for m in _SELF_MARKERS)
|
||||||
|
|
||||||
|
|
||||||
|
def is_actor_movie_credit(c: Dict[str, Any]) -> bool:
|
||||||
|
return (c.get('kind') == 'movie'
|
||||||
|
and str(c.get('department') or '') == 'Acting'
|
||||||
|
and not is_self_credit(c.get('role')))
|
||||||
|
|
||||||
|
|
||||||
|
def is_director_movie_credit(c: Dict[str, Any]) -> bool:
|
||||||
|
return c.get('kind') == 'movie' and str(c.get('role') or '').strip().lower() == 'director'
|
||||||
|
|
||||||
|
|
||||||
|
def is_relevant_movie_credit(c: Any) -> bool:
|
||||||
|
"""A movie this person ACTED in (not as themselves) or DIRECTED."""
|
||||||
|
return isinstance(c, dict) and (is_actor_movie_credit(c) or is_director_movie_credit(c))
|
||||||
|
|
||||||
|
|
||||||
|
def is_released(date_str: Any, today: str) -> bool:
|
||||||
|
"""True if the release date is on/before ``today``. No date → NOT yet released, so we
|
||||||
|
monitor it (a later scan promotes once a real, past date arrives)."""
|
||||||
|
d = str(date_str or '').strip()
|
||||||
|
return bool(d) and d[:10] <= today
|
||||||
|
|
||||||
|
|
||||||
|
def _int_set(values: Iterable) -> set:
|
||||||
|
out = set()
|
||||||
|
for x in values or []:
|
||||||
|
try:
|
||||||
|
out.add(int(x))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def select_person_movie_gaps(credits: List[Dict[str, Any]], owned_ids: Iterable,
|
||||||
|
ignored_ids: Iterable, *, today: str) -> List[Dict[str, Any]]:
|
||||||
|
"""The pure core: a followed person's un-owned actor/director MOVIE credits.
|
||||||
|
|
||||||
|
Keeps only relevant movie credits, drops owned + ignored + duplicates, ranks by
|
||||||
|
popularity (via ``filmography_gaps``), and tags each with the wishlist ``_status`` it
|
||||||
|
should get — ``'wanted'`` if released, else ``'monitored'``. No I/O."""
|
||||||
|
relevant = [c for c in (credits or []) if is_relevant_movie_credit(c)]
|
||||||
|
ignored = _int_set(ignored_ids)
|
||||||
|
out: List[Dict[str, Any]] = []
|
||||||
|
for g in filmography_gaps(owned_ids, relevant, kinds=("movie",)):
|
||||||
|
if int(g['tmdb_id']) in ignored:
|
||||||
|
continue
|
||||||
|
g = dict(g)
|
||||||
|
g['_status'] = 'wanted' if is_released(g.get('date'), today) else 'monitored'
|
||||||
|
out.append(g)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def build_detail_blob(detail: Optional[Dict[str, Any]], credit: Dict[str, Any],
|
||||||
|
person: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
"""Trim TMDB full-detail down to a rich-but-lean card blob + provenance.
|
||||||
|
|
||||||
|
Drops the heavy ``_extras`` (similar/recommendations/keywords/reviews/providers) and
|
||||||
|
keeps what a wishlist card / detail view wants. Degrades to the credit's own fields when
|
||||||
|
``detail`` is missing or a library redirect."""
|
||||||
|
via = {
|
||||||
|
'person_tmdb_id': person.get('tmdb_id'),
|
||||||
|
'person_name': person.get('title') or person.get('name'),
|
||||||
|
'role': credit.get('role'),
|
||||||
|
'as': 'director' if is_director_movie_credit(credit) else 'actor',
|
||||||
|
}
|
||||||
|
if not detail or detail.get('redirect'):
|
||||||
|
return {
|
||||||
|
'title': credit.get('title'), 'year': credit.get('year'),
|
||||||
|
'release_date': credit.get('date'), 'poster_url': credit.get('poster'),
|
||||||
|
'added_via': via,
|
||||||
|
}
|
||||||
|
director = next((p.get('name') for p in (detail.get('crew') or [])
|
||||||
|
if str(p.get('job')) == 'Director'), None)
|
||||||
|
return {
|
||||||
|
'title': detail.get('title'),
|
||||||
|
'overview': detail.get('overview'),
|
||||||
|
'tagline': detail.get('tagline'),
|
||||||
|
'status': detail.get('status'),
|
||||||
|
'rating': detail.get('rating'),
|
||||||
|
'imdb_id': detail.get('imdb_id'),
|
||||||
|
'poster_url': detail.get('poster_url') or credit.get('poster'),
|
||||||
|
'backdrop_url': detail.get('backdrop_url'),
|
||||||
|
'logo': detail.get('logo'),
|
||||||
|
'genres': detail.get('genres') or [],
|
||||||
|
'runtime_minutes': detail.get('runtime_minutes'),
|
||||||
|
'studio': detail.get('studio'),
|
||||||
|
'year': detail.get('year') or credit.get('year'),
|
||||||
|
'release_date': detail.get('release_date') or credit.get('date'),
|
||||||
|
'cast': (detail.get('cast') or [])[:15],
|
||||||
|
'director': director,
|
||||||
|
'added_via': via,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ── production seams ──────────────────────────────────────────────────────────
|
||||||
|
def _default_fetch_people() -> List[Dict[str, Any]]:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().list_watchlist('person')
|
||||||
|
|
||||||
|
|
||||||
|
def _default_fetch_credits(tmdb_id: Any) -> List[Dict[str, Any]]:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
d = get_video_enrichment_engine().person_detail(tmdb_id) or {}
|
||||||
|
return d.get('credits') or []
|
||||||
|
|
||||||
|
|
||||||
|
def _default_fetch_detail(tmdb_id: Any) -> Optional[Dict[str, Any]]:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
return get_video_enrichment_engine().tmdb_detail('movie', tmdb_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _default_owned_ids() -> Iterable:
|
||||||
|
from api.video import get_video_db
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
return get_video_db().owned_movie_tmdb_ids(resolve_video_server())
|
||||||
|
|
||||||
|
|
||||||
|
def _default_ignored_ids() -> List[Any]:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return [r.get('tmdb_id') for r in (get_video_db().list_ignored() or [])
|
||||||
|
if r.get('kind') == 'movie']
|
||||||
|
|
||||||
|
|
||||||
|
def _default_wishlisted_status() -> Dict[int, str]:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().wishlisted_movie_status()
|
||||||
|
|
||||||
|
|
||||||
|
def _default_add_movie(tmdb_id, title, *, year, poster_url, status, detail_json) -> bool:
|
||||||
|
from api.video import get_video_db
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
return get_video_db().add_movie_to_wishlist(
|
||||||
|
tmdb_id, title, year=year, poster_url=poster_url, status=status,
|
||||||
|
detail_json=detail_json, server_source=resolve_video_server())
|
||||||
|
|
||||||
|
|
||||||
|
def auto_video_scan_watchlist_people(
|
||||||
|
config: Dict[str, Any],
|
||||||
|
deps: AutomationDeps,
|
||||||
|
*,
|
||||||
|
fetch_people: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||||
|
fetch_credits: Optional[Callable[[Any], List[Dict[str, Any]]]] = None,
|
||||||
|
fetch_detail: Optional[Callable[[Any], Optional[Dict[str, Any]]]] = None,
|
||||||
|
owned_ids: Optional[Callable[[], Iterable]] = None,
|
||||||
|
ignored_ids: Optional[Callable[[], List[Any]]] = None,
|
||||||
|
wishlisted_status: Optional[Callable[[], Dict[int, str]]] = None,
|
||||||
|
add_movie: Optional[Callable[..., bool]] = None,
|
||||||
|
today_fn: Optional[Callable[[], str]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Scan every followed person's filmography and wishlist the movies the user is missing.
|
||||||
|
|
||||||
|
Returns ``{'status': 'completed', 'people': int, 'movies_added': int, 'upcoming': int,
|
||||||
|
'promoted': int, ...}`` — ``movies_added`` = released wishlisted, ``upcoming`` = monitored
|
||||||
|
(unreleased) wishlisted, ``promoted`` = monitored rows flipped to wanted now they're out."""
|
||||||
|
fetch_people = fetch_people or _default_fetch_people
|
||||||
|
fetch_credits = fetch_credits or _default_fetch_credits
|
||||||
|
fetch_detail = fetch_detail or _default_fetch_detail
|
||||||
|
owned_ids = owned_ids or _default_owned_ids
|
||||||
|
ignored_ids = ignored_ids or _default_ignored_ids
|
||||||
|
wishlisted_status = wishlisted_status or _default_wishlisted_status
|
||||||
|
add_movie = add_movie or _default_add_movie
|
||||||
|
today_fn = today_fn or (lambda: date.today().isoformat())
|
||||||
|
automation_id = config.get('_automation_id')
|
||||||
|
|
||||||
|
try:
|
||||||
|
today = today_fn()
|
||||||
|
deps.update_progress(automation_id, phase='Reading your watchlist…', progress=5,
|
||||||
|
log_line='Loading the people you follow', log_type='info')
|
||||||
|
people = fetch_people() or []
|
||||||
|
if not people:
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line='No people on the watchlist to scan', log_type='info')
|
||||||
|
return {'status': 'completed', 'people': 0, 'movies_added': 0, 'upcoming': 0,
|
||||||
|
'promoted': 0, '_manages_own_progress': True}
|
||||||
|
|
||||||
|
owned = owned_ids() or set()
|
||||||
|
ignored = ignored_ids() or []
|
||||||
|
# Snapshot of what's already wishlisted {tmdb_id: status}; updated as we add so a
|
||||||
|
# movie credited to two followed people is handled once.
|
||||||
|
wished: Dict[int, str] = dict(wishlisted_status() or {})
|
||||||
|
|
||||||
|
added = upcoming = promoted = 0
|
||||||
|
total = len(people)
|
||||||
|
for i, person in enumerate(people):
|
||||||
|
pid = person.get('tmdb_id')
|
||||||
|
pname = person.get('title') or person.get('name') or pid
|
||||||
|
deps.update_progress(automation_id, phase='Scanning filmographies…',
|
||||||
|
progress=10 + int(80 * i / max(total, 1)),
|
||||||
|
log_line="Looking up %s's movies" % pname, log_type='info')
|
||||||
|
if not pid:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
credits = fetch_credits(pid) or []
|
||||||
|
except Exception: # noqa: BLE001 - one bad lookup shouldn't abort the whole scan
|
||||||
|
deps.update_progress(automation_id, log_line="Couldn't fetch %s — skipping" % pname,
|
||||||
|
log_type='warning')
|
||||||
|
continue
|
||||||
|
|
||||||
|
for g in select_person_movie_gaps(credits, owned, ignored, today=today):
|
||||||
|
tid = int(g['tmdb_id'])
|
||||||
|
want = g['_status'] # 'wanted' | 'monitored'
|
||||||
|
existing = wished.get(tid)
|
||||||
|
if existing is not None:
|
||||||
|
# Already wishlisted — the only action left is promoting a monitored
|
||||||
|
# row that has since been released (don't re-fetch detail for it).
|
||||||
|
if existing == 'monitored' and want == 'wanted':
|
||||||
|
if add_movie(tid, g.get('title'), year=g.get('year'),
|
||||||
|
poster_url=g.get('poster'), status='wanted', detail_json=None):
|
||||||
|
wished[tid] = 'wanted'
|
||||||
|
promoted += 1
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id, log_type='success',
|
||||||
|
log_line="'%s' is out now — promoted to wanted"
|
||||||
|
% (g.get('title') or tid))
|
||||||
|
continue
|
||||||
|
|
||||||
|
# New gap — grab the rich detail (cached), build the blob, add it.
|
||||||
|
try:
|
||||||
|
detail = fetch_detail(tid)
|
||||||
|
except Exception: # noqa: BLE001 - degrade to the cheap credit fields
|
||||||
|
detail = None
|
||||||
|
blob = build_detail_blob(detail, g, person)
|
||||||
|
if add_movie(tid, g.get('title') or blob.get('title'), year=g.get('year'),
|
||||||
|
poster_url=blob.get('poster_url') or g.get('poster'),
|
||||||
|
status=want, detail_json=blob):
|
||||||
|
wished[tid] = want
|
||||||
|
if want == 'wanted':
|
||||||
|
added += 1
|
||||||
|
else:
|
||||||
|
upcoming += 1
|
||||||
|
|
||||||
|
parts = []
|
||||||
|
if added:
|
||||||
|
parts.append('%d new movie(s)' % added)
|
||||||
|
if upcoming:
|
||||||
|
parts.append('%d upcoming' % upcoming)
|
||||||
|
if promoted:
|
||||||
|
parts.append('%d promoted' % promoted)
|
||||||
|
done = ('Wishlisted ' + ', '.join(parts) + ' across %d followed person(s)' % total) \
|
||||||
|
if parts else ('Watchlist is up to date — nothing new across %d person(s)' % total)
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line=done, log_type='success')
|
||||||
|
return {'status': 'completed', 'people': total, 'movies_added': added,
|
||||||
|
'upcoming': upcoming, 'promoted': promoted, '_manages_own_progress': True}
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||||
|
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||||
179
core/automation/handlers/video_scan_watchlist_playlists.py
Normal file
179
core/automation/handlers/video_scan_watchlist_playlists.py
Normal file
|
|
@ -0,0 +1,179 @@
|
||||||
|
"""Automation handler: ``video_scan_watchlist_playlists`` action.
|
||||||
|
|
||||||
|
Sibling of the watchlist-CHANNELS scan, with a deliberately different rule. A channel is a
|
||||||
|
creator posting new stuff over time, so that scan is forward-looking (new uploads + a
|
||||||
|
last-N net). A **playlist** is a curated, finite set someone assembled — the reason you
|
||||||
|
follow it is "give me this whole list and keep it complete." So this scan MIRRORS the
|
||||||
|
playlist: it wishlists every long-form video in it you don't already have, plus anything
|
||||||
|
later added. No follow-date baseline, no last-N net.
|
||||||
|
|
||||||
|
Organisation: the playlist becomes its own "show" (playlist-as-show) — its videos are
|
||||||
|
wishlisted under the playlist's title, so the download worker files them as
|
||||||
|
``Playlist Name / Season YEAR / Playlist Name - DATE - Title`` (matches the ytdl-sub
|
||||||
|
``tv_show_name``-on-a-playlist convention). All other plumbing is shared with channels: the
|
||||||
|
same wishlist rows, the same "Process YouTube Wishlist" drain, quality + org template.
|
||||||
|
|
||||||
|
Edge: a video in both a followed channel AND a followed playlist is wishlisted once (dedup
|
||||||
|
by video id); whichever scan touched it last sets its show-name. Accepted.
|
||||||
|
|
||||||
|
Pure selection with all I/O injected; shared automation side; owns its own progress.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import date
|
||||||
|
from typing import Any, Callable, Dict, Iterable, List, Optional
|
||||||
|
|
||||||
|
from core.automation.deps import AutomationDeps
|
||||||
|
from core.automation.handlers.video_scan_watchlist_channels import _day, long_form_uploads
|
||||||
|
|
||||||
|
# how many playlist entries to read per scan (yt-dlp flat). Big enough for almost any real
|
||||||
|
# playlist; genuinely huge ones truncate (logged) rather than hammering YouTube each scan.
|
||||||
|
_PLAYLIST_FETCH_LIMIT = 1000
|
||||||
|
|
||||||
|
|
||||||
|
def select_playlist_video_gaps(
|
||||||
|
videos: List[Dict[str, Any]],
|
||||||
|
*,
|
||||||
|
wishlisted_ids: Iterable = (),
|
||||||
|
downloaded_ids: Iterable = (),
|
||||||
|
dismissed_ids: Iterable = (),
|
||||||
|
today: str,
|
||||||
|
min_seconds: int = 60,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""The pure core: every long-form video in a playlist not already wishlisted /
|
||||||
|
downloaded / dismissed (mirror the whole list). Future-dated (unaired) entries are
|
||||||
|
skipped. No baseline, no last-N — a curated playlist is wanted in full. No I/O."""
|
||||||
|
excluded = set(wishlisted_ids or ()) | set(downloaded_ids or ()) | set(dismissed_ids or ())
|
||||||
|
out: List[Dict[str, Any]] = []
|
||||||
|
for v in long_form_uploads(videos, min_seconds):
|
||||||
|
vid = v["youtube_id"]
|
||||||
|
if vid in excluded:
|
||||||
|
continue
|
||||||
|
d = _day(v.get("published_at"))
|
||||||
|
if d and today and d > today:
|
||||||
|
continue # unaired premiere — not out yet
|
||||||
|
out.append(v)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# ── production seams ──────────────────────────────────────────────────────────
|
||||||
|
def _default_fetch_playlists() -> List[Dict[str, Any]]:
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().list_watchlist_playlists()
|
||||||
|
|
||||||
|
|
||||||
|
def _default_fetch_videos(playlist_id: Any) -> List[Dict[str, Any]]:
|
||||||
|
"""A playlist's videos (flat, with durations) with upload dates merged from the cache.
|
||||||
|
Playlists have no per-channel RSS, so dates come from whatever's cached (filled over
|
||||||
|
time by the channel/InnerTube enrichers); undated entries organise cleanly."""
|
||||||
|
from api.video import get_video_db
|
||||||
|
from core.video import youtube as yt
|
||||||
|
db = get_video_db()
|
||||||
|
vids = yt.playlist_videos(str(playlist_id), limit=_PLAYLIST_FETCH_LIMIT) or []
|
||||||
|
ids = [v.get("youtube_id") for v in vids if v.get("youtube_id")]
|
||||||
|
dates = db.get_video_dates(ids) or {}
|
||||||
|
for v in vids:
|
||||||
|
if not v.get("published_at") and dates.get(v.get("youtube_id")):
|
||||||
|
v["published_at"] = dates[v["youtube_id"]]
|
||||||
|
return vids
|
||||||
|
|
||||||
|
|
||||||
|
def _default_wishlisted_ids(playlist_id: Any) -> List[Any]:
|
||||||
|
# parent_source_id holds the playlist id for playlist-sourced wishlist videos.
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().wishlisted_video_ids_for_channel(playlist_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _default_downloaded_ids(playlist_id: Any) -> List[Any]:
|
||||||
|
# Already-downloaded YouTube videos (global). A completed download leaves the wishlist,
|
||||||
|
# so without this the scan would re-add + re-download it.
|
||||||
|
from api.video import get_video_db
|
||||||
|
return get_video_db().downloaded_youtube_video_ids()
|
||||||
|
|
||||||
|
|
||||||
|
def _default_dismissed_ids(playlist_id: Any) -> List[Any]:
|
||||||
|
return [] # no youtube "dismissed" store yet (see channels-scan note)
|
||||||
|
|
||||||
|
|
||||||
|
def _default_add_videos(playlist: Dict[str, Any], videos: List[Dict[str, Any]]) -> int:
|
||||||
|
"""Wishlist under the PLAYLIST as the show (title = playlist name → playlist-as-show)."""
|
||||||
|
from api.video import get_video_db
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
return get_video_db().add_videos_to_wishlist(playlist, videos, server_source=resolve_video_server())
|
||||||
|
|
||||||
|
|
||||||
|
def auto_video_scan_watchlist_playlists(
|
||||||
|
config: Dict[str, Any],
|
||||||
|
deps: AutomationDeps,
|
||||||
|
*,
|
||||||
|
fetch_playlists: Optional[Callable[[], List[Dict[str, Any]]]] = None,
|
||||||
|
fetch_videos: Optional[Callable[[Any], List[Dict[str, Any]]]] = None,
|
||||||
|
wishlisted_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||||
|
downloaded_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||||
|
dismissed_ids: Optional[Callable[[Any], Iterable]] = None,
|
||||||
|
add_videos: Optional[Callable[[Dict[str, Any], List[Dict[str, Any]]], int]] = None,
|
||||||
|
today_fn: Optional[Callable[[], str]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Mirror every followed YouTube playlist into the wishlist (whole list + new additions).
|
||||||
|
|
||||||
|
Returns ``{'status': 'completed', 'playlists': int, 'videos_added': int, ...}``."""
|
||||||
|
fetch_playlists = fetch_playlists or _default_fetch_playlists
|
||||||
|
fetch_videos = fetch_videos or _default_fetch_videos
|
||||||
|
wishlisted_ids = wishlisted_ids or _default_wishlisted_ids
|
||||||
|
downloaded_ids = downloaded_ids or _default_downloaded_ids
|
||||||
|
dismissed_ids = dismissed_ids or _default_dismissed_ids
|
||||||
|
add_videos = add_videos or _default_add_videos
|
||||||
|
today_fn = today_fn or (lambda: date.today().isoformat())
|
||||||
|
automation_id = config.get('_automation_id')
|
||||||
|
min_seconds = max(0, int(config.get('min_seconds', 60) or 0))
|
||||||
|
|
||||||
|
try:
|
||||||
|
today = today_fn()
|
||||||
|
deps.update_progress(automation_id, phase='Reading your watchlist…', progress=5,
|
||||||
|
log_line='Loading the playlists you follow', log_type='info')
|
||||||
|
playlists = fetch_playlists() or []
|
||||||
|
if not playlists:
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line='No playlists on the watchlist to scan', log_type='info')
|
||||||
|
return {'status': 'completed', 'playlists': 0, 'videos_added': 0,
|
||||||
|
'_manages_own_progress': True}
|
||||||
|
|
||||||
|
added = 0
|
||||||
|
total = len(playlists)
|
||||||
|
for i, pl in enumerate(playlists):
|
||||||
|
pid = pl.get('playlist_id')
|
||||||
|
ptitle = pl.get('title') or pid
|
||||||
|
deps.update_progress(automation_id, phase='Scanning playlists…',
|
||||||
|
progress=10 + int(80 * i / max(total, 1)),
|
||||||
|
log_line="Mirroring '%s'" % ptitle, log_type='info')
|
||||||
|
if not pid:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
videos = fetch_videos(pid) or []
|
||||||
|
except Exception: # noqa: BLE001 - one flaky playlist shouldn't abort the scan
|
||||||
|
deps.update_progress(automation_id, log_line="Couldn't reach '%s' — skipping" % ptitle,
|
||||||
|
log_type='warning')
|
||||||
|
continue
|
||||||
|
|
||||||
|
gaps = select_playlist_video_gaps(
|
||||||
|
videos, wishlisted_ids=wishlisted_ids(pid), downloaded_ids=downloaded_ids(pid),
|
||||||
|
dismissed_ids=dismissed_ids(pid), today=today, min_seconds=min_seconds)
|
||||||
|
if gaps:
|
||||||
|
n = int(add_videos({'youtube_id': pid, 'title': ptitle,
|
||||||
|
'avatar_url': pl.get('poster_url')}, gaps) or 0)
|
||||||
|
added += n
|
||||||
|
if n:
|
||||||
|
deps.update_progress(
|
||||||
|
automation_id, log_type='success',
|
||||||
|
log_line="Wishlisted %d new video(s) from '%s'" % (n, ptitle))
|
||||||
|
|
||||||
|
done = ('Wishlisted %d new video(s) across %d playlist(s)' % (added, total)) if added \
|
||||||
|
else ('Playlists are up to date — nothing new across %d playlist(s)' % total)
|
||||||
|
deps.update_progress(automation_id, status='finished', progress=100, phase='Complete',
|
||||||
|
log_line=done, log_type='success')
|
||||||
|
return {'status': 'completed', 'playlists': total, 'videos_added': added,
|
||||||
|
'_manages_own_progress': True}
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
deps.update_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error')
|
||||||
|
return {'status': 'error', 'error': str(e), '_manages_own_progress': True}
|
||||||
|
|
@ -145,6 +145,186 @@ SYSTEM_AUTOMATIONS = [
|
||||||
'action_type': 'full_cleanup',
|
'action_type': 'full_cleanup',
|
||||||
'initial_delay': 900, # 15 min after startup
|
'initial_delay': 900, # 15 min after startup
|
||||||
},
|
},
|
||||||
|
# ── Video side (isolated app, shared engine) ──────────────────────────
|
||||||
|
# owned_by='video' keeps these OFF the music automations page (it filters
|
||||||
|
# them out) and ON the video Automations page (it shows only these).
|
||||||
|
# Schedule-based for now; a video-download-complete event trigger can
|
||||||
|
# replace the schedule once that event is wired into the engine.
|
||||||
|
# (No standalone scheduled scan — the post-download chain below keeps the library
|
||||||
|
# fresh. The 'video_scan_library' action/block still exist for a custom automation.)
|
||||||
|
# Post-download chain (video twin of music's batch_complete → scan_library →
|
||||||
|
# library_scan_completed → start_database_update). Event-based, so a finished
|
||||||
|
# video download refreshes the server then pulls the new media into video.db.
|
||||||
|
{
|
||||||
|
'name': 'Auto-Scan Video After Downloads',
|
||||||
|
'trigger_type': 'video_batch_complete',
|
||||||
|
'trigger_config': {},
|
||||||
|
'action_type': 'video_scan_server',
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'Auto-Update Video Database After Scan',
|
||||||
|
'trigger_type': 'video_library_scan_completed',
|
||||||
|
'trigger_config': {},
|
||||||
|
'action_type': 'video_update_database',
|
||||||
|
'action_config': {'mode': 'incremental'},
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
# Safety net: re-read the server hourly too, so MANUAL library additions (which Plex
|
||||||
|
# auto-scans) appear within the hour instead of waiting for the weekly deep scan. Same
|
||||||
|
# cheap incremental read as the after-scan one.
|
||||||
|
{
|
||||||
|
'name': 'Auto-Update Video Database (Hourly)',
|
||||||
|
'trigger_type': 'schedule',
|
||||||
|
'trigger_config': {'interval': 1, 'unit': 'hours'},
|
||||||
|
'action_type': 'video_update_database_hourly',
|
||||||
|
'action_config': {'mode': 'incremental'},
|
||||||
|
'initial_delay': 900,
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
# Sonarr-style: once a day at 1am (server-local), wishlist every episode airing
|
||||||
|
# today for the shows you follow (skipping ones already owned) so the day's episodes
|
||||||
|
# are queued overnight. A fixed wall-clock 'daily_time' (not a rolling 24h interval
|
||||||
|
# that drifts with restarts) — the seeder now arms timed system triggers, and
|
||||||
|
# _fix_airing_automation_schedule migrates the old 24h-interval row.
|
||||||
|
# Runs before the airing automation so the calendar it reads is current — re-pulls
|
||||||
|
# TMDB episode schedules for still-airing watchlist shows (the airing read is LOCAL).
|
||||||
|
{
|
||||||
|
'name': 'Refresh Airing TV Schedules',
|
||||||
|
'trigger_type': 'daily_time',
|
||||||
|
'trigger_config': {'time': '23:00'},
|
||||||
|
'action_type': 'video_refresh_airing_schedules',
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'Auto-Wishlist Episodes Airing Today',
|
||||||
|
'trigger_type': 'daily_time',
|
||||||
|
'trigger_config': {'time': '01:00'},
|
||||||
|
'action_type': 'video_add_airing_episodes',
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
# Video twins of the music maintenance jobs — same schedule + shared handler,
|
||||||
|
# distinct action_type + owned_by='video' so they seed as separate rows and
|
||||||
|
# show on the video Automations page (music's copies are untouched).
|
||||||
|
{
|
||||||
|
'name': 'Clean Search History',
|
||||||
|
'trigger_type': 'schedule',
|
||||||
|
'trigger_config': {'interval': 1, 'unit': 'hours'},
|
||||||
|
'action_type': 'video_clean_search_history',
|
||||||
|
'initial_delay': 600,
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'Clean Completed Downloads',
|
||||||
|
'trigger_type': 'schedule',
|
||||||
|
'trigger_config': {'interval': 5, 'unit': 'minutes'},
|
||||||
|
'action_type': 'video_clean_completed_downloads',
|
||||||
|
'initial_delay': 300,
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'Full Cleanup',
|
||||||
|
'trigger_type': 'schedule',
|
||||||
|
'trigger_config': {'interval': 12, 'unit': 'hours'},
|
||||||
|
'action_type': 'video_full_cleanup',
|
||||||
|
'initial_delay': 900,
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'Auto-Backup Database',
|
||||||
|
'trigger_type': 'schedule',
|
||||||
|
'trigger_config': {'interval': 3, 'unit': 'days'},
|
||||||
|
'action_type': 'video_backup_database',
|
||||||
|
'initial_delay': 600,
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
# Video twin of music's 'Auto-Deep Scan Library', split into TWO because Movies
|
||||||
|
# and TV are independent libraries — a TV scan never pulls in new movies and
|
||||||
|
# vice-versa. Fixed weekly deep scan (re-read + prune removed) at 02:00 server-
|
||||||
|
# local: TV Mondays, Movies Tuesdays — different days so they never overlap. The
|
||||||
|
# seeder arms timed system triggers; _fix_deep_scan_schedules migrates the
|
||||||
|
# original rolling-7-day rows. (Busy guard in the scanner is still a safety net.)
|
||||||
|
{
|
||||||
|
'name': 'Auto-Deep Scan TV Library',
|
||||||
|
'trigger_type': 'weekly_time',
|
||||||
|
'trigger_config': {'time': '02:00', 'days': ['mon']},
|
||||||
|
'action_type': 'video_deep_scan_tv',
|
||||||
|
'action_config': {'mode': 'deep', 'media_type': 'show'},
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'Auto-Deep Scan Movie Library',
|
||||||
|
'trigger_type': 'weekly_time',
|
||||||
|
'trigger_config': {'time': '02:00', 'days': ['tue']},
|
||||||
|
'action_type': 'video_deep_scan_movies',
|
||||||
|
'action_config': {'mode': 'deep', 'media_type': 'movie'},
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
# ── Watchlist → Wishlist pipeline ─────────────────────────────────────────
|
||||||
|
# Stage 1: SCANS that FILL the wishlist from what you follow. (The airing-episodes
|
||||||
|
# scan above is the show equivalent.) All no-op cleanly if you follow nobody.
|
||||||
|
{
|
||||||
|
'name': 'Auto-Scan Watchlist People', # followed actors/directors → wished movies
|
||||||
|
'trigger_type': 'daily_time', # daily; filmographies change slowly
|
||||||
|
'trigger_config': {'time': '03:00'}, # after airing/deep-scan jobs, no overlap
|
||||||
|
'action_type': 'video_scan_watchlist_people',
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'Auto-Scan Watchlist Channels', # followed YouTube channels → wished videos
|
||||||
|
'trigger_type': 'schedule',
|
||||||
|
'trigger_config': {'interval': 6, 'unit': 'hours'}, # YouTube posts at all hours
|
||||||
|
'action_type': 'video_scan_watchlist_channels',
|
||||||
|
'action_config': {'backfill_count': 10},
|
||||||
|
'initial_delay': 1200,
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'Auto-Scan Watchlist Playlists', # followed playlists (mirror-all) → wished videos
|
||||||
|
'trigger_type': 'schedule',
|
||||||
|
'trigger_config': {'interval': 6, 'unit': 'hours'},
|
||||||
|
'action_type': 'video_scan_watchlist_playlists',
|
||||||
|
'initial_delay': 1320,
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
# Stage 2: PROCESSORS that DRAIN the wishlist by downloading. Hourly; each skips
|
||||||
|
# quietly until its library folder (+ slskd, for movie/episode) is configured.
|
||||||
|
{
|
||||||
|
'name': 'Auto-Process Movie Wishlist', # wished movies → slskd search/pick/grab
|
||||||
|
'trigger_type': 'schedule',
|
||||||
|
'trigger_config': {'interval': 1, 'unit': 'hours'},
|
||||||
|
'action_type': 'video_process_movie_wishlist',
|
||||||
|
'action_config': {'max_concurrent': 3},
|
||||||
|
'initial_delay': 1620,
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'Auto-Process Episode Wishlist', # wished episodes → slskd search/pick/grab
|
||||||
|
'trigger_type': 'schedule',
|
||||||
|
'trigger_config': {'interval': 1, 'unit': 'hours'},
|
||||||
|
'action_type': 'video_process_episode_wishlist',
|
||||||
|
'action_config': {'max_concurrent': 3},
|
||||||
|
'initial_delay': 1740,
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'name': 'Auto-Process YouTube Wishlist', # wished YouTube videos → yt-dlp download
|
||||||
|
'trigger_type': 'schedule',
|
||||||
|
'trigger_config': {'interval': 1, 'unit': 'hours'},
|
||||||
|
'action_type': 'video_process_youtube_wishlist',
|
||||||
|
'action_config': {'max_concurrent': 3},
|
||||||
|
'initial_delay': 1500,
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
|
# YouTube retention: delete channel episodes outside each channel's keep window. No-op
|
||||||
|
# unless a channel opts in (cog modal → Keep); default keeps everything, so safe to seed.
|
||||||
|
{
|
||||||
|
'name': 'Auto-Clean Old YouTube Episodes',
|
||||||
|
'trigger_type': 'daily_time',
|
||||||
|
'trigger_config': {'time': '04:00'},
|
||||||
|
'action_type': 'video_clean_youtube_episodes',
|
||||||
|
'owned_by': 'video',
|
||||||
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -237,8 +417,11 @@ class AutomationEngine:
|
||||||
trigger_type=spec['trigger_type'],
|
trigger_type=spec['trigger_type'],
|
||||||
trigger_config=json.dumps(spec['trigger_config']),
|
trigger_config=json.dumps(spec['trigger_config']),
|
||||||
action_type=spec['action_type'],
|
action_type=spec['action_type'],
|
||||||
action_config='{}',
|
action_config=json.dumps(spec.get('action_config', {})),
|
||||||
profile_id=1,
|
profile_id=1,
|
||||||
|
# owned_by tags the side that owns this automation (e.g.
|
||||||
|
# 'video'), so the music page can exclude another side's rows.
|
||||||
|
owned_by=spec.get('owned_by'),
|
||||||
)
|
)
|
||||||
if aid:
|
if aid:
|
||||||
self.db.update_automation(aid, is_system=1)
|
self.db.update_automation(aid, is_system=1)
|
||||||
|
|
@ -284,7 +467,117 @@ class AutomationEngine:
|
||||||
self.db.update_automation(existing['id'], next_run=nr)
|
self.db.update_automation(existing['id'], next_run=nr)
|
||||||
logger.info(f"System automation '{spec['name']}' next_run set to {initial_delay}s from now")
|
logger.info(f"System automation '{spec['name']}' next_run set to {initial_delay}s from now")
|
||||||
else:
|
else:
|
||||||
logger.info(f"System automation '{spec['name']}' ready (event-based)")
|
# No initial_delay. A timer-based timed trigger (daily/weekly/
|
||||||
|
# monthly at a fixed wall-clock time) still needs its next_run
|
||||||
|
# armed — compute it from the schedule. Genuinely event-based
|
||||||
|
# triggers (batch_complete, scan_done) have no next_run, so
|
||||||
|
# next_run_at returns None and we leave them alone. Don't clobber
|
||||||
|
# an existing future next_run (manual edit / restart-resume).
|
||||||
|
nr_dt = next_run_at(spec['trigger_type'], spec['trigger_config'],
|
||||||
|
now_utc=_utcnow(), default_tz=self._default_tz)
|
||||||
|
if nr_dt is not None and not existing.get('next_run'):
|
||||||
|
self.db.update_automation(existing['id'], next_run=_dt_to_db_str(nr_dt))
|
||||||
|
logger.info(f"System automation '{spec['name']}' next_run armed for {_dt_to_db_str(nr_dt)} (timed)")
|
||||||
|
else:
|
||||||
|
logger.info(f"System automation '{spec['name']}' ready (event-based)")
|
||||||
|
self._fix_video_scan_default()
|
||||||
|
self._fix_airing_automation_schedule()
|
||||||
|
self._fix_deep_scan_schedules()
|
||||||
|
self._fix_wishlist_processor_rename()
|
||||||
|
|
||||||
|
def _fix_video_scan_default(self):
|
||||||
|
"""Remove the obsolete standalone 'Scan Video Library' SYSTEM automation — it's
|
||||||
|
superseded by the post-download chain (Auto-Scan Video After Downloads →
|
||||||
|
Auto-Update Video Database After Scan).
|
||||||
|
|
||||||
|
``get_system_automation_by_action`` matches ONLY a system-seeded row
|
||||||
|
(is_system=1), so a user's own scan automation is never touched. Idempotent —
|
||||||
|
safe to run on every startup; once the row is gone the lookup returns None and
|
||||||
|
it no-ops. (No flag guard: the old one could latch True without ever deleting,
|
||||||
|
which is exactly why the row survived earlier 'cleanups'.)"""
|
||||||
|
try:
|
||||||
|
auto = self.db.get_system_automation_by_action('video_scan_library')
|
||||||
|
if auto:
|
||||||
|
self.db.delete_automation(auto['id'])
|
||||||
|
logger.info("Removed superseded 'Scan Video Library' system automation (id=%s)",
|
||||||
|
auto.get('id'))
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video scan cleanup failed")
|
||||||
|
|
||||||
|
def _fix_wishlist_processor_rename(self):
|
||||||
|
"""Migrate the wishlist processors' 'Download' → 'Process' rename so a DB seeded
|
||||||
|
under the old names doesn't show stale duplicates.
|
||||||
|
|
||||||
|
The movie/episode ACTIONS were renamed (``video_download_*`` → ``video_process_*``),
|
||||||
|
so the old seeded rows are now orphaned (dead action_type) while the new ones reseed
|
||||||
|
alongside them — delete the orphans. ``delete_automation`` refuses system rows, so
|
||||||
|
clear ``is_system`` first. The YouTube action kept its type but its label changed, so
|
||||||
|
rename that row in place. Idempotent — no-ops once the DB is clean."""
|
||||||
|
try:
|
||||||
|
for dead in ('video_download_movie_wishlist', 'video_download_episode_wishlist'):
|
||||||
|
auto = self.db.get_system_automation_by_action(dead)
|
||||||
|
if auto:
|
||||||
|
self.db.update_automation(auto['id'], is_system=0) # lift the delete guard
|
||||||
|
self.db.delete_automation(auto['id'])
|
||||||
|
logger.info("Removed orphaned '%s' system automation (renamed to process, id=%s)",
|
||||||
|
auto.get('name'), auto.get('id'))
|
||||||
|
yt = self.db.get_system_automation_by_action('video_process_youtube_wishlist')
|
||||||
|
if yt and yt.get('name') == 'Auto-Download YouTube Wishlist':
|
||||||
|
self.db.update_automation(yt['id'], name='Auto-Process YouTube Wishlist')
|
||||||
|
logger.info("Renamed YouTube wishlist automation → 'Auto-Process YouTube Wishlist' (id=%s)",
|
||||||
|
yt.get('id'))
|
||||||
|
except Exception:
|
||||||
|
logger.exception("wishlist processor rename migration failed")
|
||||||
|
|
||||||
|
def _fix_airing_automation_schedule(self):
|
||||||
|
"""Migrate 'Auto-Wishlist Episodes Airing Today' from the old rolling 24h
|
||||||
|
interval to a fixed daily 1am run.
|
||||||
|
|
||||||
|
It originally shipped as a 'schedule'/24h interval because the seeder only
|
||||||
|
armed interval specs — a 'daily_time' spec sat idle and never fired. The 24h
|
||||||
|
interval fires reliably but at a time that drifts with every restart (5 min
|
||||||
|
after startup, then +24h). Now that the seeder arms timed triggers, rewrite
|
||||||
|
the live row to run at a fixed 1am (better for 'today's airings' — queues the
|
||||||
|
day overnight). Matches only the is_system row; idempotent (no-op once the row
|
||||||
|
is already daily_time)."""
|
||||||
|
try:
|
||||||
|
auto = self.db.get_system_automation_by_action('video_add_airing_episodes')
|
||||||
|
if not auto or auto.get('trigger_type') == 'daily_time':
|
||||||
|
return
|
||||||
|
cfg = {'time': '01:00'}
|
||||||
|
nr_dt = next_run_at('daily_time', cfg, now_utc=_utcnow(), default_tz=self._default_tz)
|
||||||
|
self.db.update_automation(
|
||||||
|
auto['id'], trigger_type='daily_time', trigger_config=json.dumps(cfg),
|
||||||
|
next_run=_dt_to_db_str(nr_dt) if nr_dt is not None else None)
|
||||||
|
logger.info("Migrated 'Auto-Wishlist Episodes Airing Today' to a fixed daily 01:00 (id=%s)",
|
||||||
|
auto.get('id'))
|
||||||
|
except Exception:
|
||||||
|
logger.exception("airing automation schedule migration failed")
|
||||||
|
|
||||||
|
def _fix_deep_scan_schedules(self):
|
||||||
|
"""Migrate the two video deep-scan system automations from the original
|
||||||
|
rolling 7-day interval to fixed weekly times — TV Mondays 02:00, Movies
|
||||||
|
Tuesdays 02:00 (different days so they never overlap). The seeder only
|
||||||
|
creates rows, never updates a drifted trigger; this rewrites the live rows.
|
||||||
|
Only converts the original interval rows (skips once trigger_type is already
|
||||||
|
weekly_time, so a hand-tuned day/time sticks). Idempotent."""
|
||||||
|
targets = {
|
||||||
|
'video_deep_scan_tv': {'time': '02:00', 'days': ['mon']},
|
||||||
|
'video_deep_scan_movies': {'time': '02:00', 'days': ['tue']},
|
||||||
|
}
|
||||||
|
for action_type, cfg in targets.items():
|
||||||
|
try:
|
||||||
|
auto = self.db.get_system_automation_by_action(action_type)
|
||||||
|
if not auto or auto.get('trigger_type') == 'weekly_time':
|
||||||
|
continue
|
||||||
|
nr_dt = next_run_at('weekly_time', cfg, now_utc=_utcnow(), default_tz=self._default_tz)
|
||||||
|
self.db.update_automation(
|
||||||
|
auto['id'], trigger_type='weekly_time', trigger_config=json.dumps(cfg),
|
||||||
|
next_run=_dt_to_db_str(nr_dt) if nr_dt is not None else None)
|
||||||
|
logger.info("Set '%s' to weekly %s %s (id=%s)", auto.get('name'),
|
||||||
|
cfg['days'], cfg['time'], auto.get('id'))
|
||||||
|
except Exception:
|
||||||
|
logger.exception("deep-scan schedule migration failed for %s", action_type)
|
||||||
|
|
||||||
def get_system_automation_next_run_seconds(self, action_type):
|
def get_system_automation_next_run_seconds(self, action_type):
|
||||||
"""Get seconds until next run for a system automation. Returns 0 if not found or disabled."""
|
"""Get seconds until next run for a system automation. Returns 0 if not found or disabled."""
|
||||||
|
|
|
||||||
5
core/video/__init__.py
Normal file
5
core/video/__init__.py
Normal file
|
|
@ -0,0 +1,5 @@
|
||||||
|
"""SoulSync — isolated VIDEO core package.
|
||||||
|
|
||||||
|
Holds the video library scanner and media-server adapters. Music never imports
|
||||||
|
this package; this package never imports the music database layer.
|
||||||
|
"""
|
||||||
101
core/video/discovery_gaps.py
Normal file
101
core/video/discovery_gaps.py
Normal file
|
|
@ -0,0 +1,101 @@
|
||||||
|
"""The "what am I missing?" gap engine for video discovery.
|
||||||
|
|
||||||
|
Two pure diffs that power the discover rails:
|
||||||
|
|
||||||
|
* **collection_gaps** — given the TMDB items of a franchise the user has *started*
|
||||||
|
(owns >=1 of) and the set of tmdb ids they already own, return the franchise
|
||||||
|
entries they're missing ("Complete the Matrix Collection — you're missing 2").
|
||||||
|
* **filmography_gaps** — given a person's credits (combined_credits) and the owned
|
||||||
|
set, return the titles of theirs the user doesn't have ("More from Christopher
|
||||||
|
Nolan — 3 you don't own").
|
||||||
|
|
||||||
|
Both are pure (no I/O, no TMDB, no DB) — the discover API wires them to real data
|
||||||
|
(``owned tmdb ids`` from the library, collection items from the TMDB client, person
|
||||||
|
credits from the enrichment engine), so the diff + ranking logic is unit-testable in
|
||||||
|
isolation. Items are plain dicts carrying at least ``tmdb_id``; ``kind`` /
|
||||||
|
``popularity`` / ``vote_count`` are used for filtering + ranking when present.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Dict, Iterable, List, Set
|
||||||
|
|
||||||
|
|
||||||
|
def _owned_set(owned_tmdb_ids: Iterable) -> Set[int]:
|
||||||
|
out: Set[int] = set()
|
||||||
|
for x in owned_tmdb_ids or []:
|
||||||
|
try:
|
||||||
|
out.add(int(x))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def collection_gaps(owned_tmdb_ids: Iterable, collection_items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
||||||
|
"""Franchise entries the user is missing.
|
||||||
|
|
||||||
|
``collection_items`` is the full ordered film list of a TMDB collection (each a
|
||||||
|
dict with ``tmdb_id``). Returns the entries whose tmdb id isn't in
|
||||||
|
``owned_tmdb_ids``, in the collection's original order (chronological / as TMDB
|
||||||
|
returns it), deduped. Entries without a usable tmdb id are skipped.
|
||||||
|
"""
|
||||||
|
owned = _owned_set(owned_tmdb_ids)
|
||||||
|
seen: Set[int] = set()
|
||||||
|
out: List[Dict[str, Any]] = []
|
||||||
|
for it in collection_items or []:
|
||||||
|
if not isinstance(it, dict):
|
||||||
|
continue
|
||||||
|
tid = it.get("tmdb_id")
|
||||||
|
try:
|
||||||
|
tid = int(tid)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
if tid in owned or tid in seen:
|
||||||
|
continue
|
||||||
|
seen.add(tid)
|
||||||
|
out.append(it)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def filmography_gaps(
|
||||||
|
owned_tmdb_ids: Iterable,
|
||||||
|
credits: List[Dict[str, Any]],
|
||||||
|
*,
|
||||||
|
kinds: Iterable[str] = ("movie",),
|
||||||
|
min_vote_count: int = 0,
|
||||||
|
limit: int = 0,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Titles from a person's credits the user doesn't own, ranked by popularity.
|
||||||
|
|
||||||
|
``credits`` is a person's combined credits (each a dict with ``tmdb_id`` and
|
||||||
|
``kind``). Keeps only the requested ``kinds``, drops owned + duplicate ids, and
|
||||||
|
(when ``min_vote_count`` > 0) filters out obscure entries lacking enough votes —
|
||||||
|
so the rail surfaces real films, not every uncredited cameo. Sorted by
|
||||||
|
``popularity`` (then ``vote_count``) descending; ``limit`` caps the result (0 = all).
|
||||||
|
"""
|
||||||
|
owned = _owned_set(owned_tmdb_ids)
|
||||||
|
wanted = {str(k) for k in kinds}
|
||||||
|
seen: Set[int] = set()
|
||||||
|
out: List[Dict[str, Any]] = []
|
||||||
|
for c in credits or []:
|
||||||
|
if not isinstance(c, dict):
|
||||||
|
continue
|
||||||
|
if c.get("kind") is not None and str(c.get("kind")) not in wanted:
|
||||||
|
continue
|
||||||
|
tid = c.get("tmdb_id")
|
||||||
|
try:
|
||||||
|
tid = int(tid)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
if tid in owned or tid in seen:
|
||||||
|
continue
|
||||||
|
if min_vote_count and (c.get("vote_count") or 0) < min_vote_count:
|
||||||
|
continue
|
||||||
|
seen.add(tid)
|
||||||
|
out.append(c)
|
||||||
|
|
||||||
|
out.sort(key=lambda x: ((x.get("popularity") or 0), (x.get("vote_count") or 0)), reverse=True)
|
||||||
|
return out[:limit] if limit and limit > 0 else out
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["collection_gaps", "filmography_gaps"]
|
||||||
71
core/video/discovery_recs.py
Normal file
71
core/video/discovery_recs.py
Normal file
|
|
@ -0,0 +1,71 @@
|
||||||
|
"""Blend per-title TMDB recommendations into one ranked "Recommended for you" wall.
|
||||||
|
|
||||||
|
The discover page already does per-title rails ("More like Dune"). This aggregates the
|
||||||
|
recommendations of MANY owned titles into a single personalized wall: a candidate
|
||||||
|
recommended by *more* of your titles ranks higher (consensus is a stronger signal than
|
||||||
|
any one seed), ties broken by rating then popularity. Owned titles (the engine annotates
|
||||||
|
each recommendation with ``library_id`` when owned) and the seed titles themselves are
|
||||||
|
excluded, so the wall is all stuff you don't have.
|
||||||
|
|
||||||
|
Pure + I/O-free: the discover API fetches each seed's recommendations (cached) and passes
|
||||||
|
the lists here, so the dedup/consensus ranking is unit-testable without TMDB.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Dict, Iterable, List
|
||||||
|
|
||||||
|
|
||||||
|
def blend_recommendations(
|
||||||
|
rec_lists: List[List[Dict[str, Any]]],
|
||||||
|
*,
|
||||||
|
exclude_ids: Iterable = (),
|
||||||
|
limit: int = 40,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Aggregate ``rec_lists`` (one recommendation list per seed title) into a single
|
||||||
|
ranked, deduped list of un-owned titles.
|
||||||
|
|
||||||
|
Each item is a dict with ``tmdb_id`` / ``kind`` (and optionally ``library_id`` set by
|
||||||
|
the engine when owned, plus ``rating`` / ``popularity``). A title appearing across more
|
||||||
|
seed lists scores higher; ties fall back to rating then popularity. Owned items
|
||||||
|
(``library_id`` not None) and ``exclude_ids`` (the seeds) are dropped. ``limit`` caps
|
||||||
|
the result (0 = all).
|
||||||
|
"""
|
||||||
|
exclude = set()
|
||||||
|
for x in exclude_ids or []:
|
||||||
|
try:
|
||||||
|
exclude.add(int(x))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
|
||||||
|
agg: Dict[tuple, Dict[str, Any]] = {}
|
||||||
|
for lst in rec_lists or []:
|
||||||
|
for it in lst or []:
|
||||||
|
if not isinstance(it, dict):
|
||||||
|
continue
|
||||||
|
if it.get("library_id") is not None: # owned (engine-annotated) — skip
|
||||||
|
continue
|
||||||
|
tid = it.get("tmdb_id")
|
||||||
|
try:
|
||||||
|
tid = int(tid)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
if tid in exclude:
|
||||||
|
continue
|
||||||
|
key = (it.get("kind"), tid)
|
||||||
|
entry = agg.get(key)
|
||||||
|
if entry is None:
|
||||||
|
agg[key] = {"item": it, "count": 1}
|
||||||
|
else:
|
||||||
|
entry["count"] += 1
|
||||||
|
|
||||||
|
ranked = sorted(
|
||||||
|
agg.values(),
|
||||||
|
key=lambda e: (e["count"], e["item"].get("rating") or 0, e["item"].get("popularity") or 0),
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
items = [e["item"] for e in ranked]
|
||||||
|
return items[:limit] if limit and limit > 0 else items
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["blend_recommendations"]
|
||||||
62
core/video/download_config.py
Normal file
62
core/video/download_config.py
Normal file
|
|
@ -0,0 +1,62 @@
|
||||||
|
"""Video download SOURCE config — which source(s) to download from.
|
||||||
|
|
||||||
|
Video only ever uses three sources: **soulseek / torrent / usenet** (no streaming
|
||||||
|
APIs — those are music-only). ``download_mode`` is one of those three, or
|
||||||
|
``hybrid``; in hybrid mode ``hybrid_order`` is the ordered chain of enabled sources
|
||||||
|
the (later-phase) engine tries in turn.
|
||||||
|
|
||||||
|
Pure normalize here (no DB, no network) so it's unit-tested in isolation. Stored in
|
||||||
|
video.db's ``video_settings`` (``download_mode`` + ``hybrid_order`` JSON). Isolated:
|
||||||
|
imports only json/typing, and the music side never imports it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
SOURCES = ("soulseek", "torrent", "usenet")
|
||||||
|
MODES = SOURCES + ("hybrid",)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_mode(value: Any) -> str:
|
||||||
|
v = str(value or "").strip().lower()
|
||||||
|
return v if v in MODES else "soulseek"
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_hybrid_order(value: Any) -> list:
|
||||||
|
"""Ordered, de-duped list of valid sources; defaults to ['soulseek']. Accepts a
|
||||||
|
JSON string (as stored) or a list (as posted)."""
|
||||||
|
arr = value
|
||||||
|
if isinstance(arr, str):
|
||||||
|
try:
|
||||||
|
arr = json.loads(arr)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
arr = None
|
||||||
|
out = []
|
||||||
|
if isinstance(arr, list):
|
||||||
|
for s in arr:
|
||||||
|
s = str(s or "").strip().lower()
|
||||||
|
if s in SOURCES and s not in out:
|
||||||
|
out.append(s)
|
||||||
|
return out or ["soulseek"]
|
||||||
|
|
||||||
|
|
||||||
|
def load(db) -> dict:
|
||||||
|
return {
|
||||||
|
"download_mode": normalize_mode(db.get_setting("download_mode")),
|
||||||
|
"hybrid_order": normalize_hybrid_order(db.get_setting("hybrid_order")),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def save(db, body: Any) -> dict:
|
||||||
|
"""Persist whichever of mode/hybrid_order is present in ``body``."""
|
||||||
|
body = body if isinstance(body, dict) else {}
|
||||||
|
if "download_mode" in body:
|
||||||
|
db.set_setting("download_mode", normalize_mode(body.get("download_mode")))
|
||||||
|
if "hybrid_order" in body:
|
||||||
|
db.set_setting("hybrid_order", json.dumps(normalize_hybrid_order(body.get("hybrid_order"))))
|
||||||
|
return load(db)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["SOURCES", "MODES", "normalize_mode", "normalize_hybrid_order", "load", "save"]
|
||||||
40
core/video/download_events.py
Normal file
40
core/video/download_events.py
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
"""Video download events — a tiny publish/subscribe bridge.
|
||||||
|
|
||||||
|
ISOLATION: ``core/video`` must not import the automation engine (that's music-side
|
||||||
|
shared infra). So when a batch of video downloads finishes, the monitor publishes
|
||||||
|
to this callback registry instead of touching the engine directly. The shared side
|
||||||
|
(web_server) registers a callback that forwards to
|
||||||
|
``automation_engine.emit('video_batch_complete', …)`` — the same one-way bridge the
|
||||||
|
music ``web_scan_manager`` uses for ``library_scan_completed``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Callable
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_download_events")
|
||||||
|
|
||||||
|
_batch_complete_callbacks: list[Callable[[dict], None]] = []
|
||||||
|
|
||||||
|
|
||||||
|
def register_batch_complete_callback(cb: Callable[[dict], None]) -> None:
|
||||||
|
"""Subscribe to 'a batch of video downloads just finished'. Idempotent."""
|
||||||
|
if cb not in _batch_complete_callbacks:
|
||||||
|
_batch_complete_callbacks.append(cb)
|
||||||
|
|
||||||
|
|
||||||
|
def notify_batch_complete(data: dict | None = None) -> None:
|
||||||
|
"""Fire every registered batch-complete callback (best-effort; one failing
|
||||||
|
subscriber never blocks the others or the monitor)."""
|
||||||
|
payload = data or {}
|
||||||
|
for cb in list(_batch_complete_callbacks):
|
||||||
|
try:
|
||||||
|
cb(payload)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video batch-complete callback failed")
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_for_tests() -> None:
|
||||||
|
_batch_complete_callbacks.clear()
|
||||||
468
core/video/download_monitor.py
Normal file
468
core/video/download_monitor.py
Normal file
|
|
@ -0,0 +1,468 @@
|
||||||
|
"""Background monitor that drives video downloads to completion.
|
||||||
|
|
||||||
|
A daemon thread polls slskd for the active video downloads, updates their progress,
|
||||||
|
and when one finishes MOVES the file from the shared download folder into the right
|
||||||
|
per-type library folder (Movies / TV / YouTube) and marks it completed. Simple v1:
|
||||||
|
slskd source only, flat move by basename.
|
||||||
|
|
||||||
|
The per-download decision (``process_download``) is pure — filesystem + slskd are
|
||||||
|
injected — so it's unit-tested; the thread loop is thin glue.
|
||||||
|
|
||||||
|
Isolated: stdlib + the sibling video modules + shared config_manager; no music imports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
from core.video.download_pipeline import dest_path_for, find_completed_file
|
||||||
|
from core.video.slskd_download import (
|
||||||
|
classify_state,
|
||||||
|
find_transfer,
|
||||||
|
list_downloads,
|
||||||
|
progress_pct,
|
||||||
|
start_download,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger = get_logger("video.download_monitor")
|
||||||
|
|
||||||
|
_INTERVAL = 3 # seconds between polls
|
||||||
|
_started = False
|
||||||
|
_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _complete_via_file(dl, download_dir, lister, mover, organizer):
|
||||||
|
"""Locate the finished file in the download dir and post-process it into the
|
||||||
|
library. Returns a completed/import_failed patch, or {'progress':100} if the file
|
||||||
|
isn't on disk yet. ``organizer(dl, src)`` (when supplied) runs the full Radarr-style
|
||||||
|
import (parse → templated rename → copy/replace → carry subs); otherwise we fall
|
||||||
|
back to the legacy flat move by basename."""
|
||||||
|
src = find_completed_file(download_dir, dl.get("filename"), lister)
|
||||||
|
if not src:
|
||||||
|
return {"progress": 100.0}
|
||||||
|
if organizer is not None:
|
||||||
|
return organizer(dl, src)
|
||||||
|
dest = dest_path_for(dl.get("target_dir"), src)
|
||||||
|
try:
|
||||||
|
mover(src, dest)
|
||||||
|
except Exception as e: # noqa: BLE001 - any move failure marks the download failed
|
||||||
|
return {"status": "failed", "error": "Move failed: " + str(e)}
|
||||||
|
return {"status": "completed", "progress": 100.0, "dest_path": dest}
|
||||||
|
|
||||||
|
|
||||||
|
def process_download(dl: dict, transfers: list, download_dir: str, *, lister, mover, organizer=None) -> dict | None:
|
||||||
|
"""Decide the next state for one active download given the current slskd transfers.
|
||||||
|
Returns a patch dict for the DB row, or {'_missing': True} when slskd no longer
|
||||||
|
knows the transfer (the caller decides when to give up). Robust to slskd clearing
|
||||||
|
completed transfers (the music 'Clean Completed Downloads' automation) by also
|
||||||
|
detecting completion from the file landing on disk."""
|
||||||
|
t = find_transfer(transfers, dl.get("username"), dl.get("filename"))
|
||||||
|
if not t:
|
||||||
|
# slskd forgot it — could be done+cleared. If the file's there, finish it.
|
||||||
|
done = _complete_via_file(dl, download_dir, lister, mover, organizer)
|
||||||
|
if done.get("status"):
|
||||||
|
return done
|
||||||
|
return {"_missing": True}
|
||||||
|
state = classify_state(t.get("state"))
|
||||||
|
if state == "active":
|
||||||
|
return {"status": "downloading", "progress": progress_pct(t)}
|
||||||
|
if state == "cancelled":
|
||||||
|
return {"status": "cancelled", "error": "Cancelled on Soulseek"}
|
||||||
|
if state == "failed":
|
||||||
|
return {"status": "failed", "error": "Soulseek transfer " + str(t.get("state") or "failed")}
|
||||||
|
return _complete_via_file(dl, download_dir, lister, mover, organizer) # completed
|
||||||
|
|
||||||
|
|
||||||
|
def _move(src: str, dest: str) -> None:
|
||||||
|
os.makedirs(os.path.dirname(dest) or ".", exist_ok=True)
|
||||||
|
shutil.move(src, dest)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_organizer(db):
|
||||||
|
"""A per-tick organizer closure: post-process a finished download into the library
|
||||||
|
via the importer (Radarr-style parse → ffprobe-verify → templated rename →
|
||||||
|
copy/replace → carry subs) against the real filesystem. The upgrade decision reads
|
||||||
|
the destination folder (filesystem-as-truth), so no DB/profile lookup is needed.
|
||||||
|
ffprobe is best-effort — when it isn't installed, ``probe`` returns None and the
|
||||||
|
importer falls back to scene-name parsing."""
|
||||||
|
from core.video import organization
|
||||||
|
from core.video.importer import real_fs, run_import
|
||||||
|
from core.video.mediainfo import probe
|
||||||
|
fs = real_fs()
|
||||||
|
try:
|
||||||
|
settings = organization.load(db)
|
||||||
|
except Exception: # noqa: BLE001 - a settings-load hiccup must never wedge the monitor
|
||||||
|
settings = organization.default_settings()
|
||||||
|
prober = probe if settings.get("verify_with_ffprobe", True) else None
|
||||||
|
|
||||||
|
def organize(dl, src):
|
||||||
|
# The file's down — flip to 'importing' so the UI shows the post-processing phase
|
||||||
|
# (move into the library + nfo/artwork sidecars + subtitles) instead of sitting on
|
||||||
|
# 'downloading' while this runs. Best-effort; the patch below is the real transition.
|
||||||
|
try:
|
||||||
|
db.update_video_download(dl["id"], status="importing", progress=100)
|
||||||
|
except Exception: # noqa: BLE001, S110 - a status blip must never wedge the import
|
||||||
|
pass
|
||||||
|
patch = run_import(dl, src, fs=fs, prober=prober, settings=settings)
|
||||||
|
if patch.get("status") == "completed" and patch.get("dest_path"):
|
||||||
|
if settings.get("save_artwork") or settings.get("write_nfo"):
|
||||||
|
write_sidecars(db, dl, patch["dest_path"], settings, fs)
|
||||||
|
if settings.get("download_subtitles"):
|
||||||
|
write_subtitles_for(db, dl, patch["dest_path"], settings, fs)
|
||||||
|
return patch
|
||||||
|
|
||||||
|
return organize
|
||||||
|
|
||||||
|
|
||||||
|
def _media_ids(db, dl):
|
||||||
|
"""(tmdb_id, imdb_id) for a download's title — taken directly when it was grabbed
|
||||||
|
from TMDB, or looked up from the LIBRARY row for an owned re-grab (whose ``media_id``
|
||||||
|
is the library id, not a TMDB id). (None, None) when it can't be resolved."""
|
||||||
|
dl = dl or {}
|
||||||
|
mid = dl.get("media_id")
|
||||||
|
if not mid:
|
||||||
|
return (None, None)
|
||||||
|
src = str(dl.get("media_source") or "").lower()
|
||||||
|
if src == "tmdb":
|
||||||
|
try:
|
||||||
|
return (int(mid), None)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return (None, None)
|
||||||
|
if src == "library" and db is not None:
|
||||||
|
try:
|
||||||
|
kind = "movie" if str(dl.get("kind") or "").lower() == "movie" else "show"
|
||||||
|
return db.media_tmdb_id(kind, mid)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
return (None, None)
|
||||||
|
return (None, None)
|
||||||
|
|
||||||
|
|
||||||
|
def write_sidecars(db, dl, dest_path, settings, fs):
|
||||||
|
"""Best-effort: fetch full TMDB metadata for the imported title (resolving a library
|
||||||
|
re-grab's id when needed) and write NFO + the artwork set next to it — movie folder,
|
||||||
|
or the show root for an episode. Uses the detail's ABSOLUTE image URLs (the download
|
||||||
|
row's poster is an internal/relative path, not fetchable). Never raises. Shared by
|
||||||
|
the monitor and the manual-import endpoint."""
|
||||||
|
try:
|
||||||
|
from core.video import sidecars
|
||||||
|
scope = "movie" if str(dl.get("kind") or "").lower() == "movie" else "episode"
|
||||||
|
tmdb_id, _imdb = _media_ids(db, dl)
|
||||||
|
detail = None
|
||||||
|
if tmdb_id is not None:
|
||||||
|
try:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
# full_detail (not tmdb_detail) so OWNED titles don't redirect — sidecars
|
||||||
|
# need the raw metadata + absolute image URLs regardless of ownership.
|
||||||
|
d = get_video_enrichment_engine().tmdb_full_detail(
|
||||||
|
"movie" if scope == "movie" else "show", tmdb_id)
|
||||||
|
if isinstance(d, dict):
|
||||||
|
detail = d
|
||||||
|
except Exception: # noqa: BLE001 - a metadata fetch hiccup → skip, don't fail
|
||||||
|
detail = None
|
||||||
|
sidecars.write_for(dest_path, scope, (detail or {}).get("poster_url"), detail, settings, fs)
|
||||||
|
except Exception: # noqa: BLE001 - sidecars are a nice-to-have, never fatal
|
||||||
|
logger.exception("sidecar write failed for download %s", (dl or {}).get("id"))
|
||||||
|
|
||||||
|
|
||||||
|
def write_subtitles_for(db, dl, dest_path, settings, fs):
|
||||||
|
"""Best-effort: download external .srt files (OpenSubtitles) next to the imported
|
||||||
|
video for the user's preferred languages. The .srt sits NEXT TO the file (so an
|
||||||
|
episode's subs land in its Season folder, not the show root). Never raises."""
|
||||||
|
try:
|
||||||
|
import json as _json
|
||||||
|
from core.video import subtitles
|
||||||
|
api_key = db.get_setting("opensubtitles_api_key") if db else None
|
||||||
|
fetch = subtitles.opensubtitles_fetcher(api_key)
|
||||||
|
if not fetch:
|
||||||
|
return
|
||||||
|
tmdb_id, imdb_id = _media_ids(db, dl)
|
||||||
|
identity = {}
|
||||||
|
if tmdb_id is not None:
|
||||||
|
identity["tmdb_id"] = tmdb_id
|
||||||
|
if imdb_id:
|
||||||
|
identity["imdb_id"] = imdb_id
|
||||||
|
try:
|
||||||
|
ctx = _json.loads(dl.get("search_ctx") or "{}")
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
ctx = {}
|
||||||
|
if isinstance(ctx, dict) and ctx.get("season") is not None:
|
||||||
|
identity["season"] = ctx.get("season")
|
||||||
|
identity["episode"] = ctx.get("episode")
|
||||||
|
if not (identity.get("tmdb_id") or identity.get("imdb_id")):
|
||||||
|
return
|
||||||
|
langs = subtitles.parse_langs(settings.get("subtitle_langs"))
|
||||||
|
subtitles.write_subtitles(dest_path, langs, identity, fetch, fs)
|
||||||
|
except Exception: # noqa: BLE001 - subtitle fetch is best-effort, never fatal
|
||||||
|
logger.exception("subtitle fetch failed for download %s", (dl or {}).get("id"))
|
||||||
|
|
||||||
|
|
||||||
|
def _walk(root: str):
|
||||||
|
for dirpath, _dirs, files in os.walk(str(root or ".")):
|
||||||
|
for f in files:
|
||||||
|
yield os.path.join(dirpath, f)
|
||||||
|
|
||||||
|
|
||||||
|
_GIVE_UP_AFTER = 8 # consecutive 'transfer gone, no file' polls before failing it
|
||||||
|
_misses: dict = {} # download id -> consecutive missing polls
|
||||||
|
_db_provider = None # set by ensure_started; used by the requery worker thread
|
||||||
|
_requerying: set = set() # download ids with a requery thread in flight
|
||||||
|
|
||||||
|
|
||||||
|
def _now():
|
||||||
|
return time.strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
|
||||||
|
|
||||||
|
# ── auto-retry ────────────────────────────────────────────────────────────────
|
||||||
|
def _apply_candidate(db, dl_id, row, cand, rest) -> bool:
|
||||||
|
"""Start the next candidate download and flip the row back to 'downloading'.
|
||||||
|
Returns False if slskd refused to start it."""
|
||||||
|
started = start_download(cand.get("username"), cand.get("filename"), cand.get("size_bytes") or 0)
|
||||||
|
if not started.get("ok"):
|
||||||
|
return False
|
||||||
|
tried = []
|
||||||
|
try:
|
||||||
|
tried = json.loads(row.get("tried_files") or "[]")
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
tried = []
|
||||||
|
tried.append(cand.get("filename"))
|
||||||
|
db.update_video_download(
|
||||||
|
dl_id, status="downloading", progress=0, error=None, completed_at=None,
|
||||||
|
username=cand.get("username"), filename=cand.get("filename"),
|
||||||
|
release_title=cand.get("release_title") or cand.get("filename"),
|
||||||
|
size_bytes=int(cand.get("size_bytes") or 0), quality_label=cand.get("quality_label"),
|
||||||
|
candidates=json.dumps(rest), tried_files=json.dumps(tried),
|
||||||
|
attempts=int(row.get("attempts") or 0) + 1)
|
||||||
|
_misses.pop(dl_id, None)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _archive_history(db, dl, upd) -> None:
|
||||||
|
"""Snapshot a terminal download into the permanent history. Best-effort — a
|
||||||
|
history failure must never disturb the download pipeline."""
|
||||||
|
try:
|
||||||
|
db.record_download_history({**dl, **upd})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video download %s: history snapshot failed", dl.get("id"))
|
||||||
|
|
||||||
|
|
||||||
|
def _fail_or_retry(db, dl, error_msg) -> None:
|
||||||
|
"""A download just failed/disappeared. Try the next candidate inline; if none,
|
||||||
|
hand off to a requery thread; if nothing left, mark it failed for real."""
|
||||||
|
from core.video.retry import plan_retry
|
||||||
|
plan = plan_retry(dl)
|
||||||
|
if plan["action"] == "candidate" and _apply_candidate(db, dl["id"], dl, plan["candidate"], plan["rest"]):
|
||||||
|
return
|
||||||
|
if plan["action"] in ("candidate", "requery"):
|
||||||
|
db.update_video_download(dl["id"], status="searching", error=None)
|
||||||
|
_spawn_requery(dl["id"])
|
||||||
|
return
|
||||||
|
err = error_msg or "Download failed"
|
||||||
|
completed = _now()
|
||||||
|
db.update_video_download(dl["id"], status="failed", error=err, completed_at=completed)
|
||||||
|
_archive_history(db, dl, {"status": "failed", "error": err, "completed_at": completed})
|
||||||
|
|
||||||
|
|
||||||
|
def _search_for_retry(query, max_seconds=55):
|
||||||
|
"""A bounded blocking slskd search for the retry worker + the auto-grab automation.
|
||||||
|
|
||||||
|
slskd gathers peer responses over its full search window (~60s by default), so a short
|
||||||
|
wait misses almost everything. We poll up to ``max_seconds`` but return EARLY once
|
||||||
|
results have arrived and settled — break on either plenty of hits (12+) or no new hits
|
||||||
|
for ~12s after getting some, so fast searches don't burn the whole window.
|
||||||
|
|
||||||
|
Always STOPS the slskd search when done — otherwise it keeps running its full timeout
|
||||||
|
and, since the auto-grab fires searches back-to-back, they pile up. Stopping each one
|
||||||
|
keeps concurrent slskd searches ≈ the worker pool size."""
|
||||||
|
from core.video.slskd_search import poll_search, start_search, stop_search
|
||||||
|
res = start_search(query)
|
||||||
|
sid = res.get("id")
|
||||||
|
if not sid:
|
||||||
|
# slskd didn't accept the search (not configured, errored, or rate-limited). Surface
|
||||||
|
# it as 'not started' so the caller doesn't report it as a genuine "no results".
|
||||||
|
return {"hits": [], "total_files": 0, "started": False, "error": res.get("error")}
|
||||||
|
deadline = time.monotonic() + max_seconds
|
||||||
|
last = {"hits": [], "total_files": 0}
|
||||||
|
prev, settle = 0, 0 # track hit growth; settle = consecutive no-growth polls (~1.5s each)
|
||||||
|
try:
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
last = poll_search(sid)
|
||||||
|
n = len(last.get("hits") or [])
|
||||||
|
if n >= 12:
|
||||||
|
break # plenty — stop waiting
|
||||||
|
if n > prev:
|
||||||
|
settle = 0 # still arriving — keep waiting
|
||||||
|
elif n > 0:
|
||||||
|
settle += 1
|
||||||
|
if settle >= 8: # ~12s with no new hits → results have settled
|
||||||
|
break
|
||||||
|
prev = n
|
||||||
|
time.sleep(1.5)
|
||||||
|
return last
|
||||||
|
finally:
|
||||||
|
stop_search(sid)
|
||||||
|
|
||||||
|
|
||||||
|
def _requery_worker(dl_id) -> None:
|
||||||
|
from core.video.quality_eval import evaluate_release
|
||||||
|
from core.video.quality_profile import load as load_profile
|
||||||
|
from core.video.release_parse import parse_release
|
||||||
|
from core.video.retry import merge_candidates, plan_retry
|
||||||
|
try:
|
||||||
|
db = _db_provider() if _db_provider else None
|
||||||
|
if db is None:
|
||||||
|
return
|
||||||
|
profile = load_profile(db)
|
||||||
|
for _ in range(8): # hard loop cap on top of the attempt budget
|
||||||
|
row = db.get_video_download(dl_id)
|
||||||
|
if not row or row.get("status") != "searching":
|
||||||
|
return
|
||||||
|
plan = plan_retry(row)
|
||||||
|
if plan["action"] == "candidate":
|
||||||
|
if _apply_candidate(db, dl_id, row, plan["candidate"], plan["rest"]):
|
||||||
|
return
|
||||||
|
continue
|
||||||
|
if plan["action"] != "requery":
|
||||||
|
break
|
||||||
|
query, ctx = plan["query"], plan.get("ctx") or {}
|
||||||
|
# record the attempt + the query we're about to try
|
||||||
|
tq = []
|
||||||
|
try:
|
||||||
|
tq = json.loads(row.get("tried_queries") or "[]")
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
tq = []
|
||||||
|
tq.append(query)
|
||||||
|
db.update_video_download(dl_id, tried_queries=json.dumps(tq),
|
||||||
|
attempts=int(row.get("attempts") or 0) + 1)
|
||||||
|
polled = _search_for_retry(query)
|
||||||
|
accepted = []
|
||||||
|
for hit in (polled.get("hits") or []):
|
||||||
|
v = evaluate_release(parse_release(hit.get("title")), profile,
|
||||||
|
scope=ctx.get("scope") or "movie",
|
||||||
|
want_season=ctx.get("season"), want_episode=ctx.get("episode"))
|
||||||
|
if v["accepted"]:
|
||||||
|
accepted.append(hit)
|
||||||
|
row2 = db.get_video_download(dl_id)
|
||||||
|
if not row2 or row2.get("status") != "searching":
|
||||||
|
return
|
||||||
|
tried_files = []
|
||||||
|
try:
|
||||||
|
tried_files = json.loads(row2.get("tried_files") or "[]")
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
tried_files = []
|
||||||
|
fresh = merge_candidates(accepted, tried_files)
|
||||||
|
if fresh and _apply_candidate(db, dl_id, row2, fresh[0], fresh[1:]):
|
||||||
|
return
|
||||||
|
# this query gave nothing usable → loop tries the next query (or fails)
|
||||||
|
db.update_video_download(dl_id, status="failed",
|
||||||
|
error="No working release found after retries", completed_at=_now())
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video download %s: requery worker failed", dl_id)
|
||||||
|
try:
|
||||||
|
if _db_provider:
|
||||||
|
_db_provider().update_video_download(dl_id, status="failed",
|
||||||
|
error="Retry error", completed_at=_now())
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video download %s: could not mark failed", dl_id)
|
||||||
|
finally:
|
||||||
|
_requerying.discard(dl_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _spawn_requery(dl_id) -> None:
|
||||||
|
if dl_id in _requerying:
|
||||||
|
return
|
||||||
|
_requerying.add(dl_id)
|
||||||
|
threading.Thread(target=_requery_worker, args=(dl_id,), daemon=True,
|
||||||
|
name="video-dl-requery-%s" % dl_id).start()
|
||||||
|
|
||||||
|
|
||||||
|
def _tick(db) -> None:
|
||||||
|
# 'searching' rows are owned by their requery thread; 'youtube' rows are owned by
|
||||||
|
# their yt-dlp worker thread (no slskd transfer to match) — skip both here.
|
||||||
|
active = [d for d in db.get_active_video_downloads()
|
||||||
|
if d.get("status") != "searching" and d.get("source") != "youtube"]
|
||||||
|
if not active:
|
||||||
|
_misses.clear()
|
||||||
|
return
|
||||||
|
from config.settings import config_manager
|
||||||
|
download_dir = str(config_manager.get("soulseek.download_path", "") or "")
|
||||||
|
transfers = list_downloads()
|
||||||
|
organizer = _make_organizer(db)
|
||||||
|
live_ids = set()
|
||||||
|
completed_now = 0
|
||||||
|
for dl in active:
|
||||||
|
live_ids.add(dl["id"])
|
||||||
|
upd = process_download(dl, transfers, download_dir, lister=_walk, mover=_move,
|
||||||
|
organizer=organizer)
|
||||||
|
if not upd:
|
||||||
|
continue
|
||||||
|
if upd.get("status") == "completed":
|
||||||
|
completed_now += 1
|
||||||
|
if upd.get("_missing"):
|
||||||
|
n = _misses.get(dl["id"], 0) + 1
|
||||||
|
_misses[dl["id"]] = n
|
||||||
|
if n >= _GIVE_UP_AFTER:
|
||||||
|
_misses.pop(dl["id"], None)
|
||||||
|
_fail_or_retry(db, dl, "Soulseek transfer disappeared")
|
||||||
|
continue
|
||||||
|
_misses.pop(dl["id"], None)
|
||||||
|
if upd.get("status") == "failed":
|
||||||
|
_fail_or_retry(db, dl, upd.get("error")) # auto-retry before truly failing
|
||||||
|
continue
|
||||||
|
# import_failed = the file downloaded fine but couldn't be placed (sample, wrong
|
||||||
|
# episode, not an upgrade, …). Terminal + needs manual import — NOT a download
|
||||||
|
# failure, so don't burn the retry budget re-downloading the same good file.
|
||||||
|
if upd.get("status") in ("completed", "cancelled", "import_failed"):
|
||||||
|
upd.setdefault("completed_at", _now())
|
||||||
|
try:
|
||||||
|
db.update_video_download(dl["id"], **upd)
|
||||||
|
# Snapshot terminal outcomes into the permanent history (survives the
|
||||||
|
# queue cleanup; powers the History modal + smart post-download scan).
|
||||||
|
if upd.get("status") in ("completed", "cancelled", "import_failed"):
|
||||||
|
_archive_history(db, dl, upd)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video download %s: failed to persist update", dl.get("id"))
|
||||||
|
for k in [k for k in _misses if k not in live_ids]:
|
||||||
|
_misses.pop(k, None)
|
||||||
|
# Batch complete: we finished ≥1 download this tick AND nothing is left in
|
||||||
|
# flight (queued/downloading/searching). Fires once, on the transition to
|
||||||
|
# empty — the next tick early-returns. Publishes to the event bridge so the
|
||||||
|
# 'Auto-Scan Video After Downloads' automation can refresh the server.
|
||||||
|
if completed_now and not db.get_active_video_downloads():
|
||||||
|
try:
|
||||||
|
from core.video.download_events import notify_batch_complete
|
||||||
|
notify_batch_complete({"completed": completed_now})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video monitor: batch-complete notify failed")
|
||||||
|
|
||||||
|
|
||||||
|
def _run(db_provider) -> None:
|
||||||
|
logger.info("video download monitor started")
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
db = db_provider()
|
||||||
|
if db is not None:
|
||||||
|
_tick(db)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video download monitor tick failed")
|
||||||
|
time.sleep(_INTERVAL)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_started(db_provider) -> None:
|
||||||
|
"""Start the monitor thread once (idempotent). Called when the first grab happens."""
|
||||||
|
global _started, _db_provider
|
||||||
|
with _lock:
|
||||||
|
_db_provider = db_provider
|
||||||
|
if _started:
|
||||||
|
return
|
||||||
|
_started = True
|
||||||
|
threading.Thread(target=_run, args=(db_provider,), daemon=True,
|
||||||
|
name="video-download-monitor").start()
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["process_download", "ensure_started"]
|
||||||
54
core/video/download_pipeline.py
Normal file
54
core/video/download_pipeline.py
Normal file
|
|
@ -0,0 +1,54 @@
|
||||||
|
"""Pure helpers for the video download pipeline: locate a finished file in the
|
||||||
|
download dir and work out where it should move to.
|
||||||
|
|
||||||
|
slskd writes a completed download somewhere under the shared download folder (it
|
||||||
|
mirrors the remote folder structure), so we locate the file by basename. Kept pure
|
||||||
|
(filesystem access is injected) so it's unit-tested without touching disk.
|
||||||
|
|
||||||
|
Isolated: stdlib only; no music imports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from typing import Any, Callable
|
||||||
|
|
||||||
|
|
||||||
|
def basename_of(path: Any) -> str:
|
||||||
|
"""Final path segment, handling both / and \\ separators (slskd uses Windows-style)."""
|
||||||
|
return str(path or "").replace("\\", "/").rstrip("/").rsplit("/", 1)[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def find_completed_file(download_dir: str, filename: str, lister: Callable) -> str | None:
|
||||||
|
"""Find the downloaded file on disk by basename. ``lister(dir)`` yields candidate
|
||||||
|
full paths (injected: os.walk-based in the monitor, a list in tests). Returns the
|
||||||
|
largest match (the real video, not a stray same-named bit), or None."""
|
||||||
|
base = basename_of(filename)
|
||||||
|
if not base or not download_dir:
|
||||||
|
return None
|
||||||
|
matches = [p for p in lister(download_dir) if basename_of(p) == base]
|
||||||
|
if not matches:
|
||||||
|
return None
|
||||||
|
return matches[0] if len(matches) == 1 else max(matches, key=len)
|
||||||
|
|
||||||
|
|
||||||
|
def dest_path_for(target_dir: str, src_path: str) -> str:
|
||||||
|
"""Where a finished file moves to inside its library folder (flat, by basename)."""
|
||||||
|
return os.path.join(str(target_dir or ""), basename_of(src_path))
|
||||||
|
|
||||||
|
|
||||||
|
def target_dir_for(kind: str, paths: dict) -> str:
|
||||||
|
"""Pick the library folder for a download's kind. ``paths`` = the config dict
|
||||||
|
({movies_path, tv_path, youtube_path})."""
|
||||||
|
paths = paths or {}
|
||||||
|
k = str(kind or "").lower()
|
||||||
|
if k == "movie":
|
||||||
|
return paths.get("movies_path") or ""
|
||||||
|
if k in ("show", "tv", "episode", "season", "series"):
|
||||||
|
return paths.get("tv_path") or ""
|
||||||
|
if k == "youtube":
|
||||||
|
return paths.get("youtube_path") or ""
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["basename_of", "find_completed_file", "dest_path_for", "target_dir_for"]
|
||||||
7
core/video/enrichment/__init__.py
Normal file
7
core/video/enrichment/__init__.py
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
"""SoulSync — isolated VIDEO enrichment subsystem.
|
||||||
|
|
||||||
|
Mirrors the music enrichment-worker pattern (per-source match status, a worker
|
||||||
|
loop, a registry) but operates entirely on video.db and never imports the music
|
||||||
|
enrichment code. The shared Manage-Workers modal / worker-orbs visuals are made
|
||||||
|
kind-aware separately; this package is the backend half.
|
||||||
|
"""
|
||||||
814
core/video/enrichment/backfill.py
Normal file
814
core/video/enrichment/backfill.py
Normal file
|
|
@ -0,0 +1,814 @@
|
||||||
|
"""Video BACKFILL workers — enrich an already-identified item BY id.
|
||||||
|
|
||||||
|
The matcher workers (worker.py) find an external id for a title. These workers
|
||||||
|
take items we can already address (a tmdb/imdb/tvdb id, or a YouTube video id)
|
||||||
|
and fetch SUPPLEMENTARY data for them: artwork (fanart.tv), subtitle
|
||||||
|
availability (OpenSubtitles), and the no-key YouTube extras — Return YouTube
|
||||||
|
Dislike (like/dislike estimates) and SponsorBlock (crowd segments).
|
||||||
|
|
||||||
|
Each worker presents the EXACT same lifecycle + get_stats() shape as
|
||||||
|
VideoEnrichmentWorker, so the engine registry, the /api/video/enrichment routes,
|
||||||
|
and the Manage-Workers modal (cards / animations / pause-resume) drive them
|
||||||
|
identically — a new worker is just another entry in engine.workers.
|
||||||
|
|
||||||
|
Isolated: imports only video.db helpers + requests; no music code.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_enrichment.backfill")
|
||||||
|
|
||||||
|
_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||||
|
"(KHTML, like Gecko) Chrome/124.0 Safari/537.36")
|
||||||
|
|
||||||
|
|
||||||
|
# ── tiny HTTP helper with the status semantics the workers rely on ────────────
|
||||||
|
class _RateLimited(Exception):
|
||||||
|
def __init__(self, retry_after=60):
|
||||||
|
self.retry_after = retry_after
|
||||||
|
|
||||||
|
|
||||||
|
class _Unauthorized(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _http_get_json(url, params=None, headers=None, timeout=12):
|
||||||
|
"""GET → parsed JSON (list or dict), or None for a 404 / unparseable body.
|
||||||
|
Raises _Unauthorized (401/403), _RateLimited (429), or the underlying error
|
||||||
|
so the worker can record 'error' vs back off vs mark 'not_found'."""
|
||||||
|
import requests
|
||||||
|
h = {"User-Agent": _UA}
|
||||||
|
if headers:
|
||||||
|
h.update(headers)
|
||||||
|
r = requests.get(url, params=params, headers=h, timeout=timeout)
|
||||||
|
if r.status_code == 404:
|
||||||
|
return None
|
||||||
|
if r.status_code in (401, 403):
|
||||||
|
raise _Unauthorized()
|
||||||
|
if r.status_code == 429:
|
||||||
|
try:
|
||||||
|
ra = int(r.headers.get("Retry-After") or 60)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
ra = 60
|
||||||
|
raise _RateLimited(ra)
|
||||||
|
r.raise_for_status()
|
||||||
|
try:
|
||||||
|
return r.json()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _http_post_json(url, json_body, headers=None, timeout=12):
|
||||||
|
"""POST JSON → parsed JSON, with the same status semantics as _http_get_json
|
||||||
|
(for GraphQL services like AniList). 404 → None; 401/403 → _Unauthorized;
|
||||||
|
429 → _RateLimited; other non-2xx → raises."""
|
||||||
|
import requests
|
||||||
|
h = {"User-Agent": _UA, "Content-Type": "application/json", "Accept": "application/json"}
|
||||||
|
if headers:
|
||||||
|
h.update(headers)
|
||||||
|
r = requests.post(url, json=json_body, headers=h, timeout=timeout)
|
||||||
|
if r.status_code == 404:
|
||||||
|
return None
|
||||||
|
if r.status_code in (401, 403):
|
||||||
|
raise _Unauthorized()
|
||||||
|
if r.status_code == 429:
|
||||||
|
try:
|
||||||
|
ra = int(r.headers.get("Retry-After") or 60)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
ra = 60
|
||||||
|
raise _RateLimited(ra)
|
||||||
|
r.raise_for_status()
|
||||||
|
try:
|
||||||
|
return r.json()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _norm_title(s):
|
||||||
|
"""Lowercase alphanumerics only — for conservative title matching."""
|
||||||
|
return re.sub(r"[^a-z0-9]+", "", str(s or "").lower())
|
||||||
|
|
||||||
|
|
||||||
|
# ── base worker (lifecycle + loop + status; mirrors VideoEnrichmentWorker) ────
|
||||||
|
class VideoBackfillWorker:
|
||||||
|
is_ratings = False
|
||||||
|
requires_key = False # True for workers gated on an API key (vs a keyless toggle)
|
||||||
|
|
||||||
|
def __init__(self, db, service, display_name, interval=1.0):
|
||||||
|
self.db = db
|
||||||
|
self.service = service
|
||||||
|
self.display_name = display_name
|
||||||
|
self.interval = interval
|
||||||
|
self.running = False
|
||||||
|
self.paused = False
|
||||||
|
self.should_stop = False
|
||||||
|
self._thread = None
|
||||||
|
self._stop = threading.Event()
|
||||||
|
self.current_item = None
|
||||||
|
self.stats = {"matched": 0, "not_found": 0, "errors": 0}
|
||||||
|
self.note = None
|
||||||
|
self._cooldown_until = 0.0
|
||||||
|
|
||||||
|
# ── lifecycle ─────────────────────────────────────────────────────────────
|
||||||
|
def start(self):
|
||||||
|
if self.running:
|
||||||
|
return
|
||||||
|
self.should_stop = False
|
||||||
|
self._stop.clear()
|
||||||
|
self.running = True
|
||||||
|
self._thread = threading.Thread(target=self._run, daemon=True)
|
||||||
|
self._thread.start()
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
self.should_stop = True
|
||||||
|
self._stop.set()
|
||||||
|
if self._thread:
|
||||||
|
self._thread.join(timeout=1.0)
|
||||||
|
self.running = False
|
||||||
|
|
||||||
|
def pause(self, persist=True):
|
||||||
|
self.paused = True
|
||||||
|
if persist:
|
||||||
|
self._persist_paused()
|
||||||
|
|
||||||
|
def resume(self, persist=True):
|
||||||
|
self.paused = False
|
||||||
|
if persist:
|
||||||
|
self._persist_paused()
|
||||||
|
|
||||||
|
def _persist_paused(self):
|
||||||
|
try:
|
||||||
|
self.db.set_setting(self.service + "_paused", "1" if self.paused else "0")
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video backfill: could not persist pause for %s", self.service)
|
||||||
|
|
||||||
|
def restore_paused(self):
|
||||||
|
try:
|
||||||
|
self.paused = str(self.db.get_setting(self.service + "_paused") or "") == "1"
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video backfill: could not restore pause for %s", self.service)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def enabled(self):
|
||||||
|
try:
|
||||||
|
return self._enabled()
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# ── subclass hooks ────────────────────────────────────────────────────────
|
||||||
|
def _enabled(self) -> bool:
|
||||||
|
return True
|
||||||
|
|
||||||
|
def test(self):
|
||||||
|
return (True, self.display_name + " OK")
|
||||||
|
|
||||||
|
def next_item(self):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def fetch(self, item):
|
||||||
|
"""Return data (truthy) on a hit, falsy for a genuine 'no data', or raise
|
||||||
|
on a call failure (network/rate-limit/auth)."""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def record_ok(self, item, data):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def record_empty(self, item):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def record_error(self, item):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def breakdown(self) -> dict:
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
# ── loop ──────────────────────────────────────────────────────────────────
|
||||||
|
def _run(self):
|
||||||
|
while not self.should_stop:
|
||||||
|
if self.paused or not self.enabled:
|
||||||
|
self._stop.wait(1.0)
|
||||||
|
continue
|
||||||
|
if self._cooldown_until > time.monotonic():
|
||||||
|
self.current_item = None
|
||||||
|
self._stop.wait(15.0)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
did = self.process_one()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video backfill %s loop error", self.service)
|
||||||
|
self.stats["errors"] += 1
|
||||||
|
self._stop.wait(5.0)
|
||||||
|
continue
|
||||||
|
if did:
|
||||||
|
self._stop.wait(self.interval)
|
||||||
|
else:
|
||||||
|
self.current_item = None
|
||||||
|
self._stop.wait(10.0)
|
||||||
|
|
||||||
|
def process_one(self) -> bool:
|
||||||
|
item = self.next_item()
|
||||||
|
if not item:
|
||||||
|
return False
|
||||||
|
self.current_item = {"type": item.get("kind"), "name": item.get("title")}
|
||||||
|
try:
|
||||||
|
data = self.fetch(item)
|
||||||
|
except _RateLimited as e:
|
||||||
|
self._cooldown_until = time.monotonic() + max(15, e.retry_after)
|
||||||
|
self.note = self.display_name + " rate-limited — backing off"
|
||||||
|
logger.warning("%s rate-limited; idling %ss", self.display_name, e.retry_after)
|
||||||
|
return False
|
||||||
|
except _Unauthorized:
|
||||||
|
self.note = self.display_name + " rejected the API key"
|
||||||
|
self.pause(persist=False) # transient: fixing the key rebuilds the engine
|
||||||
|
logger.warning("%s rejected the API key — pausing until fixed", self.display_name)
|
||||||
|
return False
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video backfill %s fetch failed for %s", self.service, item.get("title"))
|
||||||
|
self.stats["errors"] += 1
|
||||||
|
self.record_error(item)
|
||||||
|
return True
|
||||||
|
self.note = None
|
||||||
|
if data:
|
||||||
|
self.record_ok(item, data)
|
||||||
|
self.stats["matched"] += 1
|
||||||
|
logger.info("Enriched %s '%s' via %s", item.get("kind"), item.get("title"), self.display_name)
|
||||||
|
else:
|
||||||
|
self.record_empty(item)
|
||||||
|
self.stats["not_found"] += 1
|
||||||
|
return True
|
||||||
|
|
||||||
|
# ── status (same shape the music/video enrichment API returns) ────────────
|
||||||
|
def get_stats(self) -> dict:
|
||||||
|
breakdown = self.breakdown()
|
||||||
|
pending = sum(b["pending"] + b.get("errors", 0)
|
||||||
|
for b in breakdown.values() if not b.get("coverage_only"))
|
||||||
|
cooling = self._cooldown_until > time.monotonic()
|
||||||
|
running = self.running and not self.paused and self.enabled and not cooling
|
||||||
|
idle = running and pending == 0 and self.current_item is None
|
||||||
|
progress = {}
|
||||||
|
for kind, b in breakdown.items():
|
||||||
|
total = b["matched"] + b["not_found"] + b.get("errors", 0) + b["pending"]
|
||||||
|
done = b["matched"] + b["not_found"]
|
||||||
|
progress[kind] = {"matched": b["matched"], "total": total,
|
||||||
|
"percent": round(done / total * 100) if total else 0}
|
||||||
|
return {
|
||||||
|
"enabled": self.enabled,
|
||||||
|
"needs_key": self.requires_key, # key-gated → "Not configured"; keyless → "Disabled"
|
||||||
|
"running": running,
|
||||||
|
"paused": self.paused or cooling,
|
||||||
|
"idle": idle,
|
||||||
|
"current_item": self.current_item,
|
||||||
|
"note": self.note,
|
||||||
|
"cooldown": cooling,
|
||||||
|
"stats": {**self.stats, "pending": pending},
|
||||||
|
"progress": progress,
|
||||||
|
"breakdown": breakdown,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ── Return YouTube Dislike (no key) ───────────────────────────────────────────
|
||||||
|
class RydWorker(VideoBackfillWorker):
|
||||||
|
URL = "https://returnyoutubedislikeapi.com/votes"
|
||||||
|
|
||||||
|
def __init__(self, db):
|
||||||
|
super().__init__(db, "ryd", "Return YouTube Dislike", interval=0.6)
|
||||||
|
|
||||||
|
def _enabled(self):
|
||||||
|
return str(self.db.get_setting("ryd_enabled") or "1") != "0"
|
||||||
|
|
||||||
|
def test(self):
|
||||||
|
try:
|
||||||
|
j = _http_get_json(self.URL, {"videoId": "dQw4w9WgXcQ"})
|
||||||
|
return (j is not None, "Return YouTube Dislike reachable" if j else "No response")
|
||||||
|
except Exception as e:
|
||||||
|
return (False, str(e))
|
||||||
|
|
||||||
|
def next_item(self):
|
||||||
|
return self.db.youtube_enrich_next("ryd_status")
|
||||||
|
|
||||||
|
def fetch(self, item):
|
||||||
|
j = _http_get_json(self.URL, {"videoId": item["youtube_id"]})
|
||||||
|
if not isinstance(j, dict) or j.get("dislikes") is None:
|
||||||
|
return None
|
||||||
|
return {"likes": j.get("likes"), "dislikes": j.get("dislikes")}
|
||||||
|
|
||||||
|
def record_ok(self, item, data):
|
||||||
|
self.db.apply_youtube_votes(item["youtube_id"], data.get("likes"), data.get("dislikes"), "ok")
|
||||||
|
|
||||||
|
def record_empty(self, item):
|
||||||
|
self.db.apply_youtube_votes(item["youtube_id"], None, None, "not_found")
|
||||||
|
|
||||||
|
def record_error(self, item):
|
||||||
|
self.db.apply_youtube_votes(item["youtube_id"], None, None, "error")
|
||||||
|
|
||||||
|
def breakdown(self):
|
||||||
|
return self.db.youtube_enrich_breakdown("ryd_status")
|
||||||
|
|
||||||
|
|
||||||
|
# ── SponsorBlock (no key) ─────────────────────────────────────────────────────
|
||||||
|
_SB_CATS = ["sponsor", "selfpromo", "interaction", "intro", "outro",
|
||||||
|
"preview", "music_offtopic", "filler", "poi_highlight"]
|
||||||
|
|
||||||
|
|
||||||
|
class SponsorBlockWorker(VideoBackfillWorker):
|
||||||
|
URL = "https://sponsor.ajay.app/api/skipSegments"
|
||||||
|
|
||||||
|
def __init__(self, db):
|
||||||
|
super().__init__(db, "sponsorblock", "SponsorBlock", interval=0.6)
|
||||||
|
|
||||||
|
def _enabled(self):
|
||||||
|
return str(self.db.get_setting("sponsorblock_enabled") or "1") != "0"
|
||||||
|
|
||||||
|
def test(self):
|
||||||
|
try:
|
||||||
|
_http_get_json(self.URL, {"videoID": "dQw4w9WgXcQ", "categories": json.dumps(_SB_CATS)})
|
||||||
|
return (True, "SponsorBlock reachable")
|
||||||
|
except Exception as e:
|
||||||
|
return (False, str(e))
|
||||||
|
|
||||||
|
def next_item(self):
|
||||||
|
return self.db.youtube_enrich_next("sb_status")
|
||||||
|
|
||||||
|
def fetch(self, item):
|
||||||
|
data = _http_get_json(self.URL, {"videoID": item["youtube_id"],
|
||||||
|
"categories": json.dumps(_SB_CATS)})
|
||||||
|
if not isinstance(data, list) or not data:
|
||||||
|
return None
|
||||||
|
segs = []
|
||||||
|
for s in data:
|
||||||
|
seg = s.get("segment") or []
|
||||||
|
if len(seg) != 2 or not s.get("UUID") or not s.get("category"):
|
||||||
|
continue
|
||||||
|
segs.append({"category": s.get("category"), "start_sec": seg[0], "end_sec": seg[1],
|
||||||
|
"votes": s.get("votes"), "uuid": s.get("UUID")})
|
||||||
|
return segs or None
|
||||||
|
|
||||||
|
def record_ok(self, item, data):
|
||||||
|
self.db.apply_youtube_segments(item["youtube_id"], data, "ok")
|
||||||
|
|
||||||
|
def record_empty(self, item):
|
||||||
|
self.db.apply_youtube_segments(item["youtube_id"], None, "not_found")
|
||||||
|
|
||||||
|
def record_error(self, item):
|
||||||
|
self.db.apply_youtube_segments(item["youtube_id"], None, "error")
|
||||||
|
|
||||||
|
def breakdown(self):
|
||||||
|
return self.db.youtube_enrich_breakdown("sb_status")
|
||||||
|
|
||||||
|
|
||||||
|
# ── fanart.tv (free key) ──────────────────────────────────────────────────────
|
||||||
|
def _fa_first(j, *keys):
|
||||||
|
"""First artwork URL from the first present key, preferring English / textless
|
||||||
|
(lang '') and most-liked."""
|
||||||
|
for k in keys:
|
||||||
|
arr = j.get(k) or []
|
||||||
|
if not isinstance(arr, list) or not arr:
|
||||||
|
continue
|
||||||
|
best = sorted(arr, key=lambda a: (a.get("lang") not in ("en", ""),
|
||||||
|
-int(a.get("likes") or 0)))
|
||||||
|
url = best[0].get("url") if best else None
|
||||||
|
if url:
|
||||||
|
return url
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class FanartWorker(VideoBackfillWorker):
|
||||||
|
BASE = "https://webservice.fanart.tv/v3"
|
||||||
|
requires_key = True
|
||||||
|
|
||||||
|
def __init__(self, db):
|
||||||
|
super().__init__(db, "fanart", "fanart.tv", interval=1.0)
|
||||||
|
|
||||||
|
def _key(self):
|
||||||
|
return (self.db.get_setting("fanart_api_key") or "").strip()
|
||||||
|
|
||||||
|
def _enabled(self):
|
||||||
|
return bool(self._key())
|
||||||
|
|
||||||
|
def test(self):
|
||||||
|
key = self._key()
|
||||||
|
if not key:
|
||||||
|
return (False, "No fanart.tv API key")
|
||||||
|
try:
|
||||||
|
j = _http_get_json(self.BASE + "/movies/550", {"api_key": key})
|
||||||
|
return (j is not None, "fanart.tv key OK" if j else "No artwork returned")
|
||||||
|
except _Unauthorized:
|
||||||
|
return (False, "fanart.tv rejected the API key")
|
||||||
|
except Exception as e:
|
||||||
|
return (False, str(e))
|
||||||
|
|
||||||
|
def next_item(self):
|
||||||
|
return self.db.backfill_next("fanart")
|
||||||
|
|
||||||
|
def fetch(self, item):
|
||||||
|
key = self._key()
|
||||||
|
if not key:
|
||||||
|
return None
|
||||||
|
if item["kind"] == "movie":
|
||||||
|
ident = item.get("tmdb_id") or item.get("imdb_id")
|
||||||
|
if not ident:
|
||||||
|
return None
|
||||||
|
j = _http_get_json(self.BASE + "/movies/" + str(ident), {"api_key": key})
|
||||||
|
if not isinstance(j, dict):
|
||||||
|
return None
|
||||||
|
out = {"logo_url": _fa_first(j, "hdmovielogo", "clearlogo"),
|
||||||
|
"clearart_url": _fa_first(j, "hdmovieclearart", "movieart"),
|
||||||
|
"banner_url": _fa_first(j, "moviebanner"),
|
||||||
|
"backdrop_url": _fa_first(j, "moviebackground"),
|
||||||
|
"poster_url": _fa_first(j, "movieposter")}
|
||||||
|
else:
|
||||||
|
ident = item.get("tvdb_id")
|
||||||
|
if not ident:
|
||||||
|
return None
|
||||||
|
j = _http_get_json(self.BASE + "/tv/" + str(ident), {"api_key": key})
|
||||||
|
if not isinstance(j, dict):
|
||||||
|
return None
|
||||||
|
out = {"logo_url": _fa_first(j, "hdtvlogo", "clearlogo"),
|
||||||
|
"clearart_url": _fa_first(j, "hdclearart", "clearart"),
|
||||||
|
"banner_url": _fa_first(j, "tvbanner"),
|
||||||
|
"backdrop_url": _fa_first(j, "showbackground"),
|
||||||
|
"poster_url": _fa_first(j, "tvposter")}
|
||||||
|
out = {k: v for k, v in out.items() if v}
|
||||||
|
return out or None
|
||||||
|
|
||||||
|
def record_ok(self, item, data):
|
||||||
|
self.db.backfill_mark("fanart", item["kind"], item["id"], "ok", columns=data)
|
||||||
|
|
||||||
|
def record_empty(self, item):
|
||||||
|
self.db.backfill_mark("fanart", item["kind"], item["id"], "not_found")
|
||||||
|
|
||||||
|
def record_error(self, item):
|
||||||
|
self.db.backfill_mark("fanart", item["kind"], item["id"], "error")
|
||||||
|
|
||||||
|
def breakdown(self):
|
||||||
|
return self.db.backfill_breakdown("fanart")
|
||||||
|
|
||||||
|
|
||||||
|
# ── OpenSubtitles (free key) — subtitle-language availability ─────────────────
|
||||||
|
class OpenSubtitlesWorker(VideoBackfillWorker):
|
||||||
|
BASE = "https://api.opensubtitles.com/api/v1"
|
||||||
|
requires_key = True
|
||||||
|
|
||||||
|
def __init__(self, db):
|
||||||
|
super().__init__(db, "opensubtitles", "OpenSubtitles", interval=1.5)
|
||||||
|
|
||||||
|
def _key(self):
|
||||||
|
return (self.db.get_setting("opensubtitles_api_key") or "").strip()
|
||||||
|
|
||||||
|
def _enabled(self):
|
||||||
|
return bool(self._key())
|
||||||
|
|
||||||
|
def _headers(self):
|
||||||
|
return {"Api-Key": self._key(), "Accept": "application/json",
|
||||||
|
"User-Agent": "SoulSync v1.0"}
|
||||||
|
|
||||||
|
def test(self):
|
||||||
|
if not self._key():
|
||||||
|
return (False, "No OpenSubtitles API key")
|
||||||
|
try:
|
||||||
|
j = _http_get_json(self.BASE + "/subtitles", {"tmdb_id": 550, "languages": "en"},
|
||||||
|
headers=self._headers())
|
||||||
|
return (j is not None, "OpenSubtitles key OK" if j else "No response")
|
||||||
|
except _Unauthorized:
|
||||||
|
return (False, "OpenSubtitles rejected the API key")
|
||||||
|
except Exception as e:
|
||||||
|
return (False, str(e))
|
||||||
|
|
||||||
|
def next_item(self):
|
||||||
|
return self.db.backfill_next("opensubtitles")
|
||||||
|
|
||||||
|
def fetch(self, item):
|
||||||
|
if not self._key():
|
||||||
|
return None
|
||||||
|
params = {}
|
||||||
|
imdb = str(item.get("imdb_id") or "").lower()
|
||||||
|
if imdb.startswith("tt"):
|
||||||
|
imdb = imdb[2:]
|
||||||
|
if imdb:
|
||||||
|
params["imdb_id"] = imdb
|
||||||
|
elif item.get("tmdb_id"):
|
||||||
|
params["tmdb_id"] = item["tmdb_id"]
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
j = _http_get_json(self.BASE + "/subtitles", params, headers=self._headers())
|
||||||
|
if not isinstance(j, dict):
|
||||||
|
return None
|
||||||
|
langs = set()
|
||||||
|
for row in (j.get("data") or []):
|
||||||
|
lng = ((row.get("attributes") or {}).get("language") or "").lower()
|
||||||
|
if lng:
|
||||||
|
langs.add(lng)
|
||||||
|
if not langs:
|
||||||
|
return None
|
||||||
|
return {"subtitle_langs": json.dumps(sorted(langs))}
|
||||||
|
|
||||||
|
def record_ok(self, item, data):
|
||||||
|
self.db.backfill_mark("opensubtitles", item["kind"], item["id"], "ok", columns=data)
|
||||||
|
|
||||||
|
def record_empty(self, item):
|
||||||
|
self.db.backfill_mark("opensubtitles", item["kind"], item["id"], "not_found")
|
||||||
|
|
||||||
|
def record_error(self, item):
|
||||||
|
self.db.backfill_mark("opensubtitles", item["kind"], item["id"], "error")
|
||||||
|
|
||||||
|
def breakdown(self):
|
||||||
|
return self.db.backfill_breakdown("opensubtitles")
|
||||||
|
|
||||||
|
|
||||||
|
# ── Trakt (free API key) — community audience rating + vote count ─────────────
|
||||||
|
class TraktWorker(VideoBackfillWorker):
|
||||||
|
BASE = "https://api.trakt.tv"
|
||||||
|
requires_key = True
|
||||||
|
|
||||||
|
def __init__(self, db):
|
||||||
|
super().__init__(db, "trakt", "Trakt", interval=1.0)
|
||||||
|
|
||||||
|
def _key(self):
|
||||||
|
return (self.db.get_setting("trakt_api_key") or "").strip()
|
||||||
|
|
||||||
|
def _enabled(self):
|
||||||
|
return bool(self._key())
|
||||||
|
|
||||||
|
def _headers(self):
|
||||||
|
return {"trakt-api-key": self._key(), "trakt-api-version": "2",
|
||||||
|
"Content-Type": "application/json"}
|
||||||
|
|
||||||
|
def test(self):
|
||||||
|
if not self._key():
|
||||||
|
return (False, "No Trakt API key")
|
||||||
|
try:
|
||||||
|
j = _http_get_json(self.BASE + "/shows/trending", {"limit": 1}, headers=self._headers())
|
||||||
|
return (j is not None, "Trakt key OK" if j is not None else "No response")
|
||||||
|
except _Unauthorized:
|
||||||
|
return (False, "Trakt rejected the API key (check the Client ID)")
|
||||||
|
except Exception as e:
|
||||||
|
return (False, str(e))
|
||||||
|
|
||||||
|
def next_item(self):
|
||||||
|
return self.db.backfill_next("trakt")
|
||||||
|
|
||||||
|
def fetch(self, item):
|
||||||
|
# Trakt accepts an IMDb id directly as the {id} slug; ?extended=full carries
|
||||||
|
# the community rating + vote count on the summary.
|
||||||
|
if not self._key():
|
||||||
|
return None
|
||||||
|
imdb = str(item.get("imdb_id") or "").strip()
|
||||||
|
if not imdb.lower().startswith("tt"):
|
||||||
|
return None
|
||||||
|
typ = "movies" if item["kind"] == "movie" else "shows"
|
||||||
|
j = _http_get_json(self.BASE + "/" + typ + "/" + imdb, {"extended": "full"},
|
||||||
|
headers=self._headers())
|
||||||
|
if not isinstance(j, dict):
|
||||||
|
return None
|
||||||
|
out = {}
|
||||||
|
rating = j.get("rating")
|
||||||
|
if isinstance(rating, (int, float)) and rating > 0:
|
||||||
|
out["trakt_rating"] = round(float(rating), 1)
|
||||||
|
votes = j.get("votes")
|
||||||
|
if isinstance(votes, int) and votes > 0:
|
||||||
|
out["trakt_votes"] = votes
|
||||||
|
return out or None
|
||||||
|
|
||||||
|
def record_ok(self, item, data):
|
||||||
|
self.db.backfill_mark("trakt", item["kind"], item["id"], "ok", columns=data)
|
||||||
|
|
||||||
|
def record_empty(self, item):
|
||||||
|
self.db.backfill_mark("trakt", item["kind"], item["id"], "not_found")
|
||||||
|
|
||||||
|
def record_error(self, item):
|
||||||
|
self.db.backfill_mark("trakt", item["kind"], item["id"], "error")
|
||||||
|
|
||||||
|
def breakdown(self):
|
||||||
|
return self.db.backfill_breakdown("trakt")
|
||||||
|
|
||||||
|
|
||||||
|
# ── TVmaze (no key) — TV community rating ─────────────────────────────────────
|
||||||
|
class TVmazeWorker(VideoBackfillWorker):
|
||||||
|
BASE = "https://api.tvmaze.com"
|
||||||
|
|
||||||
|
def __init__(self, db):
|
||||||
|
super().__init__(db, "tvmaze", "TVmaze", interval=0.8)
|
||||||
|
|
||||||
|
def _enabled(self):
|
||||||
|
# Free, keyless — on by default; user can switch it off in settings.
|
||||||
|
return str(self.db.get_setting("tvmaze_enabled") or "1") != "0"
|
||||||
|
|
||||||
|
def test(self):
|
||||||
|
try:
|
||||||
|
j = _http_get_json(self.BASE + "/lookup/shows", {"imdb": "tt0903747"}) # Breaking Bad
|
||||||
|
return (j is not None, "TVmaze reachable" if j is not None else "No response")
|
||||||
|
except Exception as e:
|
||||||
|
return (False, str(e))
|
||||||
|
|
||||||
|
def next_item(self):
|
||||||
|
return self.db.backfill_next("tvmaze")
|
||||||
|
|
||||||
|
def fetch(self, item):
|
||||||
|
imdb = str(item.get("imdb_id") or "").strip()
|
||||||
|
tvdb = item.get("tvdb_id")
|
||||||
|
if imdb.lower().startswith("tt"):
|
||||||
|
params = {"imdb": imdb}
|
||||||
|
elif tvdb:
|
||||||
|
params = {"thetvdb": tvdb}
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
j = _http_get_json(self.BASE + "/lookup/shows", params)
|
||||||
|
if not isinstance(j, dict):
|
||||||
|
return None
|
||||||
|
rating = (j.get("rating") or {}).get("average")
|
||||||
|
if isinstance(rating, (int, float)) and rating > 0:
|
||||||
|
return {"tvmaze_rating": round(float(rating), 1)}
|
||||||
|
return None
|
||||||
|
|
||||||
|
def record_ok(self, item, data):
|
||||||
|
self.db.backfill_mark("tvmaze", item["kind"], item["id"], "ok", columns=data)
|
||||||
|
|
||||||
|
def record_empty(self, item):
|
||||||
|
self.db.backfill_mark("tvmaze", item["kind"], item["id"], "not_found")
|
||||||
|
|
||||||
|
def record_error(self, item):
|
||||||
|
self.db.backfill_mark("tvmaze", item["kind"], item["id"], "error")
|
||||||
|
|
||||||
|
def breakdown(self):
|
||||||
|
return self.db.backfill_breakdown("tvmaze")
|
||||||
|
|
||||||
|
|
||||||
|
# ── AniList (no key, GraphQL) — anime average score ───────────────────────────
|
||||||
|
_ANILIST_QUERY = (
|
||||||
|
"query($search:String){Media(search:$search,type:ANIME){"
|
||||||
|
"averageScore title{romaji english}}}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class AniListWorker(VideoBackfillWorker):
|
||||||
|
BASE = "https://graphql.anilist.co"
|
||||||
|
|
||||||
|
def __init__(self, db):
|
||||||
|
super().__init__(db, "anilist", "AniList", interval=1.0)
|
||||||
|
|
||||||
|
def _enabled(self):
|
||||||
|
# OFF by default — anime-only + title-search matching, so it's opt-in to
|
||||||
|
# avoid touching every show in a non-anime library.
|
||||||
|
return str(self.db.get_setting("anilist_enabled") or "0") == "1"
|
||||||
|
|
||||||
|
def test(self):
|
||||||
|
try:
|
||||||
|
j = _http_post_json(self.BASE, {"query": _ANILIST_QUERY,
|
||||||
|
"variables": {"search": "Cowboy Bebop"}})
|
||||||
|
ok = isinstance(j, dict) and (j.get("data") or {}).get("Media") is not None
|
||||||
|
return (ok, "AniList reachable" if ok else "No response")
|
||||||
|
except Exception as e:
|
||||||
|
return (False, str(e))
|
||||||
|
|
||||||
|
def next_item(self):
|
||||||
|
return self.db.backfill_next("anilist")
|
||||||
|
|
||||||
|
def fetch(self, item):
|
||||||
|
title = item.get("title")
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
j = _http_post_json(self.BASE, {"query": _ANILIST_QUERY, "variables": {"search": title}})
|
||||||
|
media = (j.get("data") or {}).get("Media") if isinstance(j, dict) else None
|
||||||
|
if not isinstance(media, dict):
|
||||||
|
return None
|
||||||
|
# Conservative guard: only accept when AniList's title actually matches ours
|
||||||
|
# (anime search is fuzzy and would otherwise score random non-anime shows).
|
||||||
|
names = media.get("title") or {}
|
||||||
|
want = _norm_title(title)
|
||||||
|
got = {_norm_title(names.get("romaji")), _norm_title(names.get("english"))}
|
||||||
|
if not any(g and (g == want or g in want or want in g) for g in got):
|
||||||
|
return None
|
||||||
|
score = media.get("averageScore")
|
||||||
|
if isinstance(score, int) and 0 < score <= 100:
|
||||||
|
return {"anilist_score": score}
|
||||||
|
return None
|
||||||
|
|
||||||
|
def record_ok(self, item, data):
|
||||||
|
self.db.backfill_mark("anilist", item["kind"], item["id"], "ok", columns=data)
|
||||||
|
|
||||||
|
def record_empty(self, item):
|
||||||
|
self.db.backfill_mark("anilist", item["kind"], item["id"], "not_found")
|
||||||
|
|
||||||
|
def record_error(self, item):
|
||||||
|
self.db.backfill_mark("anilist", item["kind"], item["id"], "error")
|
||||||
|
|
||||||
|
def breakdown(self):
|
||||||
|
return self.db.backfill_breakdown("anilist")
|
||||||
|
|
||||||
|
|
||||||
|
# ── DeArrow (no key) — crowd-sourced better titles for YouTube videos ─────────
|
||||||
|
class DeArrowWorker(VideoBackfillWorker):
|
||||||
|
URL = "https://sponsor.ajay.app/api/branding"
|
||||||
|
|
||||||
|
def __init__(self, db):
|
||||||
|
super().__init__(db, "dearrow", "DeArrow", interval=0.6)
|
||||||
|
|
||||||
|
def _enabled(self):
|
||||||
|
return str(self.db.get_setting("dearrow_enabled") or "1") != "0"
|
||||||
|
|
||||||
|
def test(self):
|
||||||
|
try:
|
||||||
|
j = _http_get_json(self.URL, {"videoID": "dQw4w9WgXcQ"})
|
||||||
|
return (j is not None, "DeArrow reachable" if j is not None else "No response")
|
||||||
|
except Exception as e:
|
||||||
|
return (False, str(e))
|
||||||
|
|
||||||
|
def next_item(self):
|
||||||
|
return self.db.youtube_enrich_next("dearrow_status")
|
||||||
|
|
||||||
|
def fetch(self, item):
|
||||||
|
j = _http_get_json(self.URL, {"videoID": item["youtube_id"]})
|
||||||
|
if not isinstance(j, dict):
|
||||||
|
return None
|
||||||
|
# DeArrow returns crowd titles in preference order; take the first that
|
||||||
|
# isn't the YouTube original.
|
||||||
|
for t in (j.get("titles") or []):
|
||||||
|
if isinstance(t, dict) and not t.get("original"):
|
||||||
|
title = (t.get("title") or "").strip()
|
||||||
|
if title:
|
||||||
|
return {"title": title}
|
||||||
|
return None
|
||||||
|
|
||||||
|
def record_ok(self, item, data):
|
||||||
|
self.db.apply_youtube_dearrow(item["youtube_id"], data.get("title"), "ok")
|
||||||
|
|
||||||
|
def record_empty(self, item):
|
||||||
|
self.db.apply_youtube_dearrow(item["youtube_id"], None, "not_found")
|
||||||
|
|
||||||
|
def record_error(self, item):
|
||||||
|
self.db.apply_youtube_dearrow(item["youtube_id"], None, "error")
|
||||||
|
|
||||||
|
def breakdown(self):
|
||||||
|
return self.db.youtube_enrich_breakdown("dearrow_status")
|
||||||
|
|
||||||
|
|
||||||
|
# ── Wikidata (no key) — the title's official website ──────────────────────────
|
||||||
|
class WikidataWorker(VideoBackfillWorker):
|
||||||
|
API = "https://www.wikidata.org/w/api.php"
|
||||||
|
|
||||||
|
def __init__(self, db):
|
||||||
|
super().__init__(db, "wikidata", "Wikidata", interval=1.0)
|
||||||
|
|
||||||
|
def _enabled(self):
|
||||||
|
return str(self.db.get_setting("wikidata_enabled") or "1") != "0"
|
||||||
|
|
||||||
|
def _entity_for_imdb(self, imdb):
|
||||||
|
s = _http_get_json(self.API, {"action": "query", "list": "search",
|
||||||
|
"srsearch": "haswbstatement:P345=" + imdb,
|
||||||
|
"srlimit": 1, "format": "json"})
|
||||||
|
hits = (((s or {}).get("query") or {}).get("search") or [])
|
||||||
|
return hits[0].get("title") if hits else None
|
||||||
|
|
||||||
|
def test(self):
|
||||||
|
try:
|
||||||
|
ok = self._entity_for_imdb("tt0137523") is not None # Fight Club
|
||||||
|
return (ok, "Wikidata reachable" if ok else "No response")
|
||||||
|
except Exception as e:
|
||||||
|
return (False, str(e))
|
||||||
|
|
||||||
|
def next_item(self):
|
||||||
|
return self.db.backfill_next("wikidata")
|
||||||
|
|
||||||
|
def fetch(self, item):
|
||||||
|
imdb = str(item.get("imdb_id") or "").strip()
|
||||||
|
if not imdb.lower().startswith("tt"):
|
||||||
|
return None
|
||||||
|
qid = self._entity_for_imdb(imdb)
|
||||||
|
if not qid:
|
||||||
|
return None
|
||||||
|
e = _http_get_json(self.API, {"action": "wbgetentities", "ids": qid,
|
||||||
|
"props": "claims", "format": "json"})
|
||||||
|
claims = (((e or {}).get("entities") or {}).get(qid) or {}).get("claims") or {}
|
||||||
|
for c in (claims.get("P856") or []): # P856 = official website
|
||||||
|
url = ((c.get("mainsnak") or {}).get("datavalue") or {}).get("value")
|
||||||
|
if isinstance(url, str) and url.startswith("http"):
|
||||||
|
return {"wikidata_url": url}
|
||||||
|
return None
|
||||||
|
|
||||||
|
def record_ok(self, item, data):
|
||||||
|
self.db.backfill_mark("wikidata", item["kind"], item["id"], "ok", columns=data)
|
||||||
|
|
||||||
|
def record_empty(self, item):
|
||||||
|
self.db.backfill_mark("wikidata", item["kind"], item["id"], "not_found")
|
||||||
|
|
||||||
|
def record_error(self, item):
|
||||||
|
self.db.backfill_mark("wikidata", item["kind"], item["id"], "error")
|
||||||
|
|
||||||
|
def breakdown(self):
|
||||||
|
return self.db.backfill_breakdown("wikidata")
|
||||||
|
|
||||||
|
|
||||||
|
def build_backfill_workers(db) -> dict:
|
||||||
|
"""All backfill workers, keyed by service id, for the engine registry."""
|
||||||
|
return {w.service: w for w in (
|
||||||
|
RydWorker(db), SponsorBlockWorker(db), FanartWorker(db), OpenSubtitlesWorker(db),
|
||||||
|
TraktWorker(db), TVmazeWorker(db), AniListWorker(db), DeArrowWorker(db),
|
||||||
|
WikidataWorker(db),
|
||||||
|
)}
|
||||||
50
core/video/enrichment/cache.py
Normal file
50
core/video/enrichment/cache.py
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
"""A small thread-safe TTL + LRU cache for the video enrichment engine.
|
||||||
|
|
||||||
|
The engine is a process-wide singleton hit concurrently by Flask request threads
|
||||||
|
AND the worker threads, so the cache must be locked. Entries expire after a TTL;
|
||||||
|
when the cache is full the LEAST-RECENTLY-USED entry is evicted (not the whole
|
||||||
|
cache wholesale). Isolated: imports only the stdlib; no music, no DB.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from collections import OrderedDict
|
||||||
|
|
||||||
|
|
||||||
|
class TTLCache:
|
||||||
|
def __init__(self, maxsize: int = 256, ttl: float = 1800.0, clock=time.monotonic):
|
||||||
|
self._max = max(1, int(maxsize))
|
||||||
|
self._ttl = float(ttl)
|
||||||
|
self._clock = clock # injectable for tests
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._data: OrderedDict = OrderedDict() # key -> (expires_at, value)
|
||||||
|
|
||||||
|
def get(self, key):
|
||||||
|
now = self._clock()
|
||||||
|
with self._lock:
|
||||||
|
hit = self._data.get(key)
|
||||||
|
if hit is None:
|
||||||
|
return None
|
||||||
|
if hit[0] <= now: # expired
|
||||||
|
del self._data[key]
|
||||||
|
return None
|
||||||
|
self._data.move_to_end(key) # mark recently used
|
||||||
|
return hit[1]
|
||||||
|
|
||||||
|
def put(self, key, value, ttl: float | None = None) -> None:
|
||||||
|
expires_at = self._clock() + (self._ttl if ttl is None else float(ttl))
|
||||||
|
with self._lock:
|
||||||
|
self._data[key] = (expires_at, value)
|
||||||
|
self._data.move_to_end(key)
|
||||||
|
while len(self._data) > self._max: # evict LRU (oldest), not everything
|
||||||
|
self._data.popitem(last=False)
|
||||||
|
|
||||||
|
def clear(self) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._data.clear()
|
||||||
|
|
||||||
|
def __len__(self) -> int:
|
||||||
|
with self._lock:
|
||||||
|
return len(self._data)
|
||||||
938
core/video/enrichment/clients.py
Normal file
938
core/video/enrichment/clients.py
Normal file
|
|
@ -0,0 +1,938 @@
|
||||||
|
"""TMDB / TVDB match clients for the video enrichment workers.
|
||||||
|
|
||||||
|
Thin adapters: ``.enabled`` (an API key is configured) and ``.match(kind, title,
|
||||||
|
year) -> {"id", "metadata"} | None``. These talk to real TMDB/TVDB APIs and are
|
||||||
|
validated against the live services; the worker LOGIC is unit-tested with a fake
|
||||||
|
client. Keys come from video_settings.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_enrichment.clients")
|
||||||
|
|
||||||
|
|
||||||
|
def _int(val):
|
||||||
|
try:
|
||||||
|
return int(val)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class TMDBClient:
|
||||||
|
BASE = "https://api.themoviedb.org/3"
|
||||||
|
IMG = "https://image.tmdb.org/t/p/original"
|
||||||
|
|
||||||
|
def __init__(self, api_key):
|
||||||
|
self.api_key = api_key or None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def enabled(self):
|
||||||
|
return bool(self.api_key)
|
||||||
|
|
||||||
|
def test(self):
|
||||||
|
if not self.api_key:
|
||||||
|
return False, "No TMDB API key set"
|
||||||
|
import requests
|
||||||
|
try:
|
||||||
|
r = requests.get(self.BASE + "/configuration", params={"api_key": self.api_key}, timeout=12)
|
||||||
|
if r.status_code == 200:
|
||||||
|
return True, "TMDB connection OK"
|
||||||
|
if r.status_code == 401:
|
||||||
|
return False, "Invalid TMDB API key"
|
||||||
|
return False, "TMDB returned HTTP " + str(r.status_code)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("TMDB test failed")
|
||||||
|
return False, "Could not reach TMDB"
|
||||||
|
|
||||||
|
def match(self, kind, title, year, known_id=None):
|
||||||
|
if not self.api_key:
|
||||||
|
return None
|
||||||
|
import requests
|
||||||
|
# The server already knows the TMDB id → go straight to the details
|
||||||
|
# fetch (accurate, one call). Otherwise fall back to a title/year search.
|
||||||
|
tmdb_id = _int(known_id)
|
||||||
|
meta = {}
|
||||||
|
if tmdb_id is None:
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
path = "/search/movie" if kind == "movie" else "/search/tv"
|
||||||
|
params = {"api_key": self.api_key, "query": title}
|
||||||
|
if year:
|
||||||
|
params["year" if kind == "movie" else "first_air_date_year"] = year
|
||||||
|
resp = requests.get(self.BASE + path, params=params, timeout=15)
|
||||||
|
# A non-200 (429 rate-limit, 5xx, timeout-as-error) is a FAILED call,
|
||||||
|
# not "no match" — raise so the worker records 'error' (retried later)
|
||||||
|
# instead of burning the item to 'not_found'.
|
||||||
|
resp.raise_for_status()
|
||||||
|
results = (resp.json() or {}).get("results") or []
|
||||||
|
if not results:
|
||||||
|
return None
|
||||||
|
tmdb_id = results[0].get("id")
|
||||||
|
meta["overview"] = results[0].get("overview")
|
||||||
|
if tmdb_id is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
detail_path = "/movie/" if kind == "movie" else "/tv/"
|
||||||
|
dr = requests.get(self.BASE + detail_path + str(tmdb_id),
|
||||||
|
params={"api_key": self.api_key,
|
||||||
|
"append_to_response": "external_ids,credits,images",
|
||||||
|
"include_image_language": "en,null"},
|
||||||
|
timeout=15).json() or {}
|
||||||
|
meta["overview"] = dr.get("overview") or meta.get("overview")
|
||||||
|
if dr.get("backdrop_path"):
|
||||||
|
meta["backdrop_url"] = self.IMG + dr["backdrop_path"]
|
||||||
|
ext = dr.get("external_ids") or {}
|
||||||
|
meta["imdb_id"] = ext.get("imdb_id") or dr.get("imdb_id")
|
||||||
|
# Everything TMDB offers (same call) — the worker backfills only the
|
||||||
|
# gaps the server left.
|
||||||
|
meta["tagline"] = dr.get("tagline")
|
||||||
|
meta["status"] = dr.get("status")
|
||||||
|
if dr.get("vote_average"):
|
||||||
|
meta["rating"] = dr.get("vote_average")
|
||||||
|
gs = [g.get("name") for g in (dr.get("genres") or []) if g.get("name")]
|
||||||
|
if gs:
|
||||||
|
meta["genres"] = gs
|
||||||
|
if kind == "movie":
|
||||||
|
meta["release_date"] = dr.get("release_date")
|
||||||
|
meta["runtime_minutes"] = dr.get("runtime")
|
||||||
|
# Franchise/collection (belongs_to_collection is a standard movie-detail
|
||||||
|
# field) — persisted so "complete your collections" gaps can diff it (#discover).
|
||||||
|
bc = dr.get("belongs_to_collection")
|
||||||
|
if bc and bc.get("id"):
|
||||||
|
meta["tmdb_collection_id"] = bc.get("id")
|
||||||
|
meta["tmdb_collection_name"] = bc.get("name")
|
||||||
|
else:
|
||||||
|
meta["first_air_date"] = dr.get("first_air_date")
|
||||||
|
meta["last_air_date"] = dr.get("last_air_date")
|
||||||
|
ert = dr.get("episode_run_time") or []
|
||||||
|
if ert:
|
||||||
|
meta["runtime_minutes"] = ert[0]
|
||||||
|
meta["tvdb_id"] = _int(ext.get("tvdb_id"))
|
||||||
|
# The FULL season list (poster may be None) — drives both the
|
||||||
|
# season-poster backfill and the episode cascade (so missing
|
||||||
|
# episodes/seasons get represented, not just what's on the server).
|
||||||
|
seasons = []
|
||||||
|
for s in (dr.get("seasons") or []):
|
||||||
|
sn = s.get("season_number")
|
||||||
|
if sn is None:
|
||||||
|
continue
|
||||||
|
seasons.append({"season_number": sn,
|
||||||
|
"poster_url": (self.IMG + s["poster_path"]) if s.get("poster_path") else None})
|
||||||
|
if seasons:
|
||||||
|
meta["seasons"] = seasons
|
||||||
|
self._add_credits(meta, dr.get("credits") or {}, dr.get("created_by") or [])
|
||||||
|
logo = self._pick_logo((dr.get("images") or {}).get("logos") or [])
|
||||||
|
if logo:
|
||||||
|
meta["logo_url"] = self.LOGO + logo
|
||||||
|
except Exception:
|
||||||
|
logger.exception("TMDB details fetch failed for %s", title or tmdb_id)
|
||||||
|
return {"id": tmdb_id, "metadata": {k: v for k, v in meta.items() if v}}
|
||||||
|
|
||||||
|
PROFILE = "https://image.tmdb.org/t/p/w185"
|
||||||
|
LOGO = "https://image.tmdb.org/t/p/w500"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _pick_logo(logos):
|
||||||
|
"""Prefer an English title logo, then a language-neutral one, then any."""
|
||||||
|
if not logos:
|
||||||
|
return None
|
||||||
|
for lang in ("en", None):
|
||||||
|
for lg in logos:
|
||||||
|
if lg.get("iso_639_1") == lang and lg.get("file_path"):
|
||||||
|
return lg["file_path"]
|
||||||
|
return logos[0].get("file_path")
|
||||||
|
|
||||||
|
def _person(self, c, job=None, character=None):
|
||||||
|
return {"name": c["name"], "tmdb_id": c.get("id"), "job": job, "character": character,
|
||||||
|
"photo_url": (self.PROFILE + c["profile_path"]) if c.get("profile_path") else None}
|
||||||
|
|
||||||
|
def _add_credits(self, meta, credits, created_by):
|
||||||
|
"""Parse TMDB cast/crew into meta['cast'] / meta['crew']."""
|
||||||
|
cast = [self._person(c, character=c.get("character"))
|
||||||
|
for c in (credits.get("cast") or [])[:20] if c.get("name")]
|
||||||
|
if cast:
|
||||||
|
meta["cast"] = cast
|
||||||
|
# Crew: headline jobs only (directors / writers); plus TV creators, which
|
||||||
|
# live in the top-level created_by, not the crew list.
|
||||||
|
wanted = {"Director", "Writer", "Screenplay"}
|
||||||
|
crew = [self._person(c, job=c.get("job")) for c in (credits.get("crew") or [])
|
||||||
|
if c.get("name") and c.get("job") in wanted]
|
||||||
|
crew += [self._person(c, job="Creator") for c in created_by if c.get("name")]
|
||||||
|
if crew:
|
||||||
|
meta["crew"] = crew
|
||||||
|
|
||||||
|
POSTER_W = "https://image.tmdb.org/t/p/w300"
|
||||||
|
BACKDROP_W = "https://image.tmdb.org/t/p/w780"
|
||||||
|
PROVIDER = "https://image.tmdb.org/t/p/original"
|
||||||
|
|
||||||
|
def extras(self, kind, tmdb_id, region="US"):
|
||||||
|
"""Live detail extras (not cached — providers change): a trailer, the
|
||||||
|
'where to watch' providers for a region, and similar titles."""
|
||||||
|
if not self.api_key or tmdb_id is None:
|
||||||
|
return {}
|
||||||
|
import requests
|
||||||
|
path = ("/movie/" if kind == "movie" else "/tv/") + str(tmdb_id)
|
||||||
|
# TV uses aggregate_credits (it carries per-actor episode counts); movies
|
||||||
|
# use credits. One call (append_to_response) fetches everything.
|
||||||
|
creds = "aggregate_credits" if kind == "show" else "credits"
|
||||||
|
r = requests.get(self.BASE + path, params={
|
||||||
|
"api_key": self.api_key, "include_image_language": "en,null",
|
||||||
|
"append_to_response": "videos,watch/providers,similar,recommendations,images,keywords,reviews," + creds},
|
||||||
|
timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
out = self._parse_extras(kind, r.json() or {}, region)
|
||||||
|
self._fill_collection(out)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _fill_collection(self, out):
|
||||||
|
"""Second call: pull the films of a movie collection (franchise)."""
|
||||||
|
coll = out.get("collection")
|
||||||
|
if coll and coll.get("id"):
|
||||||
|
try:
|
||||||
|
coll["items"] = self.collection(coll["id"])
|
||||||
|
except Exception:
|
||||||
|
logger.exception("TMDB collection fetch failed for %s", coll.get("id"))
|
||||||
|
|
||||||
|
def _parse_extras(self, kind, d, region="US"):
|
||||||
|
"""Pull trailer / where-to-watch / similar / recommendations / collection /
|
||||||
|
next-episode out of a TMDB detail body. Shared by extras() and full_detail()
|
||||||
|
so the search (preview) detail renders them too. (Collection PARTS need a
|
||||||
|
second call — see _fill_collection.)"""
|
||||||
|
out = {}
|
||||||
|
|
||||||
|
# Trailer — prefer a YouTube "Trailer", fall back to a teaser.
|
||||||
|
trailer = None
|
||||||
|
for v in (d.get("videos") or {}).get("results") or []:
|
||||||
|
if v.get("site") == "YouTube" and v.get("type") in ("Trailer", "Teaser") and v.get("key"):
|
||||||
|
trailer = {"key": v["key"], "name": v.get("name")}
|
||||||
|
if v.get("type") == "Trailer":
|
||||||
|
break
|
||||||
|
if trailer:
|
||||||
|
out["trailer"] = trailer
|
||||||
|
|
||||||
|
# Where to watch (one region; JustWatch-powered).
|
||||||
|
wp = ((d.get("watch/providers") or {}).get("results") or {}).get(region) or {}
|
||||||
|
provs, seen = [], set()
|
||||||
|
for grp in ("flatrate", "free", "ads", "rent", "buy"):
|
||||||
|
for p in (wp.get(grp) or []):
|
||||||
|
name = p.get("provider_name")
|
||||||
|
if name and name not in seen:
|
||||||
|
seen.add(name)
|
||||||
|
provs.append({"name": name,
|
||||||
|
"logo": (self.PROVIDER + p["logo_path"]) if p.get("logo_path") else None})
|
||||||
|
if provs:
|
||||||
|
out["providers"] = provs[:8]
|
||||||
|
out["providers_link"] = wp.get("link")
|
||||||
|
out["region"] = region
|
||||||
|
|
||||||
|
# More like this — recommendations (better-curated) with similar as backup.
|
||||||
|
out["recommendations"] = self._title_list((d.get("recommendations") or {}).get("results"), kind)
|
||||||
|
out["similar"] = self._title_list((d.get("similar") or {}).get("results"), kind)
|
||||||
|
if not out["recommendations"]:
|
||||||
|
out.pop("recommendations")
|
||||||
|
if not out["similar"]:
|
||||||
|
out.pop("similar")
|
||||||
|
|
||||||
|
if kind == "movie":
|
||||||
|
bc = d.get("belongs_to_collection")
|
||||||
|
if bc and bc.get("id"):
|
||||||
|
out["collection"] = {
|
||||||
|
"id": bc["id"], "name": bc.get("name"),
|
||||||
|
"poster": (self.POSTER_W + bc["poster_path"]) if bc.get("poster_path") else None,
|
||||||
|
"items": []} # parts filled by _fill_collection (2nd call)
|
||||||
|
else:
|
||||||
|
if d.get("next_episode_to_air"):
|
||||||
|
out["next_episode"] = self._episode_stub(d["next_episode_to_air"])
|
||||||
|
if d.get("last_episode_to_air"):
|
||||||
|
out["last_episode"] = self._episode_stub(d["last_episode_to_air"])
|
||||||
|
|
||||||
|
# Photos gallery (backdrops + posters) — thumb for the grid, full for the
|
||||||
|
# lightbox.
|
||||||
|
imgs = d.get("images") or {}
|
||||||
|
gallery = {}
|
||||||
|
backs = [{"thumb": self.BACKDROP_W + b["file_path"], "full": self.IMG + b["file_path"]}
|
||||||
|
for b in (imgs.get("backdrops") or [])[:14] if b.get("file_path")]
|
||||||
|
posts = [{"thumb": self.POSTER_W + p["file_path"], "full": self.IMG + p["file_path"]}
|
||||||
|
for p in (imgs.get("posters") or [])[:14] if p.get("file_path")]
|
||||||
|
if backs:
|
||||||
|
gallery["backdrops"] = backs
|
||||||
|
if posts:
|
||||||
|
gallery["posters"] = posts
|
||||||
|
if gallery:
|
||||||
|
out["gallery"] = gallery
|
||||||
|
|
||||||
|
# All videos (YouTube) — trailers / teasers / featurettes / clips / BTS.
|
||||||
|
vids = []
|
||||||
|
for v in (d.get("videos") or {}).get("results") or []:
|
||||||
|
if v.get("site") == "YouTube" and v.get("key") and v.get("type"):
|
||||||
|
vids.append({"key": v["key"], "name": v.get("name"), "type": v.get("type")})
|
||||||
|
if vids:
|
||||||
|
out["videos"] = self._order_videos(vids)
|
||||||
|
|
||||||
|
# Keywords / tags.
|
||||||
|
kw = d.get("keywords") or {}
|
||||||
|
kwlist = kw.get("keywords") or kw.get("results") or []
|
||||||
|
keywords = [k.get("name") for k in kwlist if k.get("name")][:14]
|
||||||
|
if keywords:
|
||||||
|
out["keywords"] = keywords
|
||||||
|
|
||||||
|
# Facts / box office.
|
||||||
|
facts = {}
|
||||||
|
if kind == "movie":
|
||||||
|
if d.get("budget"):
|
||||||
|
facts["budget"] = d["budget"]
|
||||||
|
if d.get("revenue"):
|
||||||
|
facts["revenue"] = d["revenue"]
|
||||||
|
if d.get("original_language"):
|
||||||
|
facts["original_language"] = d["original_language"]
|
||||||
|
countries = [c.get("name") for c in (d.get("production_countries") or []) if c.get("name")]
|
||||||
|
if not countries and d.get("origin_country"):
|
||||||
|
countries = list(d.get("origin_country") or [])
|
||||||
|
if countries:
|
||||||
|
facts["countries"] = countries[:3]
|
||||||
|
if facts:
|
||||||
|
out["facts"] = facts
|
||||||
|
|
||||||
|
# Full cast (for the "view all" expansion) — tv carries episode counts.
|
||||||
|
out["cast_full"] = self._full_cast(d, kind)
|
||||||
|
if not out["cast_full"]:
|
||||||
|
out.pop("cast_full")
|
||||||
|
|
||||||
|
# A featured review (the first/top TMDB review).
|
||||||
|
revs = (d.get("reviews") or {}).get("results") or []
|
||||||
|
for rv in revs:
|
||||||
|
if rv.get("content"):
|
||||||
|
ad = rv.get("author_details") or {}
|
||||||
|
out["review"] = {"author": rv.get("author") or ad.get("username") or "Anonymous",
|
||||||
|
"content": rv["content"], "rating": ad.get("rating"),
|
||||||
|
"created": (rv.get("created_at") or "")[:10] or None}
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
|
||||||
|
_VIDEO_ORDER = {"Trailer": 0, "Teaser": 1, "Clip": 2, "Featurette": 3, "Behind the Scenes": 4}
|
||||||
|
|
||||||
|
def _order_videos(self, vids):
|
||||||
|
return sorted(vids, key=lambda v: self._VIDEO_ORDER.get(v.get("type"), 9))[:18]
|
||||||
|
|
||||||
|
def _full_cast(self, d, kind):
|
||||||
|
if kind == "show":
|
||||||
|
src = (d.get("aggregate_credits") or {}).get("cast") \
|
||||||
|
or (d.get("credits") or {}).get("cast") or []
|
||||||
|
else:
|
||||||
|
src = (d.get("credits") or {}).get("cast") or []
|
||||||
|
out = []
|
||||||
|
for c in src:
|
||||||
|
if not c.get("name"):
|
||||||
|
continue
|
||||||
|
char = c.get("character")
|
||||||
|
if not char and c.get("roles"):
|
||||||
|
char = (c["roles"][0] or {}).get("character")
|
||||||
|
out.append({"name": c["name"], "character": char or None, "tmdb_id": c.get("id"),
|
||||||
|
"photo": (self.PROFILE + c["profile_path"]) if c.get("profile_path") else None,
|
||||||
|
"episode_count": c.get("total_episode_count")})
|
||||||
|
return out[:80]
|
||||||
|
|
||||||
|
def _title_list(self, results, kind):
|
||||||
|
out = []
|
||||||
|
for s in (results or [])[:14]:
|
||||||
|
title = s.get("title") or s.get("name")
|
||||||
|
if title and s.get("id"):
|
||||||
|
mt = s.get("media_type")
|
||||||
|
k = "movie" if mt == "movie" else "show" if mt == "tv" else kind
|
||||||
|
out.append({"title": title, "tmdb_id": s["id"], "kind": k,
|
||||||
|
"poster": (self.POSTER_W + s["poster_path"]) if s.get("poster_path") else None})
|
||||||
|
return out
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _episode_stub(e):
|
||||||
|
return {"season_number": e.get("season_number"), "episode_number": e.get("episode_number"),
|
||||||
|
"name": e.get("name"), "air_date": e.get("air_date") or None,
|
||||||
|
"overview": e.get("overview") or None}
|
||||||
|
|
||||||
|
def collection(self, collection_id):
|
||||||
|
"""The films of a movie collection (franchise), ordered by release date."""
|
||||||
|
if not self.api_key or collection_id is None:
|
||||||
|
return []
|
||||||
|
import requests
|
||||||
|
r = requests.get(self.BASE + "/collection/" + str(collection_id),
|
||||||
|
params={"api_key": self.api_key}, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
out = []
|
||||||
|
for p in ((r.json() or {}).get("parts") or []):
|
||||||
|
if not p.get("id"):
|
||||||
|
continue
|
||||||
|
out.append({"kind": "movie", "tmdb_id": p["id"], "title": p.get("title"),
|
||||||
|
"year": (p.get("release_date") or "")[:4] or None,
|
||||||
|
"date": p.get("release_date") or "",
|
||||||
|
"poster": (self.POSTER_W + p["poster_path"]) if p.get("poster_path") else None})
|
||||||
|
out.sort(key=lambda x: x["date"] or "zzzz")
|
||||||
|
return out
|
||||||
|
|
||||||
|
def season_episodes(self, tv_id, season_number):
|
||||||
|
"""Episode-level data for one season (still/overview/rating) — the show
|
||||||
|
worker cascades over a show's seasons to backfill episodes the media
|
||||||
|
server lacked. Returns {'overview', 'episodes': [...]} or None."""
|
||||||
|
if not self.api_key or tv_id is None or season_number is None:
|
||||||
|
return None
|
||||||
|
import requests
|
||||||
|
r = requests.get(self.BASE + "/tv/" + str(tv_id) + "/season/" + str(season_number),
|
||||||
|
params={"api_key": self.api_key}, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
data = r.json() or {}
|
||||||
|
out = []
|
||||||
|
for e in (data.get("episodes") or []):
|
||||||
|
en = e.get("episode_number")
|
||||||
|
if en is None:
|
||||||
|
continue
|
||||||
|
ep = {"episode_number": en, "title": e.get("name"), "overview": e.get("overview"),
|
||||||
|
"air_date": e.get("air_date") or None, "runtime_minutes": e.get("runtime"),
|
||||||
|
"rating": e.get("vote_average") or None}
|
||||||
|
if e.get("still_path"):
|
||||||
|
ep["still_url"] = self.IMG + e["still_path"]
|
||||||
|
out.append(ep)
|
||||||
|
return {"overview": data.get("overview"),
|
||||||
|
"poster_url": (self.IMG + data["poster_path"]) if data.get("poster_path") else None,
|
||||||
|
"episodes": out}
|
||||||
|
|
||||||
|
def episode_detail(self, tv_id, season_number, episode_number):
|
||||||
|
"""One episode's deeper detail (guest stars + a bigger still) for the
|
||||||
|
episode expand. Returns {guest_stars, still_url, rating, overview, ...}."""
|
||||||
|
if not self.api_key or tv_id is None:
|
||||||
|
return None
|
||||||
|
import requests
|
||||||
|
r = requests.get(self.BASE + "/tv/%s/season/%s/episode/%s" % (tv_id, season_number, episode_number),
|
||||||
|
params={"api_key": self.api_key, "append_to_response": "credits"}, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
d = r.json() or {}
|
||||||
|
guests = [{"name": g["name"], "character": g.get("character"), "tmdb_id": g.get("id"),
|
||||||
|
"photo": (self.PROFILE + g["profile_path"]) if g.get("profile_path") else None}
|
||||||
|
for g in (d.get("guest_stars") or [])[:20] if g.get("name")]
|
||||||
|
return {"guest_stars": guests,
|
||||||
|
"still_url": (self.IMG + d["still_path"]) if d.get("still_path") else None,
|
||||||
|
"rating": d.get("vote_average") or None, "overview": d.get("overview") or None,
|
||||||
|
"runtime_minutes": d.get("runtime"), "air_date": d.get("air_date") or None}
|
||||||
|
|
||||||
|
def search(self, query):
|
||||||
|
"""Multi-search (movies / TV / people) for the in-app search page. Returns
|
||||||
|
a flat list of {kind, tmdb_id, title, year, poster, ...} — no external IDs,
|
||||||
|
everything resolves back into SoulSync."""
|
||||||
|
if not self.api_key or not (query or "").strip():
|
||||||
|
return []
|
||||||
|
import requests
|
||||||
|
r = requests.get(self.BASE + "/search/multi", params={
|
||||||
|
"api_key": self.api_key, "query": query, "include_adult": "false"}, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
out = []
|
||||||
|
for it in ((r.json() or {}).get("results") or [])[:32]:
|
||||||
|
mt, tid = it.get("media_type"), it.get("id")
|
||||||
|
if not tid:
|
||||||
|
continue
|
||||||
|
if mt == "movie":
|
||||||
|
out.append({"kind": "movie", "tmdb_id": tid, "title": it.get("title"),
|
||||||
|
"year": (it.get("release_date") or "")[:4] or None,
|
||||||
|
"overview": it.get("overview"), "rating": it.get("vote_average") or None,
|
||||||
|
"poster": (self.POSTER_W + it["poster_path"]) if it.get("poster_path") else None})
|
||||||
|
elif mt == "tv":
|
||||||
|
out.append({"kind": "show", "tmdb_id": tid, "title": it.get("name"),
|
||||||
|
"year": (it.get("first_air_date") or "")[:4] or None,
|
||||||
|
"overview": it.get("overview"), "rating": it.get("vote_average") or None,
|
||||||
|
"poster": (self.POSTER_W + it["poster_path"]) if it.get("poster_path") else None})
|
||||||
|
elif mt == "person":
|
||||||
|
known = [k.get("title") or k.get("name") for k in (it.get("known_for") or [])]
|
||||||
|
out.append({"kind": "person", "tmdb_id": tid, "title": it.get("name"),
|
||||||
|
"known_for": ", ".join([k for k in known if k][:3]) or None,
|
||||||
|
"department": it.get("known_for_department"),
|
||||||
|
"poster": (self.PROFILE + it["profile_path"]) if it.get("profile_path") else None})
|
||||||
|
return out
|
||||||
|
|
||||||
|
def trending(self, window="week", kind=None):
|
||||||
|
"""Trending titles. ``kind`` None = mixed movies + shows (/trending/all — the
|
||||||
|
search-idle filler + Discover hero slideshow, hence backdrops via _disc_map).
|
||||||
|
``kind`` 'movie' / 'show' hit the dedicated single-type charts (/trending/movie,
|
||||||
|
/trending/tv) that power the split 'Top 10 Movies / TV Shows Today' rails.
|
||||||
|
Single-type endpoints omit media_type, so the kind is forced into _disc_map."""
|
||||||
|
if not self.api_key:
|
||||||
|
return []
|
||||||
|
import requests
|
||||||
|
path = ("/trending/movie/" if kind == "movie"
|
||||||
|
else "/trending/tv/" if kind == "show"
|
||||||
|
else "/trending/all/") + window
|
||||||
|
r = requests.get(self.BASE + path, params={"api_key": self.api_key}, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
forced = kind if kind in ("movie", "show") else None
|
||||||
|
return self._disc_map((r.json() or {}).get("results"), forced)[:20]
|
||||||
|
|
||||||
|
# ── discover (browse TMDB by curated list / genre / year / decade) ────────
|
||||||
|
def _disc_map(self, results, kind):
|
||||||
|
"""Flatten a TMDB movie/tv list into SoulSync items. ``kind`` forces the
|
||||||
|
type for single-type endpoints; pass None to auto-detect each row's
|
||||||
|
media_type (mixed lists like trending). Carries backdrop + overview so the
|
||||||
|
Discover hero can render a rich slide."""
|
||||||
|
out = []
|
||||||
|
for it in results or []:
|
||||||
|
tid = it.get("id")
|
||||||
|
if not tid:
|
||||||
|
continue
|
||||||
|
k = kind
|
||||||
|
if k is None:
|
||||||
|
mt = it.get("media_type")
|
||||||
|
k = "movie" if mt == "movie" else "show" if mt == "tv" else None
|
||||||
|
if k not in ("movie", "show"):
|
||||||
|
continue
|
||||||
|
is_movie = k == "movie"
|
||||||
|
out.append({
|
||||||
|
"kind": k, "tmdb_id": tid,
|
||||||
|
"title": it.get("title") if is_movie else it.get("name"),
|
||||||
|
"year": ((it.get("release_date") if is_movie else it.get("first_air_date")) or "")[:4] or None,
|
||||||
|
"rating": it.get("vote_average") or None,
|
||||||
|
"overview": it.get("overview") or None,
|
||||||
|
"original_language": it.get("original_language") or None, # for the language filter
|
||||||
|
"popularity": it.get("popularity") or None, # for blended ranking
|
||||||
|
"poster": (self.POSTER_W + it["poster_path"]) if it.get("poster_path") else None,
|
||||||
|
"backdrop": (self.BACKDROP_W + it["backdrop_path"]) if it.get("backdrop_path") else None,
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
|
||||||
|
# canned TMDB lists → (path, forced kind)
|
||||||
|
_CURATED = {
|
||||||
|
"popular_movies": ("/movie/popular", "movie"),
|
||||||
|
"top_movies": ("/movie/top_rated", "movie"),
|
||||||
|
"now_playing": ("/movie/now_playing", "movie"),
|
||||||
|
"upcoming_movies": ("/movie/upcoming", "movie"),
|
||||||
|
"popular_shows": ("/tv/popular", "show"),
|
||||||
|
"top_shows": ("/tv/top_rated", "show"),
|
||||||
|
"on_the_air": ("/tv/on_the_air", "show"),
|
||||||
|
"airing_today": ("/tv/airing_today", "show"),
|
||||||
|
}
|
||||||
|
|
||||||
|
def curated(self, key, page=1):
|
||||||
|
"""One of the canned TMDB lists (popular / top-rated / now-playing / …)."""
|
||||||
|
spec = self._CURATED.get(key)
|
||||||
|
if not spec or not self.api_key:
|
||||||
|
return []
|
||||||
|
import requests
|
||||||
|
path, kind = spec
|
||||||
|
r = requests.get(self.BASE + path,
|
||||||
|
params={"api_key": self.api_key, "page": page}, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
return self._disc_map((r.json() or {}).get("results"), kind)
|
||||||
|
|
||||||
|
def discover(self, kind, *, genre=None, year=None, decade=None, providers=None,
|
||||||
|
sort_by="popularity.desc", page=1, region="US", language=None,
|
||||||
|
keywords=None, companies=None, networks=None, cast=None, crew=None,
|
||||||
|
min_runtime=None, max_runtime=None, certification=None,
|
||||||
|
cert_country="US", vote_count_min=None, release_window=None):
|
||||||
|
"""Browse /discover/{movie,tv}. The original filters (genre / year / decade /
|
||||||
|
streaming ``providers`` + ``region`` / original ``language``) plus the
|
||||||
|
Netflix-class extensions, all optional + additive:
|
||||||
|
|
||||||
|
- ``keywords`` — TMDB keyword id(s), pipe-joined for OR ('818|9715'). Powers
|
||||||
|
mood/theme rails (feel-good, heist, time-travel …).
|
||||||
|
- ``companies`` — TMDB company id(s) — studio rails (Pixar, A24, Ghibli …).
|
||||||
|
- ``networks`` — TMDB network id(s), TV only — network rails (HBO, AMC …).
|
||||||
|
- ``cast`` / ``crew`` — person id(s), movies — "starring …" / "directed by …".
|
||||||
|
- ``min_runtime`` / ``max_runtime`` — minutes — quick-watches / epics.
|
||||||
|
- ``certification`` (+ ``cert_country``) — e.g. 'PG-13', movies — family-friendly.
|
||||||
|
- ``vote_count_min`` — override the default popularity floor (40).
|
||||||
|
- ``release_window`` — 'last_30' | 'last_90' | 'last_365' — date-windowed "new"
|
||||||
|
rails (computed relative to today)."""
|
||||||
|
if not self.api_key:
|
||||||
|
return []
|
||||||
|
import requests
|
||||||
|
is_movie = kind == "movie"
|
||||||
|
path = "/discover/movie" if is_movie else "/discover/tv"
|
||||||
|
params = {"api_key": self.api_key, "sort_by": sort_by, "page": page,
|
||||||
|
"include_adult": "false",
|
||||||
|
"vote_count.gte": vote_count_min if vote_count_min is not None else 40}
|
||||||
|
if language:
|
||||||
|
params["with_original_language"] = language
|
||||||
|
if genre:
|
||||||
|
params["with_genres"] = genre
|
||||||
|
if providers:
|
||||||
|
params["with_watch_providers"] = providers
|
||||||
|
params["watch_region"] = region or "US"
|
||||||
|
params["with_watch_monetization_types"] = "flatrate" # streaming, not rent/buy
|
||||||
|
if keywords:
|
||||||
|
params["with_keywords"] = keywords
|
||||||
|
if companies:
|
||||||
|
params["with_companies"] = companies
|
||||||
|
if networks and not is_movie:
|
||||||
|
params["with_networks"] = networks
|
||||||
|
if cast and is_movie:
|
||||||
|
params["with_cast"] = cast
|
||||||
|
if crew and is_movie:
|
||||||
|
params["with_crew"] = crew
|
||||||
|
if min_runtime:
|
||||||
|
params["with_runtime.gte"] = min_runtime
|
||||||
|
if max_runtime:
|
||||||
|
params["with_runtime.lte"] = max_runtime
|
||||||
|
if certification and is_movie:
|
||||||
|
params["certification.lte"] = certification
|
||||||
|
params["certification_country"] = cert_country or "US"
|
||||||
|
if year:
|
||||||
|
params["primary_release_year" if is_movie else "first_air_date_year"] = year
|
||||||
|
if decade:
|
||||||
|
try:
|
||||||
|
d0 = int(decade)
|
||||||
|
gte, lte = "%d-01-01" % d0, "%d-12-31" % (d0 + 9)
|
||||||
|
if is_movie:
|
||||||
|
params["primary_release_date.gte"], params["primary_release_date.lte"] = gte, lte
|
||||||
|
else:
|
||||||
|
params["first_air_date.gte"], params["first_air_date.lte"] = gte, lte
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
if release_window:
|
||||||
|
days = {"last_30": 30, "last_90": 90, "last_365": 365}.get(release_window)
|
||||||
|
if days:
|
||||||
|
import datetime as _dt
|
||||||
|
today = _dt.date.today()
|
||||||
|
start = (today - _dt.timedelta(days=days)).isoformat()
|
||||||
|
end = today.isoformat()
|
||||||
|
if is_movie:
|
||||||
|
params["primary_release_date.gte"], params["primary_release_date.lte"] = start, end
|
||||||
|
else:
|
||||||
|
params["first_air_date.gte"], params["first_air_date.lte"] = start, end
|
||||||
|
r = requests.get(self.BASE + path, params=params, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
return self._disc_map((r.json() or {}).get("results"), kind)
|
||||||
|
|
||||||
|
def genres(self, kind):
|
||||||
|
"""TMDB genre id→name list for movies or shows."""
|
||||||
|
if not self.api_key:
|
||||||
|
return []
|
||||||
|
import requests
|
||||||
|
path = "/genre/movie/list" if kind == "movie" else "/genre/tv/list"
|
||||||
|
r = requests.get(self.BASE + path, params={"api_key": self.api_key}, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
return [{"id": g["id"], "name": g["name"]}
|
||||||
|
for g in (r.json() or {}).get("genres") or [] if g.get("id")]
|
||||||
|
|
||||||
|
def recommendations(self, kind, tmdb_id, page=1):
|
||||||
|
"""TMDB 'recommended' titles for a movie/show — powers 'More like …' rails."""
|
||||||
|
if not self.api_key or tmdb_id is None:
|
||||||
|
return []
|
||||||
|
import requests
|
||||||
|
path = ("/movie/" if kind == "movie" else "/tv/") + str(tmdb_id) + "/recommendations"
|
||||||
|
r = requests.get(self.BASE + path,
|
||||||
|
params={"api_key": self.api_key, "page": page}, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
return self._disc_map((r.json() or {}).get("results"), kind)
|
||||||
|
|
||||||
|
def video_trailer(self, kind, tmdb_id):
|
||||||
|
"""The best YouTube trailer key for a title (official Trailer over Teaser).
|
||||||
|
Light — just the /videos endpoint, not the whole detail append."""
|
||||||
|
if not self.api_key or tmdb_id is None:
|
||||||
|
return None
|
||||||
|
import requests
|
||||||
|
path = ("/movie/" if kind == "movie" else "/tv/") + str(tmdb_id) + "/videos"
|
||||||
|
r = requests.get(self.BASE + path, params={"api_key": self.api_key}, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
teaser = None
|
||||||
|
for v in ((r.json() or {}).get("results") or []):
|
||||||
|
if v.get("site") != "YouTube" or not v.get("key"):
|
||||||
|
continue
|
||||||
|
t = v.get("type") or ""
|
||||||
|
if t == "Trailer":
|
||||||
|
return {"key": v["key"], "name": v.get("name")}
|
||||||
|
if t == "Teaser" and teaser is None:
|
||||||
|
teaser = {"key": v["key"], "name": v.get("name")}
|
||||||
|
return teaser
|
||||||
|
|
||||||
|
def full_detail(self, kind, tmdb_id, region="US"):
|
||||||
|
"""Complete detail for a TMDB title NOT in the library — shaped like the
|
||||||
|
library detail payload but with direct image URLs (so the same detail UI
|
||||||
|
renders it). Seasons carry counts; episodes load lazily per season."""
|
||||||
|
if not self.api_key or tmdb_id is None:
|
||||||
|
return None
|
||||||
|
import requests
|
||||||
|
path = ("/movie/" if kind == "movie" else "/tv/") + str(tmdb_id)
|
||||||
|
agg = ",aggregate_credits" if kind == "show" else ""
|
||||||
|
r = requests.get(self.BASE + path, params={
|
||||||
|
"api_key": self.api_key,
|
||||||
|
"append_to_response": "external_ids,credits,images,videos,watch/providers,similar,"
|
||||||
|
"recommendations,keywords,reviews" + agg,
|
||||||
|
"include_image_language": "en,null"}, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
dr = r.json() or {}
|
||||||
|
if not dr.get("id"):
|
||||||
|
return None
|
||||||
|
ext = dr.get("external_ids") or {}
|
||||||
|
logo = self._pick_logo((dr.get("images") or {}).get("logos") or [])
|
||||||
|
cmeta = {}
|
||||||
|
self._add_credits(cmeta, dr.get("credits") or {}, dr.get("created_by") or [])
|
||||||
|
out = {
|
||||||
|
"kind": kind, "tmdb_id": tmdb_id,
|
||||||
|
"title": dr.get("title") or dr.get("name"),
|
||||||
|
"overview": dr.get("overview"), "tagline": dr.get("tagline") or None,
|
||||||
|
"status": dr.get("status"), "rating": dr.get("vote_average") or None,
|
||||||
|
"imdb_id": ext.get("imdb_id") or dr.get("imdb_id"),
|
||||||
|
"poster_url": (self.IMG + dr["poster_path"]) if dr.get("poster_path") else None,
|
||||||
|
"backdrop_url": (self.IMG + dr["backdrop_path"]) if dr.get("backdrop_path") else None,
|
||||||
|
"logo": (self.LOGO + logo) if logo else None,
|
||||||
|
"genres": [g.get("name") for g in (dr.get("genres") or []) if g.get("name")],
|
||||||
|
"cast": [{"name": p["name"], "character": p.get("character"),
|
||||||
|
"photo": p.get("photo_url"), "tmdb_id": p.get("tmdb_id")}
|
||||||
|
for p in cmeta.get("cast") or []],
|
||||||
|
"crew": [{"name": p["name"], "job": p.get("job"), "tmdb_id": p.get("tmdb_id")}
|
||||||
|
for p in cmeta.get("crew") or []],
|
||||||
|
"_extras": self._parse_extras(kind, dr, region),
|
||||||
|
}
|
||||||
|
self._fill_collection(out["_extras"])
|
||||||
|
if kind == "movie":
|
||||||
|
out["year"] = (dr.get("release_date") or "")[:4] or None
|
||||||
|
out["release_date"] = dr.get("release_date") or None
|
||||||
|
out["runtime_minutes"] = dr.get("runtime")
|
||||||
|
out["studio"] = next((c.get("name") for c in (dr.get("production_companies") or [])), None)
|
||||||
|
else:
|
||||||
|
out["year"] = (dr.get("first_air_date") or "")[:4] or None
|
||||||
|
out["first_air_date"] = dr.get("first_air_date") or None
|
||||||
|
out["last_air_date"] = dr.get("last_air_date") or None
|
||||||
|
ert = dr.get("episode_run_time") or []
|
||||||
|
out["runtime_minutes"] = ert[0] if ert else None
|
||||||
|
out["network"] = next((n.get("name") for n in (dr.get("networks") or [])), None)
|
||||||
|
out["tvdb_id"] = _int(ext.get("tvdb_id"))
|
||||||
|
seasons = []
|
||||||
|
for s in (dr.get("seasons") or []):
|
||||||
|
num = s.get("season_number")
|
||||||
|
if num is None:
|
||||||
|
continue
|
||||||
|
seasons.append({
|
||||||
|
"season_number": num,
|
||||||
|
"title": s.get("name") or ("Specials" if num == 0 else "Season %d" % num),
|
||||||
|
"poster_url": (self.POSTER_W + s["poster_path"]) if s.get("poster_path") else None,
|
||||||
|
"episode_count": s.get("episode_count") or 0})
|
||||||
|
out["_seasons"] = sorted(seasons, key=lambda s: s["season_number"])
|
||||||
|
return out
|
||||||
|
|
||||||
|
def person(self, tmdb_id):
|
||||||
|
"""Person detail + their filmography (cast + crew credits) for the in-app
|
||||||
|
person page. Everything points back to TMDB ids we resolve in SoulSync."""
|
||||||
|
if not self.api_key or tmdb_id is None:
|
||||||
|
return None
|
||||||
|
import requests
|
||||||
|
r = requests.get(self.BASE + "/person/" + str(tmdb_id), params={
|
||||||
|
"api_key": self.api_key,
|
||||||
|
"append_to_response": "combined_credits,external_ids,images"}, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
d = r.json() or {}
|
||||||
|
if not d.get("id"):
|
||||||
|
return None
|
||||||
|
cc = d.get("combined_credits") or {}
|
||||||
|
seen, credits = set(), []
|
||||||
|
|
||||||
|
def add(c, department, role):
|
||||||
|
mt, tid = c.get("media_type"), c.get("id")
|
||||||
|
if not tid or mt not in ("movie", "tv"):
|
||||||
|
return
|
||||||
|
kind = "movie" if mt == "movie" else "show"
|
||||||
|
key = (kind, tid)
|
||||||
|
if key in seen: # same title in two roles → keep the first
|
||||||
|
return
|
||||||
|
seen.add(key)
|
||||||
|
date = c.get("release_date") or c.get("first_air_date") or ""
|
||||||
|
credits.append({
|
||||||
|
"kind": kind, "tmdb_id": tid, "title": c.get("title") or c.get("name"),
|
||||||
|
"year": (date or "")[:4] or None, "date": date or None,
|
||||||
|
"department": department, "role": role,
|
||||||
|
"popularity": c.get("popularity") or 0,
|
||||||
|
"poster": (self.POSTER_W + c["poster_path"]) if c.get("poster_path") else None})
|
||||||
|
|
||||||
|
# Cast first (so an actor-director title files under Acting), then crew.
|
||||||
|
for c in (cc.get("cast") or []):
|
||||||
|
add(c, "Acting", c.get("character") or None)
|
||||||
|
for c in (cc.get("crew") or []):
|
||||||
|
add(c, c.get("department") or "Crew", c.get("job") or None)
|
||||||
|
credits.sort(key=lambda x: x["popularity"], reverse=True)
|
||||||
|
profiles = (d.get("images") or {}).get("profiles") or []
|
||||||
|
photos = [{"thumb": self.PROFILE + p["file_path"], "full": self.IMG + p["file_path"]}
|
||||||
|
for p in profiles[:16] if p.get("file_path")]
|
||||||
|
akas = [a for a in (d.get("also_known_as") or []) if a][:6]
|
||||||
|
return {
|
||||||
|
"tmdb_id": d.get("id"), "name": d.get("name"),
|
||||||
|
"biography": d.get("biography") or None,
|
||||||
|
"known_for": d.get("known_for_department") or None,
|
||||||
|
"birthday": d.get("birthday") or None, "deathday": d.get("deathday") or None,
|
||||||
|
"place_of_birth": d.get("place_of_birth") or None,
|
||||||
|
"photo": (self.PROFILE + d["profile_path"]) if d.get("profile_path") else None,
|
||||||
|
"photos": photos, "also_known_as": akas, "credits": credits}
|
||||||
|
|
||||||
|
|
||||||
|
class TVDBClient:
|
||||||
|
BASE = "https://api4.thetvdb.com/v4"
|
||||||
|
|
||||||
|
def __init__(self, api_key):
|
||||||
|
self.api_key = api_key or None
|
||||||
|
self._token = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def enabled(self):
|
||||||
|
return bool(self.api_key)
|
||||||
|
|
||||||
|
def test(self):
|
||||||
|
if not self.api_key:
|
||||||
|
return False, "No TVDB API key set"
|
||||||
|
try:
|
||||||
|
token = self._auth()
|
||||||
|
if token:
|
||||||
|
return True, "TVDB connection OK"
|
||||||
|
return False, "TVDB login failed — check the key"
|
||||||
|
except Exception:
|
||||||
|
logger.exception("TVDB test failed")
|
||||||
|
return False, "Could not reach TVDB"
|
||||||
|
|
||||||
|
def _auth(self, force=False):
|
||||||
|
if self._token and not force:
|
||||||
|
return self._token
|
||||||
|
import requests
|
||||||
|
self._token = None
|
||||||
|
r = requests.post(self.BASE + "/login", json={"apikey": self.api_key}, timeout=15).json() or {}
|
||||||
|
self._token = (r.get("data") or {}).get("token")
|
||||||
|
return self._token
|
||||||
|
|
||||||
|
def _authed_get(self, path, params=None):
|
||||||
|
"""GET with the bearer token, transparently re-authenticating once if the
|
||||||
|
cached token has expired (401). Raises on any other non-200 so the worker
|
||||||
|
records 'error' rather than a false 'not_found'."""
|
||||||
|
import requests
|
||||||
|
token = self._auth()
|
||||||
|
if not token:
|
||||||
|
return None
|
||||||
|
r = requests.get(self.BASE + path, headers={"Authorization": "Bearer " + token},
|
||||||
|
params=params, timeout=15)
|
||||||
|
if r.status_code == 401 and self._auth(force=True): # token expired → refresh once
|
||||||
|
r = requests.get(self.BASE + path, headers={"Authorization": "Bearer " + self._token},
|
||||||
|
params=params, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json() or {}
|
||||||
|
|
||||||
|
def match(self, kind, title, year, known_id=None):
|
||||||
|
if kind != "show" or not self.api_key:
|
||||||
|
return None
|
||||||
|
tvdb_id = _int(known_id)
|
||||||
|
meta = {}
|
||||||
|
if tvdb_id is None:
|
||||||
|
if not title:
|
||||||
|
return None
|
||||||
|
r = self._authed_get("/search", {"query": title, "type": "series"})
|
||||||
|
results = (r or {}).get("data") or []
|
||||||
|
if not results:
|
||||||
|
return None
|
||||||
|
top = results[0]
|
||||||
|
tvdb_id = _int(top.get("tvdb_id") or top.get("id"))
|
||||||
|
meta["overview"] = top.get("overview")
|
||||||
|
else:
|
||||||
|
# Known id from the server → fetch the extended record (overview +
|
||||||
|
# genres + everything TVDB offers).
|
||||||
|
try:
|
||||||
|
dr = self._authed_get("/series/" + str(tvdb_id) + "/extended")
|
||||||
|
sd = (dr or {}).get("data") or {}
|
||||||
|
meta["overview"] = sd.get("overview")
|
||||||
|
gs = [g.get("name") for g in (sd.get("genres") or []) if g.get("name")]
|
||||||
|
if gs:
|
||||||
|
meta["genres"] = gs
|
||||||
|
# Show-level air time (network local time, e.g. "21:00") — drives
|
||||||
|
# the Calendar's per-day time sort. Streaming shows have none.
|
||||||
|
meta["airs_time"] = (sd.get("airsTime") or "").strip() or None
|
||||||
|
except Exception:
|
||||||
|
logger.exception("TVDB details fetch failed for %s", title or tvdb_id)
|
||||||
|
if tvdb_id is None:
|
||||||
|
return None
|
||||||
|
return {"id": tvdb_id, "metadata": {k: v for k, v in meta.items() if v}}
|
||||||
|
|
||||||
|
|
||||||
|
class OMDbAuthError(Exception):
|
||||||
|
"""OMDb rejected the API key (HTTP 401 / 'Invalid API key!'). Distinct from a
|
||||||
|
transient error or a genuine 'no rating' so the worker can pause instead of
|
||||||
|
churning the whole library on a bad key."""
|
||||||
|
|
||||||
|
|
||||||
|
class OMDBClient:
|
||||||
|
"""Ratings provider — IMDb / Rotten Tomatoes / Metacritic by imdb_id. Not a
|
||||||
|
matcher (we already have the id), so it's used as a ratings backfill, not a
|
||||||
|
worker."""
|
||||||
|
BASE = "https://www.omdbapi.com/"
|
||||||
|
|
||||||
|
def __init__(self, api_key):
|
||||||
|
self.api_key = api_key or None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def enabled(self):
|
||||||
|
return bool(self.api_key)
|
||||||
|
|
||||||
|
def test(self):
|
||||||
|
if not self.api_key:
|
||||||
|
return False, "No OMDb API key set"
|
||||||
|
import requests
|
||||||
|
try:
|
||||||
|
r = requests.get(self.BASE, params={"apikey": self.api_key, "i": "tt0111161"}, timeout=12)
|
||||||
|
# OMDb returns a JSON body even on 401 — surface its actual Error so the
|
||||||
|
# user sees WHY ("Invalid API key!" = not activated/wrong key;
|
||||||
|
# "Request limit reached!" = free-tier daily quota, resets at midnight).
|
||||||
|
try:
|
||||||
|
d = r.json() or {}
|
||||||
|
except Exception:
|
||||||
|
d = {}
|
||||||
|
if d.get("Response") == "True":
|
||||||
|
return True, "OMDb connection OK"
|
||||||
|
err = (d.get("Error") or "").strip()
|
||||||
|
if "invalid api key" in err.lower():
|
||||||
|
return False, "Invalid OMDb API key — did you click the activation link OMDb emailed you?"
|
||||||
|
if err:
|
||||||
|
return False, "OMDb: " + err
|
||||||
|
return False, "OMDb returned HTTP " + str(r.status_code)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("OMDb test failed")
|
||||||
|
return False, "Could not reach OMDb"
|
||||||
|
|
||||||
|
def ratings(self, imdb_id):
|
||||||
|
if not self.api_key or not imdb_id:
|
||||||
|
return None
|
||||||
|
import requests
|
||||||
|
r = requests.get(self.BASE, params={"apikey": self.api_key, "i": imdb_id}, timeout=12)
|
||||||
|
# A bad/expired key is a 401 (sometimes a 200 with "Invalid API key!") — a
|
||||||
|
# config problem that affects EVERY item, so flag it distinctly.
|
||||||
|
if r.status_code == 401:
|
||||||
|
err = ""
|
||||||
|
try:
|
||||||
|
err = ((r.json() or {}).get("Error") or "").strip()
|
||||||
|
except Exception: # noqa: S110 - best-effort error-body parse; we raise OMDbAuthError below regardless
|
||||||
|
pass
|
||||||
|
raise OMDbAuthError(err or "OMDb rejected the API key (HTTP 401)")
|
||||||
|
r.raise_for_status()
|
||||||
|
d = r.json() or {}
|
||||||
|
if d.get("Response") != "True":
|
||||||
|
if "invalid api key" in (d.get("Error") or "").lower():
|
||||||
|
raise OMDbAuthError(d.get("Error") or "Invalid OMDb API key")
|
||||||
|
return None # genuine "no data for this title"
|
||||||
|
out = {}
|
||||||
|
ir = d.get("imdbRating")
|
||||||
|
if ir and ir != "N/A":
|
||||||
|
try:
|
||||||
|
out["imdb_rating"] = float(ir)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
for rt in (d.get("Ratings") or []):
|
||||||
|
if rt.get("Source") == "Rotten Tomatoes":
|
||||||
|
try:
|
||||||
|
out["rt_rating"] = int((rt.get("Value") or "").rstrip("%"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
ms = d.get("Metascore")
|
||||||
|
if ms and ms != "N/A":
|
||||||
|
try:
|
||||||
|
out["metacritic"] = int(ms)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def build_clients(db) -> dict:
|
||||||
|
"""Construct the source clients from the saved API keys (in video_settings).
|
||||||
|
OMDb is included as a worker (a ratings filler) alongside the matchers."""
|
||||||
|
return {
|
||||||
|
"tmdb": TMDBClient(db.get_setting("tmdb_api_key")),
|
||||||
|
"tvdb": TVDBClient(db.get_setting("tvdb_api_key")),
|
||||||
|
"omdb": OMDBClient(db.get_setting("omdb_api_key")),
|
||||||
|
}
|
||||||
710
core/video/enrichment/engine.py
Normal file
710
core/video/enrichment/engine.py
Normal file
|
|
@ -0,0 +1,710 @@
|
||||||
|
"""Video enrichment engine — owns the per-source workers (registry).
|
||||||
|
|
||||||
|
Parallels music's enrichment registry but is isolated to the video side. Built
|
||||||
|
lazily as a process-wide singleton; starts the workers (each idles until its API
|
||||||
|
key is configured). Imports only video.db + this package.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import threading
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
from .cache import TTLCache
|
||||||
|
from .clients import OMDbAuthError
|
||||||
|
from .worker import VideoEnrichmentWorker
|
||||||
|
|
||||||
|
logger = get_logger("video_enrichment.engine")
|
||||||
|
|
||||||
|
_DISPLAY = {"tmdb": "TMDB", "tvdb": "TVDB", "omdb": "OMDb"}
|
||||||
|
|
||||||
|
|
||||||
|
class VideoEnrichmentEngine:
|
||||||
|
def __init__(self, db, clients: dict, ratings_client=None):
|
||||||
|
self.db = db
|
||||||
|
self.workers = {
|
||||||
|
service: VideoEnrichmentWorker(db, service, client, display_name=_DISPLAY.get(service))
|
||||||
|
for service, client in clients.items()
|
||||||
|
}
|
||||||
|
# Backfill workers (artwork / subtitles / no-key YouTube extras). Same
|
||||||
|
# lifecycle + get_stats() shape, so the registry/API/UI drive them too.
|
||||||
|
try:
|
||||||
|
from .backfill import build_backfill_workers
|
||||||
|
self.workers.update(build_backfill_workers(db))
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video enrichment: backfill workers unavailable")
|
||||||
|
# OMDb ratings (IMDb/RT/Metacritic) — not a matcher, so not a worker;
|
||||||
|
# backfilled on the lazy detail refresh.
|
||||||
|
self.ratings_client = ratings_client
|
||||||
|
# Thread-safe TTL+LRU cache for live TMDB detail extras / preview payloads /
|
||||||
|
# person pages / seasons / trending, so re-opening a title is instant
|
||||||
|
# instead of re-hitting TMDB. (Volatile by design — durable art/episodes/
|
||||||
|
# ratings live in video.db; we don't persist this tier.)
|
||||||
|
self._cache = TTLCache(maxsize=256, ttl=1800)
|
||||||
|
# Restore each worker's persisted pause state (survives restart).
|
||||||
|
for w in self.workers.values():
|
||||||
|
w.restore_paused()
|
||||||
|
|
||||||
|
def _region(self):
|
||||||
|
try:
|
||||||
|
return (self.db.get_setting("watch_region") or "US").upper()
|
||||||
|
except Exception:
|
||||||
|
return "US"
|
||||||
|
|
||||||
|
def _cache_get(self, key):
|
||||||
|
return self._cache.get(key)
|
||||||
|
|
||||||
|
def _cache_put(self, key, data, ttl=1800):
|
||||||
|
self._cache.put(key, data, ttl=ttl)
|
||||||
|
|
||||||
|
def _backfill_ratings(self, kind, item_id):
|
||||||
|
# The OMDb worker owns the ratings client (fallback to an injected one
|
||||||
|
# for tests that don't build a worker).
|
||||||
|
w = self.workers.get("omdb")
|
||||||
|
rc = w.client if w else self.ratings_client
|
||||||
|
# _omdb_blocked latches once the daily request limit / a bad key is hit — it affects
|
||||||
|
# EVERY item, so we stop calling OMDb for the rest of the process instead of failing
|
||||||
|
# (and logging a traceback) once per show.
|
||||||
|
if not rc or not getattr(rc, "enabled", False) or getattr(self, "_omdb_blocked", False):
|
||||||
|
return
|
||||||
|
info = (self.db.movie_match_info(item_id) if kind == "movie"
|
||||||
|
else self.db.show_match_info(item_id))
|
||||||
|
# IMDb id lives on the row — fetch it directly.
|
||||||
|
row = None
|
||||||
|
try:
|
||||||
|
with self.db.connect() as c:
|
||||||
|
tbl = "movies" if kind == "movie" else "shows"
|
||||||
|
row = c.execute(f"SELECT imdb_id FROM {tbl} WHERE id=?", (item_id,)).fetchone()
|
||||||
|
except Exception:
|
||||||
|
return
|
||||||
|
imdb_id = row["imdb_id"] if row else None
|
||||||
|
if not imdb_id:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
ratings = rc.ratings(imdb_id)
|
||||||
|
if ratings:
|
||||||
|
self.db.apply_ratings(kind, item_id, ratings)
|
||||||
|
except OMDbAuthError as e:
|
||||||
|
# daily limit / bad key — hits every item; latch off + one quiet warning, no spam.
|
||||||
|
self._omdb_blocked = True
|
||||||
|
logger.warning("OMDb ratings paused for this run: %s", e)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("ratings backfill failed for %s %s", kind, item_id)
|
||||||
|
|
||||||
|
def start_all(self):
|
||||||
|
for w in self.workers.values():
|
||||||
|
w.start()
|
||||||
|
|
||||||
|
def stop_all(self):
|
||||||
|
for w in self.workers.values():
|
||||||
|
w.stop()
|
||||||
|
|
||||||
|
# ── scan coupling ─────────────────────────────────────────────────────────
|
||||||
|
# While a library scan runs, the enrichment workers step aside to cut DB lock
|
||||||
|
# contention — exactly like the music side. We pause ONLY workers that were
|
||||||
|
# actually running (never a user's manual pause) and remember which, so the
|
||||||
|
# post-scan resume can't un-pause something the user deliberately paused. The
|
||||||
|
# auto-pause is transient (persist=False) so it never leaks into the saved
|
||||||
|
# <service>_paused flag and survives a restart as a "real" pause.
|
||||||
|
def pause_for_scan(self) -> set:
|
||||||
|
self._scan_paused = set()
|
||||||
|
for service, w in self.workers.items():
|
||||||
|
if not w.paused:
|
||||||
|
w.pause(persist=False)
|
||||||
|
self._scan_paused.add(service)
|
||||||
|
# The YouTube date enricher is a separate singleton (not in self.workers) —
|
||||||
|
# pause it too so a scan stops EVERY enricher, not just the matcher/backfill
|
||||||
|
# workers. Only if it wasn't already paused (never override a manual pause).
|
||||||
|
self._scan_paused_yt = False
|
||||||
|
try:
|
||||||
|
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||||
|
yt = get_youtube_date_enricher()
|
||||||
|
if yt and not getattr(yt, "_paused", False):
|
||||||
|
yt.pause()
|
||||||
|
self._scan_paused_yt = True
|
||||||
|
self._scan_paused.add("youtube")
|
||||||
|
except Exception:
|
||||||
|
logger.debug("video enrichment: could not pause YouTube date enricher for scan", exc_info=True)
|
||||||
|
if self._scan_paused:
|
||||||
|
logger.info("video enrichment: paused %s for library scan",
|
||||||
|
", ".join(sorted(self._scan_paused)))
|
||||||
|
return self._scan_paused
|
||||||
|
|
||||||
|
def resume_after_scan(self) -> None:
|
||||||
|
for service in getattr(self, "_scan_paused", set()):
|
||||||
|
w = self.workers.get(service)
|
||||||
|
if w:
|
||||||
|
w.resume(persist=False)
|
||||||
|
if getattr(self, "_scan_paused_yt", False):
|
||||||
|
try:
|
||||||
|
from core.video.youtube_enrichment import get_youtube_date_enricher
|
||||||
|
yt = get_youtube_date_enricher()
|
||||||
|
if yt:
|
||||||
|
yt.resume()
|
||||||
|
except Exception:
|
||||||
|
logger.debug("video enrichment: could not resume YouTube date enricher", exc_info=True)
|
||||||
|
self._scan_paused_yt = False
|
||||||
|
if getattr(self, "_scan_paused", None):
|
||||||
|
logger.info("video enrichment: resumed %s after library scan",
|
||||||
|
", ".join(sorted(self._scan_paused)))
|
||||||
|
self._scan_paused = set()
|
||||||
|
|
||||||
|
def refresh_show_art(self, show_id, *, with_ratings: bool = True) -> dict:
|
||||||
|
"""On-demand (lazy) backfill of a show's season posters + episode art from
|
||||||
|
TMDB, used when the detail page is opened and art is missing. Works
|
||||||
|
regardless of the show's match status (sidesteps 'already matched, never
|
||||||
|
re-runs'), and caches the result so it's a one-time cost per show.
|
||||||
|
|
||||||
|
``with_ratings=False`` skips the OMDb ratings backfill — used by the bulk
|
||||||
|
'Refresh Airing TV Schedules' automation, which only needs episode schedules and
|
||||||
|
would otherwise burn the daily OMDb quota one call per show."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled:
|
||||||
|
return {"ok": False, "reason": "tmdb_not_configured"}
|
||||||
|
info = self.db.show_match_info(show_id)
|
||||||
|
if not info:
|
||||||
|
return {"ok": False, "reason": "not_found"}
|
||||||
|
try:
|
||||||
|
result = w.client.match("show", info.get("title"), info.get("year"),
|
||||||
|
known_id=info.get("tmdb_id"))
|
||||||
|
except Exception:
|
||||||
|
logger.exception("refresh_show_art: match failed for show %s", show_id)
|
||||||
|
return {"ok": False, "reason": "match_error"}
|
||||||
|
if not result or not result.get("id"):
|
||||||
|
return {"ok": False, "reason": "no_match"}
|
||||||
|
# Backfills season posters + show metadata gaps (never clobbers).
|
||||||
|
self.db.enrichment_apply("tmdb", "show", show_id, matched=True,
|
||||||
|
external_id=result["id"], metadata=result.get("metadata"))
|
||||||
|
try:
|
||||||
|
nums = [s["season_number"] for s in (result.get("metadata") or {}).get("seasons") or []]
|
||||||
|
w._cascade_episodes(show_id, result["id"], nums) # full list: owned + missing
|
||||||
|
except Exception:
|
||||||
|
logger.exception("refresh_show_art: episode cascade failed for show %s", show_id)
|
||||||
|
if with_ratings:
|
||||||
|
self._backfill_ratings("show", show_id)
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
def refresh_movie_art(self, movie_id) -> dict:
|
||||||
|
"""On-demand (lazy) backfill of a movie's cast / genres / backdrop / ratings
|
||||||
|
from TMDB when the detail page is opened and they're missing. Works
|
||||||
|
regardless of match status; caches the result."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled:
|
||||||
|
return {"ok": False, "reason": "tmdb_not_configured"}
|
||||||
|
info = self.db.movie_match_info(movie_id)
|
||||||
|
if not info:
|
||||||
|
return {"ok": False, "reason": "not_found"}
|
||||||
|
try:
|
||||||
|
result = w.client.match("movie", info.get("title"), info.get("year"),
|
||||||
|
known_id=info.get("tmdb_id"))
|
||||||
|
except Exception:
|
||||||
|
logger.exception("refresh_movie_art: match failed for movie %s", movie_id)
|
||||||
|
return {"ok": False, "reason": "match_error"}
|
||||||
|
if not result or not result.get("id"):
|
||||||
|
return {"ok": False, "reason": "no_match"}
|
||||||
|
self.db.enrichment_apply("tmdb", "movie", movie_id, matched=True,
|
||||||
|
external_id=result["id"], metadata=result.get("metadata"))
|
||||||
|
self._backfill_ratings("movie", movie_id)
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
def item_extras(self, kind, item_id) -> dict:
|
||||||
|
"""Live TMDB extras (trailer / where-to-watch / similar) for the detail
|
||||||
|
page. Not cached — fetched per view so providers stay current. For an
|
||||||
|
owned item we also surface a 'watch on your server' deep link as the first
|
||||||
|
where-to-watch option."""
|
||||||
|
out = {}
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if w and w.enabled:
|
||||||
|
info = (self.db.movie_match_info(item_id) if kind == "movie"
|
||||||
|
else self.db.show_match_info(item_id))
|
||||||
|
if info and info.get("tmdb_id"):
|
||||||
|
region = self._region()
|
||||||
|
key = ("extras", kind, info["tmdb_id"], region)
|
||||||
|
cached = self._cache_get(key)
|
||||||
|
if cached is None:
|
||||||
|
try:
|
||||||
|
cached = w.client.extras(kind, info["tmdb_id"], region=region) or {}
|
||||||
|
self._cache_put(key, cached)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("item_extras failed for %s %s", kind, item_id)
|
||||||
|
cached = {}
|
||||||
|
out = dict(cached) # copy — the per-item server link isn't cached
|
||||||
|
srv = self._server_watch_link(kind, item_id)
|
||||||
|
if srv:
|
||||||
|
out["server"] = srv
|
||||||
|
return out
|
||||||
|
|
||||||
|
def _server_watch_link(self, kind, item_id) -> dict | None:
|
||||||
|
"""A 'play on your media server' deep link for an owned item, or None.
|
||||||
|
Plex → the Plex web app at the item; Jellyfin → its web detail page."""
|
||||||
|
table = "movies" if kind == "movie" else "shows"
|
||||||
|
try:
|
||||||
|
with self.db.connect() as c:
|
||||||
|
row = c.execute(
|
||||||
|
f"SELECT server_source, server_id FROM {table} WHERE id=?", (item_id,)).fetchone()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
if not row:
|
||||||
|
return None
|
||||||
|
source, sid = row["server_source"], row["server_id"]
|
||||||
|
if not source or not sid:
|
||||||
|
return None # not on a server (e.g. a wishlist row)
|
||||||
|
try:
|
||||||
|
# Use the VIDEO side's effective connection (its own creds, or inherited
|
||||||
|
# from music) — the item lives on the server the video side scanned.
|
||||||
|
from core.video.sources import video_plex_config, video_jellyfin_config
|
||||||
|
if source == "plex":
|
||||||
|
cfg = video_plex_config(self.db)
|
||||||
|
base, token = cfg.get("base_url"), cfg.get("token")
|
||||||
|
if not base or not token:
|
||||||
|
return None
|
||||||
|
mid = self._plex_machine_id(base, token)
|
||||||
|
if not mid:
|
||||||
|
return None
|
||||||
|
from urllib.parse import quote
|
||||||
|
key = quote("/library/metadata/" + str(sid), safe="")
|
||||||
|
return {"server": "Plex",
|
||||||
|
"url": "https://app.plex.tv/desktop/#!/server/%s/details?key=%s" % (mid, key)}
|
||||||
|
if source == "jellyfin":
|
||||||
|
cfg = video_jellyfin_config(self.db)
|
||||||
|
base = cfg.get("base_url")
|
||||||
|
if not base:
|
||||||
|
return None
|
||||||
|
return {"server": "Jellyfin",
|
||||||
|
"url": base.rstrip("/") + "/web/index.html#!/details?id=" + str(sid)}
|
||||||
|
except Exception:
|
||||||
|
logger.exception("server watch link failed for %s %s", kind, item_id)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _plex_machine_id(self, base, token):
|
||||||
|
"""The Plex server's machineIdentifier (needed for app.plex.tv deep links),
|
||||||
|
fetched once and cached per base URL."""
|
||||||
|
cached = getattr(self, "_plex_mid", None)
|
||||||
|
if cached and cached[0] == base:
|
||||||
|
return cached[1]
|
||||||
|
try:
|
||||||
|
import requests
|
||||||
|
r = requests.get(base.rstrip("/") + "/identity",
|
||||||
|
params={"X-Plex-Token": token},
|
||||||
|
headers={"Accept": "application/json"}, timeout=8)
|
||||||
|
mid = None
|
||||||
|
try:
|
||||||
|
mid = ((r.json() or {}).get("MediaContainer") or {}).get("machineIdentifier")
|
||||||
|
except Exception:
|
||||||
|
import re
|
||||||
|
m = re.search(r'machineIdentifier="([^"]+)"', r.text or "")
|
||||||
|
mid = m.group(1) if m else None
|
||||||
|
if mid:
|
||||||
|
self._plex_mid = (base, mid)
|
||||||
|
return mid
|
||||||
|
except Exception:
|
||||||
|
logger.exception("plex identity fetch failed")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# ── in-app search + TMDB-backed (un-owned) detail ─────────────────────────
|
||||||
|
def search(self, query) -> list:
|
||||||
|
"""Multi-search via TMDB, each movie/show annotated with the library row id
|
||||||
|
if it's already owned (so the UI links to the owned detail, not the tmdb
|
||||||
|
view)."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled:
|
||||||
|
return []
|
||||||
|
# Short TTL — identical queries within ~a minute reuse the result; ownership
|
||||||
|
# is re-stamped fresh below so 'In Library' badges stay current.
|
||||||
|
key = ("search", (query or "").strip().lower())
|
||||||
|
results = self._cache_get(key)
|
||||||
|
if results is None:
|
||||||
|
try:
|
||||||
|
results = w.client.search(query) or []
|
||||||
|
self._cache_put(key, results, ttl=60)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video search failed for %r", query)
|
||||||
|
return []
|
||||||
|
return self._stamp_owned(results)
|
||||||
|
|
||||||
|
def trending(self, window="week", kind=None) -> list:
|
||||||
|
"""Trending titles, annotated owned/not. ``window='day'`` is the real-time
|
||||||
|
daily chart (the iconic 'Top 10 today' rows); 'week' is the steadier hero/idle
|
||||||
|
feed. ``kind`` None = mixed; 'movie'/'show' = the dedicated single-type charts
|
||||||
|
(split 'Top 10 Movies / TV Shows Today'). Order is preserved for rank numbers."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled:
|
||||||
|
return []
|
||||||
|
window = "day" if window == "day" else "week"
|
||||||
|
kind = kind if kind in ("movie", "show") else None
|
||||||
|
ck = ("trending", window, kind)
|
||||||
|
cached = self._cache_get(ck)
|
||||||
|
if cached is None:
|
||||||
|
try:
|
||||||
|
cached = w.client.trending(window=window, kind=kind) or []
|
||||||
|
self._cache_put(ck, cached, ttl=3600)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video trending failed")
|
||||||
|
return []
|
||||||
|
# Re-annotate ownership fresh each call (batched) so it tracks the library.
|
||||||
|
return self._stamp_owned(cached)
|
||||||
|
|
||||||
|
# ── discover (browse TMDB lists; owned titles annotated) ──────────────────
|
||||||
|
def _stamp_owned(self, items):
|
||||||
|
"""Annotate each movie/show with its library_id for the active server —
|
||||||
|
stamped fresh (not cached with ownership) so 'In Library' tracks scans.
|
||||||
|
Batched: one query per kind for the whole list, not one per item."""
|
||||||
|
srv = self._server()
|
||||||
|
by_kind: dict = {}
|
||||||
|
for r in items or []:
|
||||||
|
if r.get("kind") in ("movie", "show") and r.get("tmdb_id"):
|
||||||
|
by_kind.setdefault(r["kind"], []).append(r["tmdb_id"])
|
||||||
|
maps = {k: self.db.library_ids_for_tmdb(k, ids, srv) for k, ids in by_kind.items()}
|
||||||
|
for r in items or []:
|
||||||
|
if r.get("kind") in ("movie", "show") and r.get("tmdb_id"):
|
||||||
|
try:
|
||||||
|
r["library_id"] = maps.get(r["kind"], {}).get(int(r["tmdb_id"]))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
r["library_id"] = None
|
||||||
|
# Drop titles the user marked 'Not interested' — one tiny indexed query, always
|
||||||
|
# fresh, so every discover surface (rails, recs, collection gaps) hides them uniformly.
|
||||||
|
try:
|
||||||
|
ignored = self.db.ignored_keys()
|
||||||
|
except Exception:
|
||||||
|
ignored = None
|
||||||
|
if ignored:
|
||||||
|
items = [r for r in (items or [])
|
||||||
|
if f"{r.get('kind')}:{r.get('tmdb_id')}" not in ignored]
|
||||||
|
return items
|
||||||
|
|
||||||
|
def discover_curated(self, key, page=1) -> list:
|
||||||
|
"""A canned TMDB list (popular / top-rated / now-playing / upcoming /
|
||||||
|
on-the-air / airing-today), cached then owned-annotated."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled:
|
||||||
|
return []
|
||||||
|
ck = ("disc-cur", key, page)
|
||||||
|
items = self._cache_get(ck)
|
||||||
|
if items is None:
|
||||||
|
try:
|
||||||
|
items = w.client.curated(key, page=page) or []
|
||||||
|
self._cache_put(ck, items, ttl=3600)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("discover curated failed (%s p%s)", key, page)
|
||||||
|
return []
|
||||||
|
return self._stamp_owned(items)
|
||||||
|
|
||||||
|
def discover_filter(self, kind, *, genre=None, year=None, decade=None, providers=None,
|
||||||
|
sort_by="popularity.desc", page=1, language=None,
|
||||||
|
keywords=None, companies=None, networks=None, cast=None, crew=None,
|
||||||
|
min_runtime=None, max_runtime=None, certification=None,
|
||||||
|
vote_count_min=None, release_window=None) -> list:
|
||||||
|
"""Browse /discover filtered by genre / year / decade / streaming provider /
|
||||||
|
original language — plus the Netflix-class extensions (keywords/mood,
|
||||||
|
companies/studio, networks, cast/crew, runtime, certification, release window).
|
||||||
|
Cached + owned-annotated; all extension params are optional/additive."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled:
|
||||||
|
return []
|
||||||
|
if kind not in ("movie", "show"):
|
||||||
|
kind = "movie"
|
||||||
|
ck = ("disc-flt", kind, genre, year, decade, providers, sort_by, page, language,
|
||||||
|
keywords, companies, networks, cast, crew, min_runtime, max_runtime,
|
||||||
|
certification, vote_count_min, release_window)
|
||||||
|
items = self._cache_get(ck)
|
||||||
|
if items is None:
|
||||||
|
try:
|
||||||
|
items = w.client.discover(
|
||||||
|
kind, genre=genre, year=year, decade=decade, providers=providers,
|
||||||
|
sort_by=sort_by, page=page, region=self._region(), language=language,
|
||||||
|
keywords=keywords, companies=companies, networks=networks, cast=cast,
|
||||||
|
crew=crew, min_runtime=min_runtime, max_runtime=max_runtime,
|
||||||
|
certification=certification, vote_count_min=vote_count_min,
|
||||||
|
release_window=release_window) or []
|
||||||
|
self._cache_put(ck, items, ttl=3600)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("discover filter failed (%s g=%s y=%s d=%s p=%s)",
|
||||||
|
kind, genre, year, decade, providers)
|
||||||
|
return []
|
||||||
|
return self._stamp_owned(items)
|
||||||
|
|
||||||
|
def genre_list(self, kind) -> list:
|
||||||
|
"""TMDB genre id→name list (long-cached — these barely change)."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled:
|
||||||
|
return []
|
||||||
|
if kind not in ("movie", "show"):
|
||||||
|
kind = "movie"
|
||||||
|
ck = ("genres", kind)
|
||||||
|
items = self._cache_get(ck)
|
||||||
|
if items is None:
|
||||||
|
try:
|
||||||
|
items = w.client.genres(kind) or []
|
||||||
|
self._cache_put(ck, items, ttl=86400)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("genre list failed (%s)", kind)
|
||||||
|
return []
|
||||||
|
return items
|
||||||
|
|
||||||
|
def recommendations(self, kind, tmdb_id, page=1) -> list:
|
||||||
|
"""'More like this' titles for a tmdb id — cached + owned-annotated."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled or not tmdb_id:
|
||||||
|
return []
|
||||||
|
ck = ("recs", kind, tmdb_id, page)
|
||||||
|
items = self._cache_get(ck)
|
||||||
|
if items is None:
|
||||||
|
try:
|
||||||
|
items = w.client.recommendations(kind, tmdb_id, page=page) or []
|
||||||
|
self._cache_put(ck, items, ttl=3600)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("recommendations failed (%s %s)", kind, tmdb_id)
|
||||||
|
return []
|
||||||
|
return self._stamp_owned(items)
|
||||||
|
|
||||||
|
def collection(self, collection_id) -> list:
|
||||||
|
"""The films of a TMDB collection (franchise) — cached + owned-annotated.
|
||||||
|
Drives the 'complete your collections' gap rails."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled or not collection_id:
|
||||||
|
return []
|
||||||
|
ck = ("collection", collection_id)
|
||||||
|
items = self._cache_get(ck)
|
||||||
|
if items is None:
|
||||||
|
try:
|
||||||
|
items = w.client.collection(collection_id) or []
|
||||||
|
self._cache_put(ck, items, ttl=86400) # franchises rarely change
|
||||||
|
except Exception:
|
||||||
|
logger.exception("collection failed (%s)", collection_id)
|
||||||
|
return []
|
||||||
|
return self._stamp_owned(items)
|
||||||
|
|
||||||
|
def movie_collection(self, tmdb_id) -> dict | None:
|
||||||
|
"""A movie's TMDB franchise membership {id, name} (belongs_to_collection), or
|
||||||
|
{id: None} when it belongs to no collection. Used to backfill already-matched
|
||||||
|
movies that predate the collection column."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled or not tmdb_id:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
meta = w.client.match("movie", None, None, known_id=tmdb_id) or {}
|
||||||
|
return {"id": meta.get("tmdb_collection_id"), "name": meta.get("tmdb_collection_name")}
|
||||||
|
except Exception:
|
||||||
|
logger.exception("movie_collection backfill failed for %s", tmdb_id)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def trailer(self, kind, tmdb_id) -> dict | None:
|
||||||
|
"""Best YouTube trailer for a title (cached a day — trailers don't move)."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled or not tmdb_id:
|
||||||
|
return None
|
||||||
|
ck = ("trailer", kind, tmdb_id)
|
||||||
|
cached = self._cache_get(ck)
|
||||||
|
if cached is None:
|
||||||
|
try:
|
||||||
|
cached = w.client.video_trailer(kind, tmdb_id) or {}
|
||||||
|
self._cache_put(ck, cached, ttl=86400)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("trailer failed (%s %s)", kind, tmdb_id)
|
||||||
|
return None
|
||||||
|
return cached or None
|
||||||
|
|
||||||
|
def tmdb_full_detail(self, kind, tmdb_id) -> dict | None:
|
||||||
|
"""Raw TMDB full detail (absolute image URLs + metadata) WITHOUT the
|
||||||
|
owned→library redirect — for sidecar / NFO writing, which needs the data even
|
||||||
|
for titles already in the library."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return w.client.full_detail(kind, tmdb_id, region=self._region())
|
||||||
|
except Exception:
|
||||||
|
logger.exception("tmdb_full_detail failed for %s %s", kind, tmdb_id)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def tmdb_detail(self, kind, tmdb_id) -> dict | None:
|
||||||
|
"""Full detail for a TMDB title not in the library — same shape as the
|
||||||
|
library detail (source='tmdb', direct image URLs, nothing owned). If it IS
|
||||||
|
in the library, returns a redirect to the owned detail instead."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled:
|
||||||
|
return None
|
||||||
|
lib_id = self.db.library_id_for_tmdb(kind, tmdb_id, self._server())
|
||||||
|
if lib_id:
|
||||||
|
return {"redirect": {"source": "library", "kind": kind, "id": lib_id}}
|
||||||
|
region = self._region()
|
||||||
|
cached = self._cache_get(("detail", kind, tmdb_id, region))
|
||||||
|
if cached is not None:
|
||||||
|
return dict(cached)
|
||||||
|
try:
|
||||||
|
d = w.client.full_detail(kind, tmdb_id, region=region)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("tmdb_detail failed for %s %s", kind, tmdb_id)
|
||||||
|
return None
|
||||||
|
if not d:
|
||||||
|
return None
|
||||||
|
d.update({"source": "tmdb", "id": tmdb_id, "owned": False, "monitored": False,
|
||||||
|
"has_poster": bool(d.get("poster_url")), "has_backdrop": bool(d.get("backdrop_url"))})
|
||||||
|
ex = d.pop("_extras", {}) or {}
|
||||||
|
d.update({"trailer": ex.get("trailer"), "providers": ex.get("providers") or [],
|
||||||
|
"providers_link": ex.get("providers_link"), "similar": ex.get("similar") or [],
|
||||||
|
"recommendations": ex.get("recommendations") or [], "collection": ex.get("collection"),
|
||||||
|
"next_episode": ex.get("next_episode"), "last_episode": ex.get("last_episode"),
|
||||||
|
"gallery": ex.get("gallery"), "videos": ex.get("videos") or [],
|
||||||
|
"keywords": ex.get("keywords") or [], "facts": ex.get("facts"),
|
||||||
|
"cast_full": ex.get("cast_full") or [], "review": ex.get("review")})
|
||||||
|
if kind == "show":
|
||||||
|
seasons = d.pop("_seasons", []) or []
|
||||||
|
for s in seasons:
|
||||||
|
s["has_poster"] = bool(s.get("poster_url"))
|
||||||
|
s["episode_total"] = s.pop("episode_count", 0) or 0
|
||||||
|
s["episode_owned"] = 0
|
||||||
|
s["episodes"] = [] # loaded lazily per season (tmdb_season)
|
||||||
|
d["seasons"] = seasons
|
||||||
|
d["season_count"] = len(seasons)
|
||||||
|
d["episode_total"] = sum(s["episode_total"] for s in seasons)
|
||||||
|
d["episode_owned"] = 0
|
||||||
|
self._fill_tmdb_ratings(d)
|
||||||
|
self._cache_put(("detail", kind, tmdb_id, region), d)
|
||||||
|
return d
|
||||||
|
|
||||||
|
def _fill_tmdb_ratings(self, d) -> None:
|
||||||
|
imdb_id = d.get("imdb_id")
|
||||||
|
ow = self.workers.get("omdb")
|
||||||
|
# share the same daily-limit latch as _backfill_ratings — once OMDb is over quota,
|
||||||
|
# every detail fetch would otherwise re-hit it and dump a traceback per title.
|
||||||
|
if (not imdb_id or not ow or not getattr(ow.client, "enabled", False)
|
||||||
|
or getattr(self, "_omdb_blocked", False)):
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
r = ow.client.ratings(imdb_id) or {}
|
||||||
|
for k in ("imdb_rating", "rt_rating", "metacritic"):
|
||||||
|
if r.get(k) is not None:
|
||||||
|
d[k] = r[k]
|
||||||
|
except OMDbAuthError as e:
|
||||||
|
self._omdb_blocked = True
|
||||||
|
logger.warning("OMDb ratings paused for this run: %s", e)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("tmdb_detail ratings failed for %s", imdb_id)
|
||||||
|
|
||||||
|
def tmdb_season(self, tv_id, season_number) -> dict | None:
|
||||||
|
"""One season's episodes for a TMDB (un-owned) show — lazy-loaded when the
|
||||||
|
season is selected on the search detail page. Nothing is owned."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled:
|
||||||
|
return None
|
||||||
|
key = ("season", tv_id, season_number)
|
||||||
|
cached = self._cache_get(key)
|
||||||
|
if cached is not None:
|
||||||
|
return cached
|
||||||
|
try:
|
||||||
|
se = w.client.season_episodes(tv_id, season_number)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("tmdb_season failed for %s S%s", tv_id, season_number)
|
||||||
|
return None
|
||||||
|
if not se:
|
||||||
|
return None
|
||||||
|
eps = [{"episode_number": e.get("episode_number"), "title": e.get("title"),
|
||||||
|
"overview": e.get("overview"), "air_date": e.get("air_date"),
|
||||||
|
"runtime_minutes": e.get("runtime_minutes"), "rating": e.get("rating"),
|
||||||
|
"still_url": e.get("still_url"), "has_still": bool(e.get("still_url")),
|
||||||
|
"owned": False}
|
||||||
|
for e in (se.get("episodes") or []) if e.get("episode_number") is not None]
|
||||||
|
out = {"season_number": season_number, "overview": se.get("overview"),
|
||||||
|
"poster_url": se.get("poster_url"), "episodes": eps}
|
||||||
|
self._cache_put(key, out)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def episode_extra(self, tmdb_id, season_number, episode_number) -> dict | None:
|
||||||
|
"""Deeper episode detail (guest stars + still) for the episode expand,
|
||||||
|
annotated owned/not + cached."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled or not tmdb_id:
|
||||||
|
return None
|
||||||
|
key = ("episode", tmdb_id, season_number, episode_number)
|
||||||
|
cached = self._cache_get(key)
|
||||||
|
if cached is None:
|
||||||
|
try:
|
||||||
|
cached = w.client.episode_detail(tmdb_id, season_number, episode_number)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("episode_extra failed for %s S%sE%s", tmdb_id, season_number, episode_number)
|
||||||
|
return None
|
||||||
|
if cached is None:
|
||||||
|
return None
|
||||||
|
self._cache_put(key, cached)
|
||||||
|
return cached
|
||||||
|
|
||||||
|
def person_detail(self, tmdb_id) -> dict | None:
|
||||||
|
"""A person (actor/director) page — bio + filmography, each credit
|
||||||
|
annotated with the library id if owned. Keeps cast clicks in-app."""
|
||||||
|
w = self.workers.get("tmdb")
|
||||||
|
if not w or not w.enabled:
|
||||||
|
return None
|
||||||
|
p = self._cache_get(("person", tmdb_id))
|
||||||
|
if p is None:
|
||||||
|
try:
|
||||||
|
p = w.client.person(tmdb_id)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("person_detail failed for %s", tmdb_id)
|
||||||
|
return None
|
||||||
|
if not p:
|
||||||
|
return None
|
||||||
|
self._cache_put(("person", tmdb_id), p)
|
||||||
|
# Re-annotate ownership fresh each call (cheap) so it tracks the library.
|
||||||
|
srv = self._server()
|
||||||
|
for c in p.get("credits") or []:
|
||||||
|
if c.get("tmdb_id"):
|
||||||
|
c["library_id"] = self.db.library_id_for_tmdb(c["kind"], c["tmdb_id"], srv)
|
||||||
|
return p
|
||||||
|
|
||||||
|
def _server(self):
|
||||||
|
"""Active video server — scopes ownership lookups so an item owned only on
|
||||||
|
the inactive server doesn't read as owned (Plex/Jellyfin stay separate)."""
|
||||||
|
try:
|
||||||
|
from core.video.sources import resolve_video_server
|
||||||
|
return resolve_video_server(self.db)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def worker(self, service):
|
||||||
|
return self.workers.get(service)
|
||||||
|
|
||||||
|
def services(self) -> list:
|
||||||
|
return [{"id": s, "display_name": w.display_name} for s, w in self.workers.items()]
|
||||||
|
|
||||||
|
|
||||||
|
_engine = None
|
||||||
|
_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def get_video_enrichment_engine():
|
||||||
|
"""Process-wide engine, created (and started) on first use."""
|
||||||
|
global _engine
|
||||||
|
if _engine is None:
|
||||||
|
with _lock:
|
||||||
|
if _engine is None:
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
from .clients import build_clients, OMDBClient
|
||||||
|
db = VideoDatabase()
|
||||||
|
eng = VideoEnrichmentEngine(db, build_clients(db),
|
||||||
|
ratings_client=OMDBClient(db.get_setting("omdb_api_key")))
|
||||||
|
eng.start_all()
|
||||||
|
_engine = eng
|
||||||
|
return _engine
|
||||||
|
|
||||||
|
|
||||||
|
def peek_video_enrichment_engine():
|
||||||
|
"""The engine ONLY if it's already running — never creates or starts it. Lets
|
||||||
|
the socket status emitter push updates without spinning up the video engine on
|
||||||
|
the music side (it stays None until something actually uses video)."""
|
||||||
|
return _engine
|
||||||
|
|
||||||
|
|
||||||
|
def rebuild_video_enrichment_engine():
|
||||||
|
"""Rebuild the engine so workers pick up changed API keys (stops the old
|
||||||
|
workers first so threads don't leak)."""
|
||||||
|
global _engine
|
||||||
|
with _lock:
|
||||||
|
if _engine is not None:
|
||||||
|
try:
|
||||||
|
_engine.stop_all()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video enrichment: stopping old engine failed")
|
||||||
|
_engine = None
|
||||||
|
return get_video_enrichment_engine()
|
||||||
340
core/video/enrichment/worker.py
Normal file
340
core/video/enrichment/worker.py
Normal file
|
|
@ -0,0 +1,340 @@
|
||||||
|
"""Video enrichment worker — one per source (TMDB, TVDB).
|
||||||
|
|
||||||
|
Mirrors the music worker: a daemon loop that pulls the next item needing
|
||||||
|
enrichment from video.db, asks its CLIENT to match it, and records the result.
|
||||||
|
The client is injected (a thin TMDB/TVDB adapter), so the worker's loop/queue/
|
||||||
|
status logic is fully testable with a fake client. Isolated: imports only
|
||||||
|
video.db helpers; no music code.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_enrichment.worker")
|
||||||
|
|
||||||
|
|
||||||
|
class VideoEnrichmentWorker:
|
||||||
|
def __init__(self, db, service, client, display_name=None, interval=2.0, retry_days=30):
|
||||||
|
self.db = db
|
||||||
|
self.service = service
|
||||||
|
self.client = client
|
||||||
|
self.display_name = display_name or service.upper()
|
||||||
|
self.interval = interval
|
||||||
|
self.retry_days = retry_days
|
||||||
|
|
||||||
|
# OMDb is a ratings filler, not a matcher — it fetches scores by imdb_id
|
||||||
|
# instead of running a match queue.
|
||||||
|
self.is_ratings = hasattr(client, "ratings") and not hasattr(client, "match")
|
||||||
|
|
||||||
|
self.running = False
|
||||||
|
self.paused = False
|
||||||
|
self.should_stop = False
|
||||||
|
self._thread = None
|
||||||
|
self._stop = threading.Event()
|
||||||
|
self.current_item = None
|
||||||
|
self.stats = {"matched": 0, "not_found": 0, "errors": 0}
|
||||||
|
self.note = None # a human reason when auto-paused (e.g. bad key)
|
||||||
|
self._rating_errors = 0 # consecutive ratings failures (transient backoff)
|
||||||
|
self._cooldown_until = 0.0 # monotonic time to idle until (daily-limit backoff)
|
||||||
|
|
||||||
|
# ── lifecycle ─────────────────────────────────────────────────────────────
|
||||||
|
def start(self):
|
||||||
|
if self.running:
|
||||||
|
return
|
||||||
|
self.should_stop = False
|
||||||
|
self._stop.clear()
|
||||||
|
self.running = True
|
||||||
|
self._thread = threading.Thread(target=self._run, daemon=True)
|
||||||
|
self._thread.start()
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
self.should_stop = True
|
||||||
|
self._stop.set()
|
||||||
|
if self._thread:
|
||||||
|
self._thread.join(timeout=1.0)
|
||||||
|
self.running = False
|
||||||
|
|
||||||
|
def pause(self, persist=True):
|
||||||
|
self.paused = True
|
||||||
|
if persist:
|
||||||
|
self._persist_paused()
|
||||||
|
|
||||||
|
def resume(self, persist=True):
|
||||||
|
self.paused = False
|
||||||
|
if persist:
|
||||||
|
self._persist_paused()
|
||||||
|
|
||||||
|
def _persist_paused(self):
|
||||||
|
# Survives restart, like music's <service>_enrichment_paused config flag.
|
||||||
|
try:
|
||||||
|
self.db.set_setting(self.service + "_paused", "1" if self.paused else "0")
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video enrichment: could not persist pause for %s", self.service)
|
||||||
|
|
||||||
|
def restore_paused(self):
|
||||||
|
try:
|
||||||
|
self.paused = str(self.db.get_setting(self.service + "_paused") or "") == "1"
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video enrichment: could not restore pause for %s", self.service)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def enabled(self):
|
||||||
|
return bool(getattr(self.client, "enabled", False))
|
||||||
|
|
||||||
|
# ── loop ──────────────────────────────────────────────────────────────────
|
||||||
|
def _run(self):
|
||||||
|
while not self.should_stop:
|
||||||
|
if self.paused or not self.enabled:
|
||||||
|
self._stop.wait(1.0)
|
||||||
|
continue
|
||||||
|
# Daily-limit cooldown: idle until the quota resets, then auto-resume
|
||||||
|
# (re-checks periodically so we pick back up shortly after midnight UTC).
|
||||||
|
if self._cooldown_until > time.monotonic():
|
||||||
|
self.current_item = None
|
||||||
|
self._stop.wait(15.0)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
did = self.process_one()
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video enrichment %s loop error", self.service)
|
||||||
|
self.stats["errors"] += 1
|
||||||
|
self._stop.wait(5.0)
|
||||||
|
continue
|
||||||
|
if did:
|
||||||
|
self._stop.wait(self.interval) # rate-limit between items
|
||||||
|
else:
|
||||||
|
self.current_item = None
|
||||||
|
self._stop.wait(10.0) # nothing to do — back off
|
||||||
|
|
||||||
|
def process_one(self) -> bool:
|
||||||
|
"""Process a single item. Returns True if one was processed."""
|
||||||
|
if self.is_ratings:
|
||||||
|
return self._process_ratings_one()
|
||||||
|
priority = None
|
||||||
|
try:
|
||||||
|
priority = self.db.get_setting("enrichment_priority") or None
|
||||||
|
except Exception:
|
||||||
|
logger.debug("enrichment priority lookup failed", exc_info=True)
|
||||||
|
item = self.db.enrichment_next(self.service, self.retry_days, priority=priority)
|
||||||
|
if not item:
|
||||||
|
# No pending matches → use idle time for the full episode-list sync
|
||||||
|
# (which also fills details for shows it touches), then the details
|
||||||
|
# backfill for everything already episode-synced but missing `status`.
|
||||||
|
return self._sync_episodes_once() or self._detail_backfill_one()
|
||||||
|
self.current_item = {"type": item["kind"], "name": item["title"]}
|
||||||
|
try:
|
||||||
|
# Prefer the provider id the server already gave us (enrich BY ID, no
|
||||||
|
# re-search); the client falls back to a title/year search if it's None.
|
||||||
|
result = self.client.match(item["kind"], item["title"], item.get("year"),
|
||||||
|
known_id=item.get("known_id"))
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video enrichment %s match failed for %s", self.service, item["title"])
|
||||||
|
self.stats["errors"] += 1
|
||||||
|
# The CALL failed (network/rate-limit/timeout) — record 'error', NOT
|
||||||
|
# 'not_found', so a transient blip isn't permanently logged as "no
|
||||||
|
# match". enrichment_next retries 'error' items after retry_days.
|
||||||
|
self.db.enrichment_apply(self.service, item["kind"], item["id"], matched=False, error=True)
|
||||||
|
return True
|
||||||
|
if result and result.get("id"):
|
||||||
|
self.db.enrichment_apply(self.service, item["kind"], item["id"], matched=True,
|
||||||
|
external_id=result["id"], metadata=result.get("metadata"))
|
||||||
|
self.stats["matched"] += 1
|
||||||
|
# Visible progress in app.log, mirroring the music workers' style.
|
||||||
|
logger.info("Matched %s '%s' -> %s ID: %s%s", item["kind"], item["title"],
|
||||||
|
self.display_name, result["id"],
|
||||||
|
" (by server id)" if item.get("known_id") else "")
|
||||||
|
# Cascade: a matched show backfills its episodes' art/overview/rating
|
||||||
|
# from the same provider (one call per season), so episodes ride along
|
||||||
|
# with their show instead of being a separate (huge) queue.
|
||||||
|
if item["kind"] == "show" and hasattr(self.client, "season_episodes"):
|
||||||
|
nums = [s["season_number"] for s in (result.get("metadata") or {}).get("seasons") or []]
|
||||||
|
self._cascade_episodes(item["id"], result["id"], nums)
|
||||||
|
else:
|
||||||
|
self.db.enrichment_apply(self.service, item["kind"], item["id"], matched=False)
|
||||||
|
self.stats["not_found"] += 1
|
||||||
|
logger.info("No %s match for %s '%s'", self.display_name, item["kind"], item["title"])
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _process_ratings_one(self) -> bool:
|
||||||
|
"""OMDb worker: fetch IMDb/RT/Metacritic for the next library item that has
|
||||||
|
an imdb_id but no ratings yet."""
|
||||||
|
from .clients import OMDbAuthError
|
||||||
|
item = self.db.ratings_next()
|
||||||
|
if not item:
|
||||||
|
self._rating_errors = 0
|
||||||
|
return False
|
||||||
|
self.current_item = {"type": item["kind"], "name": item["title"]}
|
||||||
|
try:
|
||||||
|
r = self.client.ratings(item["imdb_id"])
|
||||||
|
except OMDbAuthError as e:
|
||||||
|
msg = str(e)
|
||||||
|
if "limit" in msg.lower():
|
||||||
|
# Free-tier daily quota (1,000/req) — resets at midnight UTC. Cool
|
||||||
|
# down and AUTO-RESUME (so a big library just spreads across days)
|
||||||
|
# rather than pausing for good. Item not marked synced → retried.
|
||||||
|
if self._cooldown_until <= time.monotonic():
|
||||||
|
logger.warning("OMDb daily request limit reached — idling ratings ~30 min; "
|
||||||
|
"auto-resumes after the daily reset")
|
||||||
|
self._cooldown_until = time.monotonic() + 1800
|
||||||
|
self.note = "OMDb daily limit reached — resumes after reset"
|
||||||
|
return False
|
||||||
|
# Bad/expired/unactivated key affects EVERY item — pause instead of
|
||||||
|
# churning the whole library + flooding the log. Transient pause (not
|
||||||
|
# persisted) so fixing the key (which rebuilds the engine) resumes
|
||||||
|
# automatically. The item is NOT marked synced, so it retries once
|
||||||
|
# the key works.
|
||||||
|
if not self.paused:
|
||||||
|
logger.warning("OMDb rejected the API key — pausing ratings until it's fixed (%s)", msg)
|
||||||
|
self.note = "OMDb API key rejected"
|
||||||
|
self.pause(persist=False)
|
||||||
|
return False
|
||||||
|
except Exception as e:
|
||||||
|
# Transient (network / rate-limit / 5xx) — don't burn the item to
|
||||||
|
# 'synced'; back off, and pause after a few in a row so we don't spin.
|
||||||
|
self.stats["errors"] += 1
|
||||||
|
self._rating_errors = getattr(self, "_rating_errors", 0) + 1
|
||||||
|
logger.warning("OMDb ratings fetch failed for '%s': %s", item["title"], e)
|
||||||
|
if self._rating_errors >= 3:
|
||||||
|
logger.warning("OMDb: pausing ratings after repeated errors")
|
||||||
|
self.note = "OMDb temporarily unavailable"
|
||||||
|
self.pause(persist=False)
|
||||||
|
self._rating_errors = 0
|
||||||
|
return False
|
||||||
|
self._rating_errors = 0
|
||||||
|
self._cooldown_until = 0.0
|
||||||
|
self.note = None
|
||||||
|
if r:
|
||||||
|
self.db.apply_ratings(item["kind"], item["id"], r) # marks synced
|
||||||
|
self.stats["matched"] += 1
|
||||||
|
logger.info("Rated %s '%s' -> IMDb %s", item["kind"], item["title"], item["imdb_id"])
|
||||||
|
else:
|
||||||
|
self.db.mark_ratings_synced(item["kind"], item["id"]) # genuine no-data
|
||||||
|
self.stats["not_found"] += 1
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _sync_episodes_once(self) -> bool:
|
||||||
|
"""Background episode-sync: pull the FULL season/episode list for one
|
||||||
|
already-matched show that hasn't been synced, so library cards show real
|
||||||
|
owned/total. TMDB-only (it owns season_episodes). Returns True if it did
|
||||||
|
work (so the loop rate-limits between shows)."""
|
||||||
|
if not hasattr(self.client, "season_episodes"):
|
||||||
|
return False
|
||||||
|
show = self.db.episode_sync_next()
|
||||||
|
if not show:
|
||||||
|
return False
|
||||||
|
self.current_item = {"type": "episodes", "name": show["title"]}
|
||||||
|
try:
|
||||||
|
result = self.client.match("show", show["title"], show.get("year"),
|
||||||
|
known_id=show.get("tmdb_id"))
|
||||||
|
if result and result.get("id"):
|
||||||
|
self.db.enrichment_apply("tmdb", "show", show["id"], matched=True,
|
||||||
|
external_id=result["id"], metadata=result.get("metadata"))
|
||||||
|
nums = [s["season_number"] for s in (result.get("metadata") or {}).get("seasons") or []]
|
||||||
|
self._cascade_episodes(show["id"], result["id"], nums) # marks synced
|
||||||
|
logger.info("Synced full episode list for show '%s'", show["title"])
|
||||||
|
else:
|
||||||
|
self.db.mark_episodes_synced(show["id"]) # no match → don't re-pick
|
||||||
|
except Exception:
|
||||||
|
logger.exception("episode sync failed for show '%s'", show["title"])
|
||||||
|
self.db.mark_episodes_synced(show["id"]) # move on (never loop on one show)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _detail_backfill_one(self) -> bool:
|
||||||
|
"""Background TMDB details backfill: re-fetch ONE already-matched show/movie
|
||||||
|
that's missing details-only fields (status, network, tagline, rating…) and
|
||||||
|
gap-fill them. The matcher skips server-pre-matched items, so without this
|
||||||
|
their `status` stays blank — and the watchlist's airing-default can't see
|
||||||
|
them. Returns True if it did work (so the loop rate-limits).
|
||||||
|
|
||||||
|
TMDB-ONLY: `status` (+ network/tagline/…) come from TMDB and the queue is
|
||||||
|
keyed on tmdb_id. Running it on the TVDB worker would feed a TMDB id to
|
||||||
|
TVDB (→ 404 on every show) and double-process the queue."""
|
||||||
|
if self.service != "tmdb" or not hasattr(self.client, "match"):
|
||||||
|
return False
|
||||||
|
item = self.db.detail_backfill_next("show") or self.db.detail_backfill_next("movie")
|
||||||
|
if not item:
|
||||||
|
return False
|
||||||
|
self.current_item = {"type": item["kind"], "name": item["title"]}
|
||||||
|
try:
|
||||||
|
result = self.client.match(item["kind"], item["title"], item.get("year"),
|
||||||
|
known_id=item.get("tmdb_id"))
|
||||||
|
if result and result.get("id"):
|
||||||
|
# Gap-fill only (never clobbers server data); fills status et al.
|
||||||
|
self.db.enrichment_apply("tmdb", item["kind"], item["id"], matched=True,
|
||||||
|
external_id=result["id"], metadata=result.get("metadata"))
|
||||||
|
logger.info("Backfilled details for %s '%s'", item["kind"], item["title"])
|
||||||
|
self.db.mark_details_synced(item["kind"], item["id"]) # attempted once → don't re-pick
|
||||||
|
except Exception:
|
||||||
|
# Transient call failure — leave details_synced=0 so it retries later.
|
||||||
|
logger.exception("detail backfill failed for %s '%s'", item["kind"], item["title"])
|
||||||
|
self.stats["errors"] += 1
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _cascade_episodes(self, show_id, tv_id, season_numbers=None) -> None:
|
||||||
|
"""Backfill a show's FULL episode list from the provider (one call per
|
||||||
|
season) — owned + missing. Best-effort: a season failure never aborts the
|
||||||
|
show's enrichment. Falls back to the known seasons if none are passed."""
|
||||||
|
seasons = season_numbers
|
||||||
|
if not seasons:
|
||||||
|
try:
|
||||||
|
seasons = self.db.show_season_numbers(show_id)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("episode backfill: season list failed for show %s", show_id)
|
||||||
|
return
|
||||||
|
for snum in seasons:
|
||||||
|
try:
|
||||||
|
data = self.client.season_episodes(tv_id, snum)
|
||||||
|
if data and data.get("episodes"):
|
||||||
|
self.db.backfill_episodes(show_id, snum, data["episodes"],
|
||||||
|
data.get("overview"), data.get("poster_url"))
|
||||||
|
except Exception:
|
||||||
|
logger.exception("episode backfill failed: show %s season %s", show_id, snum)
|
||||||
|
try:
|
||||||
|
self.db.mark_episodes_synced(show_id)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("episode backfill: could not mark synced for show %s", show_id)
|
||||||
|
|
||||||
|
# ── status (same shape the music enrichment API returns) ──────────────────
|
||||||
|
def get_stats(self) -> dict:
|
||||||
|
breakdown = self.db.enrichment_breakdown(self.service)
|
||||||
|
# Errored items are outstanding (retried later), so they count as pending
|
||||||
|
# work — the worker isn't "Complete" while any remain. Episode art is a
|
||||||
|
# coverage-only cascade (no queue), so it's excluded from idle/pending.
|
||||||
|
pending = sum(b["pending"] + b.get("errors", 0)
|
||||||
|
for b in breakdown.values() if not b.get("coverage_only"))
|
||||||
|
# Shows still needing their full episode list pulled count as outstanding
|
||||||
|
# work for the TMDB worker (so it isn't "Complete" while syncing).
|
||||||
|
if hasattr(self.client, "season_episodes"):
|
||||||
|
try:
|
||||||
|
pending += self.db.episode_sync_pending_count()
|
||||||
|
except Exception:
|
||||||
|
logger.debug("episode_sync_pending_count failed", exc_info=True)
|
||||||
|
# NB: the TMDB details backfill (status/network/…) is a background gap-fill on
|
||||||
|
# ALREADY-matched items — like episode coverage, it doesn't block "Complete".
|
||||||
|
# It still runs in the idle loop; it's just not counted as blocking pending.
|
||||||
|
cooling = self._cooldown_until > time.monotonic()
|
||||||
|
running = self.running and not self.paused and self.enabled and not cooling
|
||||||
|
idle = running and pending == 0 and self.current_item is None
|
||||||
|
progress = {}
|
||||||
|
for kind, b in breakdown.items():
|
||||||
|
total = b["matched"] + b["not_found"] + b.get("errors", 0) + b["pending"]
|
||||||
|
done = b["matched"] + b["not_found"]
|
||||||
|
progress[kind] = {"matched": b["matched"], "total": total,
|
||||||
|
"percent": round(done / total * 100) if total else 0}
|
||||||
|
return {
|
||||||
|
"enabled": self.enabled,
|
||||||
|
"needs_key": True, # matchers (TMDB/TVDB/OMDb) always require an API key
|
||||||
|
"running": running,
|
||||||
|
"paused": self.paused or cooling, # cooldown reads as paused in the UI
|
||||||
|
"idle": idle,
|
||||||
|
"current_item": self.current_item,
|
||||||
|
"note": self.note,
|
||||||
|
"cooldown": cooling,
|
||||||
|
"stats": {**self.stats, "pending": pending},
|
||||||
|
"progress": progress,
|
||||||
|
"breakdown": breakdown,
|
||||||
|
}
|
||||||
386
core/video/importer.py
Normal file
386
core/video/importer.py
Normal file
|
|
@ -0,0 +1,386 @@
|
||||||
|
"""Post-process a finished video download into the library — the Radarr/Sonarr step.
|
||||||
|
|
||||||
|
On completion the monitor hands us the located file. We:
|
||||||
|
1. parse the release + SANITY-GATE it (reject non-video / samples / wrong episode /
|
||||||
|
multi-file packs we can't safely place),
|
||||||
|
2. build the canonical library path (``library_paths``),
|
||||||
|
3. decide IMPORT vs UPGRADE-replace vs not-an-upgrade by looking at what is already
|
||||||
|
in the destination folder — the filesystem is the source of truth (like Radarr),
|
||||||
|
which avoids leaning on DB columns the schema doesn't have,
|
||||||
|
4. COPY it in (renamed), carry sibling subtitles, on an upgrade delete the worse
|
||||||
|
existing file, and remove the source unless it's a torrent (preserve seeding).
|
||||||
|
|
||||||
|
``plan_import`` is pure (directory reads injected via ``list_dir``); ``run_import``
|
||||||
|
executes the plan through an injected ``fs`` facade so orchestration is unit-tested
|
||||||
|
without touching disk. Isolated — sibling video modules + stdlib only; no music imports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
from typing import Any, Callable
|
||||||
|
|
||||||
|
from core.video import organization
|
||||||
|
from core.video.download_pipeline import basename_of
|
||||||
|
from core.video.library_paths import quality_full
|
||||||
|
from core.video.quality_eval import resolution_rank
|
||||||
|
from core.video.release_parse import parse_release
|
||||||
|
|
||||||
|
VIDEO_EXTS = frozenset({
|
||||||
|
".mkv", ".mp4", ".avi", ".m4v", ".mov", ".ts", ".wmv",
|
||||||
|
".mpg", ".mpeg", ".webm", ".flv", ".m2ts",
|
||||||
|
})
|
||||||
|
SUB_EXTS = frozenset({".srt", ".sub", ".ass", ".ssa", ".idx", ".vtt", ".smi"})
|
||||||
|
|
||||||
|
_SAMPLE_MAX_BYTES = 150 * 1024 * 1024 # a "sample"-named file under this is a sample
|
||||||
|
_SAMPLE = re.compile(r"(^|[.\-_ ])sample([.\-_ ]|$)", re.I)
|
||||||
|
_SXXEXX = re.compile(r"\bS(\d{1,2})[ .]?E(\d{1,3})\b", re.I)
|
||||||
|
|
||||||
|
# A probed runtime under this (seconds) is a sample/clip, not the real thing. Movies
|
||||||
|
# get a generous floor; episodes vary wildly (shorts/cartoons) so only the absurdly
|
||||||
|
# short get caught.
|
||||||
|
_RUNTIME_FLOOR = {"movie": 15 * 60, "episode": 90}
|
||||||
|
|
||||||
|
# Source ranking for the upgrade comparison (mirrors the quality ladder order).
|
||||||
|
_SRC_RANK = {"remux": 6, "bluray": 5, "web-dl": 4, "webrip": 3, "hdtv": 2, "dvd": 1}
|
||||||
|
|
||||||
|
|
||||||
|
def ext_of(path: Any) -> str:
|
||||||
|
"""Lower-cased extension (with dot) of a path's basename, '' if none."""
|
||||||
|
return os.path.splitext(basename_of(path))[1].lower()
|
||||||
|
|
||||||
|
|
||||||
|
def is_video(path: Any) -> bool:
|
||||||
|
return ext_of(path) in VIDEO_EXTS
|
||||||
|
|
||||||
|
|
||||||
|
def is_sample(name: Any, size_bytes: Any) -> bool:
|
||||||
|
"""A 'sample'-tagged file that's also small (or of unknown size) is a sample."""
|
||||||
|
if not _SAMPLE.search(basename_of(name)):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
sz = int(size_bytes or 0)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
sz = 0
|
||||||
|
return sz == 0 or sz < _SAMPLE_MAX_BYTES
|
||||||
|
|
||||||
|
|
||||||
|
def quality_score(parsed: Any) -> int:
|
||||||
|
"""A self-contained quality score (resolution dominates, source breaks ties) used
|
||||||
|
only for the local upgrade comparison. Higher = better."""
|
||||||
|
parsed = parsed if isinstance(parsed, dict) else {}
|
||||||
|
return resolution_rank(parsed.get("resolution")) * 10 + _SRC_RANK.get(parsed.get("source"), 0)
|
||||||
|
|
||||||
|
|
||||||
|
def _scope_of(dl: dict) -> str:
|
||||||
|
"""The import scope from the search context, falling back to the download kind.
|
||||||
|
Only 'movie' and 'episode' are placeable; packs/youtube are gated out upstream."""
|
||||||
|
ctx = _search_ctx(dl)
|
||||||
|
sc = str(ctx.get("scope") or "").lower()
|
||||||
|
if sc in ("movie", "episode", "season", "series"):
|
||||||
|
return sc
|
||||||
|
k = str(dl.get("kind") or "").lower()
|
||||||
|
if k == "movie":
|
||||||
|
return "movie"
|
||||||
|
if k in ("show", "tv", "episode"):
|
||||||
|
return "episode"
|
||||||
|
return k or "movie"
|
||||||
|
|
||||||
|
|
||||||
|
def _search_ctx(dl: dict) -> dict:
|
||||||
|
try:
|
||||||
|
ctx = json.loads((dl or {}).get("search_ctx") or "{}")
|
||||||
|
return ctx if isinstance(ctx, dict) else {}
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def _ctx(dl: dict) -> dict:
|
||||||
|
sc = _search_ctx(dl)
|
||||||
|
return {
|
||||||
|
"title": sc.get("title") or (dl or {}).get("title") or "",
|
||||||
|
"year": sc.get("year") if sc.get("year") is not None else (dl or {}).get("year"),
|
||||||
|
"season": sc.get("season"),
|
||||||
|
"episode": sc.get("episode"),
|
||||||
|
"episode_title": sc.get("episode_title"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _reject(reason: str) -> dict:
|
||||||
|
return {"action": "reject", "reason": reason, "manual": True}
|
||||||
|
|
||||||
|
|
||||||
|
def _existing_match(scope: str, dest_dir: str, ctx: dict, list_dir: Callable) -> str | None:
|
||||||
|
"""The basename of a file ALREADY in the destination folder that represents this
|
||||||
|
same item (any video for a movie; a matching SxxExx for an episode), or None.
|
||||||
|
``list_dir(dir)`` yields basenames; a missing dir yields nothing."""
|
||||||
|
try:
|
||||||
|
names = [str(n) for n in (list_dir(dest_dir) or [])]
|
||||||
|
except Exception: # noqa: BLE001 - a missing/denied dir simply means "nothing there"
|
||||||
|
return None
|
||||||
|
vids = [n for n in names if ext_of(n) in VIDEO_EXTS and not is_sample(n, None)]
|
||||||
|
if scope == "movie":
|
||||||
|
return max(vids, key=len) if vids else None
|
||||||
|
if scope == "episode":
|
||||||
|
try:
|
||||||
|
ws, we = int(ctx.get("season")), int(ctx.get("episode"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
for n in vids:
|
||||||
|
m = _SXXEXX.search(n)
|
||||||
|
if m and int(m.group(1)) == ws and int(m.group(2)) == we:
|
||||||
|
return n
|
||||||
|
return None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def plan_import(dl: dict, src_path: str, *, list_dir: Callable, probe: dict | None = None,
|
||||||
|
settings: dict | None = None, force: bool = False,
|
||||||
|
override: dict | None = None) -> dict:
|
||||||
|
"""Decide what to do with a finished download. Returns one of:
|
||||||
|
|
||||||
|
{"action": "import", "dest": {...}, "quality_label": str}
|
||||||
|
{"action": "upgrade", "dest": {...}, "replace_path": str, "quality_label": str}
|
||||||
|
{"action": "reject", "reason": str, "manual": True}
|
||||||
|
|
||||||
|
Pure: all directory reads go through ``list_dir`` (injected). ``probe`` is the
|
||||||
|
ffprobe ``mediainfo`` result (or None when ffprobe is unavailable) — when present
|
||||||
|
we trust the FILE's real resolution over the scene name and reject corrupt /
|
||||||
|
too-short junk. ``settings`` are the user's organisation settings (naming templates
|
||||||
|
+ replace policy); None = defaults.
|
||||||
|
|
||||||
|
MANUAL placement: ``force=True`` with an ``override`` ({scope, title, year, season,
|
||||||
|
episode, episode_title, target_dir, media_id}) trusts the user's chosen identity —
|
||||||
|
it skips the auto sanity-gates (sample / wrong-episode / pack / not-an-upgrade) and
|
||||||
|
files the file exactly where they said, replacing any worse copy. ffprobe is still
|
||||||
|
used for the true resolution, but never to reject."""
|
||||||
|
dl = dl or {}
|
||||||
|
settings = organization.normalize(settings)
|
||||||
|
override = override or {}
|
||||||
|
scope = str(override.get("scope") or _scope_of(dl)).lower() if force else _scope_of(dl)
|
||||||
|
name = basename_of(src_path)
|
||||||
|
ext = ext_of(src_path)
|
||||||
|
parsed = parse_release(dl.get("release_title") or name)
|
||||||
|
ctx = _ctx(dl)
|
||||||
|
if force: # the user told us what it is — let their identity win
|
||||||
|
for k in ("title", "year", "season", "episode", "episode_title"):
|
||||||
|
if override.get(k) is not None:
|
||||||
|
ctx[k] = override.get(k)
|
||||||
|
|
||||||
|
if not is_video(src_path): # can't place a non-video, even on a forced import
|
||||||
|
return _reject("Not a video file (%s)" % (ext or "no extension"))
|
||||||
|
if not force:
|
||||||
|
if is_sample(name, dl.get("size_bytes")):
|
||||||
|
return _reject("Looks like a sample, not the feature")
|
||||||
|
if scope not in ("movie", "episode"):
|
||||||
|
return _reject("Season/complete packs need manual import")
|
||||||
|
if scope == "episode":
|
||||||
|
if ctx.get("season") is None or ctx.get("episode") is None:
|
||||||
|
return _reject("Missing season/episode info")
|
||||||
|
# If the release name itself names a DIFFERENT episode, don't mis-file it.
|
||||||
|
if parsed.get("episode") is not None and (
|
||||||
|
parsed.get("season") != ctx.get("season")
|
||||||
|
or parsed.get("episode") != ctx.get("episode")):
|
||||||
|
return _reject("Release is S%02dE%02d, not the episode requested"
|
||||||
|
% (parsed.get("season") or 0, parsed.get("episode") or 0))
|
||||||
|
else:
|
||||||
|
if scope not in ("movie", "episode"):
|
||||||
|
return _reject("Pick a movie or an episode to place this file")
|
||||||
|
if scope == "episode" and (ctx.get("season") is None or ctx.get("episode") is None):
|
||||||
|
return _reject("Pick a season and episode to place this file")
|
||||||
|
|
||||||
|
# ffprobe verification — best-effort; on a forced placement we use the real
|
||||||
|
# resolution but never reject on it (the user has decided).
|
||||||
|
if probe is not None:
|
||||||
|
if not force:
|
||||||
|
if not probe.get("ok"):
|
||||||
|
return _reject("No readable video stream — corrupt or fake file")
|
||||||
|
dur = probe.get("duration_sec") or 0
|
||||||
|
floor = _RUNTIME_FLOOR.get(scope)
|
||||||
|
if floor and 0 < dur < floor:
|
||||||
|
return _reject("Runtime is only %d min — looks like a sample/clip, not the %s"
|
||||||
|
% (int(dur // 60), scope))
|
||||||
|
# Trust the FILE over the (often lying) scene name: real resolution always,
|
||||||
|
# real codec only when the name didn't carry one.
|
||||||
|
parsed = dict(parsed)
|
||||||
|
if probe.get("resolution"):
|
||||||
|
parsed["resolution"] = probe["resolution"]
|
||||||
|
if probe.get("video_codec") and not parsed.get("codec"):
|
||||||
|
parsed["codec"] = probe["video_codec"]
|
||||||
|
|
||||||
|
root = (override.get("target_dir") if force else None) or dl.get("target_dir") or ""
|
||||||
|
if not root:
|
||||||
|
return _reject("No library folder configured for this type")
|
||||||
|
|
||||||
|
media_id = override.get("media_id") if force else dl.get("media_id")
|
||||||
|
quality = quality_full(parsed)
|
||||||
|
fields = {
|
||||||
|
"title": ctx.get("title"), "year": ctx.get("year"),
|
||||||
|
"series": ctx.get("title"), "season": ctx.get("season"),
|
||||||
|
"episode": ctx.get("episode"), "episode_title": ctx.get("episode_title"),
|
||||||
|
"quality": quality, "resolution": parsed.get("resolution"),
|
||||||
|
"source": parsed.get("source"), "codec": parsed.get("codec"),
|
||||||
|
"tmdbid": media_id if scope == "movie" else None,
|
||||||
|
"tvdbid": media_id if scope == "episode" else None,
|
||||||
|
}
|
||||||
|
dest = organization.render_path(scope, root, fields, settings, ext)
|
||||||
|
# Where poster.jpg goes: the movie folder, or the SHOW root for an episode
|
||||||
|
# (parent of the Season folder) — so it isn't dropped per-season.
|
||||||
|
artwork_dir = dest["dir"] if scope == "movie" else os.path.dirname(dest["dir"])
|
||||||
|
|
||||||
|
existing = _existing_match(scope, dest["dir"], ctx, list_dir)
|
||||||
|
if existing:
|
||||||
|
# A forced placement replaces whatever's there (the user chose to put it here).
|
||||||
|
if force:
|
||||||
|
return {"action": "upgrade", "dest": dest, "quality_label": quality,
|
||||||
|
"replace_path": os.path.join(dest["dir"], existing), "artwork_dir": artwork_dir}
|
||||||
|
if not settings.get("replace_existing", True):
|
||||||
|
return _reject("Already in the library (%s) — replace is turned off" % existing)
|
||||||
|
new_score = quality_score(parsed)
|
||||||
|
old_score = quality_score(parse_release(existing))
|
||||||
|
if new_score > old_score:
|
||||||
|
return {"action": "upgrade", "dest": dest, "quality_label": quality,
|
||||||
|
"replace_path": os.path.join(dest["dir"], existing), "artwork_dir": artwork_dir}
|
||||||
|
return _reject("Not an upgrade over the copy already in the library (%s)" % existing)
|
||||||
|
|
||||||
|
return {"action": "import", "dest": dest, "quality_label": quality, "artwork_dir": artwork_dir}
|
||||||
|
|
||||||
|
|
||||||
|
def plan_subs(src_path: str, dest_path: str, list_dir: Callable) -> list:
|
||||||
|
"""Sibling subtitle files to carry alongside the video, renamed to match the
|
||||||
|
destination stem (preserving any language suffix, e.g. '.en.srt'). Returns a list
|
||||||
|
of (src_abs, dest_abs) pairs. ``list_dir`` lists the SOURCE directory."""
|
||||||
|
src_dir = os.path.dirname(src_path) or "."
|
||||||
|
v_stem = os.path.splitext(basename_of(src_path))[0]
|
||||||
|
d_stem = os.path.splitext(basename_of(dest_path))[0]
|
||||||
|
d_dir = os.path.dirname(dest_path)
|
||||||
|
out = []
|
||||||
|
try:
|
||||||
|
names = [str(n) for n in (list_dir(src_dir) or [])]
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
return out
|
||||||
|
for n in names:
|
||||||
|
if ext_of(n) not in SUB_EXTS:
|
||||||
|
continue
|
||||||
|
stem, ext = os.path.splitext(n)
|
||||||
|
if stem == v_stem:
|
||||||
|
extra = "" # movie.srt → <dest>.srt
|
||||||
|
elif stem.startswith(v_stem + "."):
|
||||||
|
extra = stem[len(v_stem):] # movie.en.srt → <dest>.en.srt
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
out.append((os.path.join(src_dir, n), os.path.join(d_dir, d_stem + extra + ext)))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def run_import(dl: dict, src_path: str, *, fs: Any, prober: Callable | None = None,
|
||||||
|
settings: dict | None = None, force: bool = False,
|
||||||
|
override: dict | None = None) -> dict:
|
||||||
|
"""Execute the import and return a DB patch dict for the download row.
|
||||||
|
|
||||||
|
``fs`` is an injected facade with: ``list_dir(dir)->iterable[name]``,
|
||||||
|
``makedirs(dir)``, ``copy(src, dst)``, ``move(src, dst)``, ``remove(path)``.
|
||||||
|
``prober(path)->mediainfo`` is an optional ffprobe hook (None = skip verification).
|
||||||
|
``settings`` are the user's organisation settings (transfer mode, subtitle carry);
|
||||||
|
None = defaults. ``force``/``override`` drive a MANUAL placement (see ``plan_import``).
|
||||||
|
A reject becomes an ``import_failed`` row with ``dest_path`` pointing at the file's
|
||||||
|
current (unplaced) location so the Import page can resolve it; a success becomes a
|
||||||
|
``completed`` row with ``dest_path`` set to its final home."""
|
||||||
|
settings = organization.normalize(settings)
|
||||||
|
probe_info = None
|
||||||
|
if prober is not None:
|
||||||
|
try:
|
||||||
|
probe_info = prober(src_path)
|
||||||
|
except Exception: # noqa: BLE001 - a probe crash must not block the import
|
||||||
|
probe_info = None
|
||||||
|
plan = plan_import(dl, src_path, list_dir=fs.list_dir, probe=probe_info,
|
||||||
|
settings=settings, force=force, override=override)
|
||||||
|
if plan["action"] == "reject":
|
||||||
|
# Leave the file where it is; remember WHERE so manual import can find it.
|
||||||
|
return {"status": "import_failed", "progress": 100.0, "error": plan["reason"],
|
||||||
|
"dest_path": src_path}
|
||||||
|
|
||||||
|
dest = plan["dest"]
|
||||||
|
move_mode = settings.get("transfer_mode") == "move"
|
||||||
|
try:
|
||||||
|
fs.makedirs(dest["dir"])
|
||||||
|
if move_mode:
|
||||||
|
fs.move(src_path, dest["path"])
|
||||||
|
else:
|
||||||
|
fs.copy(src_path, dest["path"])
|
||||||
|
if settings.get("carry_subtitles", True):
|
||||||
|
for sub_src, sub_dst in plan_subs(src_path, dest["path"], fs.list_dir):
|
||||||
|
try:
|
||||||
|
fs.copy(sub_src, sub_dst)
|
||||||
|
except Exception: # noqa: BLE001 - a subtitle that won't copy isn't fatal
|
||||||
|
pass
|
||||||
|
if plan["action"] == "upgrade" and plan.get("replace_path"):
|
||||||
|
try:
|
||||||
|
fs.remove(plan["replace_path"])
|
||||||
|
except Exception: # noqa: BLE001 - failing to delete the old file isn't fatal
|
||||||
|
pass
|
||||||
|
# Copy mode reclaims the download copy UNLESS it's a torrent (keep seeding);
|
||||||
|
# move mode already relocated it.
|
||||||
|
if not move_mode and str(dl.get("source") or "").lower() != "torrent":
|
||||||
|
try:
|
||||||
|
fs.remove(src_path)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
pass
|
||||||
|
except Exception as e: # noqa: BLE001 - any copy/mkdir failure → manual import
|
||||||
|
return {"status": "import_failed", "progress": 100.0, "error": "Import failed: " + str(e),
|
||||||
|
"dest_path": src_path}
|
||||||
|
|
||||||
|
return {"status": "completed", "progress": 100.0, "dest_path": dest["path"],
|
||||||
|
"quality_label": plan.get("quality_label") or dl.get("quality_label")}
|
||||||
|
|
||||||
|
|
||||||
|
class _RealFS:
|
||||||
|
"""The production filesystem facade for ``run_import`` (os/shutil)."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def list_dir(path):
|
||||||
|
try:
|
||||||
|
return os.listdir(str(path or ""))
|
||||||
|
except OSError:
|
||||||
|
return []
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def makedirs(path):
|
||||||
|
os.makedirs(str(path or "."), exist_ok=True)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def copy(src, dst):
|
||||||
|
shutil.copy2(src, dst)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def move(src, dst):
|
||||||
|
shutil.move(src, dst)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def save_url(url, dst):
|
||||||
|
import urllib.request
|
||||||
|
req = urllib.request.Request(url, headers={"User-Agent": "SoulSync"})
|
||||||
|
with urllib.request.urlopen(req, timeout=20) as resp, open(dst, "wb") as f:
|
||||||
|
shutil.copyfileobj(resp, f)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def write_text(path, content):
|
||||||
|
with open(path, "w", encoding="utf-8") as f:
|
||||||
|
f.write(content)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def remove(path):
|
||||||
|
os.remove(path)
|
||||||
|
|
||||||
|
|
||||||
|
def real_fs() -> _RealFS:
|
||||||
|
return _RealFS()
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"VIDEO_EXTS", "SUB_EXTS", "ext_of", "is_video", "is_sample", "quality_score",
|
||||||
|
"plan_import", "plan_subs", "run_import", "real_fs",
|
||||||
|
]
|
||||||
165
core/video/library_paths.py
Normal file
165
core/video/library_paths.py
Normal file
|
|
@ -0,0 +1,165 @@
|
||||||
|
"""Organize a finished video download into a Radarr/Sonarr-standard library path.
|
||||||
|
|
||||||
|
Given the SCOPE/intent of a grab (movie vs episode, plus title/year/season/episode)
|
||||||
|
and the PARSED release quality, build the canonical folder + filename the file should
|
||||||
|
land at:
|
||||||
|
|
||||||
|
<movies_root>/The Matrix (1999)/The Matrix (1999) Bluray-1080p.mkv
|
||||||
|
<tv_root>/Breaking Bad/Season 01/Breaking Bad - S01E01 - Pilot WEBDL-1080p.mkv
|
||||||
|
|
||||||
|
These are the community-standard templates and also exactly what Plex/Jellyfin want
|
||||||
|
for reliable identification (we organise on disk; the server just refreshes).
|
||||||
|
|
||||||
|
Pure (no filesystem, no DB) so it's unit-tested in isolation. Isolated — stdlib +
|
||||||
|
the sibling ``download_pipeline.basename_of`` only; nothing from the music side.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from core.video.download_pipeline import basename_of
|
||||||
|
|
||||||
|
# Characters illegal on Windows / awkward on most filesystems, plus control chars.
|
||||||
|
_ILLEGAL = re.compile(r'[\\/:*?"<>|\x00-\x1f]')
|
||||||
|
_TRAILING_DOTSPACE = re.compile(r"[ .]+$")
|
||||||
|
|
||||||
|
# Parsed source token → the label used in the Radarr ``{Quality Full}`` tag.
|
||||||
|
_SRC_LABEL = {
|
||||||
|
"remux": "Remux", "bluray": "Bluray", "web-dl": "WEBDL",
|
||||||
|
"webrip": "WEBRip", "hdtv": "HDTV", "dvd": "DVD",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def sanitize(name: Any) -> str:
|
||||||
|
"""Filesystem-safe path COMPONENT: strip illegal chars, collapse whitespace, and
|
||||||
|
trim trailing dots/spaces (Windows rejects those). Never contains a separator."""
|
||||||
|
s = _ILLEGAL.sub("", str(name or ""))
|
||||||
|
s = re.sub(r"\s+", " ", s).strip()
|
||||||
|
s = _TRAILING_DOTSPACE.sub("", s)
|
||||||
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
def _year_suffix(year: Any) -> str:
|
||||||
|
"""' (1999)' for a plausible release year, else '' (keeps titles clean)."""
|
||||||
|
try:
|
||||||
|
y = int(year)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return ""
|
||||||
|
return " (%d)" % y if 1870 <= y <= 2999 else ""
|
||||||
|
|
||||||
|
|
||||||
|
def source_label(source: Any) -> str:
|
||||||
|
"""The display label for a parsed source token ('bluray' → 'Bluray', 'web-dl' →
|
||||||
|
'WEBDL'), '' when unknown. Used for the $source template token."""
|
||||||
|
return _SRC_LABEL.get(str(source or "").strip().lower(), "")
|
||||||
|
|
||||||
|
|
||||||
|
def quality_full(parsed: Any) -> str:
|
||||||
|
"""The Radarr-style ``{Quality Full}`` tag, e.g. 'Bluray-1080p' / 'WEBDL-1080p' /
|
||||||
|
'Remux-2160p'. '' when neither source nor resolution can be determined. A
|
||||||
|
proper/repack is appended ('… Proper') so an upgrade reads at a glance."""
|
||||||
|
parsed = parsed if isinstance(parsed, dict) else {}
|
||||||
|
src = _SRC_LABEL.get(parsed.get("source"))
|
||||||
|
res = parsed.get("resolution")
|
||||||
|
if src and res:
|
||||||
|
tag = src + "-" + res
|
||||||
|
elif res:
|
||||||
|
tag = res
|
||||||
|
elif src:
|
||||||
|
tag = src
|
||||||
|
else:
|
||||||
|
return ""
|
||||||
|
if parsed.get("proper") or parsed.get("repack"):
|
||||||
|
tag += " Proper"
|
||||||
|
return tag
|
||||||
|
|
||||||
|
|
||||||
|
def movie_folder(title: Any, year: Any) -> str:
|
||||||
|
"""'The Matrix (1999)' — the per-movie folder name."""
|
||||||
|
return (sanitize(title) or "Unknown") + _year_suffix(year)
|
||||||
|
|
||||||
|
|
||||||
|
def movie_filename(title: Any, year: Any, quality: Any, ext: Any) -> str:
|
||||||
|
"""'The Matrix (1999) Bluray-1080p.mkv' (quality tag omitted when unknown)."""
|
||||||
|
stem = (sanitize(title) or "Unknown") + _year_suffix(year)
|
||||||
|
q = sanitize(quality)
|
||||||
|
if q:
|
||||||
|
stem += " " + q
|
||||||
|
return stem + _ext(ext)
|
||||||
|
|
||||||
|
|
||||||
|
def show_folder(series: Any) -> str:
|
||||||
|
"""'Breaking Bad' — the per-series folder name."""
|
||||||
|
return sanitize(series) or "Unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def season_folder(season: Any) -> str:
|
||||||
|
"""'Season 01' (or 'Specials' for season 0, Sonarr's convention)."""
|
||||||
|
try:
|
||||||
|
n = int(season)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
n = None
|
||||||
|
if n == 0:
|
||||||
|
return "Specials"
|
||||||
|
return "Season %02d" % (n if n is not None else 0)
|
||||||
|
|
||||||
|
|
||||||
|
def episode_filename(series: Any, season: Any, episode: Any, ep_title: Any,
|
||||||
|
quality: Any, ext: Any) -> str:
|
||||||
|
"""'Breaking Bad - S01E01 - Pilot WEBDL-1080p.mkv'. The ' - {title}' segment and
|
||||||
|
the trailing quality tag are each omitted when unknown."""
|
||||||
|
try:
|
||||||
|
s = int(season)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
s = 0
|
||||||
|
try:
|
||||||
|
e = int(episode)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
e = 0
|
||||||
|
stem = "%s - S%02dE%02d" % (sanitize(series) or "Unknown", s, e)
|
||||||
|
t = sanitize(ep_title)
|
||||||
|
if t:
|
||||||
|
stem += " - " + t
|
||||||
|
q = sanitize(quality)
|
||||||
|
if q:
|
||||||
|
stem += " " + q
|
||||||
|
return stem + _ext(ext)
|
||||||
|
|
||||||
|
|
||||||
|
def _ext(ext: Any) -> str:
|
||||||
|
e = str(ext or "").strip().lower()
|
||||||
|
if not e:
|
||||||
|
return ""
|
||||||
|
return e if e.startswith(".") else "." + e
|
||||||
|
|
||||||
|
|
||||||
|
def plan_path(scope: Any, root: Any, ctx: dict, quality: Any, ext: Any) -> dict:
|
||||||
|
"""Resolve the canonical destination for a finished download.
|
||||||
|
|
||||||
|
``ctx`` = {title, year, season, episode, episode_title}. Returns
|
||||||
|
``{"dir": <abs folder>, "filename": <name>, "path": <abs file path>}``. For an
|
||||||
|
unsupported scope it falls back to a flat drop (root + sanitized basename) so a
|
||||||
|
file is never lost — the caller gates scope before relying on this."""
|
||||||
|
ctx = ctx if isinstance(ctx, dict) else {}
|
||||||
|
root = str(root or "")
|
||||||
|
sc = str(scope or "").lower()
|
||||||
|
if sc == "movie":
|
||||||
|
d = os.path.join(root, movie_folder(ctx.get("title"), ctx.get("year")))
|
||||||
|
fn = movie_filename(ctx.get("title"), ctx.get("year"), quality, ext)
|
||||||
|
elif sc == "episode":
|
||||||
|
d = os.path.join(root, show_folder(ctx.get("title")), season_folder(ctx.get("season")))
|
||||||
|
fn = episode_filename(ctx.get("title"), ctx.get("season"), ctx.get("episode"),
|
||||||
|
ctx.get("episode_title"), quality, ext)
|
||||||
|
else:
|
||||||
|
d = root
|
||||||
|
fn = sanitize(basename_of(ctx.get("src") or "")) or "download"
|
||||||
|
return {"dir": d, "filename": fn, "path": os.path.join(d, fn)}
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"sanitize", "source_label", "quality_full", "movie_folder", "movie_filename",
|
||||||
|
"show_folder", "season_folder", "episode_filename", "plan_path",
|
||||||
|
]
|
||||||
137
core/video/mediainfo.py
Normal file
137
core/video/mediainfo.py
Normal file
|
|
@ -0,0 +1,137 @@
|
||||||
|
"""Probe a finished video file with ffprobe for its TRUE media info.
|
||||||
|
|
||||||
|
We otherwise trust the release NAME for resolution/quality, and names lie: 720p
|
||||||
|
upscales labelled 1080p, trailers/samples labelled as the feature, broken muxes.
|
||||||
|
ffprobe reads the real container — duration, dimensions → resolution, codecs — so the
|
||||||
|
importer can tag the file by its actual quality and reject corrupt / too-short junk.
|
||||||
|
|
||||||
|
The parsing (``parse_ffprobe`` / ``resolution_from_dimensions``) is pure and unit-tested
|
||||||
|
on canned JSON; the subprocess runner is injected, so nothing here needs ffmpeg to be
|
||||||
|
tested. ffmpeg is OPTIONAL — when ffprobe isn't installed, or it errors, ``probe``
|
||||||
|
returns None and the caller falls back to the scene name.
|
||||||
|
|
||||||
|
Three outcomes, deliberately distinct:
|
||||||
|
- None → couldn't verify (ffprobe missing / crashed / timed out) → skip
|
||||||
|
- {"ok": False, …} → ffprobe ran and found NO video stream → corrupt / fake
|
||||||
|
- {"ok": True, …} → real media info to trust over the name
|
||||||
|
|
||||||
|
Isolated: stdlib only; no music imports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
from typing import Any, Callable
|
||||||
|
|
||||||
|
_FFPROBE = "ffprobe"
|
||||||
|
|
||||||
|
|
||||||
|
def _int(v: Any) -> int:
|
||||||
|
try:
|
||||||
|
return int(v)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def _float(v: Any) -> float:
|
||||||
|
try:
|
||||||
|
return float(v)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def resolution_from_dimensions(width: Any, height: Any) -> str | None:
|
||||||
|
"""Bucket real pixel dimensions into a resolution label. Uses the LARGER axis so
|
||||||
|
a letterboxed 1920x800 movie reads as 1080p (not 720p by its short side)."""
|
||||||
|
ref = max(_int(width), _int(height))
|
||||||
|
if ref <= 0:
|
||||||
|
return None
|
||||||
|
if ref >= 3000:
|
||||||
|
return "2160p"
|
||||||
|
if ref >= 1700:
|
||||||
|
return "1080p"
|
||||||
|
if ref >= 1100:
|
||||||
|
return "720p"
|
||||||
|
return "480p"
|
||||||
|
|
||||||
|
|
||||||
|
def _norm_codec(name: Any) -> str | None:
|
||||||
|
s = str(name or "").strip().lower()
|
||||||
|
if not s:
|
||||||
|
return None
|
||||||
|
if s in ("hevc", "h265", "x265"):
|
||||||
|
return "hevc"
|
||||||
|
if s in ("h264", "avc", "x264"):
|
||||||
|
return "x264"
|
||||||
|
if s == "av1":
|
||||||
|
return "av1"
|
||||||
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
def parse_ffprobe(data: Any) -> dict:
|
||||||
|
"""Parse ffprobe's ``-show_format -show_streams`` JSON into the fields we use.
|
||||||
|
``ok`` is True only when a video stream is present (else: corrupt / not a video)."""
|
||||||
|
data = data if isinstance(data, dict) else {}
|
||||||
|
streams = data.get("streams") or []
|
||||||
|
fmt = data.get("format") or {}
|
||||||
|
video = next((s for s in streams if s.get("codec_type") == "video"), None)
|
||||||
|
audio = next((s for s in streams if s.get("codec_type") == "audio"), None)
|
||||||
|
duration = _float(fmt.get("duration")) or _float((video or {}).get("duration"))
|
||||||
|
width = (video or {}).get("width")
|
||||||
|
height = (video or {}).get("height")
|
||||||
|
return {
|
||||||
|
"ok": video is not None,
|
||||||
|
"duration_sec": duration,
|
||||||
|
"width": _int(width),
|
||||||
|
"height": _int(height),
|
||||||
|
"resolution": resolution_from_dimensions(width, height) if video else None,
|
||||||
|
"video_codec": _norm_codec((video or {}).get("codec_name")),
|
||||||
|
"audio_codec": str((audio or {}).get("codec_name") or "") or None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def ffprobe_available() -> bool:
|
||||||
|
return shutil.which(_FFPROBE) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def _default_runner(path: str) -> str | None:
|
||||||
|
"""Run ffprobe and return its JSON stdout, or None on any failure (so a transient
|
||||||
|
ffprobe error degrades to 'unverified', never to a false 'corrupt')."""
|
||||||
|
try:
|
||||||
|
proc = subprocess.run(
|
||||||
|
[_FFPROBE, "-v", "quiet", "-print_format", "json",
|
||||||
|
"-show_format", "-show_streams", str(path)],
|
||||||
|
capture_output=True, text=True, timeout=120, check=False,
|
||||||
|
)
|
||||||
|
except Exception: # noqa: BLE001 - missing binary / timeout / OS error → unverified
|
||||||
|
return None
|
||||||
|
if proc.returncode != 0 or not (proc.stdout or "").strip():
|
||||||
|
return None
|
||||||
|
return proc.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def probe(path: Any, runner: Callable | None = None) -> dict | None:
|
||||||
|
"""Probe ``path`` and return parsed media info, or None when it can't be verified.
|
||||||
|
``runner(path)->json_str|None`` is injected (real ffprobe in prod, canned in tests).
|
||||||
|
When no runner is given and ffprobe isn't installed, returns None (skip verify)."""
|
||||||
|
use = runner if runner is not None else (_default_runner if ffprobe_available() else None)
|
||||||
|
if use is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
raw = use(path)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
return None
|
||||||
|
if not raw:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
data = json.loads(raw)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return None
|
||||||
|
return parse_ffprobe(data)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"resolution_from_dimensions", "parse_ffprobe", "ffprobe_available", "probe",
|
||||||
|
]
|
||||||
112
core/video/mock_search.py
Normal file
112
core/video/mock_search.py
Normal file
|
|
@ -0,0 +1,112 @@
|
||||||
|
"""Mock indexer — a stand-in for a real slskd/Prowlarr search while the download
|
||||||
|
engine is being built.
|
||||||
|
|
||||||
|
Given a scope (movie / episode / season / series), a title, and a SOURCE, it returns
|
||||||
|
plausible raw 'indexer hits' ({title, size_bytes, seeders}) — and each source returns
|
||||||
|
a DIFFERENT set (different release groups, seeder counts, and which qualities show up),
|
||||||
|
the way real indexers do. The real parse → evaluate → rank pipeline (release_parse +
|
||||||
|
quality_eval) runs on these exactly as it will on real hits. Deterministic (no RNG) so
|
||||||
|
results are stable for tests and reloads. THIS is the single swap-point: replace
|
||||||
|
``mock_search`` with a real indexer client and nothing downstream changes.
|
||||||
|
|
||||||
|
Pure (no DB, no network). Isolated — imports only typing; the music side never imports it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
_GB = 1024 ** 3
|
||||||
|
|
||||||
|
# Quality strings (WITHOUT a release group — the group is appended per source below)
|
||||||
|
# and an approximate size in GB. Ordered best→worst; the evaluator re-ranks anyway.
|
||||||
|
_MOVIE = [
|
||||||
|
("2160p.UHD.BluRay.REMUX.HDR.DV.TrueHD.Atmos", 58),
|
||||||
|
("2160p.WEB-DL.DDP5.1.HDR.HEVC", 19),
|
||||||
|
("1080p.BluRay.x265.10bit.DTS-HD.MA.5.1", 11),
|
||||||
|
("1080p.WEB-DL.DDP5.1.H264", 7),
|
||||||
|
("1080p.WEBRip.x264.AAC", 4),
|
||||||
|
("720p.HDTV.x264", 2),
|
||||||
|
("HDCAM.x264", 2),
|
||||||
|
]
|
||||||
|
_EPISODE = [
|
||||||
|
("2160p.WEB-DL.DDP5.1.HDR.HEVC", 5),
|
||||||
|
("1080p.BluRay.x265", 3),
|
||||||
|
("1080p.WEB-DL.H264", 2),
|
||||||
|
("720p.HDTV.x264", 1),
|
||||||
|
]
|
||||||
|
_SEASON = [
|
||||||
|
("2160p.WEB-DL.DDP5.1.HDR.HEVC", 48),
|
||||||
|
("1080p.BluRay.x265.10bit", 26),
|
||||||
|
("1080p.WEB-DL.H264", 18),
|
||||||
|
("720p.HDTV.x264", 9),
|
||||||
|
]
|
||||||
|
_SERIES = [
|
||||||
|
("COMPLETE.1080p.BluRay.x265.10bit", 120),
|
||||||
|
("COMPLETE.1080p.WEB-DL.H264", 88),
|
||||||
|
("COMPLETE.720p.WEB-DL.x264", 40),
|
||||||
|
]
|
||||||
|
|
||||||
|
# Per-source "flavour": which slice of the quality list shows up, a seeder multiplier,
|
||||||
|
# and a release-group pool. Makes Soulseek vs Torrent vs Usenet return distinct hits.
|
||||||
|
_SRC_FLAVOR = {
|
||||||
|
"soulseek": {"slice": (1, None), "seed": 0.5, "groups": ["YIFY", "GalaxyRG", "RARBG"]},
|
||||||
|
"torrent": {"slice": (0, None), "seed": 1.0, "groups": ["FraMeSToR", "FLUX", "NTb", "RARBG"]},
|
||||||
|
"usenet": {"slice": (0, -1), "seed": 1.5, "groups": ["NTb", "FLUX", "TEPES", "playWEB"]},
|
||||||
|
"youtube": {"slice": (2, None), "seed": 1.0, "groups": ["YT"]},
|
||||||
|
}
|
||||||
|
_DEFAULT_FLAVOR = _SRC_FLAVOR["torrent"]
|
||||||
|
|
||||||
|
|
||||||
|
def _slug(title: Any) -> str:
|
||||||
|
s = "".join(c if (c.isalnum() or c == " ") else " " for c in str(title or "").strip())
|
||||||
|
return ".".join(p for p in s.split() if p) or "Unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def _seeders(slug: str, i: int) -> int:
|
||||||
|
# Deterministic spread (no RNG): a stable hash of the title + index.
|
||||||
|
h = 0
|
||||||
|
for ch in slug:
|
||||||
|
h = (h * 31 + ord(ch)) & 0xFFFFFFFF
|
||||||
|
return ((h >> (i % 7)) % 240) + 3
|
||||||
|
|
||||||
|
|
||||||
|
def _ss(n) -> str:
|
||||||
|
try:
|
||||||
|
return "S%02d" % int(n)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return "S01"
|
||||||
|
|
||||||
|
|
||||||
|
def mock_search(scope: str, title: Any, *, year: Any = None, season: Any = None,
|
||||||
|
episode: Any = None, season_end: Any = None, source: Any = "torrent") -> list:
|
||||||
|
"""Return plausible raw hits for a scope + source. Replace with a real indexer later."""
|
||||||
|
slug = _slug(title)
|
||||||
|
scope = (scope or "movie").lower()
|
||||||
|
if scope == "movie":
|
||||||
|
prefix, base = slug + ("." + str(year) if year else ""), _MOVIE
|
||||||
|
elif scope == "episode":
|
||||||
|
prefix = slug + "." + _ss(season) + ("E%02d" % int(episode) if episode is not None else "E01")
|
||||||
|
base = _EPISODE
|
||||||
|
elif scope == "season":
|
||||||
|
prefix, base = slug + "." + _ss(season), _SEASON
|
||||||
|
elif scope == "series":
|
||||||
|
prefix, base = slug + ".S01-" + _ss(season_end or 5), _SERIES
|
||||||
|
else:
|
||||||
|
return []
|
||||||
|
|
||||||
|
fl = _SRC_FLAVOR.get(str(source or "").lower(), _DEFAULT_FLAVOR)
|
||||||
|
lo, hi = fl["slice"]
|
||||||
|
chosen = list(enumerate(base))[lo:hi]
|
||||||
|
hits = []
|
||||||
|
for pos, (i, (suffix, gb)) in enumerate(chosen):
|
||||||
|
group = fl["groups"][pos % len(fl["groups"])]
|
||||||
|
hits.append({
|
||||||
|
"title": prefix + "." + suffix + "-" + group,
|
||||||
|
"size_bytes": int(gb * _GB),
|
||||||
|
"seeders": max(1, int(_seeders(slug, i) * fl["seed"])),
|
||||||
|
})
|
||||||
|
return hits
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["mock_search"]
|
||||||
239
core/video/organization.py
Normal file
239
core/video/organization.py
Normal file
|
|
@ -0,0 +1,239 @@
|
||||||
|
"""Video library-organisation settings: naming templates + post-process toggles.
|
||||||
|
|
||||||
|
Mirrors the MUSIC side's file-organisation standard — editable ``$token`` path
|
||||||
|
templates with per-component sanitisation and dangling-separator cleanup — but for
|
||||||
|
video's movie/episode shape, plus the optional post-process behaviours the user can
|
||||||
|
turn on and off.
|
||||||
|
|
||||||
|
Settings (persisted as JSON in video.db ``video_settings['organization']``):
|
||||||
|
- ``movie_template`` : path template for movies (folders via '/', last = file)
|
||||||
|
- ``episode_template`` : path template for episodes
|
||||||
|
- ``verify_with_ffprobe`` : probe the real file (true quality + reject junk)
|
||||||
|
- ``replace_existing`` : upgrade-replace a worse copy already in the library
|
||||||
|
- ``transfer_mode`` : 'copy' (reclaim source unless torrent) | 'move'
|
||||||
|
- ``carry_subtitles`` : bring sibling .srt/.ass alongside the video
|
||||||
|
|
||||||
|
Template tokens
|
||||||
|
Movies: $title $titlefirst $year $quality $resolution $source $codec $edition
|
||||||
|
$tmdbid $imdbid
|
||||||
|
Episodes: $series $season $seasonraw $episode $episodetitle $year $quality
|
||||||
|
$resolution $source $codec $tvdbid
|
||||||
|
($season/$episode are zero-padded to 2; $seasonraw is the bare number.)
|
||||||
|
|
||||||
|
Pure data + a pure renderer (no DB, no FS) so it's unit-tested in isolation. Isolated —
|
||||||
|
stdlib + sibling video ``library_paths`` only; nothing from the music side.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from core.video.library_paths import sanitize, source_label
|
||||||
|
|
||||||
|
DEFAULTS = {
|
||||||
|
"version": 1,
|
||||||
|
"movie_template": "$title ($year)/$title ($year) $quality",
|
||||||
|
"episode_template": "$series/Season $season/$series - S$seasonE$episode - $episodetitle $quality",
|
||||||
|
# YouTube channels organise as a Plex "TV by date" show: channel = series,
|
||||||
|
# season = upload YEAR, episode named by upload DATE + title.
|
||||||
|
"youtube_template": "$channel/Season $year/$channel - $date - $title",
|
||||||
|
"verify_with_ffprobe": True,
|
||||||
|
"replace_existing": True,
|
||||||
|
"transfer_mode": "copy",
|
||||||
|
"carry_subtitles": True,
|
||||||
|
"save_artwork": True, # nfo + artwork sidecars on by default (cheap, local) — best-in-class
|
||||||
|
"write_nfo": True,
|
||||||
|
"download_subtitles": False, # opt-in: fetches from OpenSubtitles (external, rate-limited)
|
||||||
|
"subtitle_langs": "en",
|
||||||
|
}
|
||||||
|
|
||||||
|
_TRANSFER_MODES = ("copy", "move")
|
||||||
|
|
||||||
|
|
||||||
|
def default_settings() -> dict:
|
||||||
|
return dict(DEFAULTS)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize(raw: Any) -> dict:
|
||||||
|
"""Coerce stored/posted settings to a valid shape, filling gaps from the default.
|
||||||
|
Blank templates fall back to the default; never raises."""
|
||||||
|
d = default_settings()
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
return d
|
||||||
|
for key in ("movie_template", "episode_template", "youtube_template"):
|
||||||
|
v = raw.get(key)
|
||||||
|
if isinstance(v, str) and v.strip():
|
||||||
|
d[key] = v.strip()
|
||||||
|
for key in ("verify_with_ffprobe", "replace_existing", "carry_subtitles",
|
||||||
|
"save_artwork", "write_nfo", "download_subtitles"):
|
||||||
|
if key in raw:
|
||||||
|
d[key] = bool(raw.get(key))
|
||||||
|
tm = str(raw.get("transfer_mode") or "").strip().lower()
|
||||||
|
if tm in _TRANSFER_MODES:
|
||||||
|
d["transfer_mode"] = tm
|
||||||
|
if "subtitle_langs" in raw:
|
||||||
|
from core.video.subtitles import parse_langs
|
||||||
|
d["subtitle_langs"] = ",".join(parse_langs(raw.get("subtitle_langs")))
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
def load(db) -> dict:
|
||||||
|
raw = db.get_setting("organization")
|
||||||
|
if raw:
|
||||||
|
try:
|
||||||
|
return normalize(json.loads(raw))
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
pass
|
||||||
|
return default_settings()
|
||||||
|
|
||||||
|
|
||||||
|
def save(db, raw: Any) -> dict:
|
||||||
|
s = normalize(raw)
|
||||||
|
db.set_setting("organization", json.dumps(s))
|
||||||
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
# ── the template engine (the music $token standard, video tokens) ─────────────
|
||||||
|
def _str(v: Any) -> str:
|
||||||
|
if v is None:
|
||||||
|
return ""
|
||||||
|
return str(v)
|
||||||
|
|
||||||
|
|
||||||
|
def _pad2(v: Any) -> str:
|
||||||
|
try:
|
||||||
|
return "%02d" % int(v)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return _str(v)
|
||||||
|
|
||||||
|
|
||||||
|
def _plausible_year(v: Any) -> bool:
|
||||||
|
try:
|
||||||
|
return 1870 <= int(v) <= 2999
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _ext(ext: Any) -> str:
|
||||||
|
e = str(ext or "").strip().lower()
|
||||||
|
if not e:
|
||||||
|
return ""
|
||||||
|
return e if e.startswith(".") else "." + e
|
||||||
|
|
||||||
|
|
||||||
|
def _movie_values(f: dict) -> dict:
|
||||||
|
title = f.get("title") or "Unknown"
|
||||||
|
return {
|
||||||
|
"title": title,
|
||||||
|
"titlefirst": (str(title)[:1] or "U").upper(),
|
||||||
|
"year": _str(f.get("year")) if _plausible_year(f.get("year")) else "",
|
||||||
|
"quality": _str(f.get("quality")),
|
||||||
|
"resolution": _str(f.get("resolution")),
|
||||||
|
"source": source_label(f.get("source")),
|
||||||
|
"codec": _str(f.get("codec")).upper(),
|
||||||
|
"edition": _str(f.get("edition")),
|
||||||
|
"tmdbid": _str(f.get("tmdbid")),
|
||||||
|
"imdbid": _str(f.get("imdbid")),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _episode_values(f: dict) -> dict:
|
||||||
|
series = f.get("series") or f.get("title") or "Unknown"
|
||||||
|
return {
|
||||||
|
"series": series,
|
||||||
|
"season": _pad2(f.get("season")),
|
||||||
|
"seasonraw": _str(f.get("season")),
|
||||||
|
"episode": _pad2(f.get("episode")),
|
||||||
|
"episodetitle": _str(f.get("episode_title")),
|
||||||
|
"year": _str(f.get("year")) if _plausible_year(f.get("year")) else "",
|
||||||
|
"quality": _str(f.get("quality")),
|
||||||
|
"resolution": _str(f.get("resolution")),
|
||||||
|
"source": source_label(f.get("source")),
|
||||||
|
"codec": _str(f.get("codec")).upper(),
|
||||||
|
"tvdbid": _str(f.get("tvdbid")),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _youtube_values(f: dict) -> dict:
|
||||||
|
"""Template values for a YouTube upload — channel-as-show, season=year, date-named
|
||||||
|
episode (Plex 'TV by date'). ``published_at``/``date`` is 'YYYY-MM-DD'."""
|
||||||
|
channel = f.get("channel") or f.get("series") or f.get("title") or "Unknown"
|
||||||
|
pub = str(f.get("published_at") or f.get("date") or "")[:10]
|
||||||
|
y = m = d = ""
|
||||||
|
if len(pub) == 10 and pub[4] == "-" and pub[7] == "-":
|
||||||
|
y, m, d = pub[0:4], pub[5:7], pub[8:10]
|
||||||
|
has_year = _plausible_year(y)
|
||||||
|
return {
|
||||||
|
"channel": channel,
|
||||||
|
"title": _str(f.get("title")) or "Unknown",
|
||||||
|
"year": y if has_year else "",
|
||||||
|
"date": pub if has_year else "", # only a trustworthy full date
|
||||||
|
"month": m if has_year else "",
|
||||||
|
"day": d if has_year else "",
|
||||||
|
"videoid": _str(f.get("youtube_id")),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def render_template(template: Any, values: dict) -> str:
|
||||||
|
"""Substitute ``$token`` / ``${token}`` from ``values`` into ``template``. Each
|
||||||
|
value is path-sanitised first (so a title with '/' can't spawn a folder), and
|
||||||
|
tokens are replaced longest-name-first ($episodetitle before $episode)."""
|
||||||
|
clean = {k: sanitize(v) for k, v in (values or {}).items()}
|
||||||
|
out = str(template or "")
|
||||||
|
for tok in sorted(clean, key=len, reverse=True):
|
||||||
|
out = out.replace("${" + tok + "}", clean[tok])
|
||||||
|
for tok in sorted(clean, key=len, reverse=True):
|
||||||
|
out = out.replace("$" + tok, clean[tok])
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _tidy_component(part: str) -> str:
|
||||||
|
"""Clean one path segment: drop a ' - ' left dangling by an empty token, remove
|
||||||
|
empty ()/[] left by an empty $year, collapse whitespace, trim stray dashes and
|
||||||
|
Windows-hostile trailing dots/spaces."""
|
||||||
|
p = re.sub(r"\s+-\s+(?=(\s|$))", " ", part) # ' - ' before an empty token
|
||||||
|
p = re.sub(r"\(\s*\)", "", p) # empty ( ) from a missing $year
|
||||||
|
p = re.sub(r"\[\s*\]", "", p)
|
||||||
|
p = re.sub(r"\s+", " ", p).strip()
|
||||||
|
p = p.strip("-").strip()
|
||||||
|
return p.rstrip(". ")
|
||||||
|
|
||||||
|
|
||||||
|
def render_path(scope: Any, root: Any, fields: dict, settings: Any, ext: Any) -> dict:
|
||||||
|
"""Render the destination for a finished download from the user's templates.
|
||||||
|
Returns ``{"dir", "filename", "path"}`` (same shape as ``library_paths.plan_path``).
|
||||||
|
An unsupported scope falls back to a flat drop so a file is never lost."""
|
||||||
|
settings = settings if isinstance(settings, dict) else {}
|
||||||
|
fields = fields if isinstance(fields, dict) else {}
|
||||||
|
root = str(root or "")
|
||||||
|
sc = str(scope or "").lower()
|
||||||
|
|
||||||
|
if sc == "movie":
|
||||||
|
tmpl = settings.get("movie_template") or DEFAULTS["movie_template"]
|
||||||
|
values = _movie_values(fields)
|
||||||
|
elif sc == "episode":
|
||||||
|
tmpl = settings.get("episode_template") or DEFAULTS["episode_template"]
|
||||||
|
values = _episode_values(fields)
|
||||||
|
elif sc == "youtube":
|
||||||
|
tmpl = settings.get("youtube_template") or DEFAULTS["youtube_template"]
|
||||||
|
values = _youtube_values(fields)
|
||||||
|
else:
|
||||||
|
base = (sanitize(fields.get("title")) or "download") + _ext(ext)
|
||||||
|
return {"dir": root, "filename": base, "path": os.path.join(root, base)}
|
||||||
|
|
||||||
|
rendered = render_template(tmpl, values)
|
||||||
|
parts = [p for p in (_tidy_component(seg) for seg in rendered.split("/")) if p]
|
||||||
|
if not parts:
|
||||||
|
parts = ["download"]
|
||||||
|
d = os.path.join(root, *parts[:-1]) if len(parts) > 1 else root
|
||||||
|
filename = parts[-1] + _ext(ext)
|
||||||
|
return {"dir": d, "filename": filename, "path": os.path.join(d, filename)}
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"DEFAULTS", "default_settings", "normalize", "load", "save",
|
||||||
|
"render_template", "render_path",
|
||||||
|
]
|
||||||
215
core/video/quality_eval.py
Normal file
215
core/video/quality_eval.py
Normal file
|
|
@ -0,0 +1,215 @@
|
||||||
|
"""Evaluate a video file/release against the quality profile.
|
||||||
|
|
||||||
|
Two consumers, one source of truth:
|
||||||
|
- the Download modal — to tell the user whether the copy they already own meets
|
||||||
|
their quality target (or is eligible for an upgrade), and
|
||||||
|
- the (later-phase) download engine — to filter/score search results.
|
||||||
|
|
||||||
|
Pure functions (no DB, no network) so they're unit-tested in isolation. Isolated —
|
||||||
|
imports only the sibling video ``quality_profile`` constants; nothing from music.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
# Resolution ranking (higher = better). The loose cutoff and the owned-vs-target
|
||||||
|
# check both compare on this rank, so "1920x1080", "1080p" and "1080" all agree.
|
||||||
|
_RES_RANK = (("2160", 4), ("4k", 4), ("1440", 3), ("1080", 3),
|
||||||
|
("720", 2), ("576", 1), ("480", 1), ("sd", 1))
|
||||||
|
_RES_LABEL = {4: "4K", 3: "1080p", 2: "720p", 1: "SD", 0: ""}
|
||||||
|
|
||||||
|
|
||||||
|
def resolution_rank(res: Any) -> int:
|
||||||
|
"""Map a raw resolution token to a rank int (4=4K … 1=SD, 0=unknown)."""
|
||||||
|
s = str(res or "").strip().lower()
|
||||||
|
for token, rank in _RES_RANK:
|
||||||
|
if token in s:
|
||||||
|
return rank
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def resolution_label(res: Any) -> str:
|
||||||
|
"""A friendly resolution label ('4K' / '1080p' / '720p' / 'SD' / '')."""
|
||||||
|
return _RES_LABEL.get(resolution_rank(res), "")
|
||||||
|
|
||||||
|
|
||||||
|
def _cutoff_label(cutoff: str) -> str:
|
||||||
|
return _RES_LABEL.get(resolution_rank(cutoff), "best")
|
||||||
|
|
||||||
|
|
||||||
|
def _codec_family(codec: Any) -> str:
|
||||||
|
"""Normalise a stored video codec to a reject-list key ('x264'/'hevc'/'av1')."""
|
||||||
|
s = str(codec or "").strip().lower()
|
||||||
|
if not s:
|
||||||
|
return ""
|
||||||
|
if "av1" in s:
|
||||||
|
return "av1"
|
||||||
|
if "265" in s or "hevc" in s:
|
||||||
|
return "hevc"
|
||||||
|
if "264" in s or "avc" in s:
|
||||||
|
return "x264"
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
def meets_cutoff(resolution: Any, profile: dict) -> bool:
|
||||||
|
"""Does an owned item's resolution already satisfy the loose cutoff target?
|
||||||
|
An empty cutoff ('always upgrade') is never 'good enough'."""
|
||||||
|
cut = (profile or {}).get("cutoff_resolution", "")
|
||||||
|
if not cut:
|
||||||
|
return False
|
||||||
|
return resolution_rank(resolution) >= resolution_rank(cut)
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_owned(file: Any, profile: Any) -> dict:
|
||||||
|
"""Verdict for a copy the user already owns, vs their quality profile.
|
||||||
|
|
||||||
|
Returns ``{"meets": bool, "resolution_label": str, "reasons": [{ok, text}]}``
|
||||||
|
— ``meets`` False means it's eligible for an upgrade. ``reasons`` is an ordered,
|
||||||
|
render-ready list of the checks (ok=True is reassuring, ok=False explains why
|
||||||
|
an upgrade would help)."""
|
||||||
|
file = file if isinstance(file, dict) else {}
|
||||||
|
profile = profile if isinstance(profile, dict) else {}
|
||||||
|
reasons: list = []
|
||||||
|
meets = True
|
||||||
|
|
||||||
|
res = file.get("resolution")
|
||||||
|
cut = profile.get("cutoff_resolution", "")
|
||||||
|
if not cut:
|
||||||
|
meets = False
|
||||||
|
reasons.append({"ok": False, "text": "You're set to always chase the best — eligible for an upgrade."})
|
||||||
|
elif resolution_rank(res) >= resolution_rank(cut):
|
||||||
|
reasons.append({"ok": True, "text": "Meets your " + _cutoff_label(cut) + " target."})
|
||||||
|
else:
|
||||||
|
meets = False
|
||||||
|
reasons.append({"ok": False, "text": "Below your " + _cutoff_label(cut) + " target — eligible for an upgrade."})
|
||||||
|
|
||||||
|
fam = _codec_family(file.get("video_codec"))
|
||||||
|
if fam and fam in (profile.get("rejects") or []):
|
||||||
|
meets = False
|
||||||
|
reasons.append({"ok": False, "text": "Its " + fam + " codec is on your reject list."})
|
||||||
|
|
||||||
|
return {
|
||||||
|
"meets": meets,
|
||||||
|
"resolution_label": resolution_label(res),
|
||||||
|
"reasons": reasons,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ── release (search hit) evaluation ───────────────────────────────────────────
|
||||||
|
# Map a parsed source → the tier-key prefix used in the quality profile's ladder.
|
||||||
|
_SRC_TIER = {"remux": "remux", "bluray": "bluray", "web-dl": "web",
|
||||||
|
"webrip": "webrip", "hdtv": "hdtv", "dvd": "dvd"}
|
||||||
|
_RES_SCORE = {"2160p": 400, "1080p": 300, "720p": 200, "480p": 100}
|
||||||
|
_SRC_SCORE = {"remux": 90, "bluray": 70, "web-dl": 55, "webrip": 40, "hdtv": 25, "dvd": 10}
|
||||||
|
|
||||||
|
|
||||||
|
def tier_key(source, resolution) -> str:
|
||||||
|
"""The quality-ladder key for a parsed (source, resolution), or '' if it isn't a
|
||||||
|
ladder tier (junk sources like cam/screener have no tier)."""
|
||||||
|
pre = _SRC_TIER.get(source)
|
||||||
|
if not pre:
|
||||||
|
# A loosely-named release with a known resolution but NO recognised source
|
||||||
|
# (very common — lots of releases tag '1080p' but not the source) → assume web
|
||||||
|
# so it still lands on a tier instead of being rejected as 'unknown quality'.
|
||||||
|
# ffprobe verifies the real quality after download.
|
||||||
|
if resolution and not source:
|
||||||
|
pre = "web"
|
||||||
|
else:
|
||||||
|
return ""
|
||||||
|
if pre == "dvd":
|
||||||
|
return "dvd"
|
||||||
|
return (pre + "-" + resolution) if resolution else ""
|
||||||
|
|
||||||
|
|
||||||
|
def _scope_ok(parsed, scope, want_season, want_episode):
|
||||||
|
"""Validate a hit actually matches what was searched (Sonarr-style): an episode
|
||||||
|
search wants SxxExx, a season search wants the whole season PACK, a show search
|
||||||
|
wants a complete-series pack."""
|
||||||
|
season, episode = parsed.get("season"), parsed.get("episode")
|
||||||
|
if scope == "movie":
|
||||||
|
return (None, None) if season is None else (None, "This is a TV release, not the movie")
|
||||||
|
if scope == "episode":
|
||||||
|
if episode is None:
|
||||||
|
return None, "Not a single episode"
|
||||||
|
if want_season is not None and season != want_season:
|
||||||
|
return None, "Wrong season"
|
||||||
|
if want_episode is not None and episode != want_episode:
|
||||||
|
return None, "Wrong episode"
|
||||||
|
return None, None
|
||||||
|
if scope == "season":
|
||||||
|
if not parsed.get("is_season_pack"):
|
||||||
|
return None, "Not a full-season pack"
|
||||||
|
if want_season is not None and season != want_season:
|
||||||
|
return None, "Wrong season"
|
||||||
|
return None, None
|
||||||
|
if scope == "series":
|
||||||
|
return (None, None) if parsed.get("is_series_pack") else (None, "Not a complete-series pack")
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_release(parsed, profile, *, scope="movie", want_season=None,
|
||||||
|
want_episode=None, size_gb=None) -> dict:
|
||||||
|
"""Judge a parsed search hit against the quality profile + the search scope.
|
||||||
|
|
||||||
|
Returns ``{accepted, score, rejected, tier, quality_label}`` — ``accepted`` False
|
||||||
|
means it's filtered out (``rejected`` says why); ``score`` ranks the keepers."""
|
||||||
|
parsed = parsed if isinstance(parsed, dict) else {}
|
||||||
|
profile = profile if isinstance(profile, dict) else {}
|
||||||
|
res, source = parsed.get("resolution"), parsed.get("source")
|
||||||
|
rejects = profile.get("rejects") or []
|
||||||
|
rejected = None
|
||||||
|
|
||||||
|
# 1) hard rejects — junk source / 3D / rejected codec
|
||||||
|
if source in ("cam", "screener", "workprint") and source in rejects:
|
||||||
|
rejected = source + " is on your reject list"
|
||||||
|
fam = _codec_family(parsed.get("codec"))
|
||||||
|
if not rejected and fam and fam in rejects:
|
||||||
|
rejected = fam + " codec is on your reject list"
|
||||||
|
if not rejected and parsed.get("three_d") and "3d" in rejects:
|
||||||
|
rejected = "3D is on your reject list"
|
||||||
|
|
||||||
|
# 2) must be an enabled ladder tier
|
||||||
|
tier = tier_key(source, res)
|
||||||
|
if not rejected:
|
||||||
|
enabled = {t.get("key") for t in (profile.get("tiers") or []) if t.get("enabled")}
|
||||||
|
if not tier:
|
||||||
|
rejected = "Unknown / unsupported quality"
|
||||||
|
elif tier not in enabled:
|
||||||
|
rejected = (resolution_label(res) or "This quality") + " " + (source or "") + " isn't in your enabled tiers"
|
||||||
|
|
||||||
|
# 3) HDR required (a real filter when set)
|
||||||
|
if not rejected and profile.get("prefer_hdr") == "require" and not parsed.get("hdr"):
|
||||||
|
rejected = "HDR required but this is SDR"
|
||||||
|
|
||||||
|
# 4) scope validation (episode vs season pack vs series pack)
|
||||||
|
if not rejected:
|
||||||
|
_, scope_reason = _scope_ok(parsed, scope, want_season, want_episode)
|
||||||
|
if scope_reason:
|
||||||
|
rejected = scope_reason
|
||||||
|
|
||||||
|
# 5) size guard (movie/episode only — packs are legitimately large)
|
||||||
|
if not rejected and size_gb:
|
||||||
|
cap = profile.get("max_movie_gb") if scope == "movie" else (profile.get("max_episode_gb") if scope == "episode" else 0)
|
||||||
|
if cap and size_gb > cap:
|
||||||
|
rejected = "Over your " + str(cap) + " GB size cap"
|
||||||
|
|
||||||
|
# score the keepers (higher = better)
|
||||||
|
score = _RES_SCORE.get(res, 0) + _SRC_SCORE.get(source, 0)
|
||||||
|
if profile.get("prefer_codec") not in (None, "any") and fam == profile.get("prefer_codec"):
|
||||||
|
score += 40
|
||||||
|
if parsed.get("hdr") and profile.get("prefer_hdr") in ("prefer", "require"):
|
||||||
|
score += 30
|
||||||
|
if parsed.get("audio") in ("atmos", "truehd", "dts-hd"):
|
||||||
|
score += 15
|
||||||
|
if profile.get("prefer_repack") and (parsed.get("repack") or parsed.get("proper")):
|
||||||
|
score += 10
|
||||||
|
|
||||||
|
label = " · ".join([x for x in [resolution_label(res),
|
||||||
|
(source or "").upper() if source else "", fam.upper() if fam else ""] if x])
|
||||||
|
return {"accepted": rejected is None, "score": score, "rejected": rejected,
|
||||||
|
"tier": tier, "quality_label": label}
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["resolution_rank", "resolution_label", "meets_cutoff", "evaluate_owned",
|
||||||
|
"tier_key", "evaluate_release"]
|
||||||
169
core/video/quality_profile.py
Normal file
169
core/video/quality_profile.py
Normal file
|
|
@ -0,0 +1,169 @@
|
||||||
|
"""Video quality profile — ONE unified, Radarr/Sonarr-class profile applied to
|
||||||
|
every video download source (slskd / torrent / usenet).
|
||||||
|
|
||||||
|
Unlike the music side (where quality is bitrate density), video quality is a
|
||||||
|
**source×resolution tier** parsed from a release title, refined by codec / HDR /
|
||||||
|
audio preferences. The (later-phase) download engine uses this profile to pick the
|
||||||
|
best candidate and to decide when a library item is "good enough" (the cutoff).
|
||||||
|
|
||||||
|
The model (rich-curated — Radarr-competitive without a full custom-formats engine):
|
||||||
|
|
||||||
|
- ``tiers`` : the source×resolution quality ladder (Remux-2160p … SDTV)
|
||||||
|
as ONE ranked list; each tier enabled + ordered best→worst.
|
||||||
|
- ``cutoff`` : once the library holds a tier at/above this rank, stop
|
||||||
|
upgrading (prevents endless re-grabbing).
|
||||||
|
- ``rejects`` : hard blocks — never grab these (cam / screener / workprint /
|
||||||
|
3d / optionally x264).
|
||||||
|
- preferences (SOFT — they score/tie-break, they never reject on their own):
|
||||||
|
``prefer_codec`` (any|hevc|av1), ``prefer_hdr`` (off|prefer|require),
|
||||||
|
``prefer_audio`` (any|surround|lossless|atmos), ``prefer_repack`` (bool).
|
||||||
|
- ``min_size_gb`` / ``max_size_gb`` : size guard per item (0 = no limit).
|
||||||
|
|
||||||
|
Pure data + normalize/validate here (no DB, no network) so it's unit-tested in
|
||||||
|
isolation. Persisted as a JSON blob in video.db's ``video_settings['quality_profile']``.
|
||||||
|
This module is isolated — it imports nothing from the music side.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
# The quality ladder, ordered best→worst. ``key`` = ``<source>-<resolution>`` (plus
|
||||||
|
# the two resolution-less SD tiers). This is the default ranking the UI renders.
|
||||||
|
TIERS = (
|
||||||
|
"remux-2160p", "bluray-2160p", "web-2160p",
|
||||||
|
"remux-1080p", "bluray-1080p", "web-1080p", "webrip-1080p", "hdtv-1080p",
|
||||||
|
"bluray-720p", "web-720p", "hdtv-720p",
|
||||||
|
"dvd", "sdtv",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Default-enabled tiers: solid 1080p + 720p coverage. 4K tiers off (size) and the
|
||||||
|
# SD tiers (dvd/sdtv) off — users opt into those deliberately.
|
||||||
|
_DEFAULT_ON = frozenset({
|
||||||
|
"remux-1080p", "bluray-1080p", "web-1080p", "webrip-1080p", "hdtv-1080p",
|
||||||
|
"bluray-720p", "web-720p", "hdtv-720p",
|
||||||
|
})
|
||||||
|
|
||||||
|
# Hard rejects (never grabbed). x264 is offered but OFF by default (rejecting it
|
||||||
|
# would drop most releases) — power users who only want HEVC/AV1 can enable it.
|
||||||
|
REJECTS = ("cam", "screener", "workprint", "3d", "x264")
|
||||||
|
|
||||||
|
CODECS = ("any", "hevc", "av1") # SOFT codec preference (tie-breaker)
|
||||||
|
HDR_MODES = ("off", "prefer", "require") # require = HDR-only (a real filter)
|
||||||
|
AUDIO_MODES = ("any", "surround", "lossless", "atmos")
|
||||||
|
MAX_SIZE_CAP_GB = 200 # slider ceiling; 0 means "no limit"
|
||||||
|
|
||||||
|
# The cutoff is a LOOSE resolution target (Radarr-style "upgrade until"): once the
|
||||||
|
# library holds an item at this resolution or better, stop chasing upgrades. ""
|
||||||
|
# (empty) means "best available — always upgrade". Always offered in full, regardless
|
||||||
|
# of which specific tiers are toggled on.
|
||||||
|
RESOLUTIONS = ("2160p", "1080p", "720p", "480p")
|
||||||
|
|
||||||
|
_TIER_SET = frozenset(TIERS)
|
||||||
|
|
||||||
|
|
||||||
|
def default_profile() -> dict:
|
||||||
|
"""A sensible best-in-class default: full 1080p/720p ladder, loose cutoff at
|
||||||
|
1080p, junk rejected, HEVC + HDR preferred (soft)."""
|
||||||
|
return {
|
||||||
|
"version": 2,
|
||||||
|
"tiers": [{"key": k, "enabled": k in _DEFAULT_ON} for k in TIERS],
|
||||||
|
"cutoff_resolution": "1080p",
|
||||||
|
"rejects": ["cam", "screener", "workprint", "3d"],
|
||||||
|
"prefer_codec": "hevc",
|
||||||
|
"prefer_hdr": "prefer",
|
||||||
|
"prefer_audio": "any",
|
||||||
|
"prefer_repack": True,
|
||||||
|
"max_movie_gb": 0, # per-item size guard, split by runtime (0 = no limit)
|
||||||
|
"max_episode_gb": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _coerce_int(value: Any, default: int) -> int:
|
||||||
|
try:
|
||||||
|
return int(value)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
def _clamp_size(value: Any) -> int:
|
||||||
|
return min(MAX_SIZE_CAP_GB, max(0, _coerce_int(value, 0)))
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_tiers(value: Any) -> list:
|
||||||
|
"""Rebuild the ranked tier ladder: keep the caller's order for known tiers,
|
||||||
|
coerce each ``enabled``, drop junk/dupes, then append any missing tiers in
|
||||||
|
canonical order so the ladder is always complete. Defaults from ``_DEFAULT_ON``."""
|
||||||
|
enabled = {k: (k in _DEFAULT_ON) for k in TIERS}
|
||||||
|
order: list = []
|
||||||
|
if isinstance(value, list):
|
||||||
|
for item in value:
|
||||||
|
if isinstance(item, dict):
|
||||||
|
k = str(item.get("key") or "").strip().lower()
|
||||||
|
else:
|
||||||
|
k = str(item or "").strip().lower()
|
||||||
|
if k in _TIER_SET and k not in order:
|
||||||
|
order.append(k)
|
||||||
|
if isinstance(item, dict) and "enabled" in item:
|
||||||
|
enabled[k] = bool(item.get("enabled"))
|
||||||
|
for k in TIERS: # complete the ladder, canonical order
|
||||||
|
if k not in order:
|
||||||
|
order.append(k)
|
||||||
|
return [{"key": k, "enabled": enabled[k]} for k in order]
|
||||||
|
|
||||||
|
|
||||||
|
def normalize(raw: Any) -> dict:
|
||||||
|
"""Coerce a stored/posted profile to a valid shape, filling gaps from the
|
||||||
|
default. Unknown keys dropped; invalid values fall back. Never raises."""
|
||||||
|
d = default_profile()
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
return d
|
||||||
|
|
||||||
|
d["tiers"] = normalize_tiers(raw.get("tiers"))
|
||||||
|
|
||||||
|
if "cutoff_resolution" in raw:
|
||||||
|
cr = str(raw.get("cutoff_resolution") or "").strip().lower()
|
||||||
|
if cr in RESOLUTIONS or cr == "": # "" = best available / always upgrade
|
||||||
|
d["cutoff_resolution"] = cr
|
||||||
|
|
||||||
|
rj = raw.get("rejects")
|
||||||
|
if isinstance(rj, list):
|
||||||
|
chosen = {str(x or "").strip().lower() for x in rj}
|
||||||
|
d["rejects"] = [r for r in REJECTS if r in chosen] # canonical order, valid only
|
||||||
|
|
||||||
|
if raw.get("prefer_codec") in CODECS:
|
||||||
|
d["prefer_codec"] = raw["prefer_codec"]
|
||||||
|
if raw.get("prefer_hdr") in HDR_MODES:
|
||||||
|
d["prefer_hdr"] = raw["prefer_hdr"]
|
||||||
|
if raw.get("prefer_audio") in AUDIO_MODES:
|
||||||
|
d["prefer_audio"] = raw["prefer_audio"]
|
||||||
|
d["prefer_repack"] = bool(raw.get("prefer_repack", d["prefer_repack"]))
|
||||||
|
|
||||||
|
d["max_movie_gb"] = _clamp_size(raw.get("max_movie_gb"))
|
||||||
|
d["max_episode_gb"] = _clamp_size(raw.get("max_episode_gb"))
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
def load(db) -> dict:
|
||||||
|
"""Read + normalize the stored profile, or the default if none/garbage."""
|
||||||
|
raw = db.get_setting("quality_profile")
|
||||||
|
if raw:
|
||||||
|
try:
|
||||||
|
return normalize(json.loads(raw))
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
pass
|
||||||
|
return default_profile()
|
||||||
|
|
||||||
|
|
||||||
|
def save(db, raw: Any) -> dict:
|
||||||
|
"""Normalize + persist; returns the normalized profile that was stored."""
|
||||||
|
prof = normalize(raw)
|
||||||
|
db.set_setting("quality_profile", json.dumps(prof))
|
||||||
|
return prof
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"TIERS", "REJECTS", "CODECS", "HDR_MODES", "AUDIO_MODES", "RESOLUTIONS",
|
||||||
|
"MAX_SIZE_CAP_GB", "default_profile", "normalize", "normalize_tiers", "load", "save",
|
||||||
|
]
|
||||||
119
core/video/release_parse.py
Normal file
119
core/video/release_parse.py
Normal file
|
|
@ -0,0 +1,119 @@
|
||||||
|
"""Parse a scene/p2p release title into structured quality + scope fields.
|
||||||
|
|
||||||
|
This is the video equivalent of Sonarr/Radarr's release parser: given a raw name
|
||||||
|
like ``The Wire S02 1080p BluRay x265 DDP5.1-GROUP`` it pulls out resolution,
|
||||||
|
source, codec, HDR, audio, group, repack/proper, and the SEASON/EPISODE scope
|
||||||
|
(single episode vs season pack vs complete-series pack). The download engine and
|
||||||
|
the search UI both rely on this to validate that a hit actually matches what was
|
||||||
|
searched (a season search must return the *whole* season, etc.).
|
||||||
|
|
||||||
|
Pure + regex-only (no DB, no network) so it's unit-tested in isolation. Isolated —
|
||||||
|
imports only re/typing; the music side never imports it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
# Resolution — first match wins (most specific first).
|
||||||
|
_RES = [
|
||||||
|
(re.compile(r"\b(2160p|4k|uhd)\b", re.I), "2160p"),
|
||||||
|
(re.compile(r"\b1080[pi]\b", re.I), "1080p"),
|
||||||
|
(re.compile(r"\b720[pi]\b", re.I), "720p"),
|
||||||
|
(re.compile(r"\b(480[pi]|576[pi])\b", re.I), "480p"),
|
||||||
|
]
|
||||||
|
# Source — order matters (remux before bluray; web-dl before webrip).
|
||||||
|
_SOURCE = [
|
||||||
|
(re.compile(r"\bremux\b", re.I), "remux"),
|
||||||
|
(re.compile(r"\b(blu-?ray|bdrip|brrip|bd25|bd50)\b", re.I), "bluray"),
|
||||||
|
(re.compile(r"\b(web-?dl|web\.?dl|amzn|nf|dsnp|hmax|atvp)\b", re.I), "web-dl"),
|
||||||
|
(re.compile(r"\bweb-?rip\b", re.I), "webrip"),
|
||||||
|
(re.compile(r"\bweb\b", re.I), "web-dl"), # plain "WEB" (very common) — treat as WEB-DL
|
||||||
|
(re.compile(r"\bhdtv\b", re.I), "hdtv"),
|
||||||
|
(re.compile(r"\b(dvdrip|dvd)\b", re.I), "dvd"),
|
||||||
|
(re.compile(r"\b(cam|hdcam|ts|telesync|hdts)\b", re.I), "cam"),
|
||||||
|
(re.compile(r"\b(scr|screener|dvdscr|bdscr)\b", re.I), "screener"),
|
||||||
|
(re.compile(r"\bworkprint\b", re.I), "workprint"),
|
||||||
|
]
|
||||||
|
_CODEC = [
|
||||||
|
(re.compile(r"\b(x265|h\.?265|hevc)\b", re.I), "hevc"),
|
||||||
|
(re.compile(r"\b(x264|h\.?264|avc)\b", re.I), "x264"),
|
||||||
|
(re.compile(r"\bav1\b", re.I), "av1"),
|
||||||
|
]
|
||||||
|
_HDR = [
|
||||||
|
(re.compile(r"\b(dolby\s?vision|do?vi|\bdv\b)\b", re.I), "dv"),
|
||||||
|
(re.compile(r"\bhdr10\+\b", re.I), "hdr10"),
|
||||||
|
(re.compile(r"\bhdr10\b", re.I), "hdr10"),
|
||||||
|
(re.compile(r"\bhdr\b", re.I), "hdr"),
|
||||||
|
]
|
||||||
|
_AUDIO = [
|
||||||
|
(re.compile(r"\batmos\b", re.I), "atmos"),
|
||||||
|
(re.compile(r"\b(truehd|true-hd)\b", re.I), "truehd"),
|
||||||
|
(re.compile(r"\b(dts-?hd|dts-?x)\b", re.I), "dts-hd"),
|
||||||
|
(re.compile(r"\bdts\b", re.I), "dts"),
|
||||||
|
(re.compile(r"\b(ddp|eac3|dd\+)\b", re.I), "eac3"),
|
||||||
|
(re.compile(r"\b(ac3|dd5\.?1|dd2\.?0)\b", re.I), "ac3"),
|
||||||
|
(re.compile(r"\baac\b", re.I), "aac"),
|
||||||
|
]
|
||||||
|
|
||||||
|
_RANGE = re.compile(r"\bS(\d{1,2})\s*[-–]\s*S?(\d{1,2})\b", re.I) # S01-S05
|
||||||
|
_SXXEXX = re.compile(r"\bS(\d{1,2})[\s.]?E(\d{1,3})\b", re.I) # S02E03
|
||||||
|
_SXX = re.compile(r"\bS(\d{1,2})\b", re.I) # S02 (pack)
|
||||||
|
_SEASON_WORD = re.compile(r"\bseason[\s.]?(\d{1,2})\b", re.I) # Season 2
|
||||||
|
_COMPLETE = re.compile(r"\b(complete|collection|all\s?seasons)\b", re.I)
|
||||||
|
_GROUP = re.compile(r"-([A-Za-z0-9]{2,})\s*$")
|
||||||
|
|
||||||
|
|
||||||
|
def _first(table, text) -> Any:
|
||||||
|
for rx, val in table:
|
||||||
|
if rx.search(text):
|
||||||
|
return val
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_release(title: Any) -> dict:
|
||||||
|
"""Parse a release name into quality + scope fields. Never raises."""
|
||||||
|
t = str(title or "")
|
||||||
|
out = {
|
||||||
|
"title": t,
|
||||||
|
"resolution": _first(_RES, t),
|
||||||
|
"source": _first(_SOURCE, t),
|
||||||
|
"codec": _first(_CODEC, t),
|
||||||
|
"hdr": _first(_HDR, t),
|
||||||
|
"audio": _first(_AUDIO, t),
|
||||||
|
"group": None,
|
||||||
|
"repack": bool(re.search(r"\brepack\b", t, re.I)),
|
||||||
|
"proper": bool(re.search(r"\bproper\b", t, re.I)),
|
||||||
|
"three_d": bool(re.search(r"\b3d\b", t, re.I)),
|
||||||
|
"season": None,
|
||||||
|
"season_end": None,
|
||||||
|
"episode": None,
|
||||||
|
"is_season_pack": False,
|
||||||
|
"is_series_pack": False,
|
||||||
|
}
|
||||||
|
g = _GROUP.search(t)
|
||||||
|
if g:
|
||||||
|
out["group"] = g.group(1)
|
||||||
|
|
||||||
|
rng = _RANGE.search(t)
|
||||||
|
m = _SXXEXX.search(t)
|
||||||
|
if rng:
|
||||||
|
out["season"] = int(rng.group(1))
|
||||||
|
out["season_end"] = int(rng.group(2))
|
||||||
|
out["is_series_pack"] = True
|
||||||
|
elif m:
|
||||||
|
out["season"] = int(m.group(1))
|
||||||
|
out["episode"] = int(m.group(2))
|
||||||
|
else:
|
||||||
|
sm = _SXX.search(t) or _SEASON_WORD.search(t)
|
||||||
|
if sm:
|
||||||
|
out["season"] = int(sm.group(1))
|
||||||
|
out["is_season_pack"] = True
|
||||||
|
if _COMPLETE.search(t):
|
||||||
|
out["is_series_pack"] = True
|
||||||
|
out["is_season_pack"] = False
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["parse_release"]
|
||||||
60
core/video/retention.py
Normal file
60
core/video/retention.py
Normal file
|
|
@ -0,0 +1,60 @@
|
||||||
|
"""Pure retention math for the YouTube channel auto-clean.
|
||||||
|
|
||||||
|
A channel's retention setting is a small string the cog modal stores:
|
||||||
|
``all`` — keep everything (default; nothing is ever deleted)
|
||||||
|
``count_<n>`` — keep the newest N episodes by upload date
|
||||||
|
``days_<n>`` — keep episodes uploaded within the last N days
|
||||||
|
|
||||||
|
Episodes are aged by their UPLOAD date (``published_at``), falling back to the date in the
|
||||||
|
filename (the youtube template embeds it) so downloads from before the column existed still
|
||||||
|
work. An episode with no derivable upload date is NEVER pruned (safe). All file I/O lives in
|
||||||
|
the handler; this module just decides which episodes fall outside the keep window.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from datetime import date, timedelta
|
||||||
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
_DATE_RE = re.compile(r"(\d{4}-\d{2}-\d{2})")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_retention(value: Any) -> Optional[Tuple[str, int]]:
|
||||||
|
"""``'count_30'`` → ``('count', 30)``; ``'all'`` / blank / junk → None (keep everything)."""
|
||||||
|
if not value or value == "all":
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
mode, raw = str(value).split("_", 1)
|
||||||
|
n = int(raw)
|
||||||
|
except (ValueError, AttributeError):
|
||||||
|
return None
|
||||||
|
return (mode, n) if (mode in ("count", "days") and n > 0) else None
|
||||||
|
|
||||||
|
|
||||||
|
def episode_date(ep: Dict[str, Any]) -> str:
|
||||||
|
"""The episode's upload date (``YYYY-MM-DD``): ``published_at`` if stored, else parsed
|
||||||
|
from the filename. ``''`` when neither yields one."""
|
||||||
|
p = str(ep.get("published_at") or "")[:10]
|
||||||
|
if len(p) == 10 and _DATE_RE.fullmatch(p):
|
||||||
|
return p
|
||||||
|
m = _DATE_RE.search(str(ep.get("filename") or ""))
|
||||||
|
return m.group(1) if m else ""
|
||||||
|
|
||||||
|
|
||||||
|
def episodes_to_prune(episodes: List[Dict[str, Any]], retention: Any, *, today: str) -> List[Dict[str, Any]]:
|
||||||
|
"""The episodes to DELETE under ``retention`` (newest upload kept first). Pure — episodes
|
||||||
|
with no derivable upload date are kept. ``today`` is an ISO date for the days-based cutoff."""
|
||||||
|
parsed = parse_retention(retention)
|
||||||
|
if not parsed:
|
||||||
|
return []
|
||||||
|
mode, n = parsed
|
||||||
|
dated = [(episode_date(e), e) for e in episodes]
|
||||||
|
dated = sorted([(d, e) for d, e in dated if d], key=lambda t: t[0], reverse=True)
|
||||||
|
if mode == "count":
|
||||||
|
return [e for _, e in dated[n:]] # everything beyond the newest n
|
||||||
|
try:
|
||||||
|
cutoff = (date.fromisoformat(today) - timedelta(days=n)).isoformat()
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return []
|
||||||
|
return [e for d, e in dated if d < cutoff] # uploaded before the cutoff
|
||||||
91
core/video/retry.py
Normal file
91
core/video/retry.py
Normal file
|
|
@ -0,0 +1,91 @@
|
||||||
|
"""Auto-retry logic for video downloads (the music-style depth).
|
||||||
|
|
||||||
|
When a grabbed release fails (transfer error / peer cancel / never lands), the engine
|
||||||
|
shouldn't just give up — it should try the NEXT-best candidate from the same search,
|
||||||
|
and when those run out, RE-SEARCH with a different query (e.g. a movie without its
|
||||||
|
year) and try those. This module is the pure decision engine; the monitor performs
|
||||||
|
the I/O (start the download / run the requery).
|
||||||
|
|
||||||
|
Pure (json + stdlib only); unit-tested. Isolated — no music imports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
MAX_ATTEMPTS = 6 # total tries (candidate hops + requeries) before giving up
|
||||||
|
|
||||||
|
|
||||||
|
def next_query(ctx: dict, tried: Any) -> str | None:
|
||||||
|
"""The next alternate slskd query to try for a search context, or None when
|
||||||
|
exhausted. Movie: 'Title Year' then 'Title'. TV keeps the SxxExx/Sxx identity but
|
||||||
|
offers a couple of numbering variants."""
|
||||||
|
ctx = ctx or {}
|
||||||
|
triedset = set(tried or [])
|
||||||
|
scope = str(ctx.get("scope") or "movie").lower()
|
||||||
|
title = str(ctx.get("title") or "").strip()
|
||||||
|
cands = []
|
||||||
|
if scope == "movie":
|
||||||
|
if ctx.get("year"):
|
||||||
|
cands.append(("%s %s" % (title, ctx["year"])).strip())
|
||||||
|
cands.append(title)
|
||||||
|
elif scope == "episode" and ctx.get("season") is not None and ctx.get("episode") is not None:
|
||||||
|
s, e = int(ctx["season"]), int(ctx["episode"])
|
||||||
|
cands.append("%s S%02dE%02d" % (title, s, e))
|
||||||
|
cands.append("%s %dx%02d" % (title, s, e))
|
||||||
|
elif scope == "season" and ctx.get("season") is not None:
|
||||||
|
s = int(ctx["season"])
|
||||||
|
cands.append("%s S%02d" % (title, s))
|
||||||
|
cands.append("%s Season %d" % (title, s))
|
||||||
|
else:
|
||||||
|
cands.append(title)
|
||||||
|
for q in cands:
|
||||||
|
if q and q not in triedset:
|
||||||
|
return q
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _loads(s, default):
|
||||||
|
try:
|
||||||
|
v = json.loads(s) if s else default
|
||||||
|
return v if v is not None else default
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
def plan_retry(row: dict, max_attempts: int = MAX_ATTEMPTS) -> dict:
|
||||||
|
"""Decide what to do for a failed download row. Returns one of:
|
||||||
|
{action: 'candidate', candidate: {...}, rest: [...]} — try the next stored hit
|
||||||
|
{action: 'requery', query: str, ctx: {...}} — re-search a new query
|
||||||
|
{action: 'fail', reason: str} — genuinely out of options
|
||||||
|
Pure: reads the row's JSON columns (candidates / tried_files / search_ctx / tried_queries)."""
|
||||||
|
if int(row.get("attempts") or 0) >= max_attempts:
|
||||||
|
return {"action": "fail", "reason": "retry budget reached"}
|
||||||
|
tried_files = set(_loads(row.get("tried_files"), []))
|
||||||
|
fresh = [c for c in _loads(row.get("candidates"), []) if c.get("filename") not in tried_files]
|
||||||
|
if fresh:
|
||||||
|
return {"action": "candidate", "candidate": fresh[0], "rest": fresh[1:]}
|
||||||
|
ctx = _loads(row.get("search_ctx"), {})
|
||||||
|
q = next_query(ctx, _loads(row.get("tried_queries"), []))
|
||||||
|
if q:
|
||||||
|
return {"action": "requery", "query": q, "ctx": ctx}
|
||||||
|
return {"action": "fail", "reason": "no candidates or queries left"}
|
||||||
|
|
||||||
|
|
||||||
|
def merge_candidates(new_accepted: Any, tried_files: Any) -> list:
|
||||||
|
"""Turn fresh accepted search results into candidate dicts, dropping anything
|
||||||
|
already attempted (so a requery never re-tries the same failing release)."""
|
||||||
|
seen = set(tried_files or [])
|
||||||
|
out = []
|
||||||
|
for r in (new_accepted or []):
|
||||||
|
fn = r.get("filename")
|
||||||
|
if not fn or fn in seen:
|
||||||
|
continue
|
||||||
|
seen.add(fn)
|
||||||
|
out.append({"username": r.get("username"), "filename": fn, "size_bytes": r.get("size_bytes"),
|
||||||
|
"quality_label": r.get("quality_label"), "release_title": r.get("title") or r.get("release_title")})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["MAX_ATTEMPTS", "next_query", "plan_retry", "merge_candidates"]
|
||||||
305
core/video/scanner.py
Normal file
305
core/video/scanner.py
Normal file
|
|
@ -0,0 +1,305 @@
|
||||||
|
"""SoulSync — video library scanner.
|
||||||
|
|
||||||
|
The media SERVER (Plex/Jellyfin) is the source of truth, exactly like the music
|
||||||
|
side: we ask the server what it has and mirror it into video.db. This module is
|
||||||
|
server-agnostic — it consumes a "video media source" (duck-typed) that yields
|
||||||
|
normalized dicts, so it never touches a media-server SDK directly. The Plex /
|
||||||
|
Jellyfin adapters live in core/video/sources.py.
|
||||||
|
|
||||||
|
A source must provide:
|
||||||
|
source.server_name -> 'plex' | 'jellyfin'
|
||||||
|
source.iter_movies(incremental=False) -> iterable of normalized movie dicts
|
||||||
|
source.iter_shows(incremental=False) -> iterable of normalized show dicts
|
||||||
|
|
||||||
|
Scan MODES (mirroring the music side's full_refresh / incremental / deep_scan):
|
||||||
|
'incremental' - only recently-added items from the server; upsert; no prune.
|
||||||
|
'full' - every item; upsert all (refresh metadata + add new); no prune.
|
||||||
|
'deep' - every item; upsert; PRUNE what the server no longer has.
|
||||||
|
|
||||||
|
ISOLATION: imports only video.db + shared infra; music never imports this.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_scanner")
|
||||||
|
|
||||||
|
VALID_MODES = ("incremental", "full", "deep")
|
||||||
|
# Which library to scan. Movies and TV are independent libraries, so a TV scan
|
||||||
|
# must never touch movies and vice-versa. 'all' (default) does both.
|
||||||
|
VALID_MEDIA_TYPES = ("all", "movie", "show")
|
||||||
|
|
||||||
|
# Incremental stops after this many consecutive already-known items (recent
|
||||||
|
# first), mirroring music's "25 consecutive complete albums" early-stop.
|
||||||
|
INCREMENTAL_STOP_AFTER = 25
|
||||||
|
# Below this library size, an incremental scan falls back to a full pass (music
|
||||||
|
# does the same when the DB is too small to be worth an incremental).
|
||||||
|
INCREMENTAL_MIN_LIBRARY = 50
|
||||||
|
|
||||||
|
|
||||||
|
class VideoLibraryScanner:
|
||||||
|
"""Reads the active media server and upserts movies/shows into video.db."""
|
||||||
|
|
||||||
|
def __init__(self, db, pause_workers=None, resume_workers=None):
|
||||||
|
self.db = db
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._status = {"state": "idle"}
|
||||||
|
self._thread = None
|
||||||
|
self._cancel = False
|
||||||
|
# Optional hooks: pause enrichment workers while a scan runs, resume after
|
||||||
|
# (injected by get_video_scanner; left None in tests so no engine spins up).
|
||||||
|
self._pause_workers = pause_workers
|
||||||
|
self._resume_workers = resume_workers
|
||||||
|
|
||||||
|
def get_status(self) -> dict:
|
||||||
|
with self._lock:
|
||||||
|
return dict(self._status)
|
||||||
|
|
||||||
|
def _set(self, **kw) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._status.update(kw)
|
||||||
|
|
||||||
|
def cancel(self) -> dict:
|
||||||
|
"""Request the running scan to stop after the current item."""
|
||||||
|
with self._lock:
|
||||||
|
if self._status.get("state") == "scanning":
|
||||||
|
self._cancel = True
|
||||||
|
self._status["phase"] = "cancelling"
|
||||||
|
return {"status": "cancelling"}
|
||||||
|
return {"status": "idle"}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _norm_mode(mode) -> str:
|
||||||
|
return mode if mode in VALID_MODES else "full"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _norm_media_type(media_type) -> str:
|
||||||
|
"""'movie'|'show'|'all', accepting the friendly aliases the UI/config use."""
|
||||||
|
m = str(media_type or "all").lower()
|
||||||
|
if m in ("movie", "movies", "film", "films"):
|
||||||
|
return "movie"
|
||||||
|
if m in ("show", "shows", "tv", "series", "episode", "episodes"):
|
||||||
|
return "show"
|
||||||
|
return "all"
|
||||||
|
|
||||||
|
def request_scan(self, source_factory, mode: str = "full", media_type: str = "all") -> dict:
|
||||||
|
"""Kick off a background scan. ``source_factory()`` returns a media
|
||||||
|
source (or None if no video-capable server is connected). ``media_type``
|
||||||
|
limits it to one library ('movie' / 'show'); 'all' does both."""
|
||||||
|
mode = self._norm_mode(mode)
|
||||||
|
media_type = self._norm_media_type(media_type)
|
||||||
|
with self._lock:
|
||||||
|
if self._status.get("state") == "scanning":
|
||||||
|
return {"status": "in_progress"}
|
||||||
|
self._cancel = False
|
||||||
|
self._status = {"state": "scanning", "phase": "starting", "mode": mode,
|
||||||
|
"media_type": media_type, "started_at": time.time(),
|
||||||
|
"percent": None, "movies": 0, "shows": 0, "episodes": 0}
|
||||||
|
self._thread = threading.Thread(
|
||||||
|
target=self._run, args=(source_factory, mode, media_type), daemon=True)
|
||||||
|
self._thread.start()
|
||||||
|
return {"status": "started", "mode": mode, "media_type": media_type}
|
||||||
|
|
||||||
|
def scan_sync(self, source_factory, mode: str = "full", media_type: str = "all") -> dict:
|
||||||
|
"""Run a scan inline (used by tests / callers that want to block).
|
||||||
|
|
||||||
|
``media_type`` limits the scan to one library: 'movie' or 'show' (TV);
|
||||||
|
'all' (default) does both. Because the scanner is a process singleton, a
|
||||||
|
scan that starts while another is running returns ``state='in_progress'``
|
||||||
|
without stomping the live one — the caller should skip."""
|
||||||
|
mode = self._norm_mode(mode)
|
||||||
|
media_type = self._norm_media_type(media_type)
|
||||||
|
with self._lock:
|
||||||
|
if self._status.get("state") == "scanning":
|
||||||
|
return {"state": "in_progress", "phase": "a video scan is already running"}
|
||||||
|
self._cancel = False
|
||||||
|
self._status = {"state": "scanning", "phase": "starting", "mode": mode,
|
||||||
|
"media_type": media_type, "started_at": time.time(),
|
||||||
|
"percent": None, "movies": 0, "shows": 0, "episodes": 0}
|
||||||
|
self._run(source_factory, mode, media_type)
|
||||||
|
return self.get_status()
|
||||||
|
|
||||||
|
def _finish_cancelled(self, movies, shows, episodes) -> None:
|
||||||
|
self._set(state="cancelled", phase="cancelled", finished_at=time.time(),
|
||||||
|
movies=movies, shows=shows, episodes=episodes)
|
||||||
|
logger.info("Video scan cancelled at %d movies, %d shows", movies, shows)
|
||||||
|
|
||||||
|
def _pause_for_scan(self) -> bool:
|
||||||
|
"""Pause enrichment workers for the duration of the scan. Best-effort —
|
||||||
|
a failure here must never abort the scan."""
|
||||||
|
if not self._pause_workers:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
self._pause_workers()
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
logger.debug("video scan: pausing enrichment workers failed", exc_info=True)
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _resume_after_scan(self) -> None:
|
||||||
|
if not self._resume_workers:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
self._resume_workers()
|
||||||
|
except Exception:
|
||||||
|
logger.debug("video scan: resuming enrichment workers failed", exc_info=True)
|
||||||
|
|
||||||
|
def _run(self, source_factory, mode: str = "full", media_type: str = "all") -> None:
|
||||||
|
# Enrichment steps aside for the scan (all modes, both entry points), and
|
||||||
|
# the finally guarantees it resumes on success, cancel, or error.
|
||||||
|
paused = self._pause_for_scan()
|
||||||
|
try:
|
||||||
|
source = source_factory()
|
||||||
|
if source is None:
|
||||||
|
self._set(state="error", phase="no video server",
|
||||||
|
error="No connected Plex/Jellyfin video server")
|
||||||
|
return
|
||||||
|
server = source.server_name
|
||||||
|
incremental = mode == "incremental"
|
||||||
|
do_prune = mode == "deep"
|
||||||
|
# Movies and TV are independent libraries — scan only the requested
|
||||||
|
# kind(s) so a TV scan never pulls in (or prunes) movies, and vice-versa.
|
||||||
|
do_movies = media_type in ("all", "movie")
|
||||||
|
do_shows = media_type in ("all", "show")
|
||||||
|
# FULL = a clean reset (clobber enrichment-owned fields). Incremental/deep
|
||||||
|
# PRESERVE them, so a routine re-scan never wipes the TMDB-backfilled
|
||||||
|
# `status` the airing watchlist relies on. (Only an explicit full resets.)
|
||||||
|
preserve = mode != "full"
|
||||||
|
|
||||||
|
# Incremental on a near-empty library is pointless — fall back to a
|
||||||
|
# full pass so the first scan actually populates (music does this).
|
||||||
|
if incremental and (self.db.table_count("movies") + self.db.table_count("shows")) < INCREMENTAL_MIN_LIBRARY:
|
||||||
|
incremental = False
|
||||||
|
|
||||||
|
# Totals up front so the progress bar shows a REAL percentage
|
||||||
|
# (movies + shows are the unit; episodes ride along under each show).
|
||||||
|
total = 0
|
||||||
|
try:
|
||||||
|
c = source.counts(incremental=incremental) or {}
|
||||||
|
total = (int(c.get("movies", 0) or 0) if do_movies else 0) + \
|
||||||
|
(int(c.get("shows", 0) or 0) if do_shows else 0)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("video scan: counts() unavailable; progress will be indeterminate")
|
||||||
|
processed = 0
|
||||||
|
|
||||||
|
def pct():
|
||||||
|
return round(processed / total * 100) if total else None
|
||||||
|
|
||||||
|
known_movies = self.db.server_ids("movies", server) if (incremental and do_movies) else set()
|
||||||
|
known_shows = self.db.server_ids("shows", server) if (incremental and do_shows) else set()
|
||||||
|
|
||||||
|
# ── Movies ── (skipped entirely on a TV-only scan)
|
||||||
|
seen_movies: set[str] = set()
|
||||||
|
movies = 0
|
||||||
|
removed_m = 0
|
||||||
|
if do_movies:
|
||||||
|
self._set(phase="scanning movies", total=total, percent=pct())
|
||||||
|
consec = 0
|
||||||
|
for item in source.iter_movies(incremental=incremental):
|
||||||
|
if self._cancel:
|
||||||
|
return self._finish_cancelled(movies, 0, 0)
|
||||||
|
sid = str(item["server_id"])
|
||||||
|
# Incremental early-stop: skip already-known items and bail after
|
||||||
|
# a run of consecutive known ones (server lists recent first).
|
||||||
|
if incremental and sid in known_movies:
|
||||||
|
consec += 1
|
||||||
|
if consec >= INCREMENTAL_STOP_AFTER:
|
||||||
|
break
|
||||||
|
continue
|
||||||
|
consec = 0
|
||||||
|
try:
|
||||||
|
self.db.upsert_movie(server, item, preserve_enrichment=preserve)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video scan: skipping movie %s", sid)
|
||||||
|
continue
|
||||||
|
seen_movies.add(sid)
|
||||||
|
movies += 1
|
||||||
|
processed += 1
|
||||||
|
if movies % 10 == 0:
|
||||||
|
self._set(movies=movies, percent=pct())
|
||||||
|
self._set(movies=movies, percent=pct())
|
||||||
|
# Prune ONLY on a deep scan, and only when we actually saw items —
|
||||||
|
# so a transient empty response can never wipe the library. The prune
|
||||||
|
# runs AFTER the bar fills, and a big cleanup (many orphaned rows +
|
||||||
|
# cascades) takes a few seconds — surface a phase so the UI shows
|
||||||
|
# "cleaning up", not a stuck 100%.
|
||||||
|
if do_prune and seen_movies:
|
||||||
|
self._set(phase="cleaning up removed movies", percent=pct())
|
||||||
|
removed_m = (self.db.prune_missing("movies", server, seen_movies)
|
||||||
|
if do_prune and seen_movies else 0)
|
||||||
|
|
||||||
|
# ── Shows ── (skipped entirely on a movie-only scan)
|
||||||
|
seen_shows: set[str] = set()
|
||||||
|
shows = 0
|
||||||
|
episodes = 0
|
||||||
|
removed_s = 0
|
||||||
|
if do_shows:
|
||||||
|
self._set(phase="scanning shows")
|
||||||
|
consec = 0
|
||||||
|
for show in source.iter_shows(incremental=incremental):
|
||||||
|
if self._cancel:
|
||||||
|
return self._finish_cancelled(movies, shows, episodes)
|
||||||
|
sid = str(show["server_id"])
|
||||||
|
if incremental and sid in known_shows:
|
||||||
|
consec += 1
|
||||||
|
if consec >= INCREMENTAL_STOP_AFTER:
|
||||||
|
break
|
||||||
|
continue
|
||||||
|
consec = 0
|
||||||
|
try:
|
||||||
|
self.db.upsert_show_tree(server, show, preserve_enrichment=preserve)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video scan: skipping show %s", sid)
|
||||||
|
continue
|
||||||
|
seen_shows.add(sid)
|
||||||
|
shows += 1
|
||||||
|
episodes += sum(len(s.get("episodes", [])) for s in show.get("seasons", []))
|
||||||
|
processed += 1
|
||||||
|
self._set(shows=shows, episodes=episodes, percent=pct())
|
||||||
|
# Final prune (the one that delays "done" on a deep scan) — show it.
|
||||||
|
if do_prune and seen_shows:
|
||||||
|
self._set(phase="cleaning up removed shows", percent=100)
|
||||||
|
removed_s = (self.db.prune_missing("shows", server, seen_shows)
|
||||||
|
if do_prune and seen_shows else 0)
|
||||||
|
|
||||||
|
self._set(state="done", phase="complete", finished_at=time.time(),
|
||||||
|
movies=movies, shows=shows, episodes=episodes, percent=100,
|
||||||
|
removed=removed_m + removed_s)
|
||||||
|
logger.info("Video scan (%s) complete: %d movies, %d shows, %d episodes (%d pruned)",
|
||||||
|
mode, movies, shows, episodes, removed_m + removed_s)
|
||||||
|
except Exception as e: # noqa: BLE001 - report any failure to the UI
|
||||||
|
logger.exception("Video library scan failed")
|
||||||
|
self._set(state="error", phase="failed", error=str(e))
|
||||||
|
finally:
|
||||||
|
if paused:
|
||||||
|
self._resume_after_scan()
|
||||||
|
|
||||||
|
|
||||||
|
# Module-level singleton, bound to the (single) video DB.
|
||||||
|
_scanner = None
|
||||||
|
_scanner_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _engine_pause_for_scan() -> None:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
get_video_enrichment_engine().pause_for_scan()
|
||||||
|
|
||||||
|
|
||||||
|
def _engine_resume_after_scan() -> None:
|
||||||
|
from core.video.enrichment.engine import get_video_enrichment_engine
|
||||||
|
get_video_enrichment_engine().resume_after_scan()
|
||||||
|
|
||||||
|
|
||||||
|
def get_video_scanner(db) -> VideoLibraryScanner:
|
||||||
|
global _scanner
|
||||||
|
if _scanner is None:
|
||||||
|
with _scanner_lock:
|
||||||
|
if _scanner is None:
|
||||||
|
_scanner = VideoLibraryScanner(
|
||||||
|
db, pause_workers=_engine_pause_for_scan,
|
||||||
|
resume_workers=_engine_resume_after_scan)
|
||||||
|
return _scanner
|
||||||
229
core/video/sidecars.py
Normal file
229
core/video/sidecars.py
Normal file
|
|
@ -0,0 +1,229 @@
|
||||||
|
"""Write Kodi/Jellyfin-style sidecars next to imported video — NFO metadata + the full
|
||||||
|
artwork set — so the library is self-describing on disk (server-agnostic + portable).
|
||||||
|
|
||||||
|
The metadata comes from the on-demand TMDB detail fetch (``engine.tmdb_detail``), passed
|
||||||
|
in as ``meta``. This module is pure — NFO XML building + a write plan — with the
|
||||||
|
filesystem injected, so it's unit-tested without disk or network. Best-effort BY
|
||||||
|
CONTRACT: the caller treats every failure as non-fatal (a missing poster or a flaky
|
||||||
|
fetch never breaks an import).
|
||||||
|
|
||||||
|
Layout written:
|
||||||
|
Movie folder: movie.nfo, poster.jpg, fanart.jpg, clearlogo.png
|
||||||
|
Show root: tvshow.nfo, poster.jpg, fanart.jpg, clearlogo.png, seasonNN-poster.jpg
|
||||||
|
|
||||||
|
Isolated: stdlib only; no music imports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
from xml.sax.saxutils import escape as _xesc
|
||||||
|
|
||||||
|
_HEAD = '<?xml version="1.0" encoding="UTF-8" standalone="yes"?>\n'
|
||||||
|
|
||||||
|
|
||||||
|
def _t(tag: str, value: Any) -> str:
|
||||||
|
"""A single XML element, or '' when the value is empty (so absent fields are
|
||||||
|
simply omitted rather than written blank)."""
|
||||||
|
if value is None:
|
||||||
|
return ""
|
||||||
|
s = str(value).strip()
|
||||||
|
if not s:
|
||||||
|
return ""
|
||||||
|
return " <%s>%s</%s>\n" % (tag, _xesc(s), tag)
|
||||||
|
|
||||||
|
|
||||||
|
def _year(meta: dict) -> str:
|
||||||
|
y = meta.get("year")
|
||||||
|
return str(y) if y else ""
|
||||||
|
|
||||||
|
|
||||||
|
def _uniqueids(meta: dict, *, tmdb_default: bool) -> str:
|
||||||
|
out = ""
|
||||||
|
tmdb = meta.get("tmdb_id")
|
||||||
|
if tmdb:
|
||||||
|
out += ' <uniqueid type="tmdb"%s>%s</uniqueid>\n' % (
|
||||||
|
' default="true"' if tmdb_default else "", _xesc(str(tmdb)))
|
||||||
|
if meta.get("tvdb_id"):
|
||||||
|
out += ' <uniqueid type="tvdb"%s>%s</uniqueid>\n' % (
|
||||||
|
' default="true"' if not tmdb_default else "", _xesc(str(meta["tvdb_id"])))
|
||||||
|
if meta.get("imdb_id"):
|
||||||
|
out += ' <uniqueid type="imdb">%s</uniqueid>\n' % _xesc(str(meta["imdb_id"]))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _genres(meta: dict) -> str:
|
||||||
|
return "".join(_t("genre", g) for g in (meta.get("genres") or []) if g)
|
||||||
|
|
||||||
|
|
||||||
|
def _actors(meta: dict, limit: int = 20) -> str:
|
||||||
|
out = ""
|
||||||
|
for i, c in enumerate((meta.get("cast") or [])[:limit]):
|
||||||
|
if not isinstance(c, dict) or not c.get("name"):
|
||||||
|
continue
|
||||||
|
out += " <actor>\n <name>%s</name>\n" % _xesc(str(c["name"]))
|
||||||
|
if c.get("character"):
|
||||||
|
out += " <role>%s</role>\n" % _xesc(str(c["character"]))
|
||||||
|
out += " <order>%d</order>\n </actor>\n" % i
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _artwork_tags(meta: dict) -> str:
|
||||||
|
out = ""
|
||||||
|
if meta.get("poster_url"):
|
||||||
|
out += ' <thumb aspect="poster">%s</thumb>\n' % _xesc(str(meta["poster_url"]))
|
||||||
|
if meta.get("logo"):
|
||||||
|
out += ' <thumb aspect="clearlogo">%s</thumb>\n' % _xesc(str(meta["logo"]))
|
||||||
|
if meta.get("backdrop_url"):
|
||||||
|
out += " <fanart>\n <thumb>%s</thumb>\n </fanart>\n" % _xesc(str(meta["backdrop_url"]))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def nfo_movie(meta: dict) -> str:
|
||||||
|
"""A Kodi/Jellyfin-compatible ``movie.nfo`` from a TMDB detail dict."""
|
||||||
|
meta = meta if isinstance(meta, dict) else {}
|
||||||
|
body = (
|
||||||
|
_t("title", meta.get("title"))
|
||||||
|
+ _t("originaltitle", meta.get("original_title"))
|
||||||
|
+ _t("year", _year(meta))
|
||||||
|
+ _t("plot", meta.get("overview"))
|
||||||
|
+ _t("outline", meta.get("overview"))
|
||||||
|
+ _t("tagline", meta.get("tagline"))
|
||||||
|
+ _t("runtime", meta.get("runtime_minutes"))
|
||||||
|
+ _t("mpaa", meta.get("content_rating"))
|
||||||
|
+ _t("studio", meta.get("studio"))
|
||||||
|
+ _t("premiered", meta.get("release_date"))
|
||||||
|
+ _t("status", meta.get("status"))
|
||||||
|
+ _t("rating", meta.get("rating"))
|
||||||
|
+ _genres(meta)
|
||||||
|
+ _uniqueids(meta, tmdb_default=True)
|
||||||
|
+ _artwork_tags(meta)
|
||||||
|
+ _actors(meta)
|
||||||
|
)
|
||||||
|
return _HEAD + "<movie>\n" + body + "</movie>\n"
|
||||||
|
|
||||||
|
|
||||||
|
def nfo_tvshow(meta: dict) -> str:
|
||||||
|
"""A Kodi/Jellyfin-compatible ``tvshow.nfo`` from a TMDB detail dict."""
|
||||||
|
meta = meta if isinstance(meta, dict) else {}
|
||||||
|
body = (
|
||||||
|
_t("title", meta.get("title"))
|
||||||
|
+ _t("year", _year(meta))
|
||||||
|
+ _t("plot", meta.get("overview"))
|
||||||
|
+ _t("outline", meta.get("overview"))
|
||||||
|
+ _t("tagline", meta.get("tagline"))
|
||||||
|
+ _t("runtime", meta.get("runtime_minutes"))
|
||||||
|
+ _t("mpaa", meta.get("content_rating"))
|
||||||
|
+ _t("studio", meta.get("network"))
|
||||||
|
+ _t("premiered", meta.get("first_air_date"))
|
||||||
|
+ _t("status", meta.get("status"))
|
||||||
|
+ _t("rating", meta.get("rating"))
|
||||||
|
+ _genres(meta)
|
||||||
|
+ _uniqueids(meta, tmdb_default=True)
|
||||||
|
+ _artwork_tags(meta)
|
||||||
|
+ _actors(meta)
|
||||||
|
)
|
||||||
|
return _HEAD + "<tvshow>\n" + body + "</tvshow>\n"
|
||||||
|
|
||||||
|
|
||||||
|
def _season_art(meta: dict) -> list:
|
||||||
|
"""(url, filename) pairs for season posters — seasonNN-poster.jpg (Specials → 00)."""
|
||||||
|
out = []
|
||||||
|
for s in (meta.get("_seasons") or []):
|
||||||
|
if not isinstance(s, dict) or not s.get("poster_url"):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
n = int(s.get("season_number"))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
out.append((s["poster_url"], "season%02d-poster.jpg" % n))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def plan_sidecars(scope: Any, meta: dict, settings: dict) -> dict:
|
||||||
|
"""Decide what to write for one imported item. Returns
|
||||||
|
``{"nfo": (filename, content) | None, "art": [(url, filename), ...]}`` — gated by
|
||||||
|
the ``write_nfo`` / ``save_artwork`` settings. Pure."""
|
||||||
|
meta = meta if isinstance(meta, dict) else {}
|
||||||
|
settings = settings if isinstance(settings, dict) else {}
|
||||||
|
sc = str(scope or "").lower()
|
||||||
|
nfo = None
|
||||||
|
art = []
|
||||||
|
|
||||||
|
if settings.get("write_nfo"):
|
||||||
|
if sc == "movie":
|
||||||
|
nfo = ("movie.nfo", nfo_movie(meta))
|
||||||
|
elif sc == "episode":
|
||||||
|
nfo = ("tvshow.nfo", nfo_tvshow(meta))
|
||||||
|
|
||||||
|
if settings.get("save_artwork"):
|
||||||
|
if meta.get("poster_url"):
|
||||||
|
art.append((meta["poster_url"], "poster.jpg"))
|
||||||
|
if meta.get("backdrop_url"):
|
||||||
|
art.append((meta["backdrop_url"], "fanart.jpg"))
|
||||||
|
if meta.get("logo"):
|
||||||
|
art.append((meta["logo"], "clearlogo.png"))
|
||||||
|
if sc == "episode":
|
||||||
|
art.extend(_season_art(meta))
|
||||||
|
|
||||||
|
return {"nfo": nfo, "art": art}
|
||||||
|
|
||||||
|
|
||||||
|
def write(folder: str, scope: Any, meta: dict, settings: dict, fs: Any) -> None:
|
||||||
|
"""Write the planned sidecars into ``folder`` via the injected ``fs``
|
||||||
|
(``list_dir``, ``makedirs``, ``write_text(path, str)``, ``save_url(url, path)``).
|
||||||
|
Idempotent: skips any file already present so upgrades/re-imports don't refetch.
|
||||||
|
Best-effort: each file is independent and a failure is swallowed."""
|
||||||
|
import os
|
||||||
|
plan = plan_sidecars(scope, meta, settings)
|
||||||
|
if not plan["nfo"] and not plan["art"]:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
existing = {str(n).lower() for n in (fs.list_dir(folder) or [])}
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
existing = set()
|
||||||
|
try:
|
||||||
|
fs.makedirs(folder)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
return
|
||||||
|
|
||||||
|
if plan["nfo"]:
|
||||||
|
name, content = plan["nfo"]
|
||||||
|
if name.lower() not in existing:
|
||||||
|
try:
|
||||||
|
fs.write_text(os.path.join(folder, name), content)
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
pass
|
||||||
|
for url, name in plan["art"]:
|
||||||
|
if name.lower() in existing:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
fs.save_url(url, os.path.join(folder, name))
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def write_for(dest_path: str, scope: Any, poster_url: Any, detail: Any,
|
||||||
|
settings: dict, fs: Any) -> None:
|
||||||
|
"""Resolve the sidecar folder from a finished file's path and write into it. The
|
||||||
|
folder is the movie folder (parent of the file) or, for an episode, the SHOW root
|
||||||
|
(parent of the Season folder) so show-level art/NFO land once. ``poster_url`` is the
|
||||||
|
baseline from the download row; ``detail`` (the TMDB fetch, or None) fills the rest —
|
||||||
|
so a poster still lands even when the detail fetch is unavailable."""
|
||||||
|
import os
|
||||||
|
sc = str(scope or "").lower()
|
||||||
|
if sc == "movie":
|
||||||
|
folder = os.path.dirname(dest_path)
|
||||||
|
elif sc == "episode":
|
||||||
|
folder = os.path.dirname(os.path.dirname(dest_path)) # show root, above Season NN
|
||||||
|
else:
|
||||||
|
return
|
||||||
|
if not folder:
|
||||||
|
return
|
||||||
|
meta = {"poster_url": poster_url} if poster_url else {}
|
||||||
|
if isinstance(detail, dict):
|
||||||
|
meta.update(detail)
|
||||||
|
write(folder, sc, meta, settings, fs)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["nfo_movie", "nfo_tvshow", "plan_sidecars", "write", "write_for"]
|
||||||
127
core/video/slskd_download.py
Normal file
127
core/video/slskd_download.py
Normal file
|
|
@ -0,0 +1,127 @@
|
||||||
|
"""Start + track Soulseek (slskd) downloads for the video pipeline.
|
||||||
|
|
||||||
|
Thin wrapper over slskd's transfer API (the same shared instance the music side uses):
|
||||||
|
- ``start_download(username, filename, size)`` → POST /transfers/downloads/{username}
|
||||||
|
- ``list_downloads()`` → GET /transfers/downloads (flattened)
|
||||||
|
The flatten + state-classification helpers are pure (unit-tested); the HTTP is glue.
|
||||||
|
|
||||||
|
Isolated: stdlib + requests + shared ``config_manager``; no music imports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
def _conn():
|
||||||
|
from config.settings import config_manager
|
||||||
|
base = str(config_manager.get("soulseek.slskd_url", "") or "").rstrip("/")
|
||||||
|
key = config_manager.get("soulseek.api_key", "") or ""
|
||||||
|
return base, ({"X-API-Key": key} if key else {})
|
||||||
|
|
||||||
|
|
||||||
|
def start_download(username: str, filename: str, size_bytes: int = 0) -> dict:
|
||||||
|
"""Ask slskd to download one file from a user. Returns {ok[, error]}."""
|
||||||
|
base, headers = _conn()
|
||||||
|
if not base:
|
||||||
|
return {"ok": False, "error": "slskd isn't configured"}
|
||||||
|
try:
|
||||||
|
r = requests.post(base + "/api/v0/transfers/downloads/" + quote(str(username or "")),
|
||||||
|
json=[{"filename": filename, "size": int(size_bytes or 0)}],
|
||||||
|
headers=headers, timeout=15)
|
||||||
|
r.raise_for_status()
|
||||||
|
return {"ok": True}
|
||||||
|
except Exception as e: # noqa: BLE001 - surface any slskd/network failure to the caller
|
||||||
|
return {"ok": False, "error": str(e)}
|
||||||
|
|
||||||
|
|
||||||
|
def list_downloads() -> list:
|
||||||
|
"""Current slskd downloads, flattened to one dict per file."""
|
||||||
|
base, headers = _conn()
|
||||||
|
if not base:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
r = requests.get(base + "/api/v0/transfers/downloads", headers=headers, timeout=15)
|
||||||
|
if not r.ok:
|
||||||
|
return []
|
||||||
|
data = r.json()
|
||||||
|
except Exception: # noqa: BLE001, S110 - transient slskd error → no downloads this tick
|
||||||
|
return []
|
||||||
|
return flatten_downloads(data)
|
||||||
|
|
||||||
|
|
||||||
|
def flatten_downloads(data: Any) -> list:
|
||||||
|
"""slskd's nested users→directories→files → a flat list of file transfers. Pure."""
|
||||||
|
out = []
|
||||||
|
for user in (data if isinstance(data, list) else []):
|
||||||
|
if not isinstance(user, dict):
|
||||||
|
continue
|
||||||
|
un = user.get("username", "")
|
||||||
|
for d in (user.get("directories") or []):
|
||||||
|
for f in (d.get("files") or []):
|
||||||
|
out.append({
|
||||||
|
"username": un, "filename": f.get("filename", ""), "id": f.get("id", ""),
|
||||||
|
"state": f.get("state", ""), "size": f.get("size", 0) or 0,
|
||||||
|
"transferred": f.get("bytesTransferred", 0) or 0,
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def classify_state(state: Any) -> str:
|
||||||
|
"""slskd state string → 'completed' | 'cancelled' | 'failed' | 'active'. Pure."""
|
||||||
|
s = str(state or "").lower()
|
||||||
|
if "completed" in s and "succeed" in s:
|
||||||
|
return "completed"
|
||||||
|
if "cancel" in s:
|
||||||
|
return "cancelled"
|
||||||
|
if any(x in s for x in ("error", "timed", "failed", "reject")):
|
||||||
|
return "failed"
|
||||||
|
if "completed" in s: # completed but not succeeded → treat as failed
|
||||||
|
return "failed"
|
||||||
|
return "active"
|
||||||
|
|
||||||
|
|
||||||
|
def cancel_download(username: str, filename: str) -> dict:
|
||||||
|
"""Cancel (and remove) a slskd transfer matching username+filename. Returns
|
||||||
|
{ok[, gone][, error]}. 'gone' = the transfer was already absent."""
|
||||||
|
base, headers = _conn()
|
||||||
|
if not base:
|
||||||
|
return {"ok": False, "error": "slskd isn't configured"}
|
||||||
|
tid = None
|
||||||
|
for t in list_downloads():
|
||||||
|
if t.get("username") == username and t.get("filename") == filename:
|
||||||
|
tid = t.get("id")
|
||||||
|
break
|
||||||
|
if not tid:
|
||||||
|
return {"ok": True, "gone": True}
|
||||||
|
try:
|
||||||
|
requests.delete(base + "/api/v0/transfers/downloads/%s/%s" % (quote(str(username or "")), tid),
|
||||||
|
headers=headers, params={"remove": "true"}, timeout=10)
|
||||||
|
return {"ok": True}
|
||||||
|
except Exception as e: # noqa: BLE001 - surface the failure; the row is marked cancelled anyway
|
||||||
|
return {"ok": False, "error": str(e)}
|
||||||
|
|
||||||
|
|
||||||
|
def progress_pct(transfer: dict) -> float:
|
||||||
|
"""0–100 from bytesTransferred/size (100 once completed). Pure."""
|
||||||
|
transfer = transfer or {}
|
||||||
|
if classify_state(transfer.get("state")) == "completed":
|
||||||
|
return 100.0
|
||||||
|
size = transfer.get("size", 0) or 0
|
||||||
|
tr = transfer.get("transferred", 0) or 0
|
||||||
|
return round(min(99.0, (tr / size * 100.0) if size else 0.0), 1)
|
||||||
|
|
||||||
|
|
||||||
|
def find_transfer(transfers: list, username: str, filename: str) -> dict:
|
||||||
|
"""The transfer matching a started download (by username + filename). Pure."""
|
||||||
|
for t in (transfers or []):
|
||||||
|
if t.get("username") == username and t.get("filename") == filename:
|
||||||
|
return t
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["start_download", "list_downloads", "flatten_downloads", "classify_state",
|
||||||
|
"progress_pct", "find_transfer", "cancel_download"]
|
||||||
336
core/video/slskd_search.py
Normal file
336
core/video/slskd_search.py
Normal file
|
|
@ -0,0 +1,336 @@
|
||||||
|
"""Real Soulseek (slskd) search for the video Download view.
|
||||||
|
|
||||||
|
Replaces the mock for the Soulseek source: it POSTs a search to the slskd instance
|
||||||
|
(the shared ``soulseek.*`` config used by the music side too), polls for responses,
|
||||||
|
keeps the video files, and GROUPS them by release folder so each card is one release
|
||||||
|
(with a peer count) rather than one row per user. The parse → evaluate → rank pipeline
|
||||||
|
downstream is unchanged — this just returns the same ``{title, size_bytes, …}`` shape.
|
||||||
|
|
||||||
|
Isolated: imports only stdlib + requests + the shared ``config_manager`` (app config,
|
||||||
|
not music code). The pure helpers (build_query / group_video_files) are unit-tested;
|
||||||
|
the HTTP poll is thin I/O glue.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
# slskd rate-limits search CREATION and returns 429 when exceeded. The music side caps at
|
||||||
|
# ~35 searches / 220s; we mirror that, plus a small min-gap between creations (the auto-grab
|
||||||
|
# fires from a thread pool, so without it a burst — or a 429 that returns instantly — storms
|
||||||
|
# slskd). A 429 sets a cooldown so we back off instead of hammering.
|
||||||
|
_SEARCH_LOCK = threading.Lock()
|
||||||
|
_SEARCH_TIMES: list = [] # reserved creation times (monotonic), pruned to the window
|
||||||
|
_COOLDOWN_UNTIL = [0.0]
|
||||||
|
_MAX_PER_WINDOW = 35
|
||||||
|
_WINDOW_SECONDS = 220.0
|
||||||
|
_MIN_GAP_SECONDS = 2.0
|
||||||
|
|
||||||
|
|
||||||
|
def _reserve_search_slot() -> float:
|
||||||
|
"""Reserve the next allowed search-creation time under the rate limit; returns it."""
|
||||||
|
with _SEARCH_LOCK:
|
||||||
|
now = time.monotonic()
|
||||||
|
while _SEARCH_TIMES and _SEARCH_TIMES[0] <= now - _WINDOW_SECONDS:
|
||||||
|
_SEARCH_TIMES.pop(0)
|
||||||
|
at = now
|
||||||
|
if _SEARCH_TIMES:
|
||||||
|
at = max(at, _SEARCH_TIMES[-1] + _MIN_GAP_SECONDS) # space from last
|
||||||
|
if len(_SEARCH_TIMES) >= _MAX_PER_WINDOW:
|
||||||
|
at = max(at, _SEARCH_TIMES[0] + _WINDOW_SECONDS) # window full → wait
|
||||||
|
at = max(at, _COOLDOWN_UNTIL[0]) # honor a 429 cooldown
|
||||||
|
_SEARCH_TIMES.append(at)
|
||||||
|
return at
|
||||||
|
|
||||||
|
|
||||||
|
def _throttle_search() -> None:
|
||||||
|
"""Block until we're allowed to create the next slskd search."""
|
||||||
|
wait = _reserve_search_slot() - time.monotonic()
|
||||||
|
if wait > 0:
|
||||||
|
time.sleep(wait)
|
||||||
|
|
||||||
|
|
||||||
|
def _note_rate_limited(retry_after: Any = None) -> None:
|
||||||
|
"""slskd returned 429 — back off before the next search."""
|
||||||
|
try:
|
||||||
|
secs = float(retry_after) if retry_after else 30.0
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
secs = 30.0
|
||||||
|
with _SEARCH_LOCK:
|
||||||
|
_COOLDOWN_UNTIL[0] = time.monotonic() + max(5.0, min(secs, 120.0))
|
||||||
|
|
||||||
|
# Container extensions we treat as the actual video (everything else — subs, nfo,
|
||||||
|
# art, samples — is ignored for quality purposes).
|
||||||
|
VIDEO_EXTS = frozenset((
|
||||||
|
"mkv", "mp4", "avi", "m4v", "mov", "wmv", "ts", "m2ts", "mpg", "mpeg",
|
||||||
|
"webm", "flv", "vob", "divx", "mk3d",
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
def _conn():
|
||||||
|
from config.settings import config_manager
|
||||||
|
base = str(config_manager.get("soulseek.slskd_url", "") or "").rstrip("/")
|
||||||
|
key = config_manager.get("soulseek.api_key", "") or ""
|
||||||
|
headers = {"Accept": "application/json"} # make slskd answer in JSON, not whatever default
|
||||||
|
if key:
|
||||||
|
headers["X-API-Key"] = key
|
||||||
|
return base, headers
|
||||||
|
|
||||||
|
|
||||||
|
def build_query(scope: str, title: Any, *, year: Any = None, season: Any = None,
|
||||||
|
episode: Any = None) -> str:
|
||||||
|
"""The text we hand slskd for a given scope (movie / episode / season / series)."""
|
||||||
|
t = str(title or "").strip()
|
||||||
|
scope = (scope or "movie").lower()
|
||||||
|
try:
|
||||||
|
s_i = int(season) if season is not None else None
|
||||||
|
e_i = int(episode) if episode is not None else None
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
s_i = e_i = None
|
||||||
|
if scope == "episode" and s_i is not None and e_i is not None:
|
||||||
|
return "%s S%02dE%02d" % (t, s_i, e_i)
|
||||||
|
if scope == "season" and s_i is not None:
|
||||||
|
return "%s S%02d" % (t, s_i)
|
||||||
|
if scope == "movie" and year:
|
||||||
|
return ("%s %s" % (t, year)).strip()
|
||||||
|
return t # series / fallback
|
||||||
|
|
||||||
|
|
||||||
|
def _is_video(filename: str) -> bool:
|
||||||
|
fn = str(filename or "").replace("\\", "/")
|
||||||
|
base = fn.rsplit("/", 1)[-1]
|
||||||
|
if "sample" in base.lower():
|
||||||
|
return False
|
||||||
|
ext = base.rsplit(".", 1)[-1].lower() if "." in base else ""
|
||||||
|
return ext in VIDEO_EXTS
|
||||||
|
|
||||||
|
|
||||||
|
def _release_name(filename: str) -> str:
|
||||||
|
"""The release a file belongs to — its parent folder (where scene/p2p put the
|
||||||
|
quality), falling back to the bare filename (sans extension)."""
|
||||||
|
fn = str(filename or "").replace("\\", "/").strip("/")
|
||||||
|
parts = [p for p in fn.split("/") if p]
|
||||||
|
if len(parts) >= 2:
|
||||||
|
return parts[-2]
|
||||||
|
base = parts[-1] if parts else ""
|
||||||
|
return base.rsplit(".", 1)[0] if "." in base else base
|
||||||
|
|
||||||
|
|
||||||
|
def peer_availability(free_slots: Any, upload_speed: Any, queue_length: Any) -> float:
|
||||||
|
"""How DOWNLOADABLE a peer is right now (higher = grabs sooner). Mirrors the music
|
||||||
|
side's availability scoring: a free upload slot, the upload speed (tiered), and the
|
||||||
|
queue length (graduated penalty). A fast peer with a free slot and an empty queue beats
|
||||||
|
a faster one stuck behind a 1500-deep queue. Pure."""
|
||||||
|
try:
|
||||||
|
slots, speed, queue = int(free_slots or 0), int(upload_speed or 0), int(queue_length or 0)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
slots, speed, queue = 0, 0, 0
|
||||||
|
s = 0.05 if slots > 0 else -0.15
|
||||||
|
if speed >= 5_000_000:
|
||||||
|
s += 0.15
|
||||||
|
elif speed >= 1_000_000:
|
||||||
|
s += 0.10
|
||||||
|
elif speed >= 500_000:
|
||||||
|
s += 0.05
|
||||||
|
elif speed < 100_000:
|
||||||
|
s -= 0.05
|
||||||
|
if queue > 50:
|
||||||
|
s -= 0.25
|
||||||
|
elif queue > 20:
|
||||||
|
s -= 0.15
|
||||||
|
elif queue > 10:
|
||||||
|
s -= 0.10
|
||||||
|
return round(s, 4)
|
||||||
|
|
||||||
|
|
||||||
|
def group_video_files(responses: Any) -> list:
|
||||||
|
"""Flatten slskd responses → one hit per release folder. The chosen source per release
|
||||||
|
is the most DOWNLOADABLE peer (free slot → low queue → speed), not just the fastest, and
|
||||||
|
hits are ranked by that availability — so we grab a free-slot/empty-queue release over a
|
||||||
|
high-spec one stuck behind a huge queue. Pure — drives the unit tests."""
|
||||||
|
groups: dict = {}
|
||||||
|
for resp in (responses if isinstance(responses, list) else []):
|
||||||
|
if not isinstance(resp, dict):
|
||||||
|
continue
|
||||||
|
user = resp.get("username")
|
||||||
|
speed = resp.get("uploadSpeed", 0) or 0
|
||||||
|
slots = resp.get("freeUploadSlots", 0) or 0
|
||||||
|
queue = resp.get("queueLength", 0) or 0
|
||||||
|
avail = peer_availability(slots, speed, queue)
|
||||||
|
for f in (resp.get("files") or []):
|
||||||
|
fn = f.get("filename", "")
|
||||||
|
if not _is_video(fn):
|
||||||
|
continue
|
||||||
|
rel = _release_name(fn)
|
||||||
|
g = groups.get(rel)
|
||||||
|
if g is None:
|
||||||
|
g = groups[rel] = {"title": rel, "size_bytes": 0, "users": set(),
|
||||||
|
"best": (-99.0, -1), "username": None, "slots": 0,
|
||||||
|
"queue": 0, "speed": 0, "availability": -99.0, "filename": fn}
|
||||||
|
g["size_bytes"] = max(g["size_bytes"], f.get("size", 0) or 0)
|
||||||
|
if user:
|
||||||
|
g["users"].add(user)
|
||||||
|
if (avail, speed) > g["best"]: # most available, then fastest, peer wins
|
||||||
|
g["best"] = (avail, speed)
|
||||||
|
g["username"], g["slots"], g["queue"] = user, slots, queue
|
||||||
|
g["speed"], g["availability"], g["filename"] = speed, avail, fn
|
||||||
|
out = []
|
||||||
|
for g in groups.values():
|
||||||
|
out.append({"title": g["title"], "size_bytes": g["size_bytes"], "peers": len(g["users"]),
|
||||||
|
"username": g["username"], "slots": g["slots"], "queue": g["queue"],
|
||||||
|
"speed": g["speed"], "availability": g["availability"], "filename": g["filename"]})
|
||||||
|
# availability first, then more peers, then bigger; the quality profile still gates the
|
||||||
|
# final pick downstream (_evaluate_hits), this just orders within a quality tier.
|
||||||
|
out.sort(key=lambda h: (h["availability"], h["peers"], h["size_bytes"]), reverse=True)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _min_speed_bytes() -> int:
|
||||||
|
from config.settings import config_manager
|
||||||
|
try:
|
||||||
|
return int(config_manager.get("soulseek.min_peer_upload_speed", 0) or 0) * 125000
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def search_timeout_ms() -> int:
|
||||||
|
"""How long to ask slskd to keep searching — the SAME ``soulseek.search_timeout``
|
||||||
|
the music side uses (default 60s), so results have time to arrive."""
|
||||||
|
from config.settings import config_manager
|
||||||
|
try:
|
||||||
|
secs = int(config_manager.get("soulseek.search_timeout", 60) or 60)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
secs = 60
|
||||||
|
return max(10, min(120, secs)) * 1000
|
||||||
|
|
||||||
|
|
||||||
|
def start_search(query: str) -> dict:
|
||||||
|
"""Kick off a slskd search (don't wait). Returns {configured[, id][, error]}.
|
||||||
|
The caller polls ``poll_responses(id)`` until satisfied (like the music side).
|
||||||
|
|
||||||
|
We generate the search id OURSELVES and pass it to slskd — it honors a client-supplied
|
||||||
|
``id`` — so we never depend on parsing it back out of the POST response. Some slskd builds
|
||||||
|
return the id in a Location header / a non-dict body, which made us think the search
|
||||||
|
'didn't run' even though slskd created it (the bug behind the fast 'no results')."""
|
||||||
|
import uuid
|
||||||
|
base, headers = _conn()
|
||||||
|
if not base:
|
||||||
|
return {"configured": False}
|
||||||
|
_throttle_search() # stay under slskd's search-creation rate limit
|
||||||
|
search_id = str(uuid.uuid4())
|
||||||
|
payload = {"id": search_id, "searchText": query, "timeout": search_timeout_ms(),
|
||||||
|
"filterResponses": True, "minimumResponseFileCount": 1,
|
||||||
|
"minimumPeerUploadSpeed": _min_speed_bytes()}
|
||||||
|
try:
|
||||||
|
r = requests.post(base + "/api/v0/searches", json=payload, headers=headers, timeout=15)
|
||||||
|
if r.status_code == 429: # rate limited — back off, don't cascade
|
||||||
|
_note_rate_limited(r.headers.get("Retry-After"))
|
||||||
|
return {"configured": True, "error": "429 rate limited (backing off)"}
|
||||||
|
r.raise_for_status()
|
||||||
|
except Exception as e: # noqa: BLE001 - surface any slskd/network failure to the UI
|
||||||
|
return {"configured": True, "error": str(e)}
|
||||||
|
# Prefer the id slskd echoes back if present; otherwise the one we supplied (it's honored).
|
||||||
|
sid = None
|
||||||
|
try:
|
||||||
|
data = r.json()
|
||||||
|
if isinstance(data, dict):
|
||||||
|
sid = data.get("id")
|
||||||
|
elif isinstance(data, str) and data.strip():
|
||||||
|
sid = data.strip().strip('"')
|
||||||
|
elif isinstance(data, list) and data and isinstance(data[0], dict):
|
||||||
|
sid = data[0].get("id")
|
||||||
|
except Exception: # noqa: BLE001, S110 - non-JSON / empty body → fall back to our id
|
||||||
|
pass
|
||||||
|
return {"configured": True, "id": sid or search_id}
|
||||||
|
|
||||||
|
|
||||||
|
def stop_search(search_id) -> None:
|
||||||
|
"""Stop + forget a search on slskd once we're done polling it. slskd otherwise keeps
|
||||||
|
every search running its full ``search_timeout`` (default 60s), so the bounded automation
|
||||||
|
searches — which finish fast when results come in — pile up dozens-deep and swamp slskd.
|
||||||
|
Best-effort; a failed cleanup never matters."""
|
||||||
|
base, headers = _conn()
|
||||||
|
if not base or not search_id:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
requests.delete(base + "/api/v0/searches/%s" % search_id, headers=headers, timeout=10)
|
||||||
|
except Exception: # noqa: BLE001, S110 - cleanup is best-effort
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def poll_responses(search_id: str) -> list:
|
||||||
|
"""Current grouped video hits for an in-flight search (cheap; call repeatedly)."""
|
||||||
|
return poll_search(search_id)["hits"]
|
||||||
|
|
||||||
|
|
||||||
|
def poll_search(search_id: str) -> dict:
|
||||||
|
"""Poll an in-flight search → {hits (grouped video releases), total_files (every
|
||||||
|
file slskd returned, incl. non-video)}. total_files lets the UI distinguish
|
||||||
|
'nothing back yet' from 'plenty back but it's all audio/junk, no video'."""
|
||||||
|
base, headers = _conn()
|
||||||
|
if not base or not search_id:
|
||||||
|
return {"hits": [], "total_files": 0}
|
||||||
|
try:
|
||||||
|
r = requests.get(base + "/api/v0/searches/%s/responses" % search_id, headers=headers, timeout=15)
|
||||||
|
if not r.ok:
|
||||||
|
return {"hits": [], "total_files": 0}
|
||||||
|
data = r.json()
|
||||||
|
except Exception: # noqa: BLE001, S110 - transient error → no new hits this poll
|
||||||
|
return {"hits": [], "total_files": 0}
|
||||||
|
total = 0
|
||||||
|
for u in (data if isinstance(data, list) else []):
|
||||||
|
if isinstance(u, dict):
|
||||||
|
for d in (u.get("directories") or []):
|
||||||
|
total += len(d.get("files") or [])
|
||||||
|
return {"hits": group_video_files(data), "total_files": total}
|
||||||
|
|
||||||
|
|
||||||
|
def slskd_search(query: str, *, max_seconds: int = 8, slskd_timeout_ms: int = 4500) -> dict:
|
||||||
|
"""POST a search to slskd, poll responses, return {configured, hits[, error]}.
|
||||||
|
Thin I/O glue around ``group_video_files``."""
|
||||||
|
from config.settings import config_manager
|
||||||
|
base = str(config_manager.get("soulseek.slskd_url", "") or "").rstrip("/")
|
||||||
|
if not base:
|
||||||
|
return {"configured": False, "hits": []}
|
||||||
|
key = config_manager.get("soulseek.api_key", "") or ""
|
||||||
|
headers = {"X-API-Key": key} if key else {}
|
||||||
|
try:
|
||||||
|
min_mbps = int(config_manager.get("soulseek.min_peer_upload_speed", 0) or 0)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
min_mbps = 0
|
||||||
|
payload = {"searchText": query, "timeout": slskd_timeout_ms, "filterResponses": True,
|
||||||
|
"minimumResponseFileCount": 1, "minimumPeerUploadSpeed": min_mbps * 125000}
|
||||||
|
try:
|
||||||
|
r = requests.post(base + "/api/v0/searches", json=payload, headers=headers, timeout=10)
|
||||||
|
r.raise_for_status()
|
||||||
|
data = r.json()
|
||||||
|
except Exception as e: # noqa: BLE001 - surface any slskd/network failure to the UI
|
||||||
|
return {"configured": True, "hits": [], "error": str(e)}
|
||||||
|
sid = data.get("id") if isinstance(data, dict) else (
|
||||||
|
data[0].get("id") if isinstance(data, list) and data and isinstance(data[0], dict) else None)
|
||||||
|
if not sid:
|
||||||
|
return {"configured": True, "hits": [], "error": "slskd returned no search id"}
|
||||||
|
|
||||||
|
responses: list = []
|
||||||
|
deadline = time.monotonic() + max_seconds
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
rr = requests.get(base + "/api/v0/searches/%s/responses" % sid, headers=headers, timeout=10)
|
||||||
|
if rr.ok:
|
||||||
|
body = rr.json()
|
||||||
|
if isinstance(body, list):
|
||||||
|
responses = body
|
||||||
|
except Exception: # noqa: BLE001, S110 - keep polling through transient errors
|
||||||
|
pass
|
||||||
|
if len(responses) >= 25:
|
||||||
|
break
|
||||||
|
time.sleep(1)
|
||||||
|
return {"configured": True, "hits": group_video_files(responses)}
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["VIDEO_EXTS", "build_query", "group_video_files", "slskd_search",
|
||||||
|
"start_search", "poll_responses", "poll_search", "search_timeout_ms"]
|
||||||
908
core/video/sources.py
Normal file
908
core/video/sources.py
Normal file
|
|
@ -0,0 +1,908 @@
|
||||||
|
"""SoulSync — video media-server adapters (Plex / Jellyfin).
|
||||||
|
|
||||||
|
Turn the live, already-connected media clients (owned by the shared
|
||||||
|
MediaServerEngine) into normalized dicts the scanner understands. We REUSE the
|
||||||
|
shared connection/auth (don't reinvent it) but keep all video-section logic here
|
||||||
|
so music code is untouched.
|
||||||
|
|
||||||
|
NOTE: these talk to real Plex/Jellyfin servers and can only be fully validated
|
||||||
|
against a live server. The scanner itself is server-agnostic and unit-tested
|
||||||
|
with a fake source; bugs found here against a real library are localized to
|
||||||
|
these adapters.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_sources")
|
||||||
|
|
||||||
|
# Library scans are bulk operations — a far longer per-request timeout than the
|
||||||
|
# shared client's interactive one, so big libraries don't read-timeout mid-scan.
|
||||||
|
PLEX_SCAN_TIMEOUT = 120
|
||||||
|
|
||||||
|
|
||||||
|
def _to_int(val):
|
||||||
|
if val is None:
|
||||||
|
return None
|
||||||
|
m = re.match(r"\d+", str(val))
|
||||||
|
return int(m.group()) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_plex_guids(obj) -> dict:
|
||||||
|
"""tmdb/imdb/tvdb ids from a Plex item's guids — Plex already matched them."""
|
||||||
|
out = {"tmdb_id": None, "imdb_id": None, "tvdb_id": None}
|
||||||
|
try:
|
||||||
|
for g in (getattr(obj, "guids", None) or []):
|
||||||
|
gid = getattr(g, "id", "") or ""
|
||||||
|
if "://" not in gid:
|
||||||
|
continue
|
||||||
|
scheme, value = gid.split("://", 1)
|
||||||
|
scheme = scheme.lower()
|
||||||
|
if scheme == "imdb":
|
||||||
|
out["imdb_id"] = (value.split("?")[0] or None)
|
||||||
|
elif scheme == "tmdb":
|
||||||
|
out["tmdb_id"] = _to_int(value)
|
||||||
|
elif scheme == "tvdb":
|
||||||
|
out["tvdb_id"] = _to_int(value)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_jf_providers(item) -> dict:
|
||||||
|
"""tmdb/imdb/tvdb ids from a Jellyfin item's ProviderIds."""
|
||||||
|
providers = item.get("ProviderIds") or {}
|
||||||
|
low = {(k or "").lower(): v for k, v in providers.items()}
|
||||||
|
return {
|
||||||
|
"imdb_id": low.get("imdb") or None,
|
||||||
|
"tmdb_id": _to_int(low.get("tmdb")),
|
||||||
|
"tvdb_id": _to_int(low.get("tvdb")),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _vdb(db=None):
|
||||||
|
"""The video DB (the given one, or a fresh handle). None if unavailable."""
|
||||||
|
if db is not None:
|
||||||
|
return db
|
||||||
|
try:
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
return VideoDatabase()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def video_plex_config(db=None):
|
||||||
|
"""VIDEO's effective Plex connection: the video side's OWN stored creds
|
||||||
|
(video.db) when set, otherwise INHERITED read-only from the music config.
|
||||||
|
The video side never writes the music config, so this is one-way."""
|
||||||
|
db = _vdb(db)
|
||||||
|
try:
|
||||||
|
url = (db.get_setting("video_plex_url") or "").strip() if db else ""
|
||||||
|
token = (db.get_setting("video_plex_token") or "").strip() if db else ""
|
||||||
|
except Exception:
|
||||||
|
url, token = "", ""
|
||||||
|
if url and token:
|
||||||
|
return {"base_url": url, "token": token, "source": "video"}
|
||||||
|
try:
|
||||||
|
from config.settings import config_manager
|
||||||
|
cfg = config_manager.get_plex_config() or {}
|
||||||
|
return {"base_url": cfg.get("base_url") or "", "token": cfg.get("token") or "",
|
||||||
|
"source": "music"}
|
||||||
|
except Exception:
|
||||||
|
return {"base_url": "", "token": "", "source": "music"}
|
||||||
|
|
||||||
|
|
||||||
|
def video_jellyfin_config(db=None):
|
||||||
|
"""VIDEO's effective Jellyfin connection: the video side's OWN stored creds
|
||||||
|
when set, otherwise INHERITED read-only from the music config."""
|
||||||
|
db = _vdb(db)
|
||||||
|
try:
|
||||||
|
url = (db.get_setting("video_jellyfin_url") or "").strip() if db else ""
|
||||||
|
key = (db.get_setting("video_jellyfin_key") or "").strip() if db else ""
|
||||||
|
except Exception:
|
||||||
|
url, key = "", ""
|
||||||
|
if url and key:
|
||||||
|
return {"base_url": url, "api_key": key, "source": "video"}
|
||||||
|
try:
|
||||||
|
from config.settings import config_manager
|
||||||
|
cfg = config_manager.get_jellyfin_config() or {}
|
||||||
|
return {"base_url": cfg.get("base_url") or "", "api_key": cfg.get("api_key") or "",
|
||||||
|
"source": "music"}
|
||||||
|
except Exception:
|
||||||
|
return {"base_url": "", "api_key": "", "source": "music"}
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_video_server(db=None):
|
||||||
|
"""The server the VIDEO side uses — a configured Plex/Jellyfin, resolved
|
||||||
|
INDEPENDENTLY of the music 'active server' pointer (so e.g. Navidrome-for-music
|
||||||
|
+ Plex-for-video works, and music-only servers never apply here). Returns
|
||||||
|
'plex' | 'jellyfin' | None. Order: explicit video pick → the single configured
|
||||||
|
one → Plex when both → None. 'Configured' means video's EFFECTIVE config
|
||||||
|
(its own creds, or inherited from music) has a base_url."""
|
||||||
|
db = _vdb(db)
|
||||||
|
plex_ok = bool(video_plex_config(db).get("base_url"))
|
||||||
|
jelly_ok = bool(video_jellyfin_config(db).get("base_url"))
|
||||||
|
|
||||||
|
pref = None
|
||||||
|
if db is not None:
|
||||||
|
try:
|
||||||
|
pref = db.get_setting("video_server")
|
||||||
|
except Exception:
|
||||||
|
pref = None
|
||||||
|
if pref == "plex" and plex_ok:
|
||||||
|
return "plex"
|
||||||
|
if pref == "jellyfin" and jelly_ok:
|
||||||
|
return "jellyfin"
|
||||||
|
# Fully INDEPENDENT of the music 'active server' — only an explicit video pick
|
||||||
|
# or the configured server(s) decide, so changing the music server NEVER
|
||||||
|
# changes video (and vice-versa). Auto-pick the single configured one; default
|
||||||
|
# to Plex when both are set (the user picks Jellyfin via the Video Source panel).
|
||||||
|
if plex_ok and not jelly_ok:
|
||||||
|
return "plex"
|
||||||
|
if jelly_ok and not plex_ok:
|
||||||
|
return "jellyfin"
|
||||||
|
if plex_ok and jelly_ok:
|
||||||
|
return "plex"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _video_jellyfin_source(cfg, movies_lib=None, tv_lib=None):
|
||||||
|
"""A JellyfinVideoSource connected with VIDEO's OWN config — independent of
|
||||||
|
music's shared singleton client. The video source only needs base_url/api_key
|
||||||
|
(for _make_request) and any valid user_id (for /Users/{id}/Items browsing)."""
|
||||||
|
base = (cfg.get("base_url") or "").rstrip("/")
|
||||||
|
key = cfg.get("api_key") or ""
|
||||||
|
if not base or not key:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
from core.jellyfin_client import JellyfinClient
|
||||||
|
client = JellyfinClient()
|
||||||
|
client.base_url = base
|
||||||
|
client.api_key = key
|
||||||
|
users = client._make_request("/Users") or []
|
||||||
|
if not users:
|
||||||
|
return None
|
||||||
|
# Jellyfin scopes /Users/{id}/Views to that user's library access, so honor
|
||||||
|
# the user the operator explicitly picked (stored video_jellyfin_user). Until
|
||||||
|
# they pick, default to an ADMIN (full visibility) so nothing's hidden.
|
||||||
|
pref = ""
|
||||||
|
try:
|
||||||
|
_db = _vdb()
|
||||||
|
pref = (_db.get_setting("video_jellyfin_user") or "") if _db else ""
|
||||||
|
except Exception:
|
||||||
|
pref = ""
|
||||||
|
chosen = next((u for u in users if u.get("Id") == pref), None)
|
||||||
|
if chosen is None:
|
||||||
|
admins = [u for u in users if (u.get("Policy") or {}).get("IsAdministrator")]
|
||||||
|
chosen = (admins or users)[0]
|
||||||
|
uid = chosen.get("Id")
|
||||||
|
if not uid:
|
||||||
|
return None
|
||||||
|
client.user_id = uid
|
||||||
|
return JellyfinVideoSource(client, movies_lib=movies_lib, tv_lib=tv_lib)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video sources: Jellyfin connect failed")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def video_jellyfin_test(cfg):
|
||||||
|
"""Diagnose the video Jellyfin connection precisely (for the Test button).
|
||||||
|
Returns (ok: bool, message: str). Distinguishes 'can't reach the server',
|
||||||
|
'API key rejected', and 'no users' instead of one vague failure — reuses the
|
||||||
|
same X-Emby-Token header the music client uses (_make_request)."""
|
||||||
|
base = (cfg.get("base_url") or "").rstrip("/")
|
||||||
|
key = cfg.get("api_key") or ""
|
||||||
|
if not base or not key:
|
||||||
|
return False, "Jellyfin URL/API key not set"
|
||||||
|
import requests
|
||||||
|
headers = {"X-Emby-Token": key}
|
||||||
|
try:
|
||||||
|
info = requests.get(base + "/System/Info", headers=headers, timeout=8)
|
||||||
|
except requests.exceptions.ConnectionError:
|
||||||
|
return False, "Can't reach Jellyfin at %s — is it running and reachable on that host/port?" % base
|
||||||
|
except requests.exceptions.RequestException as e:
|
||||||
|
return False, "Couldn't connect to Jellyfin: %s" % (e,)
|
||||||
|
if info.status_code in (401, 403):
|
||||||
|
return False, "Jellyfin rejected the API key (HTTP %d). Check the key." % info.status_code
|
||||||
|
if info.status_code != 200:
|
||||||
|
return False, "Jellyfin returned HTTP %d for /System/Info." % info.status_code
|
||||||
|
try:
|
||||||
|
users = requests.get(base + "/Users", headers=headers, timeout=8).json()
|
||||||
|
except Exception:
|
||||||
|
users = None
|
||||||
|
if not users:
|
||||||
|
return False, "Connected, but Jellyfin returned no users for this API key."
|
||||||
|
name = (info.json() or {}).get("ServerName") or "Jellyfin"
|
||||||
|
return True, "Connected to %s" % name
|
||||||
|
|
||||||
|
|
||||||
|
def _build_source(movies_lib=None, tv_lib=None):
|
||||||
|
"""Build a media source for the VIDEO server (see resolve_video_server),
|
||||||
|
restricted to the named Movies/TV libraries when given. Uses VIDEO's OWN
|
||||||
|
effective connection config (its creds, or inherited from music) — never the
|
||||||
|
music side's live connection, so the two stay independent."""
|
||||||
|
db = _vdb()
|
||||||
|
server = resolve_video_server(db)
|
||||||
|
|
||||||
|
if server == "plex":
|
||||||
|
cfg = video_plex_config(db)
|
||||||
|
base_url, token = cfg.get("base_url"), cfg.get("token")
|
||||||
|
if not base_url or not token:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
from plexapi.server import PlexServer
|
||||||
|
srv = PlexServer(base_url, token, timeout=PLEX_SCAN_TIMEOUT)
|
||||||
|
return PlexVideoSource(srv, movies_lib=movies_lib, tv_lib=tv_lib)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video sources: Plex connect failed")
|
||||||
|
return None
|
||||||
|
|
||||||
|
if server == "jellyfin":
|
||||||
|
return _video_jellyfin_source(video_jellyfin_config(db), movies_lib, tv_lib)
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _load_selection():
|
||||||
|
"""The user's Movies/TV library choice for the VIDEO server (or {})."""
|
||||||
|
try:
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
server = resolve_video_server()
|
||||||
|
if not server:
|
||||||
|
return {}
|
||||||
|
return VideoDatabase().get_library_selection(server)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video sources: could not load library selection")
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def get_active_video_source():
|
||||||
|
"""Source for SCANNING — restricted to the user-mapped Movies/TV libraries.
|
||||||
|
Falls back to all libraries when nothing is mapped yet."""
|
||||||
|
sel = _load_selection() or {}
|
||||||
|
return _build_source(sel.get("movies") or None, sel.get("tv") or None)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_media_type(media_type) -> str:
|
||||||
|
"""'movie'|'show'|'all' — accepts the friendly aliases (movies/tv/series/…) the
|
||||||
|
UI and automation configs use. Movies and TV are independent libraries, so the
|
||||||
|
scan family is scoped by this everywhere."""
|
||||||
|
m = str(media_type or "all").lower()
|
||||||
|
if m in ("movie", "movies", "film", "films"):
|
||||||
|
return "movie"
|
||||||
|
if m in ("show", "shows", "tv", "series", "episode", "episodes"):
|
||||||
|
return "show"
|
||||||
|
return "all"
|
||||||
|
|
||||||
|
|
||||||
|
def refresh_video_server_sections(media_type="all"):
|
||||||
|
"""Tell the active media server to rescan its selected VIDEO sections (so newly
|
||||||
|
downloaded files get indexed) — the video twin of music's 'Scan Library'.
|
||||||
|
``media_type`` scopes it to one library ('movie' / 'show'); 'all' (default) nudges
|
||||||
|
both. Returns {ok, sections} or {ok: False, error}."""
|
||||||
|
media_type = normalize_media_type(media_type)
|
||||||
|
src = get_active_video_source()
|
||||||
|
if src is None:
|
||||||
|
return {"ok": False, "error": "No video server configured"}
|
||||||
|
if not hasattr(src, "refresh_sections"):
|
||||||
|
return {"ok": False, "error": "This server doesn't support a scan trigger"}
|
||||||
|
try:
|
||||||
|
return src.refresh_sections(media_type)
|
||||||
|
except Exception as e: # noqa: BLE001 - surface any server error to the automation
|
||||||
|
logger.exception("video sources: refresh failed")
|
||||||
|
return {"ok": False, "error": str(e)}
|
||||||
|
|
||||||
|
|
||||||
|
def video_server_scan_in_progress(media_type="all"):
|
||||||
|
"""True if the active video server is mid-scan for the given library (or either,
|
||||||
|
for 'all'); False if idle; None if it can't be determined — no server, or an
|
||||||
|
adapter that can't report scan state. Callers fall back to a fixed wait on None."""
|
||||||
|
media_type = normalize_media_type(media_type)
|
||||||
|
src = get_active_video_source()
|
||||||
|
if src is None or not hasattr(src, "is_scanning"):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return bool(src.is_scanning(media_type))
|
||||||
|
except Exception:
|
||||||
|
logger.debug("video sources: scan-status check failed", exc_info=True)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def video_server_has_item(media_type, item) -> bool:
|
||||||
|
"""True if the active server already has this specific grab indexed — the signal
|
||||||
|
for the post-download scan to skip a library's expensive crawl. Conservative: any
|
||||||
|
uncertainty (no server, unsupported, error, no match) → False, so we scan."""
|
||||||
|
media_type = normalize_media_type(media_type)
|
||||||
|
if media_type not in ("movie", "show") or not item:
|
||||||
|
return False
|
||||||
|
src = get_active_video_source()
|
||||||
|
if src is None or not hasattr(src, "has_item"):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
return bool(src.has_item(media_type, item))
|
||||||
|
except Exception:
|
||||||
|
logger.debug("video sources: has_item probe failed", exc_info=True)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def list_video_libraries():
|
||||||
|
"""Discover the active server's video libraries for the mapping UI:
|
||||||
|
{'server', 'movies': [{'title'}], 'tv': [{'title'}]} or None."""
|
||||||
|
src = _build_source()
|
||||||
|
if src is None:
|
||||||
|
return None
|
||||||
|
out = src.available_libraries()
|
||||||
|
out["server"] = src.server_name
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# ── Plex ──────────────────────────────────────────────────────────────────────
|
||||||
|
class PlexVideoSource:
|
||||||
|
server_name = "plex"
|
||||||
|
|
||||||
|
def __init__(self, server, movies_lib=None, tv_lib=None):
|
||||||
|
self._server = server
|
||||||
|
self._movies_lib = movies_lib
|
||||||
|
self._tv_lib = tv_lib
|
||||||
|
|
||||||
|
def _sections(self, kind: str, name=None):
|
||||||
|
secs = [s for s in self._server.library.sections() if s.type == kind]
|
||||||
|
if name:
|
||||||
|
secs = [s for s in secs if s.title == name]
|
||||||
|
return secs
|
||||||
|
|
||||||
|
def _scan_sections(self, kind: str, name):
|
||||||
|
"""Sections to SCAN for a kind. UNLIKE _sections, an empty name means
|
||||||
|
'this kind isn't mapped' → scan NOTHING (never fall back to all sections).
|
||||||
|
Prevents a missing selection from silently pulling every library."""
|
||||||
|
return self._sections(kind, name) if name else []
|
||||||
|
|
||||||
|
def available_libraries(self) -> dict:
|
||||||
|
return {
|
||||||
|
"movies": [{"title": s.title} for s in self._sections("movie")],
|
||||||
|
"tv": [{"title": s.title} for s in self._sections("show")],
|
||||||
|
}
|
||||||
|
|
||||||
|
def counts(self, incremental=False) -> dict:
|
||||||
|
"""Cheap item totals (no full fetch) for the progress bar."""
|
||||||
|
m = sum(int(getattr(s, "totalSize", 0) or 0) for s in self._scan_sections("movie", self._movies_lib))
|
||||||
|
sh = sum(int(getattr(s, "totalSize", 0) or 0) for s in self._scan_sections("show", self._tv_lib))
|
||||||
|
if incremental:
|
||||||
|
m, sh = min(m, 100), min(sh, 50)
|
||||||
|
return {"movies": m, "shows": sh}
|
||||||
|
|
||||||
|
def iter_movies(self, incremental=False):
|
||||||
|
for section in self._scan_sections("movie", self._movies_lib):
|
||||||
|
items = section.search(sort="addedAt:desc", maxresults=100) if incremental else section.all()
|
||||||
|
for m in items:
|
||||||
|
try:
|
||||||
|
yield self._movie(m)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Plex: skipping movie %s", getattr(m, "title", "?"))
|
||||||
|
|
||||||
|
def iter_shows(self, incremental=False):
|
||||||
|
for section in self._scan_sections("show", self._tv_lib):
|
||||||
|
items = section.search(sort="addedAt:desc", maxresults=50) if incremental else section.all()
|
||||||
|
for sh in items:
|
||||||
|
try:
|
||||||
|
yield self._show(sh)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Plex: skipping show %s", getattr(sh, "title", "?"))
|
||||||
|
|
||||||
|
def refresh_sections(self, media_type="all") -> dict:
|
||||||
|
"""Tell Plex to rescan the selected video sections so freshly-downloaded files
|
||||||
|
get indexed. (plexapi ``section.update()`` triggers the library scan.)
|
||||||
|
``media_type`` scopes it to the Movie or TV section; 'all' does both."""
|
||||||
|
n = 0
|
||||||
|
for kind, name in (("movie", self._movies_lib), ("show", self._tv_lib)):
|
||||||
|
if media_type != "all" and media_type != kind:
|
||||||
|
continue
|
||||||
|
for s in self._scan_sections(kind, name):
|
||||||
|
try:
|
||||||
|
s.update()
|
||||||
|
n += 1
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Plex: refresh failed for section %s", getattr(s, "title", "?"))
|
||||||
|
return {"ok": n > 0, "sections": n}
|
||||||
|
|
||||||
|
def is_scanning(self, media_type="all") -> bool:
|
||||||
|
"""True if any SELECTED video section (scoped by media_type) is currently
|
||||||
|
being scanned by Plex. Checks the per-section refreshing flag, then the
|
||||||
|
server activity feed (real-time) — mirrors the music PlexClient check."""
|
||||||
|
sections = []
|
||||||
|
for kind, name in (("movie", self._movies_lib), ("show", self._tv_lib)):
|
||||||
|
if media_type != "all" and media_type != kind:
|
||||||
|
continue
|
||||||
|
sections.extend(self._scan_sections(kind, name))
|
||||||
|
for s in sections:
|
||||||
|
if getattr(s, "refreshing", False):
|
||||||
|
return True
|
||||||
|
titles = {(getattr(s, "title", "") or "").lower() for s in sections}
|
||||||
|
for act in self._server.activities():
|
||||||
|
if getattr(act, "type", "") in ("library.scan", "library.refresh"):
|
||||||
|
at = (getattr(act, "title", "") or "").lower()
|
||||||
|
if any(t and t in at for t in titles):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def has_item(self, media_type, item) -> bool:
|
||||||
|
"""True if Plex ALREADY has this specific grab indexed (so the post-download
|
||||||
|
scan can skip the crawl). Conservative — only True when we can positively match
|
||||||
|
the exact movie (title + year) or episode (show + SxE)."""
|
||||||
|
title = (item or {}).get("title")
|
||||||
|
if not title:
|
||||||
|
return False
|
||||||
|
if media_type == "movie":
|
||||||
|
year = (item or {}).get("year")
|
||||||
|
for sec in self._scan_sections("movie", self._movies_lib):
|
||||||
|
try:
|
||||||
|
hits = sec.search(title=title, maxresults=5)
|
||||||
|
except Exception:
|
||||||
|
hits = []
|
||||||
|
for h in hits:
|
||||||
|
hy = getattr(h, "year", None)
|
||||||
|
if year and hy and abs(int(hy) - int(year)) > 1:
|
||||||
|
continue
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
if media_type == "show":
|
||||||
|
sn, en = (item or {}).get("season_number"), (item or {}).get("episode_number")
|
||||||
|
for sec in self._scan_sections("show", self._tv_lib):
|
||||||
|
try:
|
||||||
|
hits = sec.search(title=title, maxresults=5)
|
||||||
|
except Exception:
|
||||||
|
hits = []
|
||||||
|
for show in hits:
|
||||||
|
if sn is None or en is None:
|
||||||
|
return True # show present, no episode to pin
|
||||||
|
try:
|
||||||
|
if show.episode(season=int(sn), episode=int(en)):
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
return False
|
||||||
|
return False
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _part_file(obj):
|
||||||
|
try:
|
||||||
|
media = obj.media[0]
|
||||||
|
part = media.parts[0]
|
||||||
|
return {
|
||||||
|
"relative_path": part.file,
|
||||||
|
"size_bytes": getattr(part, "size", None),
|
||||||
|
"resolution": getattr(media, "videoResolution", None),
|
||||||
|
"video_codec": getattr(media, "videoCodec", None),
|
||||||
|
"audio_codec": getattr(media, "audioCodec", None),
|
||||||
|
"runtime_seconds": int(obj.duration / 1000) if getattr(obj, "duration", None) else None,
|
||||||
|
}
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _tags(seq) -> list:
|
||||||
|
"""Tag names from a Plex tag list (genres/etc.)."""
|
||||||
|
out = []
|
||||||
|
for t in (seq or []):
|
||||||
|
tag = getattr(t, "tag", None)
|
||||||
|
if tag:
|
||||||
|
out.append(tag)
|
||||||
|
return out
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _date(val):
|
||||||
|
try:
|
||||||
|
return val.date().isoformat() if val else None
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _movie(self, m) -> dict:
|
||||||
|
dur = getattr(m, "duration", None)
|
||||||
|
d = {
|
||||||
|
"server_id": str(m.ratingKey),
|
||||||
|
"title": m.title,
|
||||||
|
"year": getattr(m, "year", None),
|
||||||
|
"overview": getattr(m, "summary", None),
|
||||||
|
"poster_url": getattr(m, "thumb", None),
|
||||||
|
"content_rating": getattr(m, "contentRating", None),
|
||||||
|
"studio": getattr(m, "studio", None),
|
||||||
|
"tagline": getattr(m, "tagline", None),
|
||||||
|
"rating": getattr(m, "audienceRating", None),
|
||||||
|
"rating_critic": getattr(m, "rating", None),
|
||||||
|
"genres": self._tags(getattr(m, "genres", None)),
|
||||||
|
"runtime_minutes": int(dur / 60000) if dur else None,
|
||||||
|
"file": self._part_file(m),
|
||||||
|
}
|
||||||
|
d.update(_parse_plex_guids(m))
|
||||||
|
return d
|
||||||
|
|
||||||
|
def _episode(self, ep, snum, enum) -> dict:
|
||||||
|
dur = getattr(ep, "duration", None)
|
||||||
|
aired = getattr(ep, "originallyAvailableAt", None)
|
||||||
|
return {
|
||||||
|
"server_id": str(ep.ratingKey),
|
||||||
|
"season_number": snum,
|
||||||
|
"episode_number": enum,
|
||||||
|
"title": ep.title,
|
||||||
|
"overview": getattr(ep, "summary", None),
|
||||||
|
"air_date": aired.date().isoformat() if aired else None,
|
||||||
|
"runtime_minutes": int(dur / 60000) if dur else None,
|
||||||
|
"still_url": getattr(ep, "thumb", None),
|
||||||
|
"rating": getattr(ep, "audienceRating", None),
|
||||||
|
"tvdb_id": _parse_plex_guids(ep).get("tvdb_id"),
|
||||||
|
"file": self._part_file(ep),
|
||||||
|
}
|
||||||
|
|
||||||
|
def _show(self, sh) -> dict:
|
||||||
|
# One episodes() call for the whole show (grouped by season) instead of a
|
||||||
|
# request per season — far fewer round-trips, much less timeout-prone.
|
||||||
|
seasons_map = {}
|
||||||
|
try:
|
||||||
|
for ep in sh.episodes():
|
||||||
|
enum = getattr(ep, "index", None)
|
||||||
|
if enum is None:
|
||||||
|
# No episode number (unmatched/special) — can't key it; skip.
|
||||||
|
continue
|
||||||
|
snum = ep.parentIndex if getattr(ep, "parentIndex", None) is not None else 0
|
||||||
|
seasons_map.setdefault(snum, []).append(self._episode(ep, snum, enum))
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Plex: failed reading episodes for %s", getattr(sh, "title", "?"))
|
||||||
|
# Season metadata (poster/title/overview) — one extra call per show gives
|
||||||
|
# real per-season art for the detail page.
|
||||||
|
season_meta = {}
|
||||||
|
try:
|
||||||
|
for se in sh.seasons():
|
||||||
|
sidx = getattr(se, "index", None)
|
||||||
|
if sidx is None:
|
||||||
|
continue
|
||||||
|
season_meta[sidx] = {
|
||||||
|
"server_id": str(getattr(se, "ratingKey", "")) or None,
|
||||||
|
"title": getattr(se, "title", None),
|
||||||
|
"overview": getattr(se, "summary", None),
|
||||||
|
"poster_url": getattr(se, "thumb", None),
|
||||||
|
}
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Plex: failed reading seasons for %s", getattr(sh, "title", "?"))
|
||||||
|
seasons = []
|
||||||
|
for n, eps in sorted(seasons_map.items()):
|
||||||
|
meta = season_meta.get(n, {})
|
||||||
|
seasons.append({"server_id": meta.get("server_id"), "season_number": n,
|
||||||
|
"title": meta.get("title"), "overview": meta.get("overview"),
|
||||||
|
"poster_url": meta.get("poster_url"), "episodes": eps})
|
||||||
|
d = {
|
||||||
|
"server_id": str(sh.ratingKey),
|
||||||
|
"title": sh.title,
|
||||||
|
"year": getattr(sh, "year", None),
|
||||||
|
"overview": getattr(sh, "summary", None),
|
||||||
|
"poster_url": getattr(sh, "thumb", None),
|
||||||
|
"status": None,
|
||||||
|
"network": getattr(sh, "network", None),
|
||||||
|
"content_rating": getattr(sh, "contentRating", None),
|
||||||
|
"tagline": getattr(sh, "tagline", None),
|
||||||
|
"rating": getattr(sh, "audienceRating", None),
|
||||||
|
"first_air_date": self._date(getattr(sh, "originallyAvailableAt", None)),
|
||||||
|
"last_air_date": None,
|
||||||
|
"genres": self._tags(getattr(sh, "genres", None)),
|
||||||
|
"seasons": seasons,
|
||||||
|
}
|
||||||
|
d.update(_parse_plex_guids(sh))
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
# ── Jellyfin ────────────────────────────────────────────────────────────────
|
||||||
|
_JF_MOVIE_FIELDS = ("Overview,Path,MediaSources,ProductionYear,OfficialRating,RunTimeTicks,Studios,"
|
||||||
|
"ProviderIds,Genres,Taglines,CommunityRating,CriticRating")
|
||||||
|
_JF_EP_FIELDS = ("Overview,Path,MediaSources,PremiereDate,RunTimeTicks,IndexNumber,ParentIndexNumber,"
|
||||||
|
"ProviderIds,CommunityRating")
|
||||||
|
_JF_SHOW_FIELDS = ("Overview,ProductionYear,OfficialRating,ProviderIds,Genres,Taglines,CommunityRating,"
|
||||||
|
"PremiereDate,EndDate")
|
||||||
|
|
||||||
|
|
||||||
|
class JellyfinVideoSource:
|
||||||
|
server_name = "jellyfin"
|
||||||
|
|
||||||
|
def __init__(self, client, movies_lib=None, tv_lib=None):
|
||||||
|
self._c = client
|
||||||
|
self.uid = client.user_id
|
||||||
|
self._movies_lib = movies_lib
|
||||||
|
self._tv_lib = tv_lib
|
||||||
|
|
||||||
|
def _req(self, path, params=None):
|
||||||
|
return self._c._make_request(path, params=params)
|
||||||
|
|
||||||
|
def _views(self, collection_type: str, name=None):
|
||||||
|
resp = self._req(f"/Users/{self.uid}/Views") or {}
|
||||||
|
views = [v for v in resp.get("Items", [])
|
||||||
|
if (v.get("CollectionType") or "").lower() == collection_type]
|
||||||
|
if name:
|
||||||
|
views = [v for v in views if v.get("Name") == name]
|
||||||
|
return views
|
||||||
|
|
||||||
|
def _scan_views(self, collection_type: str, name):
|
||||||
|
"""Views to SCAN. An empty name means this kind isn't mapped → scan NOTHING
|
||||||
|
(never fall back to all views), so a missing selection can't pull every
|
||||||
|
library. (available_libraries still lists all via _views.)"""
|
||||||
|
return self._views(collection_type, name) if name else []
|
||||||
|
|
||||||
|
def available_libraries(self) -> dict:
|
||||||
|
return {
|
||||||
|
"movies": [{"title": v.get("Name")} for v in self._views("movies")],
|
||||||
|
"tv": [{"title": v.get("Name")} for v in self._views("tvshows")],
|
||||||
|
}
|
||||||
|
|
||||||
|
def refresh_sections(self, media_type="all") -> dict:
|
||||||
|
"""Ask Jellyfin to rescan the selected video libraries (POST /Items/{id}/Refresh)
|
||||||
|
so freshly-downloaded files get indexed. _make_request is GET-only, so POST direct.
|
||||||
|
``media_type`` scopes it to the Movie or TV view; 'all' does both."""
|
||||||
|
import requests
|
||||||
|
base = (self._c.base_url or "").rstrip("/")
|
||||||
|
if not base:
|
||||||
|
return {"ok": False, "sections": 0}
|
||||||
|
headers = {"X-Emby-Token": self._c.api_key or ""}
|
||||||
|
views = []
|
||||||
|
if media_type in ("all", "movie"):
|
||||||
|
views += list(self._scan_views("movies", self._movies_lib))
|
||||||
|
if media_type in ("all", "show"):
|
||||||
|
views += list(self._scan_views("tvshows", self._tv_lib))
|
||||||
|
n = 0
|
||||||
|
for v in views:
|
||||||
|
vid = v.get("Id")
|
||||||
|
if not vid:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
requests.post(base + "/Items/" + str(vid) + "/Refresh", headers=headers,
|
||||||
|
params={"Recursive": "true", "MetadataRefreshMode": "Default",
|
||||||
|
"ImageRefreshMode": "Default"}, timeout=15)
|
||||||
|
n += 1
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Jellyfin: refresh failed for view %s", vid)
|
||||||
|
return {"ok": n > 0, "sections": n}
|
||||||
|
|
||||||
|
def is_scanning(self, media_type="all") -> bool:
|
||||||
|
"""True if Jellyfin's library-scan scheduled task is running. Jellyfin's
|
||||||
|
scan isn't per-library, so media_type is ignored (any scan counts)."""
|
||||||
|
import requests
|
||||||
|
base = (self._c.base_url or "").rstrip("/")
|
||||||
|
if not base:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
r = requests.get(base + "/ScheduledTasks",
|
||||||
|
headers={"X-Emby-Token": self._c.api_key or ""}, timeout=10)
|
||||||
|
tasks = r.json() if r.ok else []
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
for task in tasks or []:
|
||||||
|
name = (task.get("Name") or "").lower()
|
||||||
|
if (("scan" in name or "refresh" in name or "library" in name)
|
||||||
|
and task.get("State") in ("Running", "Cancelling")):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def has_item(self, media_type, item) -> bool:
|
||||||
|
"""True if Jellyfin already has this grab indexed. Conservative — matches the
|
||||||
|
exact movie (name + year) or episode (series + SxE); any uncertainty → False."""
|
||||||
|
import requests
|
||||||
|
title = (item or {}).get("title")
|
||||||
|
base = (self._c.base_url or "").rstrip("/")
|
||||||
|
if not title or not base:
|
||||||
|
return False
|
||||||
|
headers = {"X-Emby-Token": self._c.api_key or ""}
|
||||||
|
|
||||||
|
def _find(item_type):
|
||||||
|
try:
|
||||||
|
r = requests.get(base + "/Items", headers=headers, timeout=10, params={
|
||||||
|
"searchTerm": title, "IncludeItemTypes": item_type, "Recursive": "true",
|
||||||
|
"Fields": "ProductionYear", "Limit": 5})
|
||||||
|
return (r.json() or {}).get("Items", []) if r.ok else []
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
|
||||||
|
if media_type == "movie":
|
||||||
|
year = (item or {}).get("year")
|
||||||
|
for it in _find("Movie"):
|
||||||
|
py = it.get("ProductionYear")
|
||||||
|
if year and py and abs(int(py) - int(year)) > 1:
|
||||||
|
continue
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
if media_type == "show":
|
||||||
|
sn, en = (item or {}).get("season_number"), (item or {}).get("episode_number")
|
||||||
|
for series in _find("Series"):
|
||||||
|
sid = series.get("Id")
|
||||||
|
if not sid:
|
||||||
|
continue
|
||||||
|
if sn is None or en is None:
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
r = requests.get(base + "/Shows/" + str(sid) + "/Episodes", headers=headers,
|
||||||
|
timeout=10, params={"season": int(sn)})
|
||||||
|
eps = (r.json() or {}).get("Items", []) if r.ok else []
|
||||||
|
if any(int(e.get("IndexNumber") or -1) == int(en) for e in eps):
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
return False
|
||||||
|
return False
|
||||||
|
|
||||||
|
def counts(self, incremental=False) -> dict:
|
||||||
|
def total(view, itype):
|
||||||
|
resp = self._req(f"/Users/{self.uid}/Items", {
|
||||||
|
"ParentId": view["Id"], "IncludeItemTypes": itype,
|
||||||
|
"Recursive": "true", "Limit": "0"}) or {}
|
||||||
|
return int(resp.get("TotalRecordCount", 0) or 0)
|
||||||
|
m = sum(total(v, "Movie") for v in self._scan_views("movies", self._movies_lib))
|
||||||
|
sh = sum(total(v, "Series") for v in self._scan_views("tvshows", self._tv_lib))
|
||||||
|
if incremental:
|
||||||
|
m, sh = min(m, 100), min(sh, 50)
|
||||||
|
return {"movies": m, "shows": sh}
|
||||||
|
|
||||||
|
def _paged(self, path, params, page_size=500):
|
||||||
|
"""Yield items across pages so large libraries aren't capped/truncated."""
|
||||||
|
start = 0
|
||||||
|
while True:
|
||||||
|
p = dict(params)
|
||||||
|
p.update({"StartIndex": str(start), "Limit": str(page_size)})
|
||||||
|
resp = self._req(path, p) or {}
|
||||||
|
batch = resp.get("Items", [])
|
||||||
|
for it in batch:
|
||||||
|
yield it
|
||||||
|
start += len(batch)
|
||||||
|
total = resp.get("TotalRecordCount")
|
||||||
|
if not batch or len(batch) < page_size or (total is not None and start >= total):
|
||||||
|
break
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _ticks_to_seconds(ticks):
|
||||||
|
return int(ticks / 10_000_000) if ticks else None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _file(item):
|
||||||
|
sources = item.get("MediaSources") or []
|
||||||
|
if not sources:
|
||||||
|
path = item.get("Path")
|
||||||
|
return {"relative_path": path} if path else None
|
||||||
|
src = sources[0]
|
||||||
|
streams = src.get("MediaStreams") or []
|
||||||
|
vid = next((s for s in streams if s.get("Type") == "Video"), {})
|
||||||
|
aud = next((s for s in streams if s.get("Type") == "Audio"), {})
|
||||||
|
return {
|
||||||
|
"relative_path": src.get("Path") or item.get("Path") or "",
|
||||||
|
"size_bytes": src.get("Size"),
|
||||||
|
"resolution": (str(vid.get("Height")) + "p") if vid.get("Height") else None,
|
||||||
|
"video_codec": vid.get("Codec"),
|
||||||
|
"audio_codec": aud.get("Codec"),
|
||||||
|
"runtime_seconds": JellyfinVideoSource._ticks_to_seconds(item.get("RunTimeTicks")),
|
||||||
|
}
|
||||||
|
|
||||||
|
def iter_movies(self, incremental=False):
|
||||||
|
path = f"/Users/{self.uid}/Items"
|
||||||
|
for view in self._scan_views("movies", self._movies_lib):
|
||||||
|
params = {"ParentId": view["Id"], "IncludeItemTypes": "Movie",
|
||||||
|
"Recursive": "true", "Fields": _JF_MOVIE_FIELDS}
|
||||||
|
if incremental:
|
||||||
|
params.update({"SortBy": "DateCreated", "SortOrder": "Descending", "Limit": "100"})
|
||||||
|
items = (self._req(path, params) or {}).get("Items", [])
|
||||||
|
else:
|
||||||
|
items = self._paged(path, params)
|
||||||
|
for it in items:
|
||||||
|
try:
|
||||||
|
yield self._movie(it)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Jellyfin: skipping movie %s", it.get("Name", "?"))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _first(seq):
|
||||||
|
seq = seq or []
|
||||||
|
return seq[0] if seq else None
|
||||||
|
|
||||||
|
def _movie(self, it) -> dict:
|
||||||
|
studios = it.get("Studios") or []
|
||||||
|
ticks = it.get("RunTimeTicks")
|
||||||
|
d = {
|
||||||
|
"server_id": str(it["Id"]),
|
||||||
|
"title": it.get("Name"),
|
||||||
|
"year": it.get("ProductionYear"),
|
||||||
|
"overview": it.get("Overview"),
|
||||||
|
"poster_url": (it.get("ImageTags") or {}).get("Primary"),
|
||||||
|
"content_rating": it.get("OfficialRating"),
|
||||||
|
"studio": studios[0].get("Name") if studios else None,
|
||||||
|
"tagline": self._first(it.get("Taglines")),
|
||||||
|
"rating": it.get("CommunityRating"),
|
||||||
|
"rating_critic": it.get("CriticRating"),
|
||||||
|
"genres": it.get("Genres") or [],
|
||||||
|
"runtime_minutes": int(ticks / 600_000_000) if ticks else None,
|
||||||
|
"file": self._file(it),
|
||||||
|
}
|
||||||
|
d.update(_parse_jf_providers(it))
|
||||||
|
return d
|
||||||
|
|
||||||
|
def iter_shows(self, incremental=False):
|
||||||
|
path = f"/Users/{self.uid}/Items"
|
||||||
|
for view in self._scan_views("tvshows", self._tv_lib):
|
||||||
|
params = {"ParentId": view["Id"], "IncludeItemTypes": "Series",
|
||||||
|
"Recursive": "true", "Fields": _JF_SHOW_FIELDS}
|
||||||
|
if incremental:
|
||||||
|
params.update({"SortBy": "DateCreated", "SortOrder": "Descending", "Limit": "50"})
|
||||||
|
items = (self._req(path, params) or {}).get("Items", [])
|
||||||
|
else:
|
||||||
|
items = self._paged(path, params)
|
||||||
|
for it in items:
|
||||||
|
try:
|
||||||
|
yield self._show(it)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Jellyfin: skipping show %s", it.get("Name", "?"))
|
||||||
|
|
||||||
|
def _show(self, it) -> dict:
|
||||||
|
series_id = str(it["Id"])
|
||||||
|
seasons = []
|
||||||
|
try:
|
||||||
|
eps_resp = self._req(f"/Shows/{series_id}/Episodes", {
|
||||||
|
"UserId": self.uid, "Fields": _JF_EP_FIELDS}) or {}
|
||||||
|
by_season: dict[int, list] = {}
|
||||||
|
for ep in eps_resp.get("Items", []):
|
||||||
|
enum = ep.get("IndexNumber")
|
||||||
|
if enum is None:
|
||||||
|
continue # unnumbered/special — can't key it
|
||||||
|
snum = ep.get("ParentIndexNumber") or 0
|
||||||
|
aired = ep.get("PremiereDate")
|
||||||
|
ticks = ep.get("RunTimeTicks")
|
||||||
|
by_season.setdefault(snum, []).append({
|
||||||
|
"server_id": str(ep["Id"]),
|
||||||
|
"season_number": snum,
|
||||||
|
"episode_number": enum,
|
||||||
|
"title": ep.get("Name"),
|
||||||
|
"overview": ep.get("Overview"),
|
||||||
|
"air_date": aired[:10] if aired else None,
|
||||||
|
"runtime_minutes": int(ticks / 600_000_000) if ticks else None,
|
||||||
|
"still_url": (ep.get("ImageTags") or {}).get("Primary"),
|
||||||
|
"rating": ep.get("CommunityRating"),
|
||||||
|
"tvdb_id": _parse_jf_providers(ep).get("tvdb_id"),
|
||||||
|
"file": self._file(ep),
|
||||||
|
})
|
||||||
|
# Season metadata (poster/title/overview) — one extra call per show.
|
||||||
|
season_meta = {}
|
||||||
|
try:
|
||||||
|
seas = self._req(f"/Shows/{series_id}/Seasons",
|
||||||
|
{"UserId": self.uid, "Fields": "Overview"}) or {}
|
||||||
|
for se in seas.get("Items", []):
|
||||||
|
sidx = se.get("IndexNumber")
|
||||||
|
if sidx is None:
|
||||||
|
continue
|
||||||
|
season_meta[sidx] = {
|
||||||
|
"server_id": str(se["Id"]) if se.get("Id") else None,
|
||||||
|
"title": se.get("Name"),
|
||||||
|
"overview": se.get("Overview"),
|
||||||
|
"poster_url": (se.get("ImageTags") or {}).get("Primary"),
|
||||||
|
}
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Jellyfin: failed reading seasons for %s", it.get("Name", "?"))
|
||||||
|
for snum, eps in sorted(by_season.items()):
|
||||||
|
meta = season_meta.get(snum, {})
|
||||||
|
seasons.append({"server_id": meta.get("server_id"), "season_number": snum,
|
||||||
|
"title": meta.get("title"), "overview": meta.get("overview"),
|
||||||
|
"poster_url": meta.get("poster_url"), "episodes": eps})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Jellyfin: failed reading episodes for %s", it.get("Name", "?"))
|
||||||
|
premiere = it.get("PremiereDate")
|
||||||
|
end = it.get("EndDate")
|
||||||
|
d = {
|
||||||
|
"server_id": series_id,
|
||||||
|
"title": it.get("Name"),
|
||||||
|
"year": it.get("ProductionYear"),
|
||||||
|
"overview": it.get("Overview"),
|
||||||
|
"poster_url": (it.get("ImageTags") or {}).get("Primary"),
|
||||||
|
"status": None,
|
||||||
|
"network": None,
|
||||||
|
"content_rating": it.get("OfficialRating"),
|
||||||
|
"tagline": self._first(it.get("Taglines")),
|
||||||
|
"rating": it.get("CommunityRating"),
|
||||||
|
"first_air_date": premiere[:10] if premiere else None,
|
||||||
|
"last_air_date": end[:10] if end else None,
|
||||||
|
"genres": it.get("Genres") or [],
|
||||||
|
"seasons": seasons,
|
||||||
|
}
|
||||||
|
d.update(_parse_jf_providers(it))
|
||||||
|
return d
|
||||||
153
core/video/subtitles.py
Normal file
153
core/video/subtitles.py
Normal file
|
|
@ -0,0 +1,153 @@
|
||||||
|
"""Download external subtitle .srt files from OpenSubtitles and drop them next to the
|
||||||
|
imported video as ``<video stem>.<lang>.srt``.
|
||||||
|
|
||||||
|
The enrichment worker only records subtitle-language AVAILABILITY; this fetches the
|
||||||
|
actual file: search → pick the most-downloaded subtitle for the language → request the
|
||||||
|
(time-limited) download link → save it. Parsing (``parse_langs`` / ``pick_best_file``)
|
||||||
|
is pure; the HTTP and filesystem are injected, so it's unit-tested without network or
|
||||||
|
disk. Best-effort BY CONTRACT — OpenSubtitles' free tier is daily-quota-limited, so a
|
||||||
|
miss is normal and never breaks an import.
|
||||||
|
|
||||||
|
Isolated: stdlib only; no music imports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
from typing import Any, Callable
|
||||||
|
|
||||||
|
BASE = "https://api.opensubtitles.com/api/v1"
|
||||||
|
_UA = "SoulSync v1.0"
|
||||||
|
|
||||||
|
|
||||||
|
def parse_langs(raw: Any) -> list:
|
||||||
|
"""'en, es ; fr' → ['en','es','fr'] (lower-cased, de-duped). Defaults to ['en']."""
|
||||||
|
out = []
|
||||||
|
for tok in re.split(r"[,\s;]+", str(raw or "")):
|
||||||
|
t = tok.strip().lower()
|
||||||
|
if t and t not in out:
|
||||||
|
out.append(t)
|
||||||
|
return out or ["en"]
|
||||||
|
|
||||||
|
|
||||||
|
def pick_best_file(search_json: Any, lang: str) -> Any:
|
||||||
|
"""The ``file_id`` of the most-downloaded subtitle for ``lang`` in a /subtitles
|
||||||
|
response, or None. Pure."""
|
||||||
|
if not isinstance(search_json, dict):
|
||||||
|
return None
|
||||||
|
best, best_dl = None, -1
|
||||||
|
want = str(lang or "").lower()
|
||||||
|
for row in (search_json.get("data") or []):
|
||||||
|
attrs = row.get("attributes") or {}
|
||||||
|
if str(attrs.get("language") or "").lower() != want:
|
||||||
|
continue
|
||||||
|
files = attrs.get("files") or []
|
||||||
|
if not files or files[0].get("file_id") is None:
|
||||||
|
continue
|
||||||
|
dl = attrs.get("download_count") or 0
|
||||||
|
if dl > best_dl:
|
||||||
|
best, best_dl = files[0]["file_id"], dl
|
||||||
|
return best
|
||||||
|
|
||||||
|
|
||||||
|
def srt_name(video_path: Any, lang: str) -> str:
|
||||||
|
"""'<video stem>.<lang>.srt' (no directory)."""
|
||||||
|
stem = os.path.splitext(os.path.basename(str(video_path or "")))[0]
|
||||||
|
return "%s.%s.srt" % (stem, lang)
|
||||||
|
|
||||||
|
|
||||||
|
def search_params(identity: dict, lang: str) -> dict | None:
|
||||||
|
"""OpenSubtitles /subtitles query for a title identity. For an episode the show's
|
||||||
|
id is used with season/episode; for a movie, the imdb/tmdb id. None if unidentified."""
|
||||||
|
identity = identity if isinstance(identity, dict) else {}
|
||||||
|
params = {"languages": lang}
|
||||||
|
if identity.get("season") is not None and identity.get("tmdb_id"):
|
||||||
|
params["parent_tmdb_id"] = identity["tmdb_id"]
|
||||||
|
params["season_number"] = identity["season"]
|
||||||
|
params["episode_number"] = identity.get("episode")
|
||||||
|
return params
|
||||||
|
if identity.get("imdb_id"):
|
||||||
|
params["imdb_id"] = re.sub(r"^tt", "", str(identity["imdb_id"]).lower())
|
||||||
|
return params
|
||||||
|
if identity.get("tmdb_id"):
|
||||||
|
params["tmdb_id"] = identity["tmdb_id"]
|
||||||
|
return params
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# ── real HTTP fetcher (injected into write_subtitles) ─────────────────────────
|
||||||
|
def _headers(key: str) -> dict:
|
||||||
|
return {"Api-Key": key, "User-Agent": _UA, "Accept": "application/json"}
|
||||||
|
|
||||||
|
|
||||||
|
def _get_json(url: str, params: dict, headers: dict) -> Any:
|
||||||
|
req = urllib.request.Request(url + "?" + urllib.parse.urlencode(params), headers=headers)
|
||||||
|
with urllib.request.urlopen(req, timeout=20) as r:
|
||||||
|
return json.loads(r.read().decode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def _post_json(url: str, body: dict, headers: dict) -> Any:
|
||||||
|
h = dict(headers)
|
||||||
|
h["Content-Type"] = "application/json"
|
||||||
|
req = urllib.request.Request(url, data=json.dumps(body).encode("utf-8"), headers=h, method="POST")
|
||||||
|
with urllib.request.urlopen(req, timeout=20) as r:
|
||||||
|
return json.loads(r.read().decode("utf-8"))
|
||||||
|
|
||||||
|
|
||||||
|
def _get_text(url: str) -> str:
|
||||||
|
req = urllib.request.Request(url, headers={"User-Agent": _UA})
|
||||||
|
with urllib.request.urlopen(req, timeout=30) as r:
|
||||||
|
return r.read().decode("utf-8", "replace")
|
||||||
|
|
||||||
|
|
||||||
|
def opensubtitles_fetcher(api_key: Any) -> Callable | None:
|
||||||
|
"""Return ``fetch(identity, lang) -> srt_text | None`` backed by OpenSubtitles, or
|
||||||
|
None when there's no key. ``identity`` = {imdb_id?, tmdb_id?, season?, episode?}."""
|
||||||
|
key = str(api_key or "").strip()
|
||||||
|
if not key:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def fetch(identity, lang):
|
||||||
|
params = search_params(identity, lang)
|
||||||
|
if params is None:
|
||||||
|
return None
|
||||||
|
found = _get_json(BASE + "/subtitles", params, _headers(key))
|
||||||
|
file_id = pick_best_file(found, lang)
|
||||||
|
if file_id is None:
|
||||||
|
return None
|
||||||
|
dl = _post_json(BASE + "/download", {"file_id": file_id}, _headers(key))
|
||||||
|
link = (dl or {}).get("link")
|
||||||
|
return _get_text(link) if link else None
|
||||||
|
|
||||||
|
return fetch
|
||||||
|
|
||||||
|
|
||||||
|
def write_subtitles(video_path: str, langs: list, identity: dict, fetch: Callable, fs: Any) -> None:
|
||||||
|
"""For each language not already present as ``<stem>.<lang>.srt`` next to the video,
|
||||||
|
fetch and write it via the injected ``fetch`` + ``fs`` (``list_dir``, ``write_text``).
|
||||||
|
Idempotent + best-effort."""
|
||||||
|
if not fetch:
|
||||||
|
return
|
||||||
|
folder = os.path.dirname(str(video_path or ""))
|
||||||
|
try:
|
||||||
|
existing = {str(n).lower() for n in (fs.list_dir(folder) or [])}
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
existing = set()
|
||||||
|
for lang in (langs or []):
|
||||||
|
name = srt_name(video_path, lang)
|
||||||
|
if name.lower() in existing:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
text = fetch(identity, lang)
|
||||||
|
if text:
|
||||||
|
fs.write_text(os.path.join(folder, name), text)
|
||||||
|
except Exception: # noqa: BLE001 - a quota miss / network blip is expected, never fatal
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["parse_langs", "pick_best_file", "srt_name", "search_params",
|
||||||
|
"opensubtitles_fetcher", "write_subtitles"]
|
||||||
812
core/video/youtube.py
Normal file
812
core/video/youtube.py
Normal file
|
|
@ -0,0 +1,812 @@
|
||||||
|
"""Resolve a YouTube *channel* reference into a followable channel + its recent
|
||||||
|
uploads — the data source behind "follow a YouTube channel" on the video side.
|
||||||
|
|
||||||
|
yt-dlp only (no API key). The upload list is fetched with ``extract_flat`` so it
|
||||||
|
stays cheap even for channels with thousands of videos; the trade-off is that
|
||||||
|
per-video publish dates are sparse in flat mode, so they get backfilled later
|
||||||
|
(same pattern as the wishlist art backfill). Real *downloads* will additionally
|
||||||
|
need Deno + ffmpeg; flat *listing* does not.
|
||||||
|
|
||||||
|
The URL parsing and the yt-dlp-dict → our-shape mapping are pure functions so
|
||||||
|
they're unit-testable without touching the network; the one network call goes
|
||||||
|
through ``_extract`` which accepts an injectable factory for tests.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yt_dlp
|
||||||
|
except Exception: # pragma: no cover - yt-dlp is a hard dep, but stay import-safe
|
||||||
|
yt_dlp = None
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Path prefixes that identify a channel (everything after is the channel key).
|
||||||
|
# Tabs like /videos, /streams, /shorts, /featured, /playlists are stripped.
|
||||||
|
_CHANNEL_TABS = {"videos", "streams", "shorts", "featured", "playlists", "community", "about"}
|
||||||
|
_HANDLE_RE = re.compile(r"^@?[A-Za-z0-9._-]{2,}$")
|
||||||
|
_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||||
|
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_channel_url(raw):
|
||||||
|
"""Normalize a pasted channel reference to a canonical uploads URL, or return
|
||||||
|
None if it isn't a channel (a /watch video, a playlist, the YT home, etc.).
|
||||||
|
|
||||||
|
Accepts: ``https://www.youtube.com/@PlayStation`` (+ /videos etc.),
|
||||||
|
``/channel/UC...``, ``/c/Name``, ``/user/Name``, a bare ``@handle``, or a
|
||||||
|
bare ``handle``. Returns ``https://www.youtube.com/<base>/videos``.
|
||||||
|
"""
|
||||||
|
if not raw or not str(raw).strip():
|
||||||
|
return None
|
||||||
|
raw = str(raw).strip()
|
||||||
|
|
||||||
|
# Bare handle (no scheme, no slash) → treat as @handle.
|
||||||
|
if "/" not in raw and "." not in raw and " " not in raw:
|
||||||
|
if _HANDLE_RE.match(raw):
|
||||||
|
handle = raw if raw.startswith("@") else "@" + raw
|
||||||
|
return "https://www.youtube.com/" + handle + "/videos"
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Give bare "youtube.com/..." a scheme so urlparse populates netloc.
|
||||||
|
parse_target = raw if "://" in raw else "https://" + raw
|
||||||
|
try:
|
||||||
|
u = urlparse(parse_target)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
host = (u.netloc or "").lower()
|
||||||
|
if host.startswith("www."):
|
||||||
|
host = host[4:]
|
||||||
|
if host not in ("youtube.com", "m.youtube.com", "music.youtube.com"):
|
||||||
|
return None
|
||||||
|
|
||||||
|
segs = [s for s in (u.path or "").split("/") if s]
|
||||||
|
if not segs:
|
||||||
|
return None
|
||||||
|
|
||||||
|
first = segs[0]
|
||||||
|
if first.startswith("@"):
|
||||||
|
base = first # /@handle
|
||||||
|
elif first in ("channel", "c", "user") and len(segs) >= 2:
|
||||||
|
base = first + "/" + segs[1] # /channel/UC.., /c/Name, /user/Name
|
||||||
|
else:
|
||||||
|
return None # /watch, /playlist, /shorts/<id>, home…
|
||||||
|
|
||||||
|
return "https://www.youtube.com/" + base + "/videos"
|
||||||
|
|
||||||
|
|
||||||
|
def _best_thumb(thumbs):
|
||||||
|
"""Pick the highest-resolution thumbnail URL from a yt-dlp thumbnails list."""
|
||||||
|
if not thumbs:
|
||||||
|
return None
|
||||||
|
best, best_area = None, -1
|
||||||
|
for t in thumbs:
|
||||||
|
if not isinstance(t, dict) or not t.get("url"):
|
||||||
|
continue
|
||||||
|
area = (t.get("width") or 0) * (t.get("height") or 0)
|
||||||
|
# preference value as a tie-breaker when dimensions are absent
|
||||||
|
score = area if area else (t.get("preference") or 0)
|
||||||
|
if score >= best_area:
|
||||||
|
best, best_area = t.get("url"), score
|
||||||
|
return best
|
||||||
|
|
||||||
|
|
||||||
|
def _thumb_by_id(thumbs, keyword):
|
||||||
|
"""The highest-res thumbnail whose yt-dlp id mentions ``keyword`` (e.g.
|
||||||
|
'avatar' / 'banner'), or None — channels expose both in one thumbnails list."""
|
||||||
|
hits = [t for t in (thumbs or []) if isinstance(t, dict) and t.get("url")
|
||||||
|
and keyword in str(t.get("id", "")).lower()]
|
||||||
|
return _best_thumb(hits)
|
||||||
|
|
||||||
|
|
||||||
|
def _entry_date(e):
|
||||||
|
"""ISO date (YYYY-MM-DD) for a flat entry, or None — flat mode often omits it."""
|
||||||
|
ts = e.get("timestamp")
|
||||||
|
if isinstance(ts, (int, float)):
|
||||||
|
try:
|
||||||
|
return datetime.fromtimestamp(ts, tz=timezone.utc).date().isoformat()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
ud = e.get("upload_date") # 'YYYYMMDD'
|
||||||
|
if isinstance(ud, str) and len(ud) == 8 and ud.isdigit():
|
||||||
|
return f"{ud[0:4]}-{ud[4:6]}-{ud[6:8]}"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _shape_entries(entries, limit):
|
||||||
|
"""Flat playlist/channel entries → our lightweight video shape (id-less and
|
||||||
|
null entries dropped)."""
|
||||||
|
out = []
|
||||||
|
for e in [x for x in (entries or []) if isinstance(x, dict)][:limit]:
|
||||||
|
vid = e.get("id")
|
||||||
|
if not vid:
|
||||||
|
continue
|
||||||
|
dur = e.get("duration")
|
||||||
|
out.append({
|
||||||
|
"youtube_id": vid,
|
||||||
|
"title": e.get("title") or "",
|
||||||
|
"published_at": _entry_date(e),
|
||||||
|
"duration_seconds": int(dur) if isinstance(dur, (int, float)) else None,
|
||||||
|
"thumbnail_url": _best_thumb(e.get("thumbnails")) or e.get("thumbnail"),
|
||||||
|
"view_count": e.get("view_count"),
|
||||||
|
"description": e.get("description"),
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def shape_channel(info, limit=30):
|
||||||
|
"""Map a yt-dlp channel info dict to our followable-channel shape."""
|
||||||
|
info = info or {}
|
||||||
|
videos = _shape_entries(info.get("entries"), limit)
|
||||||
|
handle = info.get("uploader_id") or info.get("channel_id_handle")
|
||||||
|
if handle and not str(handle).startswith("@"):
|
||||||
|
handle = None # uploader_id is sometimes the UC id, not a handle
|
||||||
|
thumbs = info.get("thumbnails")
|
||||||
|
return {
|
||||||
|
"youtube_id": info.get("channel_id") or info.get("id"),
|
||||||
|
"title": info.get("channel") or info.get("uploader") or info.get("title") or "",
|
||||||
|
"handle": handle,
|
||||||
|
"avatar_url": _thumb_by_id(thumbs, "avatar") or _best_thumb(thumbs),
|
||||||
|
"banner_url": _thumb_by_id(thumbs, "banner"),
|
||||||
|
"description": info.get("description"),
|
||||||
|
"subscriber_count": info.get("channel_follower_count"),
|
||||||
|
"view_count": info.get("view_count"),
|
||||||
|
"tags": (info.get("tags") or [])[:12],
|
||||||
|
"video_count": info.get("playlist_count") or len(videos),
|
||||||
|
"videos": videos,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _cookie_opts():
|
||||||
|
"""The music client's cookie convention so age/region-gated content works."""
|
||||||
|
try:
|
||||||
|
from config.settings import config_manager
|
||||||
|
cb = config_manager.get("youtube.cookies_browser", "")
|
||||||
|
# 'custom' = the paste-cookies.txt sentinel (not a yt-dlp browser); skip it here.
|
||||||
|
if cb and cb != "custom":
|
||||||
|
return {"cookiesfrombrowser": (cb,)}
|
||||||
|
except Exception: # noqa: S110 - cookie config is best-effort; extraction still works without it
|
||||||
|
pass
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def _ydl_opts(limit, db=None):
|
||||||
|
opts = {
|
||||||
|
"quiet": True,
|
||||||
|
"no_warnings": True,
|
||||||
|
"extract_flat": True, # fast: enumerate uploads without per-video format probing
|
||||||
|
"skip_download": True,
|
||||||
|
"playlistend": int(limit),
|
||||||
|
# NB: do NOT pin user_agent here — yt-dlp manages its own (current) client
|
||||||
|
# identity; a static UA trips YouTube's heuristics and truncates large
|
||||||
|
# playlist pagination (the reason a fresh ytdl-sub pages further than us).
|
||||||
|
}
|
||||||
|
opts.update(_cookie_opts())
|
||||||
|
return opts
|
||||||
|
|
||||||
|
|
||||||
|
def shape_video(info):
|
||||||
|
"""Map yt-dlp's FULL single-video info dict to our rich video shape (the data
|
||||||
|
flat-listing can't give: description, views, likes, duration, tags)."""
|
||||||
|
info = info or {}
|
||||||
|
dur = info.get("duration")
|
||||||
|
return {
|
||||||
|
"youtube_id": info.get("id"),
|
||||||
|
"title": info.get("title") or "",
|
||||||
|
"description": info.get("description"),
|
||||||
|
"duration_seconds": int(dur) if isinstance(dur, (int, float)) else None,
|
||||||
|
"view_count": info.get("view_count"),
|
||||||
|
"like_count": info.get("like_count"),
|
||||||
|
"published_at": _entry_date(info),
|
||||||
|
"thumbnail_url": _best_thumb(info.get("thumbnails")) or info.get("thumbnail"),
|
||||||
|
"channel_title": info.get("channel") or info.get("uploader"),
|
||||||
|
"channel_id": info.get("channel_id"),
|
||||||
|
"webpage_url": info.get("webpage_url") or ("https://www.youtube.com/watch?v=" + (info.get("id") or "")),
|
||||||
|
"tags": info.get("tags") or [],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def video_detail(video_id, ydl_factory=None):
|
||||||
|
"""Full metadata for ONE video (non-flat extract) — done lazily on click, the
|
||||||
|
way the TV nebula lazy-loads guest stars. Accepts a raw id or a watch URL."""
|
||||||
|
if not video_id:
|
||||||
|
return None
|
||||||
|
vid = str(video_id).strip()
|
||||||
|
url = vid if vid.startswith("http") else "https://www.youtube.com/watch?v=" + vid
|
||||||
|
opts = {"quiet": True, "no_warnings": True, "skip_download": True, "user_agent": _UA}
|
||||||
|
opts.update(_cookie_opts())
|
||||||
|
info = _extract(url, opts, ydl_factory)
|
||||||
|
if not info or not info.get("id"):
|
||||||
|
return None
|
||||||
|
return shape_video(info)
|
||||||
|
|
||||||
|
|
||||||
|
def _extract(url, opts, ydl_factory=None):
|
||||||
|
"""Run yt-dlp's extract_info; isolated for test injection. Returns a dict or None."""
|
||||||
|
factory = ydl_factory or (yt_dlp.YoutubeDL if yt_dlp else None)
|
||||||
|
if factory is None:
|
||||||
|
logger.warning("yt-dlp unavailable; cannot resolve YouTube channel")
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
with factory(opts) as ydl:
|
||||||
|
return ydl.extract_info(url, download=False)
|
||||||
|
except Exception as e:
|
||||||
|
logger.info("YouTube channel resolve failed for %s: %s", url, e)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_channel(raw, limit=30, ydl_factory=None, db=None):
|
||||||
|
"""Resolve a pasted channel reference to ``{youtube_id, title, handle,
|
||||||
|
avatar_url, videos:[...], ...}``, or None if it isn't a resolvable channel."""
|
||||||
|
url = parse_channel_url(raw)
|
||||||
|
if not url:
|
||||||
|
return None
|
||||||
|
info = _extract(url, _ydl_opts(limit, db), ydl_factory)
|
||||||
|
if not info:
|
||||||
|
return None
|
||||||
|
shaped = shape_channel(info, limit)
|
||||||
|
return shaped if shaped.get("youtube_id") else None
|
||||||
|
|
||||||
|
|
||||||
|
# No-key bulk date sources: community Piped/Invidious instances. Flaky (they go
|
||||||
|
# up/down), so we try several and fall back to per-video yt-dlp if none answer.
|
||||||
|
_PROXY_INSTANCES = [
|
||||||
|
("piped", "https://pipedapi.kavin.rocks"),
|
||||||
|
("piped", "https://pipedapi.adminforge.de"),
|
||||||
|
("piped", "https://api.piped.private.coffee"),
|
||||||
|
("invidious", "https://invidious.nerdvpn.de"),
|
||||||
|
("invidious", "https://inv.nadeko.net"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _epoch_to_date(v):
|
||||||
|
"""epoch seconds OR milliseconds → 'YYYY-MM-DD', or None."""
|
||||||
|
try:
|
||||||
|
n = float(v)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
if n <= 0:
|
||||||
|
return None
|
||||||
|
if n > 1e12: # milliseconds
|
||||||
|
n /= 1000.0
|
||||||
|
try:
|
||||||
|
return datetime.fromtimestamp(n, tz=timezone.utc).date().isoformat()
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _vid_from_url(u):
|
||||||
|
m = re.search(r"[?&]v=([\w-]+)", u or "")
|
||||||
|
return m.group(1) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_proxy_dates(obj):
|
||||||
|
"""{video_id: 'YYYY-MM-DD'} from a Piped (relatedStreams) or Invidious (videos)
|
||||||
|
channel JSON response — handles both shapes."""
|
||||||
|
out = {}
|
||||||
|
if not isinstance(obj, dict):
|
||||||
|
return out
|
||||||
|
for s in (obj.get("relatedStreams") or []): # Piped
|
||||||
|
if not isinstance(s, dict):
|
||||||
|
continue
|
||||||
|
vid = _vid_from_url(s.get("url")) or s.get("videoId")
|
||||||
|
d = _epoch_to_date(s.get("uploaded"))
|
||||||
|
if vid and d:
|
||||||
|
out[vid] = d
|
||||||
|
for v in (obj.get("videos") or []): # Invidious
|
||||||
|
if not isinstance(v, dict):
|
||||||
|
continue
|
||||||
|
vid = v.get("videoId")
|
||||||
|
d = _epoch_to_date(v.get("published"))
|
||||||
|
if vid and d:
|
||||||
|
out[vid] = d
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _proxy_get(url, fetch):
|
||||||
|
if fetch is not None:
|
||||||
|
return fetch(url)
|
||||||
|
import requests
|
||||||
|
# Short timeout so a dead instance (most public ones are flaky) fails fast and
|
||||||
|
# we fall through to the next / to the yt-dlp fallback instead of hanging.
|
||||||
|
r = requests.get(url, timeout=4, headers={"User-Agent": _UA, "Accept": "application/json"})
|
||||||
|
return r.json() if r.status_code == 200 else None
|
||||||
|
|
||||||
|
|
||||||
|
def _harvest(kind, base, cid, pages, fetch):
|
||||||
|
"""Walk one instance's channel pages, accumulating {video_id: date}. Follows
|
||||||
|
pagination even when an early page is empty-but-has-a-next-token (some Piped
|
||||||
|
instances return an empty first page), stopping when there's no token left."""
|
||||||
|
from urllib.parse import quote
|
||||||
|
out, token = {}, None
|
||||||
|
for i in range(max(1, pages)):
|
||||||
|
if kind == "piped":
|
||||||
|
url = (base + "/channel/" + cid) if token is None \
|
||||||
|
else (base + "/nextpage/channel/" + cid + "?nextpage=" + quote(token, safe=""))
|
||||||
|
else: # invidious
|
||||||
|
url = base + "/api/v1/channels/" + cid + "/videos" + \
|
||||||
|
(("?continuation=" + quote(token, safe="")) if token else "")
|
||||||
|
obj = _proxy_get(url, fetch)
|
||||||
|
if not isinstance(obj, dict):
|
||||||
|
break
|
||||||
|
out.update(parse_proxy_dates(obj))
|
||||||
|
token = obj.get("nextpage") if kind == "piped" else obj.get("continuation")
|
||||||
|
if not token:
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def proxy_channel_dates(channel_id, pages=6, fetch=None, instances=None):
|
||||||
|
"""Bulk upload dates for a whole channel via a no-key proxy (Piped/Invidious),
|
||||||
|
paginated. Tries instances until one answers. {video_id: 'YYYY-MM-DD'} (empty
|
||||||
|
if all are down → caller falls back to per-video yt-dlp)."""
|
||||||
|
cid = str(channel_id or "").strip()
|
||||||
|
if not cid:
|
||||||
|
return {}
|
||||||
|
for kind, base in (instances or _PROXY_INSTANCES):
|
||||||
|
try:
|
||||||
|
dates = _harvest(kind, base, cid, pages, fetch)
|
||||||
|
if dates:
|
||||||
|
return dates
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def parse_rss_dates(xml_text):
|
||||||
|
"""{video_id: 'YYYY-MM-DD'} from a YouTube channel RSS feed (pure, testable)."""
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
out = {}
|
||||||
|
try:
|
||||||
|
root = ET.fromstring(xml_text)
|
||||||
|
except Exception:
|
||||||
|
return out
|
||||||
|
ns = {"a": "http://www.w3.org/2005/Atom", "yt": "http://www.youtube.com/xml/schemas/2015"}
|
||||||
|
for entry in root.findall("a:entry", ns):
|
||||||
|
vid = entry.find("yt:videoId", ns)
|
||||||
|
pub = entry.find("a:published", ns)
|
||||||
|
if vid is not None and vid.text and pub is not None and pub.text:
|
||||||
|
out[vid.text.strip()] = pub.text.strip()[:10] # ISO datetime → YYYY-MM-DD
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def channel_recent_dates(channel_id, fetch=None):
|
||||||
|
"""Real upload dates for a channel's ~15 most-recent videos via its public RSS
|
||||||
|
feed — one cheap GET, no yt-dlp, no bot risk. {video_id: 'YYYY-MM-DD'}."""
|
||||||
|
cid = str(channel_id or "").strip()
|
||||||
|
if not cid:
|
||||||
|
return {}
|
||||||
|
url = "https://www.youtube.com/feeds/videos.xml?channel_id=" + cid
|
||||||
|
try:
|
||||||
|
if fetch is not None:
|
||||||
|
return parse_rss_dates(fetch(url) or "")
|
||||||
|
import requests
|
||||||
|
r = requests.get(url, timeout=10, headers={"User-Agent": _UA})
|
||||||
|
return parse_rss_dates(r.text) if r.status_code == 200 else {}
|
||||||
|
except Exception as e:
|
||||||
|
logger.info("YouTube RSS dates failed for %s: %s", cid, e)
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
# ── InnerTube: YouTube's own browse API (no key/Java/proxy) ──────────────────
|
||||||
|
# Same technique NewPipe/yt-dlp use. We call the channel "Videos" tab and read the
|
||||||
|
# lockupViewModel items: contentId (video id) + a relative "N units ago" string.
|
||||||
|
# Relative → approximate date (fine for YEAR grouping). One request per ~30 videos,
|
||||||
|
# paginated via continuation tokens (light on rate limits). Parsing is pure +
|
||||||
|
# resilient (recursive search rather than a brittle fixed path).
|
||||||
|
_INNERTUBE_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8" # public WEB key (stable for years)
|
||||||
|
_INNERTUBE_CTX = {"client": {"clientName": "WEB", "clientVersion": "2.20240304.00.00", "hl": "en", "gl": "US"}}
|
||||||
|
_VIDEOS_PARAMS = "EgZ2aWRlb3PyBgQKAjoA" # selects a channel's "Videos" tab
|
||||||
|
_INNERTUBE_PAGES = 8
|
||||||
|
_INNERTUBE_DELAY = 0.6 # politeness pause between pages
|
||||||
|
_REL_UNIT_DAYS = {"second": 0, "minute": 0, "hour": 0, "day": 1, "week": 7, "month": 30.44, "year": 365.25}
|
||||||
|
_REL_RE = re.compile(r"(\d+)\s+(second|minute|hour|day|week|month|year)s?\s+ago", re.I)
|
||||||
|
|
||||||
|
|
||||||
|
def _json_find_all(obj, key, acc):
|
||||||
|
if isinstance(obj, dict):
|
||||||
|
if key in obj:
|
||||||
|
acc.append(obj[key])
|
||||||
|
for v in obj.values():
|
||||||
|
_json_find_all(v, key, acc)
|
||||||
|
elif isinstance(obj, list):
|
||||||
|
for v in obj:
|
||||||
|
_json_find_all(v, key, acc)
|
||||||
|
return acc
|
||||||
|
|
||||||
|
|
||||||
|
def _json_content_strings(obj, acc):
|
||||||
|
if isinstance(obj, dict):
|
||||||
|
c = obj.get("content")
|
||||||
|
if isinstance(c, str):
|
||||||
|
acc.append(c)
|
||||||
|
for v in obj.values():
|
||||||
|
_json_content_strings(v, acc)
|
||||||
|
elif isinstance(obj, list):
|
||||||
|
for v in obj:
|
||||||
|
_json_content_strings(v, acc)
|
||||||
|
return acc
|
||||||
|
|
||||||
|
|
||||||
|
def relative_to_date(text, now=None):
|
||||||
|
"""'2 years ago' / '9 hours ago' → approximate 'YYYY-MM-DD', or None."""
|
||||||
|
m = _REL_RE.search(text or "")
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
if now is None:
|
||||||
|
now = datetime.now(timezone.utc).date()
|
||||||
|
days = int(m.group(1)) * _REL_UNIT_DAYS[m.group(2).lower()]
|
||||||
|
return (now - timedelta(days=round(days))).isoformat()
|
||||||
|
|
||||||
|
|
||||||
|
def innertube_parse_videos(obj):
|
||||||
|
"""[(video_id, relative_text)] for VIDEO lockups in an InnerTube browse response."""
|
||||||
|
out, seen = [], set()
|
||||||
|
for lk in _json_find_all(obj, "lockupViewModel", []):
|
||||||
|
if not isinstance(lk, dict) or lk.get("contentType") != "LOCKUP_CONTENT_TYPE_VIDEO":
|
||||||
|
continue
|
||||||
|
vid = lk.get("contentId")
|
||||||
|
if not vid or vid in seen:
|
||||||
|
continue
|
||||||
|
rel = next((t for t in _json_content_strings(lk.get("metadata"), []) if _REL_RE.search(t)), None)
|
||||||
|
if rel:
|
||||||
|
seen.add(vid)
|
||||||
|
out.append((vid, rel))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def innertube_continuation(obj):
|
||||||
|
"""The pagination continuation token (from a continuationItemRenderer), or None."""
|
||||||
|
for cir in _json_find_all(obj, "continuationItemRenderer", []):
|
||||||
|
if isinstance(cir, dict):
|
||||||
|
tok = (cir.get("continuationEndpoint") or {}).get("continuationCommand", {}).get("token")
|
||||||
|
if tok:
|
||||||
|
return tok
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _innertube_post(payload, post):
|
||||||
|
if post is not None:
|
||||||
|
return post(payload)
|
||||||
|
import requests
|
||||||
|
r = requests.post("https://www.youtube.com/youtubei/v1/browse", params={"key": _INNERTUBE_KEY},
|
||||||
|
json=payload, timeout=10,
|
||||||
|
headers={"User-Agent": _UA, "Content-Type": "application/json", "Accept-Language": "en-US"})
|
||||||
|
return r.json() if (r.status_code == 200 and r.headers.get("content-type", "").startswith("application/json")) else None
|
||||||
|
|
||||||
|
|
||||||
|
def innertube_channel_dates(channel_id, pages=_INNERTUBE_PAGES, now=None, post=None):
|
||||||
|
"""{video_id: 'YYYY-MM-DD'} for a channel's videos via YouTube's own InnerTube
|
||||||
|
browse API — no key/Java/proxy, ~1 request per 30 videos. Dates are APPROXIMATE
|
||||||
|
(from relative text), which is fine for year-seasons; the exact yt-dlp path can
|
||||||
|
refine specific videos later. Bounded + throttled. {} on any failure (→ caller
|
||||||
|
falls back)."""
|
||||||
|
cid = str(channel_id or "").strip()
|
||||||
|
if not cid.startswith("UC"):
|
||||||
|
return {}
|
||||||
|
if now is None:
|
||||||
|
now = datetime.now(timezone.utc).date()
|
||||||
|
out = {}
|
||||||
|
payload = {"context": _INNERTUBE_CTX, "browseId": cid, "params": _VIDEOS_PARAMS}
|
||||||
|
for _ in range(max(1, pages)):
|
||||||
|
try:
|
||||||
|
j = _innertube_post(payload, post)
|
||||||
|
except Exception:
|
||||||
|
break
|
||||||
|
if not isinstance(j, dict):
|
||||||
|
break
|
||||||
|
for vid, rel in innertube_parse_videos(j):
|
||||||
|
if vid not in out:
|
||||||
|
d = relative_to_date(rel, now)
|
||||||
|
if d:
|
||||||
|
out[vid] = d
|
||||||
|
token = innertube_continuation(j)
|
||||||
|
if not token:
|
||||||
|
break
|
||||||
|
payload = {"context": _INNERTUBE_CTX, "continuation": token}
|
||||||
|
if post is None:
|
||||||
|
time.sleep(_INNERTUBE_DELAY) # rate-limit politeness
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _lockup_title(lk):
|
||||||
|
"""The video title from a lockupViewModel (metadata.title.content)."""
|
||||||
|
for md in _json_find_all(lk, "lockupMetadataViewModel", []):
|
||||||
|
if isinstance(md, dict) and isinstance(md.get("title"), dict):
|
||||||
|
c = md["title"].get("content")
|
||||||
|
if isinstance(c, str) and c:
|
||||||
|
return c
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _lockup_thumb(lk):
|
||||||
|
"""The first thumbnail url from a lockupViewModel (contentImage…sources[0].url)."""
|
||||||
|
for srcs in _json_find_all(lk.get("contentImage") or {}, "sources", []):
|
||||||
|
if isinstance(srcs, list) and srcs and isinstance(srcs[0], dict) and srcs[0].get("url"):
|
||||||
|
return srcs[0]["url"]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
_DUR_RE = re.compile(r"^\d{1,2}:\d{2}(?::\d{2})?$") # 12:34 / 1:02:03 (the overlay badge)
|
||||||
|
_VIEWS_RE = re.compile(r"([\d.,]+)\s*([KMB]?)\s+views?", re.I) # "2.6M views", "1,234 views"
|
||||||
|
_VIEW_MULT = {"": 1, "K": 1e3, "M": 1e6, "B": 1e9}
|
||||||
|
|
||||||
|
|
||||||
|
def _json_all_strings(obj, acc):
|
||||||
|
if isinstance(obj, dict):
|
||||||
|
for v in obj.values():
|
||||||
|
_json_all_strings(v, acc)
|
||||||
|
elif isinstance(obj, list):
|
||||||
|
for v in obj:
|
||||||
|
_json_all_strings(v, acc)
|
||||||
|
elif isinstance(obj, str):
|
||||||
|
acc.append(obj)
|
||||||
|
return acc
|
||||||
|
|
||||||
|
|
||||||
|
def parse_view_count(text):
|
||||||
|
"""'2.6M views' → 2600000, '1,234 views' → 1234, else None (approximate)."""
|
||||||
|
m = _VIEWS_RE.search(text or "")
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return int(float(m.group(1).replace(",", "")) * _VIEW_MULT.get(m.group(2).upper(), 1))
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _lockup_duration(lk):
|
||||||
|
"""The duration overlay badge ('12:34') from a lockupViewModel, or None."""
|
||||||
|
return next((s for s in _json_all_strings(lk, []) if _DUR_RE.match(s)), None)
|
||||||
|
|
||||||
|
|
||||||
|
def _lockup_views(lk):
|
||||||
|
"""Approximate view count parsed from the lockup's metadata text, or None."""
|
||||||
|
return next((v for v in (parse_view_count(s) for s in _json_content_strings(lk.get("metadata"), []))
|
||||||
|
if v is not None), None)
|
||||||
|
|
||||||
|
|
||||||
|
def innertube_parse_video_items(obj):
|
||||||
|
"""Full per-video metadata for VIDEO lockups in an InnerTube browse/continuation
|
||||||
|
response: [{youtube_id, title, thumbnail_url, duration, view_count, relative}].
|
||||||
|
Same source as innertube_parse_videos but keeps the title/thumbnail/duration/
|
||||||
|
views so the channel page can list the whole catalog (not just date them)."""
|
||||||
|
out, seen = [], set()
|
||||||
|
for lk in _json_find_all(obj, "lockupViewModel", []):
|
||||||
|
if not isinstance(lk, dict) or lk.get("contentType") != "LOCKUP_CONTENT_TYPE_VIDEO":
|
||||||
|
continue
|
||||||
|
vid = lk.get("contentId")
|
||||||
|
if not vid or vid in seen:
|
||||||
|
continue
|
||||||
|
seen.add(vid)
|
||||||
|
rel = next((t for t in _json_content_strings(lk.get("metadata"), []) if _REL_RE.search(t)), None)
|
||||||
|
out.append({"youtube_id": vid, "title": _lockup_title(lk), "thumbnail_url": _lockup_thumb(lk),
|
||||||
|
"duration": _lockup_duration(lk), "view_count": _lockup_views(lk), "relative": rel})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def innertube_channel_videos_page(channel_id, continuation=None, now=None, post=None):
|
||||||
|
"""ONE InnerTube page of a channel's videos (with metadata) + the next token:
|
||||||
|
{"videos": [{youtube_id, title, thumbnail_url, published_at}], "continuation": token|None}.
|
||||||
|
Stateless — hand the returned token back to fetch the next page, so the caller
|
||||||
|
can stream the FULL catalog in batches (O(n), each page fetched once). {} fields
|
||||||
|
/ None token on any failure."""
|
||||||
|
cid = str(channel_id or "").strip()
|
||||||
|
if not continuation and not cid.startswith("UC"):
|
||||||
|
return {"videos": [], "continuation": None}
|
||||||
|
if now is None:
|
||||||
|
now = datetime.now(timezone.utc).date()
|
||||||
|
payload = ({"context": _INNERTUBE_CTX, "continuation": continuation} if continuation
|
||||||
|
else {"context": _INNERTUBE_CTX, "browseId": cid, "params": _VIDEOS_PARAMS})
|
||||||
|
try:
|
||||||
|
j = _innertube_post(payload, post)
|
||||||
|
except Exception:
|
||||||
|
return {"videos": [], "continuation": None}
|
||||||
|
if not isinstance(j, dict):
|
||||||
|
return {"videos": [], "continuation": None}
|
||||||
|
videos = []
|
||||||
|
for it in innertube_parse_video_items(j):
|
||||||
|
it["published_at"] = relative_to_date(it.pop("relative", None), now)
|
||||||
|
videos.append(it)
|
||||||
|
return {"videos": videos, "continuation": innertube_continuation(j)}
|
||||||
|
|
||||||
|
|
||||||
|
def _innertube_playlist_total(obj):
|
||||||
|
"""The playlist's TRUE video count from its browse header ('512 videos'), or None.
|
||||||
|
Reliable even when yt-dlp's playlist_count isn't populated."""
|
||||||
|
for t in _json_find_all(obj, "numVideosText", []):
|
||||||
|
m = re.search(r"([\d,]+)", "".join(_json_all_strings(t, [])))
|
||||||
|
if m:
|
||||||
|
try:
|
||||||
|
return int(m.group(1).replace(",", ""))
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def innertube_playlist_page(playlist_id, continuation=None, now=None, post=None):
|
||||||
|
"""One InnerTube page of a PLAYLIST's videos (browseId VL<id>) in the curator's
|
||||||
|
order: {"videos": [...], "continuation": token|None, "total": N|None}. Fixes
|
||||||
|
yt-dlp flat's ~100-item cap (browse pages to ~200) and exposes the real count.
|
||||||
|
Same item shape as innertube_channel_videos_page."""
|
||||||
|
pid = str(playlist_id or "").strip()
|
||||||
|
if not continuation and not pid:
|
||||||
|
return {"videos": [], "continuation": None, "total": None}
|
||||||
|
if now is None:
|
||||||
|
now = datetime.now(timezone.utc).date()
|
||||||
|
payload = ({"context": _INNERTUBE_CTX, "continuation": continuation} if continuation
|
||||||
|
else {"context": _INNERTUBE_CTX, "browseId": "VL" + pid})
|
||||||
|
try:
|
||||||
|
j = _innertube_post(payload, post)
|
||||||
|
except Exception:
|
||||||
|
return {"videos": [], "continuation": None, "total": None}
|
||||||
|
if not isinstance(j, dict):
|
||||||
|
return {"videos": [], "continuation": None, "total": None}
|
||||||
|
videos = []
|
||||||
|
for it in innertube_parse_video_items(j):
|
||||||
|
it["published_at"] = relative_to_date(it.pop("relative", None), now)
|
||||||
|
videos.append(it)
|
||||||
|
return {"videos": videos, "continuation": innertube_continuation(j), "total": _innertube_playlist_total(j)}
|
||||||
|
|
||||||
|
|
||||||
|
def innertube_playlist_catalog(playlist_id, pages=40, now=None, post=None):
|
||||||
|
"""A playlist's video list (curator order) + TRUE total via InnerTube:
|
||||||
|
{"videos": [...], "total": N|None}. yt-dlp flat caps at ~100; the browse pages
|
||||||
|
to ~200 and the header carries the real count. {} on failure."""
|
||||||
|
pid = str(playlist_id or "").strip()
|
||||||
|
if not pid:
|
||||||
|
return {"videos": [], "total": None}
|
||||||
|
if now is None:
|
||||||
|
now = datetime.now(timezone.utc).date()
|
||||||
|
out, seen, token, total = [], set(), None, None
|
||||||
|
for _ in range(max(1, pages)):
|
||||||
|
page = innertube_playlist_page(pid, continuation=token, now=now, post=post)
|
||||||
|
if total is None:
|
||||||
|
total = page.get("total")
|
||||||
|
for v in page.get("videos") or []:
|
||||||
|
if v.get("youtube_id") and v["youtube_id"] not in seen:
|
||||||
|
seen.add(v["youtube_id"])
|
||||||
|
out.append(v)
|
||||||
|
token = page.get("continuation")
|
||||||
|
if not token:
|
||||||
|
break
|
||||||
|
if post is None:
|
||||||
|
time.sleep(_INNERTUBE_DELAY)
|
||||||
|
return {"videos": out, "total": total}
|
||||||
|
|
||||||
|
|
||||||
|
def innertube_channel_catalog(channel_id, pages=_INNERTUBE_PAGES, now=None, post=None):
|
||||||
|
"""A channel's video catalog (list + approximate dates) via InnerTube, up to
|
||||||
|
``pages`` pages: [{youtube_id, title, thumbnail_url, published_at}]. Like
|
||||||
|
innertube_channel_dates but keeps title/thumbnail so callers can REMEMBER the
|
||||||
|
whole list, not just date it. Bounded + throttled; [] on any failure."""
|
||||||
|
cid = str(channel_id or "").strip()
|
||||||
|
if not cid.startswith("UC"):
|
||||||
|
return []
|
||||||
|
if now is None:
|
||||||
|
now = datetime.now(timezone.utc).date()
|
||||||
|
out, seen, token = [], set(), None
|
||||||
|
for _ in range(max(1, pages)):
|
||||||
|
page = innertube_channel_videos_page(cid, continuation=token, now=now, post=post)
|
||||||
|
for v in page.get("videos") or []:
|
||||||
|
if v.get("youtube_id") and v["youtube_id"] not in seen:
|
||||||
|
seen.add(v["youtube_id"])
|
||||||
|
out.append(v)
|
||||||
|
token = page.get("continuation")
|
||||||
|
if not token:
|
||||||
|
break
|
||||||
|
if post is None:
|
||||||
|
time.sleep(_INNERTUBE_DELAY) # rate-limit politeness
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def search_channels(query, limit=6, ydl_factory=None):
|
||||||
|
"""Search YouTube for CHANNELS (the results page filtered to channels) → a few
|
||||||
|
{youtube_id, title, handle, avatar_url, subscriber_count} for the search page.
|
||||||
|
Best-effort; entries that aren't channels are skipped."""
|
||||||
|
from urllib.parse import quote
|
||||||
|
q = (query or "").strip()
|
||||||
|
if not q:
|
||||||
|
return []
|
||||||
|
# sp=EgIQAg%3D%3D is YouTube's "Type: Channel" search filter.
|
||||||
|
url = "https://www.youtube.com/results?search_query=" + quote(q) + "&sp=EgIQAg%3D%3D"
|
||||||
|
info = _extract(url, _ydl_opts(limit * 3), ydl_factory) # over-fetch; some entries may be videos
|
||||||
|
out = []
|
||||||
|
for e in [x for x in ((info or {}).get("entries") or []) if isinstance(x, dict)]:
|
||||||
|
cid = e.get("channel_id")
|
||||||
|
if not cid and str(e.get("id", "")).startswith("UC"):
|
||||||
|
cid = e.get("id")
|
||||||
|
if not cid:
|
||||||
|
m = re.search(r"/channel/(UC[\w-]+)", e.get("url") or "")
|
||||||
|
if m:
|
||||||
|
cid = m.group(1)
|
||||||
|
if not cid or not str(cid).startswith("UC"):
|
||||||
|
continue
|
||||||
|
title = e.get("channel") or e.get("title") or e.get("uploader") or ""
|
||||||
|
if not title:
|
||||||
|
continue
|
||||||
|
uid = e.get("uploader_id")
|
||||||
|
out.append({
|
||||||
|
"youtube_id": cid,
|
||||||
|
"title": title,
|
||||||
|
"handle": uid if str(uid or "").startswith("@") else None,
|
||||||
|
"avatar_url": _best_thumb(e.get("thumbnails")) or e.get("thumbnail"),
|
||||||
|
"subscriber_count": e.get("channel_follower_count"),
|
||||||
|
"video_count": e.get("playlist_count"),
|
||||||
|
})
|
||||||
|
if len(out) >= limit:
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def channel_playlists(channel_id, limit=50, ydl_factory=None):
|
||||||
|
"""The channel's playlists (flat) → [{playlist_id, title, video_count,
|
||||||
|
thumbnail_url}] — rendered as "seasons" on the channel page. Lazy-loaded."""
|
||||||
|
if not channel_id:
|
||||||
|
return []
|
||||||
|
url = "https://www.youtube.com/channel/" + str(channel_id) + "/playlists"
|
||||||
|
info = _extract(url, _ydl_opts(limit), ydl_factory)
|
||||||
|
out = []
|
||||||
|
for e in [x for x in ((info or {}).get("entries") or []) if isinstance(x, dict)][:limit]:
|
||||||
|
pid = e.get("id")
|
||||||
|
if not pid:
|
||||||
|
continue
|
||||||
|
out.append({
|
||||||
|
"playlist_id": pid,
|
||||||
|
"title": e.get("title") or "",
|
||||||
|
"video_count": e.get("playlist_count") or e.get("video_count"),
|
||||||
|
"thumbnail_url": _best_thumb(e.get("thumbnails")) or e.get("thumbnail"),
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def playlist_videos(playlist_id, limit=50, ydl_factory=None):
|
||||||
|
"""A playlist's videos (flat), in the same shape as channel uploads."""
|
||||||
|
if not playlist_id:
|
||||||
|
return []
|
||||||
|
url = "https://www.youtube.com/playlist?list=" + str(playlist_id)
|
||||||
|
info = _extract(url, _ydl_opts(limit), ydl_factory)
|
||||||
|
return _shape_entries((info or {}).get("entries"), limit)
|
||||||
|
|
||||||
|
|
||||||
|
_PLAYLIST_RE = re.compile(r"[?&]list=([A-Za-z0-9_-]+)")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_playlist_id(raw):
|
||||||
|
"""A followable playlist id from a URL or bare id, or None. Rejects mixes/radios
|
||||||
|
(RD…) and the personal Watch-Later/Liked lists, which aren't real playlists."""
|
||||||
|
raw = (raw or "").strip()
|
||||||
|
m = _PLAYLIST_RE.search(raw)
|
||||||
|
pid = m.group(1) if m else (raw if re.match(r"^(PL|OL|FL|UU)[A-Za-z0-9_-]{8,}$", raw) else None)
|
||||||
|
if not pid or pid.startswith(("RD", "UL", "LL", "WL")):
|
||||||
|
return None
|
||||||
|
return pid
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_playlist(raw, limit=100, ydl_factory=None, db=None):
|
||||||
|
"""Resolve a pasted playlist reference → ``{playlist_id, title, channel_title,
|
||||||
|
video_count, thumbnail_url, videos:[...]}``, or None if it isn't a real
|
||||||
|
followable playlist. Order is the curator's (NOT date) — playlists are
|
||||||
|
deliberately a partial, ordered set."""
|
||||||
|
pid = parse_playlist_id(raw)
|
||||||
|
if not pid:
|
||||||
|
return None
|
||||||
|
info = _extract("https://www.youtube.com/playlist?list=" + pid, _ydl_opts(limit, db), ydl_factory)
|
||||||
|
if not info:
|
||||||
|
return None
|
||||||
|
videos = _shape_entries(info.get("entries"), limit)
|
||||||
|
return {
|
||||||
|
"playlist_id": pid,
|
||||||
|
"title": info.get("title") or "Playlist",
|
||||||
|
"channel_title": info.get("channel") or info.get("uploader") or "",
|
||||||
|
"channel_id": info.get("channel_id") or info.get("uploader_id"),
|
||||||
|
"video_count": info.get("playlist_count") or len(videos),
|
||||||
|
"thumbnail_url": _best_thumb(info.get("thumbnails")) or (videos[0]["thumbnail_url"] if videos else None),
|
||||||
|
"videos": videos,
|
||||||
|
}
|
||||||
453
core/video/youtube_download.py
Normal file
453
core/video/youtube_download.py
Normal file
|
|
@ -0,0 +1,453 @@
|
||||||
|
"""YouTube download worker — the fulfillment lane for wished YouTube videos.
|
||||||
|
|
||||||
|
Model B (Boulder): YouTube grabs flow through the SAME ``video_downloads`` queue +
|
||||||
|
history as movies/TV, so the Downloads page, live progress, and History modal all work
|
||||||
|
for YouTube for free. But the mechanism is different — there's no slskd transfer to poll;
|
||||||
|
yt-dlp fetches the stream directly. So this worker owns a YouTube download end to end:
|
||||||
|
|
||||||
|
pick stream (quality profile → yt-dlp format) → download into the library, organised
|
||||||
|
as a Plex "TV by date" show (channel/Season YEAR/channel - DATE - title) → mark the
|
||||||
|
row completed + archive it to history → remove the video from the wishlist.
|
||||||
|
|
||||||
|
The slskd ``download_monitor`` simply SKIPS ``source='youtube'`` rows (they have no
|
||||||
|
transfer to match), so this lane never disturbs the movie/TV pipeline.
|
||||||
|
|
||||||
|
The orchestration (``process_youtube_download``) is PURE — the actual yt-dlp run and all
|
||||||
|
DB writes are injected seams — so the lifecycle (dest planning, completion → archive +
|
||||||
|
unwish, failure → archive, no unwish) is unit-tested without a network or a DB. Production
|
||||||
|
(``run_youtube_download``) lazily binds the real calls and runs it on a worker thread.
|
||||||
|
|
||||||
|
Isolated: imports only sibling ``core.video`` modules; nothing from the music side.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from typing import Any, Callable, Dict, Optional
|
||||||
|
|
||||||
|
from core.video import organization, youtube_quality
|
||||||
|
from core.video.youtube_quality import format_selection
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
try:
|
||||||
|
import yt_dlp
|
||||||
|
except Exception: # noqa: BLE001 - optional at import; absence handled at call time
|
||||||
|
yt_dlp = None
|
||||||
|
|
||||||
|
|
||||||
|
def youtube_fields_from_download(dl: Dict[str, Any]) -> Dict[str, Any]:
|
||||||
|
"""Organising fields for a YouTube download row. The channel/video-title/date that
|
||||||
|
``render_path('youtube', …)`` needs ride in ``search_ctx`` (a generic JSON column);
|
||||||
|
fall back to the row's own columns when absent."""
|
||||||
|
ctx = dl.get("search_ctx")
|
||||||
|
if isinstance(ctx, str):
|
||||||
|
try:
|
||||||
|
ctx = json.loads(ctx)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
ctx = {}
|
||||||
|
if not isinstance(ctx, dict):
|
||||||
|
ctx = {}
|
||||||
|
return {
|
||||||
|
"channel": ctx.get("channel") or dl.get("title"),
|
||||||
|
"title": ctx.get("video_title") or dl.get("title"),
|
||||||
|
"published_at": ctx.get("published_at") or dl.get("year"),
|
||||||
|
"youtube_id": dl.get("media_id"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def quality_override_from_download(dl: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||||
|
"""A per-channel quality override stashed in the row's ``search_ctx`` at enqueue time,
|
||||||
|
or None (use the global YouTube quality profile)."""
|
||||||
|
ctx = dl.get("search_ctx")
|
||||||
|
if isinstance(ctx, str):
|
||||||
|
try:
|
||||||
|
ctx = json.loads(ctx)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
ctx = {}
|
||||||
|
ctx = ctx if isinstance(ctx, dict) else {}
|
||||||
|
q = ctx.get("quality")
|
||||||
|
return q if isinstance(q, dict) else None
|
||||||
|
|
||||||
|
|
||||||
|
def plan_destination(dl: Dict[str, Any], settings: Dict[str, Any], container: str) -> Dict[str, str]:
|
||||||
|
"""Where this video lands in the library: ``{dir, filename, path}`` under the youtube
|
||||||
|
root (``target_dir``), organised by the youtube template. Pure."""
|
||||||
|
ext = "." + str(container or "mp4").lstrip(".")
|
||||||
|
return organization.render_path("youtube", dl.get("target_dir"),
|
||||||
|
youtube_fields_from_download(dl), settings, ext)
|
||||||
|
|
||||||
|
|
||||||
|
def ydl_download_opts(profile: Any, dest_dir: str, dest_stem: str,
|
||||||
|
*, progress_hook: Optional[Callable] = None, postprocess_hook: Optional[Callable] = None,
|
||||||
|
cookie_opts: Optional[dict] = None) -> dict:
|
||||||
|
"""The yt-dlp options dict for one download: format selection from the quality profile,
|
||||||
|
a fixed output path (dir + stem + yt-dlp's own ext), polite defaults. Pure."""
|
||||||
|
sel = format_selection(profile)
|
||||||
|
opts = {
|
||||||
|
"quiet": True,
|
||||||
|
"no_warnings": True,
|
||||||
|
"noplaylist": True,
|
||||||
|
"retries": 3,
|
||||||
|
"format": sel["format"],
|
||||||
|
"format_sort": sel["format_sort"],
|
||||||
|
"merge_output_format": sel["merge_output_format"],
|
||||||
|
"paths": {"home": str(dest_dir or "")},
|
||||||
|
"outtmpl": dest_stem + ".%(ext)s",
|
||||||
|
# sidecars: the episode thumbnail (→ '<name>-thumb.jpg' on import) + the metadata
|
||||||
|
# json we mine for the .nfo (description / duration). ffmpeg (already needed to merge)
|
||||||
|
# normalises the thumbnail to jpg.
|
||||||
|
"writethumbnail": True,
|
||||||
|
"writeinfojson": True,
|
||||||
|
"postprocessors": [{"key": "FFmpegThumbnailsConvertor", "format": "jpg"}],
|
||||||
|
}
|
||||||
|
if cookie_opts:
|
||||||
|
opts.update(cookie_opts)
|
||||||
|
if progress_hook:
|
||||||
|
opts["progress_hooks"] = [progress_hook]
|
||||||
|
if postprocess_hook:
|
||||||
|
opts["postprocessor_hooks"] = [postprocess_hook] # fires while ffmpeg merges/converts
|
||||||
|
return opts
|
||||||
|
|
||||||
|
|
||||||
|
def _stem_and_container(dest: Dict[str, str], container: str) -> tuple:
|
||||||
|
"""(filename stem, final ext) — strip the ext render_path put on the filename so
|
||||||
|
yt-dlp can own the extension during the merge."""
|
||||||
|
fn = dest.get("filename") or "download"
|
||||||
|
cont = str(container or "mp4").lstrip(".")
|
||||||
|
stem = fn[:-(len(cont) + 1)] if fn.lower().endswith("." + cont.lower()) else os.path.splitext(fn)[0]
|
||||||
|
return stem or "download", cont
|
||||||
|
|
||||||
|
|
||||||
|
def download_one(video_id: Any, dest_dir: str, dest_stem: str, profile: Any, container: str,
|
||||||
|
*, ydl_factory=None, progress_hook=None, postprocess_hook=None, cookie_opts=None) -> Dict[str, Any]:
|
||||||
|
"""Run yt-dlp for ONE video into ``dest_dir/dest_stem.ext``. Returns
|
||||||
|
``{ok, dest_path|None, error|None}``. The yt-dlp class is injectable for tests."""
|
||||||
|
vid = str(video_id or "").strip()
|
||||||
|
if not vid:
|
||||||
|
return {"ok": False, "dest_path": None, "error": "No video id"}
|
||||||
|
factory = ydl_factory or (yt_dlp.YoutubeDL if yt_dlp else None)
|
||||||
|
if factory is None:
|
||||||
|
return {"ok": False, "dest_path": None, "error": "yt-dlp unavailable"}
|
||||||
|
opts = ydl_download_opts(profile, dest_dir, dest_stem, progress_hook=progress_hook,
|
||||||
|
postprocess_hook=postprocess_hook, cookie_opts=cookie_opts)
|
||||||
|
url = vid if vid.startswith("http") else "https://www.youtube.com/watch?v=" + vid
|
||||||
|
try:
|
||||||
|
with factory(opts) as ydl:
|
||||||
|
ydl.download([url])
|
||||||
|
except Exception as e: # noqa: BLE001 - any yt-dlp failure → a failed download, not a crash
|
||||||
|
logger.info("youtube download failed for %s: %s", vid, e)
|
||||||
|
return {"ok": False, "dest_path": None, "error": str(e)}
|
||||||
|
dest_path = os.path.join(str(dest_dir or ""), dest_stem + "." + str(container or "mp4").lstrip("."))
|
||||||
|
return {"ok": True, "dest_path": dest_path, "error": None}
|
||||||
|
|
||||||
|
|
||||||
|
def _default_move(src: str, dest: str) -> None:
|
||||||
|
"""Move a finished staged file into the library, creating the target folders."""
|
||||||
|
os.makedirs(os.path.dirname(dest) or ".", exist_ok=True)
|
||||||
|
shutil.move(src, dest)
|
||||||
|
|
||||||
|
|
||||||
|
def build_episode_nfo(fields: Dict[str, Any], *, description: Any = None, runtime: Any = None) -> str:
|
||||||
|
"""A Jellyfin/Kodi/Plex ``<episodedetails>`` sidecar for a YouTube 'episode'. Pure — the
|
||||||
|
description/runtime come from yt-dlp's info json. season = upload year, episode = MMDD
|
||||||
|
(Plex 'by date' matches on ``<aired>``; the numbers help Jellyfin/Kodi)."""
|
||||||
|
from xml.sax.saxutils import escape
|
||||||
|
date = str(fields.get("published_at") or "")[:10]
|
||||||
|
year = date[:4]
|
||||||
|
out = ['<?xml version="1.0" encoding="UTF-8"?>', '<episodedetails>',
|
||||||
|
' <title>%s</title>' % escape(str(fields.get("title") or fields.get("channel") or "Video"))]
|
||||||
|
if year.isdigit():
|
||||||
|
out.append(' <season>%s</season>' % year)
|
||||||
|
if len(date) == 10 and date[5:7].isdigit() and date[8:10].isdigit():
|
||||||
|
out.append(' <episode>%d</episode>' % int(date[5:7] + date[8:10]))
|
||||||
|
if description:
|
||||||
|
out.append(' <plot>%s</plot>' % escape(str(description)))
|
||||||
|
if date:
|
||||||
|
out.append(' <aired>%s</aired>' % escape(date))
|
||||||
|
if fields.get("channel"):
|
||||||
|
out.append(' <studio>%s</studio>' % escape(str(fields["channel"])))
|
||||||
|
if fields.get("youtube_id"):
|
||||||
|
out.append(' <uniqueid type="youtube" default="true">%s</uniqueid>' % escape(str(fields["youtube_id"])))
|
||||||
|
try:
|
||||||
|
if runtime:
|
||||||
|
out.append(' <runtime>%d</runtime>' % round(float(runtime) / 60))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
out.append('</episodedetails>')
|
||||||
|
return "\n".join(out) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def _silent_remove(path: str) -> None:
|
||||||
|
try:
|
||||||
|
os.remove(path)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _default_sidecars(staged_video: str, final_video: str, fields: Dict[str, Any],
|
||||||
|
settings: Dict[str, Any]) -> None:
|
||||||
|
"""Place the YouTube episode's sidecars next to the imported video — gated by the SAME
|
||||||
|
post-processing toggles as the movie/TV side: ``save_artwork`` → ``<name>-thumb.jpg``
|
||||||
|
(server episode art), ``write_nfo`` → ``<name>.nfo`` (metadata). yt-dlp dropped a
|
||||||
|
thumbnail + ``.info.json`` next to the staged video; we always clean those up (move the
|
||||||
|
wanted ones into the library, delete the rest), so nothing litters the download folder
|
||||||
|
when a toggle is off. Best-effort — never fails the grab."""
|
||||||
|
settings = settings if isinstance(settings, dict) else {}
|
||||||
|
want_thumb, want_nfo = bool(settings.get("save_artwork")), bool(settings.get("write_nfo"))
|
||||||
|
try:
|
||||||
|
src_dir, src_stem = os.path.dirname(staged_video), os.path.splitext(os.path.basename(staged_video))[0]
|
||||||
|
dst_dir, dst_stem = os.path.dirname(final_video), os.path.splitext(os.path.basename(final_video))[0]
|
||||||
|
# thumbnail: keep as -thumb when wanted, else discard the staged copy
|
||||||
|
for ext in (".jpg", ".jpeg", ".png", ".webp"):
|
||||||
|
src_thumb = os.path.join(src_dir, src_stem + ext)
|
||||||
|
if os.path.exists(src_thumb):
|
||||||
|
if want_thumb:
|
||||||
|
os.makedirs(dst_dir or ".", exist_ok=True)
|
||||||
|
shutil.move(src_thumb, os.path.join(dst_dir, dst_stem + "-thumb" + (".jpg" if ext == ".jpeg" else ext)))
|
||||||
|
else:
|
||||||
|
_silent_remove(src_thumb)
|
||||||
|
break
|
||||||
|
# info json → mine for the nfo (when wanted), then always drop it
|
||||||
|
info = {}
|
||||||
|
info_path = os.path.join(src_dir, src_stem + ".info.json")
|
||||||
|
if os.path.exists(info_path):
|
||||||
|
try:
|
||||||
|
with open(info_path, encoding="utf-8") as f:
|
||||||
|
info = json.load(f)
|
||||||
|
except (ValueError, OSError):
|
||||||
|
info = {}
|
||||||
|
_silent_remove(info_path)
|
||||||
|
if want_nfo:
|
||||||
|
os.makedirs(dst_dir or ".", exist_ok=True)
|
||||||
|
with open(os.path.join(dst_dir, dst_stem + ".nfo"), "w", encoding="utf-8") as f:
|
||||||
|
f.write(build_episode_nfo(fields, description=info.get("description"), runtime=info.get("duration")))
|
||||||
|
except Exception: # noqa: BLE001 - sidecars are a nice-to-have, never fatal to the grab
|
||||||
|
logger.exception("youtube sidecars failed for %s", final_video)
|
||||||
|
|
||||||
|
|
||||||
|
def process_youtube_download(
|
||||||
|
dl: Dict[str, Any],
|
||||||
|
*,
|
||||||
|
profile: Any,
|
||||||
|
settings: Dict[str, Any],
|
||||||
|
download: Callable = download_one,
|
||||||
|
update_row: Callable[..., Any],
|
||||||
|
archive: Callable[[Dict[str, Any], Dict[str, Any]], Any],
|
||||||
|
clear_wishlist: Callable[[Any], Any],
|
||||||
|
stage_dir: Optional[str] = None,
|
||||||
|
move: Callable[[str, str], Any] = _default_move,
|
||||||
|
sidecars: Callable[[str, str, Dict[str, Any], Dict[str, Any]], Any] = _default_sidecars,
|
||||||
|
progress_hook: Optional[Callable] = None,
|
||||||
|
postprocess_hook: Optional[Callable] = None,
|
||||||
|
cookie_opts: Optional[dict] = None,
|
||||||
|
now: Optional[Callable[[], str]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Fulfil one queued YouTube download. PURE — all I/O injected.
|
||||||
|
|
||||||
|
Pipeline (same shape as the movie/TV lane): download into ``stage_dir`` (the shared
|
||||||
|
download folder) → flip to 'importing' → MOVE into the organised library path → completed.
|
||||||
|
When ``stage_dir`` is None it downloads straight into the library (legacy fallback, e.g.
|
||||||
|
no download folder configured), skipping the move.
|
||||||
|
|
||||||
|
On success: row → completed (with dest_path), snapshot to history, and remove the video
|
||||||
|
from the wishlist (it's in the library now; history is the permanent record). On
|
||||||
|
failure: row → failed + a history snapshot; the wishlist row is LEFT so a later scan/run
|
||||||
|
can retry. Returns a small result dict."""
|
||||||
|
now = now or (lambda: "")
|
||||||
|
settings = settings if isinstance(settings, dict) else {}
|
||||||
|
container = format_selection(profile)["merge_output_format"]
|
||||||
|
dest = plan_destination(dl, settings, container) # the FINAL organised library path
|
||||||
|
stem, cont = _stem_and_container(dest, container)
|
||||||
|
# Download target: the staging folder when set, else straight to the library. Either way
|
||||||
|
# do NOT write the organised DIR back to target_dir — it's the youtube ROOT, and
|
||||||
|
# plan_destination re-derives the channel/season folders under it; clobbering it re-nests
|
||||||
|
# on a re-run (the orphan reaper re-queues an interrupted download).
|
||||||
|
dl_dir = stage_dir if stage_dir else dest.get("dir")
|
||||||
|
update_row(dl.get("id"), status="downloading", progress=0, filename=dest.get("filename"))
|
||||||
|
|
||||||
|
res = download(dl.get("media_id"), dl_dir, stem, profile, cont,
|
||||||
|
progress_hook=progress_hook, postprocess_hook=postprocess_hook, cookie_opts=cookie_opts)
|
||||||
|
|
||||||
|
if not res.get("ok"):
|
||||||
|
err = res.get("error") or "Download failed"
|
||||||
|
completed = now()
|
||||||
|
update_row(dl.get("id"), status="failed", error=err, completed_at=completed)
|
||||||
|
archive(dl, {"status": "failed", "error": err, "completed_at": completed})
|
||||||
|
return {"status": "failed", "error": err}
|
||||||
|
|
||||||
|
staged_path = res.get("dest_path") or os.path.join(dl_dir or "", stem + "." + cont)
|
||||||
|
final_path = dest.get("path") or staged_path
|
||||||
|
|
||||||
|
# Staged build → post-process into the library (the visible 'importing' phase).
|
||||||
|
if stage_dir and staged_path and staged_path != final_path:
|
||||||
|
update_row(dl.get("id"), status="importing", progress=100, filename=dest.get("filename"))
|
||||||
|
try:
|
||||||
|
move(staged_path, final_path)
|
||||||
|
except Exception as e: # noqa: BLE001 - downloaded fine but couldn't be placed
|
||||||
|
err = "Import failed: " + str(e)
|
||||||
|
completed = now()
|
||||||
|
update_row(dl.get("id"), status="import_failed", error=err, completed_at=completed)
|
||||||
|
archive(dl, {"status": "import_failed", "error": err, "completed_at": completed})
|
||||||
|
logger.exception("youtube download %s: import move failed", dl.get("id"))
|
||||||
|
return {"status": "import_failed", "error": err}
|
||||||
|
dest_path = final_path
|
||||||
|
else:
|
||||||
|
dest_path = staged_path or final_path
|
||||||
|
|
||||||
|
# episode sidecars (-thumb.jpg + .nfo) next to the imported video — gated by the
|
||||||
|
# save_artwork / write_nfo post-processing toggles (shared with the movie/TV side).
|
||||||
|
sidecars(staged_path, dest_path, youtube_fields_from_download(dl), settings)
|
||||||
|
|
||||||
|
completed = now()
|
||||||
|
update_row(dl.get("id"), status="completed", progress=100,
|
||||||
|
dest_path=dest_path, completed_at=completed)
|
||||||
|
archive(dl, {"status": "completed", "dest_path": dest_path, "completed_at": completed})
|
||||||
|
try:
|
||||||
|
clear_wishlist(dl.get("media_id"))
|
||||||
|
except Exception: # noqa: BLE001 - unwish is best-effort; the file is already in place
|
||||||
|
logger.exception("youtube download %s: unwish failed", dl.get("id"))
|
||||||
|
return {"status": "completed", "dest_path": dest_path}
|
||||||
|
|
||||||
|
|
||||||
|
# ── concurrency + pacing (the music side's lesson: cap concurrency AND space starts) ──
|
||||||
|
# yt-dlp 429s if hammered, so fetch STARTS are spaced ≥ this far apart across all workers.
|
||||||
|
_DELAY_SECONDS = 3.0
|
||||||
|
_pace_lock = threading.Lock()
|
||||||
|
_last_start = [0.0]
|
||||||
|
|
||||||
|
|
||||||
|
def _pace(delay: float) -> None:
|
||||||
|
"""Block until at least ``delay`` seconds after the previous fetch start, then reserve
|
||||||
|
this start slot. Reserving under the lock (sleeping outside it) staggers concurrent
|
||||||
|
workers without serialising them past the delay."""
|
||||||
|
if delay <= 0:
|
||||||
|
return
|
||||||
|
with _pace_lock:
|
||||||
|
now = time.monotonic()
|
||||||
|
start_at = max(now, _last_start[0] + delay) if _last_start[0] else now
|
||||||
|
_last_start[0] = start_at
|
||||||
|
wait = start_at - time.monotonic()
|
||||||
|
if wait > 0:
|
||||||
|
time.sleep(wait)
|
||||||
|
|
||||||
|
|
||||||
|
# Download ids with a live worker thread right now. After a restart this is empty, so any
|
||||||
|
# row still marked 'downloading' is an orphan (its thread died) → the reaper re-queues it.
|
||||||
|
_active_worker_ids: set = set()
|
||||||
|
|
||||||
|
|
||||||
|
def _spawn_worker(dl_id: Any, db_provider: Callable) -> None:
|
||||||
|
threading.Thread(target=run_youtube_download, args=(dl_id, db_provider),
|
||||||
|
daemon=True, name="yt-dl-%s" % dl_id).start()
|
||||||
|
|
||||||
|
|
||||||
|
def requeue_orphaned_youtube(db_provider: Callable) -> int:
|
||||||
|
"""Recover YouTube downloads stuck in 'downloading' with no live worker (e.g. after a
|
||||||
|
restart killed the threads) by putting them back to 'queued' so the pump re-runs them.
|
||||||
|
A download whose worker is alive is in ``_active_worker_ids`` and is left untouched.
|
||||||
|
Returns the count recovered."""
|
||||||
|
n = 0
|
||||||
|
for d in (db_provider().get_active_video_downloads() or []):
|
||||||
|
if (d.get("source") == "youtube" and d.get("status") == "downloading"
|
||||||
|
and d.get("id") not in _active_worker_ids):
|
||||||
|
db_provider().update_video_download(d["id"], status="queued", progress=0)
|
||||||
|
n += 1
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
|
def start_next_queued(db_provider: Callable) -> Any:
|
||||||
|
"""Claim the next queued YouTube download and start it. Returns its id, or None if the
|
||||||
|
queue is empty. The wishlist pump uses this to fill slots; each finished worker calls it
|
||||||
|
once (one-out-one-in) so the queue drains continuously at the established concurrency."""
|
||||||
|
row = db_provider().claim_next_youtube_queued()
|
||||||
|
if not row:
|
||||||
|
return None
|
||||||
|
_spawn_worker(row["id"], db_provider)
|
||||||
|
return row["id"]
|
||||||
|
|
||||||
|
|
||||||
|
# ── production wiring ─────────────────────────────────────────────────────────
|
||||||
|
def run_youtube_download(dl_id: Any, db_provider: Callable) -> None:
|
||||||
|
"""Production entry: fetch the row, bind real seams, fulfil it. Called on a worker
|
||||||
|
thread by the pump; on finish it starts the next queued download (one-out-one-in)."""
|
||||||
|
_active_worker_ids.add(dl_id) # mark this download as having a live worker
|
||||||
|
db = db_provider()
|
||||||
|
dl = db.get_video_download(dl_id)
|
||||||
|
if not dl:
|
||||||
|
_active_worker_ids.discard(dl_id)
|
||||||
|
start_next_queued(db_provider) # keep the queue moving even on a stale id
|
||||||
|
return
|
||||||
|
# Per-channel quality override (stashed in search_ctx at enqueue) wins over the global.
|
||||||
|
override = quality_override_from_download(dl)
|
||||||
|
profile = youtube_quality.normalize(override) if override else youtube_quality.load(db)
|
||||||
|
settings = organization.load(db)
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
def _now():
|
||||||
|
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
||||||
|
|
||||||
|
def _progress(d):
|
||||||
|
# yt-dlp progress hook → row progress %. Best-effort; never raises into yt-dlp.
|
||||||
|
try:
|
||||||
|
if d.get("status") == "downloading":
|
||||||
|
total = d.get("total_bytes") or d.get("total_bytes_estimate") or 0
|
||||||
|
got = d.get("downloaded_bytes") or 0
|
||||||
|
if total:
|
||||||
|
db.update_video_download(dl_id, progress=int(got * 100 / total))
|
||||||
|
except Exception: # noqa: BLE001, S110 - a progress glitch must not abort the download
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _postprocess(d):
|
||||||
|
# yt-dlp finished the bytes and is now merging audio+video / converting (ffmpeg) — flip
|
||||||
|
# to 'importing' so the card stops sitting on a stuck-looking 100% 'Downloading' until
|
||||||
|
# it (and the library move that follows) are done. Best-effort.
|
||||||
|
try:
|
||||||
|
if d.get("status") in ("started", "processing"):
|
||||||
|
db.update_video_download(dl_id, status="importing", progress=100)
|
||||||
|
except Exception: # noqa: BLE001, S110 - a hook glitch must not abort the download
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _archive(row, upd):
|
||||||
|
try:
|
||||||
|
db.record_download_history({**row, **upd})
|
||||||
|
except Exception:
|
||||||
|
logger.exception("youtube download %s: history snapshot failed", dl_id)
|
||||||
|
|
||||||
|
cookie_opts = None
|
||||||
|
try:
|
||||||
|
from core.video.youtube import _cookie_opts
|
||||||
|
cookie_opts = _cookie_opts()
|
||||||
|
except Exception: # noqa: BLE001 - cookies are optional
|
||||||
|
cookie_opts = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
_pace(_DELAY_SECONDS) # space fetch starts to avoid yt-dlp 429s
|
||||||
|
# Stage into the shared download folder (a 'youtube' subfolder), then transfer to the
|
||||||
|
# library — same pipeline as movies/TV. Falls back to straight-to-library if no
|
||||||
|
# download folder is configured.
|
||||||
|
from config.settings import config_manager
|
||||||
|
dl_root = str(config_manager.get("soulseek.download_path", "") or "").strip()
|
||||||
|
stage_dir = os.path.join(dl_root, "youtube") if dl_root else None
|
||||||
|
|
||||||
|
process_youtube_download(
|
||||||
|
dl, profile=profile, settings=settings,
|
||||||
|
update_row=db.update_video_download, archive=_archive,
|
||||||
|
clear_wishlist=lambda vid: db.remove_youtube_from_wishlist("video", vid),
|
||||||
|
stage_dir=stage_dir,
|
||||||
|
progress_hook=_progress, postprocess_hook=_postprocess, cookie_opts=cookie_opts, now=_now)
|
||||||
|
finally:
|
||||||
|
_active_worker_ids.discard(dl_id) # worker done — no longer protects this row
|
||||||
|
start_next_queued(db_provider) # one out, one in — drain the queue
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"youtube_fields_from_download", "plan_destination", "ydl_download_opts",
|
||||||
|
"download_one", "process_youtube_download", "run_youtube_download",
|
||||||
|
"start_next_queued", "requeue_orphaned_youtube", "quality_override_from_download",
|
||||||
|
]
|
||||||
239
core/video/youtube_enrichment.py
Normal file
239
core/video/youtube_enrichment.py
Normal file
|
|
@ -0,0 +1,239 @@
|
||||||
|
"""Background YouTube date-enricher (video side, isolated).
|
||||||
|
|
||||||
|
Followed channels get their full upload-date catalog fetched in the background so
|
||||||
|
the channel page's year-seasons populate fully (the fast flat listing has no
|
||||||
|
dates). Cheap no-key bulk source first (Piped/Invidious proxy via
|
||||||
|
``proxy_channel_dates``); per-video yt-dlp only as a throttled fallback for the
|
||||||
|
channel's wished videos when every proxy is down. Everything is cached in
|
||||||
|
``youtube_video_dates`` so it's a one-time cost per channel and instant after.
|
||||||
|
|
||||||
|
A single daemon thread drains an enqueue() queue. Enqueue a channel when it's
|
||||||
|
followed (or its page opened while followed). Reads/writes only video_library.db.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import queue
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
logger = get_logger("video_enrichment.youtube") # same namespace as the other video workers
|
||||||
|
|
||||||
|
# Per-video fallback (used when the bulk proxy is down): dated in a small thread
|
||||||
|
# pool so a channel finishes in ~30s instead of minutes, without bursting too hard.
|
||||||
|
_FALLBACK_CAP = 60
|
||||||
|
_FALLBACK_WORKERS = 3
|
||||||
|
|
||||||
|
|
||||||
|
class YoutubeDateEnricher:
|
||||||
|
def __init__(self, db_factory=None):
|
||||||
|
self._db_factory = db_factory or self._default_db
|
||||||
|
self._q: "queue.Queue[str]" = queue.Queue()
|
||||||
|
self._inflight = set()
|
||||||
|
self._titles = {}
|
||||||
|
self._thread = None
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._current = None # channel being enriched right now (for the orb)
|
||||||
|
self._paused = False
|
||||||
|
self._channels_done = 0
|
||||||
|
self._dates_total = 0
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _default_db():
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
return VideoDatabase()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _proxy_instances(db):
|
||||||
|
"""Parse the optional youtube_proxy_instances setting into [(kind, url), …].
|
||||||
|
Empty (the default) → proxy skipped entirely. Format is comma/newline
|
||||||
|
separated 'kind|url' (kind inferred from the url if omitted)."""
|
||||||
|
try:
|
||||||
|
raw = db.get_setting("youtube_proxy_instances") or ""
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
out = []
|
||||||
|
for part in raw.replace("\n", ",").split(","):
|
||||||
|
part = part.strip()
|
||||||
|
if not part:
|
||||||
|
continue
|
||||||
|
if "|" in part:
|
||||||
|
kind, url = part.split("|", 1)
|
||||||
|
kind, url = kind.strip().lower(), url.strip()
|
||||||
|
else:
|
||||||
|
url = part
|
||||||
|
kind = "invidious" if "invidious" in url or "/api/v1" in url else "piped"
|
||||||
|
if url.startswith("http"):
|
||||||
|
out.append((kind, url.rstrip("/")))
|
||||||
|
return out
|
||||||
|
|
||||||
|
def enqueue(self, channel_id, title=None):
|
||||||
|
"""Queue a followed channel for full date enrichment (deduped; starts the
|
||||||
|
worker thread on first use)."""
|
||||||
|
cid = str(channel_id or "").strip()
|
||||||
|
if not cid:
|
||||||
|
return
|
||||||
|
# Never spawn the background daemon (network + the default DB) under tests;
|
||||||
|
# the enricher's logic is exercised directly via _enrich() with a tmp DB.
|
||||||
|
if os.environ.get("PYTEST_CURRENT_TEST"):
|
||||||
|
return
|
||||||
|
with self._lock:
|
||||||
|
if title:
|
||||||
|
self._titles[cid] = title
|
||||||
|
if cid in self._inflight:
|
||||||
|
return
|
||||||
|
self._inflight.add(cid)
|
||||||
|
self._q.put(cid)
|
||||||
|
if self._thread is None or not self._thread.is_alive():
|
||||||
|
self._thread = threading.Thread(target=self._run, name="yt-date-enricher", daemon=True)
|
||||||
|
self._thread.start()
|
||||||
|
|
||||||
|
def pause(self):
|
||||||
|
self._paused = True
|
||||||
|
|
||||||
|
def resume(self):
|
||||||
|
self._paused = False
|
||||||
|
|
||||||
|
def stats(self):
|
||||||
|
"""Dashboard-orb telemetry — same shape the enrichment workers report."""
|
||||||
|
queued = self._q.qsize()
|
||||||
|
running = bool(self._current) and not self._paused
|
||||||
|
cur = self._current
|
||||||
|
return {
|
||||||
|
"enabled": True,
|
||||||
|
"idle": False, # this worker idles, it never "completes"
|
||||||
|
"running": running,
|
||||||
|
"paused": self._paused,
|
||||||
|
"current_item": {"type": "channel", "name": cur} if cur else None,
|
||||||
|
"progress": {"channels": {"matched": self._channels_done,
|
||||||
|
"total": self._channels_done + queued + (1 if cur else 0)}},
|
||||||
|
"queued": queued,
|
||||||
|
"dates_cached": self._dates_total,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _run(self):
|
||||||
|
while True:
|
||||||
|
if self._paused:
|
||||||
|
time.sleep(0.5)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
cid = self._q.get(timeout=45)
|
||||||
|
except queue.Empty:
|
||||||
|
return # idle → let the thread die; re-spawned on next enqueue
|
||||||
|
try:
|
||||||
|
self._enrich(cid)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("YouTube date enrichment failed for %s", cid)
|
||||||
|
finally:
|
||||||
|
self._current = None
|
||||||
|
with self._lock:
|
||||||
|
self._inflight.discard(cid)
|
||||||
|
self._q.task_done()
|
||||||
|
|
||||||
|
def _enrich(self, channel_id):
|
||||||
|
"""REMEMBER a channel: cache its video catalog (list) + metadata + upload
|
||||||
|
dates so re-opening it is instant. InnerTube bulk; per-video yt-dlp fallback."""
|
||||||
|
from core.video import youtube as yt
|
||||||
|
db = self._db_factory()
|
||||||
|
cid = str(channel_id or "").strip()
|
||||||
|
if not cid or db.channel_dates_enriched_recently(cid):
|
||||||
|
return
|
||||||
|
self._current = self._titles.get(cid) or cid
|
||||||
|
logger.debug("Enriching dates for %s (%s)", self._current, cid)
|
||||||
|
|
||||||
|
# Remember channel metadata (avatar/subs/tags) for an instant header re-open.
|
||||||
|
try:
|
||||||
|
meta = yt.resolve_channel("https://www.youtube.com/channel/" + cid, limit=1)
|
||||||
|
if meta and meta.get("youtube_id"):
|
||||||
|
db.cache_channel_meta(cid, meta)
|
||||||
|
except Exception:
|
||||||
|
logger.debug("channel meta fetch failed for %s", cid, exc_info=True)
|
||||||
|
|
||||||
|
# PRIMARY: InnerTube — the whole videos tab (list + approximate dates) in a
|
||||||
|
# handful of requests. Remember the LIST too (not just dates) so the page
|
||||||
|
# can render the full catalog cache-first on the next open.
|
||||||
|
dates = {}
|
||||||
|
try:
|
||||||
|
catalog = yt.innertube_channel_catalog(cid) or []
|
||||||
|
except Exception:
|
||||||
|
catalog = []
|
||||||
|
logger.debug("innertube catalog fetch failed for %s", cid, exc_info=True)
|
||||||
|
if catalog:
|
||||||
|
db.cache_channel_videos(cid, catalog)
|
||||||
|
dates = {v["youtube_id"]: v["published_at"] for v in catalog if v.get("published_at")}
|
||||||
|
logger.debug("innertube remembered %d videos (%d dated) for %s", len(catalog), len(dates), cid)
|
||||||
|
# SECONDARY (opt-in): a configured Piped/Invidious proxy — only if InnerTube
|
||||||
|
# came up empty (the public instances are unreliable/API-disabled).
|
||||||
|
if not dates:
|
||||||
|
instances = self._proxy_instances(db)
|
||||||
|
if instances:
|
||||||
|
try:
|
||||||
|
dates = yt.proxy_channel_dates(cid, instances=instances) or {}
|
||||||
|
except Exception:
|
||||||
|
logger.debug("proxy date fetch failed for %s", cid, exc_info=True)
|
||||||
|
if dates:
|
||||||
|
db.cache_video_dates([{"youtube_id": k, "published_at": v} for k, v in dates.items()])
|
||||||
|
|
||||||
|
# FALLBACK (the basic method): exact dates per-video via yt-dlp, only for
|
||||||
|
# the channel's videos still UNDATED — cheap when the bulk pass worked.
|
||||||
|
ids = set(db.wishlisted_video_ids_for_channel(cid))
|
||||||
|
if not dates:
|
||||||
|
try:
|
||||||
|
ch = yt.resolve_channel("https://www.youtube.com/channel/" + cid, limit=60)
|
||||||
|
for v in (ch or {}).get("videos") or []:
|
||||||
|
if v.get("youtube_id"):
|
||||||
|
ids.add(v["youtube_id"])
|
||||||
|
except Exception:
|
||||||
|
logger.debug("flat resolve for date fallback failed for %s", cid, exc_info=True)
|
||||||
|
ids = list(ids)
|
||||||
|
have = db.get_video_dates(ids)
|
||||||
|
missing = [i for i in ids if i not in have and i not in dates][:_FALLBACK_CAP]
|
||||||
|
filled = 0
|
||||||
|
if missing:
|
||||||
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
|
|
||||||
|
cur = self._current
|
||||||
|
|
||||||
|
def fetch_date(vid):
|
||||||
|
try:
|
||||||
|
v = yt.video_detail(vid) or {}
|
||||||
|
d = v.get("published_at")
|
||||||
|
if d:
|
||||||
|
# Per-item line, matching the other workers' "Matched … -> …".
|
||||||
|
logger.info("Dated %s '%s' -> %s", cur, (v.get("title") or vid)[:70], d)
|
||||||
|
else:
|
||||||
|
logger.info("No date for %s '%s'", cur, (v.get("title") or vid)[:70])
|
||||||
|
return vid, d
|
||||||
|
except Exception:
|
||||||
|
return vid, None
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=_FALLBACK_WORKERS) as ex:
|
||||||
|
for fut in as_completed([ex.submit(fetch_date, v) for v in missing]):
|
||||||
|
vid, d = fut.result()
|
||||||
|
if d:
|
||||||
|
db.cache_video_dates([{"youtube_id": vid, "published_at": d}])
|
||||||
|
filled += 1
|
||||||
|
self._channels_done += 1
|
||||||
|
self._dates_total += len(dates) + filled
|
||||||
|
# Tag the source so legacy (pre-InnerTube) rows are recognisable and upgrade.
|
||||||
|
method = "innertube" if (catalog or dates) else "fallback"
|
||||||
|
db.mark_channel_dates_enriched(cid, len(dates) + filled, method=method)
|
||||||
|
# Terse per-channel summary (like the worker's "Synced full episode list…").
|
||||||
|
logger.info("Remembered %d videos for %s (%d dated, %d per-video)",
|
||||||
|
len(catalog) or (len(dates) + filled), self._current, len(dates), filled)
|
||||||
|
|
||||||
|
|
||||||
|
_enricher = None
|
||||||
|
_enricher_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def get_youtube_date_enricher():
|
||||||
|
global _enricher
|
||||||
|
if _enricher is None:
|
||||||
|
with _enricher_lock:
|
||||||
|
if _enricher is None:
|
||||||
|
_enricher = YoutubeDateEnricher()
|
||||||
|
return _enricher
|
||||||
116
core/video/youtube_quality.py
Normal file
116
core/video/youtube_quality.py
Normal file
|
|
@ -0,0 +1,116 @@
|
||||||
|
"""YouTube download quality profile — deliberately SEPARATE from the main video
|
||||||
|
quality profile (``core/video/quality_profile.py``).
|
||||||
|
|
||||||
|
YouTube is fetched with yt-dlp, not from scene/p2p releases, so the Radarr-style
|
||||||
|
ladder (Remux / BluRay / WEB-DL, HDR/audio tiers, scene rejects) is meaningless
|
||||||
|
here. yt-dlp just picks a stream by **resolution + codec + container**, so this
|
||||||
|
profile is small: a resolution ceiling, a codec preference, an output container,
|
||||||
|
and two flags (60fps / HDR). The (later-phase) downloader maps these to a yt-dlp
|
||||||
|
``format`` / ``format_sort`` selection.
|
||||||
|
|
||||||
|
Pure normalize/load/save (no DB, no network) so it's unit-tested in isolation.
|
||||||
|
Persisted as a JSON blob in video.db's ``video_settings['youtube_quality_profile']``.
|
||||||
|
Isolated — imports only json/typing; the music side never imports it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
# Resolution ceiling (yt-dlp height filter). "best" = no cap (take the top stream).
|
||||||
|
RESOLUTIONS = ("best", "4320p", "2160p", "1440p", "1080p", "720p", "480p", "360p")
|
||||||
|
CODECS = ("any", "av1", "vp9", "h264") # SOFT preference; "any" = let yt-dlp pick best
|
||||||
|
CONTAINERS = ("mp4", "mkv", "webm") # yt-dlp --merge-output-format
|
||||||
|
|
||||||
|
|
||||||
|
def default_profile() -> dict:
|
||||||
|
"""Sensible default: 1080p ceiling, yt-dlp's best codec, mp4 (most compatible),
|
||||||
|
prefer 60fps, SDR (HDR off — washes out on non-HDR displays)."""
|
||||||
|
return {
|
||||||
|
"version": 1,
|
||||||
|
"max_resolution": "1080p",
|
||||||
|
"video_codec": "any",
|
||||||
|
"container": "mp4",
|
||||||
|
"prefer_60fps": True,
|
||||||
|
"allow_hdr": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def normalize(raw: Any) -> dict:
|
||||||
|
"""Coerce a stored/posted profile to a valid shape, filling gaps from the
|
||||||
|
default. Unknown keys dropped; invalid values fall back. Never raises."""
|
||||||
|
d = default_profile()
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
return d
|
||||||
|
if raw.get("max_resolution") in RESOLUTIONS:
|
||||||
|
d["max_resolution"] = raw["max_resolution"]
|
||||||
|
if raw.get("video_codec") in CODECS:
|
||||||
|
d["video_codec"] = raw["video_codec"]
|
||||||
|
if raw.get("container") in CONTAINERS:
|
||||||
|
d["container"] = raw["container"]
|
||||||
|
d["prefer_60fps"] = bool(raw.get("prefer_60fps", d["prefer_60fps"]))
|
||||||
|
d["allow_hdr"] = bool(raw.get("allow_hdr", d["allow_hdr"]))
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
# Resolution → pixel-height ceiling for yt-dlp's ``height<=`` filter. "best" = no cap.
|
||||||
|
_RES_HEIGHT = {"best": None, "4320p": 4320, "2160p": 2160, "1440p": 1440,
|
||||||
|
"1080p": 1080, "720p": 720, "480p": 480, "360p": 360}
|
||||||
|
|
||||||
|
|
||||||
|
def format_selection(profile: Any) -> dict:
|
||||||
|
"""Map a profile to the yt-dlp options the (download) engine passes through:
|
||||||
|
``{format, format_sort, merge_output_format}``. Pure — no DB, no network.
|
||||||
|
|
||||||
|
* ``format`` takes the best video+audio capped to the resolution ceiling, falling
|
||||||
|
back to an uncapped best so a video that only exists above the cap still grabs.
|
||||||
|
* ``format_sort`` is an ordered soft-preference list (codec → resolution → fps →
|
||||||
|
SDR) — yt-dlp picks the top match, so these never *exclude* a stream, they rank it.
|
||||||
|
* ``merge_output_format`` is the container yt-dlp muxes into.
|
||||||
|
|
||||||
|
NB: the exact yt-dlp tokens are tunable against the live yt-dlp version when the
|
||||||
|
downloader is wired; the shape (one capped format expr + a ranked sort list) is the
|
||||||
|
contract the tests pin.
|
||||||
|
"""
|
||||||
|
p = normalize(profile)
|
||||||
|
height = _RES_HEIGHT.get(p["max_resolution"])
|
||||||
|
if height:
|
||||||
|
fmt = "bv*[height<=%d]+ba/b[height<=%d]/bv*+ba/b" % (height, height)
|
||||||
|
else:
|
||||||
|
fmt = "bv*+ba/b"
|
||||||
|
|
||||||
|
sort: list[str] = []
|
||||||
|
if p["video_codec"] != "any":
|
||||||
|
sort.append("vcodec:%s" % p["video_codec"]) # soft codec preference
|
||||||
|
sort.append("res:%d" % height if height else "res") # prefer the ceiling, else highest
|
||||||
|
if p["prefer_60fps"]:
|
||||||
|
sort.append("fps") # prefer higher frame rate
|
||||||
|
if not p["allow_hdr"]:
|
||||||
|
sort.append("hdr:SDR") # rank SDR above HDR (don't exclude)
|
||||||
|
|
||||||
|
return {"format": fmt, "format_sort": sort, "merge_output_format": p["container"]}
|
||||||
|
|
||||||
|
|
||||||
|
def load(db) -> dict:
|
||||||
|
"""Read + normalize the stored profile, or the default if none/garbage."""
|
||||||
|
raw = db.get_setting("youtube_quality_profile")
|
||||||
|
if raw:
|
||||||
|
try:
|
||||||
|
return normalize(json.loads(raw))
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
pass
|
||||||
|
return default_profile()
|
||||||
|
|
||||||
|
|
||||||
|
def save(db, raw: Any) -> dict:
|
||||||
|
"""Normalize + persist; returns the normalized profile that was stored."""
|
||||||
|
prof = normalize(raw)
|
||||||
|
db.set_setting("youtube_quality_profile", json.dumps(prof))
|
||||||
|
return prof
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"RESOLUTIONS", "CODECS", "CONTAINERS",
|
||||||
|
"default_profile", "normalize", "format_selection", "load", "save",
|
||||||
|
]
|
||||||
|
|
@ -222,7 +222,7 @@ class YouTubeClient(DownloadSourcePlugin):
|
||||||
|
|
||||||
# Cookie support — use browser cookies for YouTube auth
|
# Cookie support — use browser cookies for YouTube auth
|
||||||
from config.settings import config_manager
|
from config.settings import config_manager
|
||||||
cookies_browser = config_manager.get('youtube.cookies_browser', '')
|
cookies_browser = ('' if (_cb := config_manager.get('youtube.cookies_browser', '')) == 'custom' else _cb)
|
||||||
if cookies_browser:
|
if cookies_browser:
|
||||||
self.download_opts['cookiesfrombrowser'] = (cookies_browser,)
|
self.download_opts['cookiesfrombrowser'] = (cookies_browser,)
|
||||||
|
|
||||||
|
|
@ -309,7 +309,7 @@ class YouTubeClient(DownloadSourcePlugin):
|
||||||
"""Reload YouTube settings from config (called when settings are saved)."""
|
"""Reload YouTube settings from config (called when settings are saved)."""
|
||||||
from config.settings import config_manager
|
from config.settings import config_manager
|
||||||
self._download_delay = config_manager.get('youtube.download_delay', 3)
|
self._download_delay = config_manager.get('youtube.download_delay', 3)
|
||||||
cookies_browser = config_manager.get('youtube.cookies_browser', '')
|
cookies_browser = ('' if (_cb := config_manager.get('youtube.cookies_browser', '')) == 'custom' else _cb)
|
||||||
if cookies_browser:
|
if cookies_browser:
|
||||||
self.download_opts['cookiesfrombrowser'] = (cookies_browser,)
|
self.download_opts['cookiesfrombrowser'] = (cookies_browser,)
|
||||||
elif 'cookiesfrombrowser' in self.download_opts:
|
elif 'cookiesfrombrowser' in self.download_opts:
|
||||||
|
|
@ -736,7 +736,7 @@ class YouTubeClient(DownloadSourcePlugin):
|
||||||
'default_search': 'ytsearch',
|
'default_search': 'ytsearch',
|
||||||
'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||||
}
|
}
|
||||||
cookies_browser = config_manager.get('youtube.cookies_browser', '')
|
cookies_browser = ('' if (_cb := config_manager.get('youtube.cookies_browser', '')) == 'custom' else _cb)
|
||||||
if cookies_browser:
|
if cookies_browser:
|
||||||
ydl_opts['cookiesfrombrowser'] = (cookies_browser,)
|
ydl_opts['cookiesfrombrowser'] = (cookies_browser,)
|
||||||
|
|
||||||
|
|
@ -816,7 +816,7 @@ class YouTubeClient(DownloadSourcePlugin):
|
||||||
}
|
}
|
||||||
|
|
||||||
# Add cookie support for search (avoids bot detection)
|
# Add cookie support for search (avoids bot detection)
|
||||||
cookies_browser = config_manager.get('youtube.cookies_browser', '')
|
cookies_browser = ('' if (_cb := config_manager.get('youtube.cookies_browser', '')) == 'custom' else _cb)
|
||||||
if cookies_browser:
|
if cookies_browser:
|
||||||
ydl_opts['cookiesfrombrowser'] = (cookies_browser,)
|
ydl_opts['cookiesfrombrowser'] = (cookies_browser,)
|
||||||
|
|
||||||
|
|
@ -1180,7 +1180,7 @@ class YouTubeClient(DownloadSourcePlugin):
|
||||||
'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||||
}
|
}
|
||||||
|
|
||||||
cookies_browser = config_manager.get('youtube.cookies_browser', '')
|
cookies_browser = ('' if (_cb := config_manager.get('youtube.cookies_browser', '')) == 'custom' else _cb)
|
||||||
if cookies_browser:
|
if cookies_browser:
|
||||||
download_opts['cookiesfrombrowser'] = (cookies_browser,)
|
download_opts['cookiesfrombrowser'] = (cookies_browser,)
|
||||||
|
|
||||||
|
|
|
||||||
3604
database/video_database.py
Normal file
3604
database/video_database.py
Normal file
File diff suppressed because it is too large
Load diff
607
database/video_schema.sql
Normal file
607
database/video_schema.sql
Normal file
|
|
@ -0,0 +1,607 @@
|
||||||
|
-- ============================================================================
|
||||||
|
-- SoulSync — VIDEO side schema (database/video_library.db)
|
||||||
|
--
|
||||||
|
-- ISOLATION: this is a SEPARATE SQLite file from the music library. The video
|
||||||
|
-- code owns it exclusively; music never opens it and it never references music
|
||||||
|
-- tables. A bug, migration, or reset here cannot touch music data, and the two
|
||||||
|
-- never contend for the same write lock.
|
||||||
|
--
|
||||||
|
-- DESIGN PRINCIPLES (deliberately avoiding the music DB's known pain points):
|
||||||
|
-- * No polymorphic (entity_type, entity_id) keys. Where a row can belong to a
|
||||||
|
-- movie OR an episode OR a youtube video, we use separate nullable FKs with
|
||||||
|
-- a CHECK that exactly one is set — real foreign keys, real cascades.
|
||||||
|
-- * No "source id" blob / naming spaghetti. External ids are a few explicit,
|
||||||
|
-- well-named, indexed columns (tmdb_id, tvdb_id, imdb_id, youtube_id).
|
||||||
|
-- * No metadata dumping-ground column. Structured config that is genuinely a
|
||||||
|
-- list (quality profile contents) is small, ordered JSON; everything else
|
||||||
|
-- is a real column.
|
||||||
|
-- * Watchlist / Wishlist / Calendar are DERIVED VIEWS over monitored + file
|
||||||
|
-- state, not standalone tables — so they can't drift out of sync with the
|
||||||
|
-- library the way a duplicated table does. (See note in design summary.)
|
||||||
|
--
|
||||||
|
-- Run order matters (FKs reference earlier tables). The init module executes
|
||||||
|
-- this whole file inside one transaction with foreign_keys ON.
|
||||||
|
-- ============================================================================
|
||||||
|
|
||||||
|
-- ── Meta ────────────────────────────────────────────────────────────────────
|
||||||
|
CREATE TABLE IF NOT EXISTS meta (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ── Configuration ───────────────────────────────────────────────────────────
|
||||||
|
-- Root folders: where each kind of library content is stored on disk.
|
||||||
|
CREATE TABLE IF NOT EXISTS root_folders (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
path TEXT NOT NULL UNIQUE,
|
||||||
|
content_kind TEXT NOT NULL CHECK (content_kind IN ('movie', 'show', 'youtube')),
|
||||||
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Quality profiles: an ordered, named acceptance ladder (Radarr/Sonarr-style).
|
||||||
|
-- `items` is a small JSON array of allowed quality names, best-first; `cutoff`
|
||||||
|
-- is the name we stop upgrading at. JSON is appropriate here — it is genuinely
|
||||||
|
-- an ordered list of config, not a metadata grab-bag.
|
||||||
|
CREATE TABLE IF NOT EXISTS quality_profiles (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
name TEXT NOT NULL UNIQUE,
|
||||||
|
cutoff TEXT,
|
||||||
|
items TEXT NOT NULL DEFAULT '[]',
|
||||||
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Video-side settings. KEY/VALUE for now; at the end-of-branch settings.db
|
||||||
|
-- consolidation these migrate into the shared config store. Value is JSON.
|
||||||
|
CREATE TABLE IF NOT EXISTS video_settings (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT,
|
||||||
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ── Content: Movies ─────────────────────────────────────────────────────────
|
||||||
|
CREATE TABLE IF NOT EXISTS movies (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
server_source TEXT, -- 'plex' | 'jellyfin' (NULL = not on a server yet, e.g. wishlist)
|
||||||
|
server_id TEXT, -- media server native id (Plex ratingKey / Jellyfin Item Id)
|
||||||
|
tmdb_id INTEGER, -- not unique: same film can sit in >1 library
|
||||||
|
imdb_id TEXT,
|
||||||
|
tmdb_match_status TEXT, -- enrichment: NULL=pending | matched | not_found | error
|
||||||
|
tmdb_last_attempted TEXT,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
sort_title TEXT,
|
||||||
|
year INTEGER,
|
||||||
|
overview TEXT,
|
||||||
|
runtime_minutes INTEGER,
|
||||||
|
status TEXT, -- announced | in_production | released
|
||||||
|
release_date TEXT, -- primary/theatrical (ISO date)
|
||||||
|
digital_release_date TEXT,
|
||||||
|
studio TEXT,
|
||||||
|
content_rating TEXT, -- e.g. PG-13
|
||||||
|
tagline TEXT,
|
||||||
|
tmdb_collection_id INTEGER, -- TMDB belongs_to_collection id (franchise); for "complete your collections" gaps
|
||||||
|
tmdb_collection_name TEXT, -- collection display name (e.g. "The Matrix Collection")
|
||||||
|
rating REAL, -- TMDB audience score (0-10)
|
||||||
|
rating_critic REAL, -- critic score (0-100) when offered
|
||||||
|
imdb_rating REAL, -- IMDb (0-10, via OMDb)
|
||||||
|
rt_rating INTEGER, -- Rotten Tomatoes (0-100)
|
||||||
|
metacritic INTEGER, -- Metacritic (0-100)
|
||||||
|
ratings_synced INTEGER NOT NULL DEFAULT 0, -- OMDb ratings fetched?
|
||||||
|
poster_url TEXT,
|
||||||
|
backdrop_url TEXT,
|
||||||
|
logo_url TEXT, -- transparent title logo (clearlogo)
|
||||||
|
monitored INTEGER NOT NULL DEFAULT 1, -- tracked for acquisition
|
||||||
|
has_file INTEGER NOT NULL DEFAULT 0, -- owned? (denormalized)
|
||||||
|
quality_profile_id INTEGER REFERENCES quality_profiles(id) ON DELETE SET NULL,
|
||||||
|
root_folder_id INTEGER REFERENCES root_folders(id) ON DELETE SET NULL,
|
||||||
|
path TEXT, -- folder on disk once owned
|
||||||
|
added_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_movies_tmdb ON movies(tmdb_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_movies_monitored ON movies(monitored, has_file);
|
||||||
|
-- NOTE: idx on tmdb_collection_id (a migration-added column) lives in _POST_INDEXES,
|
||||||
|
-- not here — this schema runs BEFORE the ALTER that adds the column on upgraded DBs.
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_movies_release ON movies(release_date);
|
||||||
|
-- Upsert/stale-removal key: the server's native id. Multiple NULLs are allowed
|
||||||
|
-- (wishlist items not yet on a server), so this never blocks non-server rows.
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS ux_movies_server ON movies(server_source, server_id);
|
||||||
|
|
||||||
|
-- ── Content: TV (shows → seasons → episodes) ────────────────────────────────
|
||||||
|
CREATE TABLE IF NOT EXISTS shows (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
server_source TEXT, -- 'plex' | 'jellyfin' (NULL = not on a server yet)
|
||||||
|
server_id TEXT, -- media server native id
|
||||||
|
tvdb_id INTEGER, -- not unique (same series can sit in >1 library)
|
||||||
|
tmdb_id INTEGER,
|
||||||
|
imdb_id TEXT,
|
||||||
|
tmdb_match_status TEXT, -- enrichment match state per source
|
||||||
|
tmdb_last_attempted TEXT,
|
||||||
|
tvdb_match_status TEXT,
|
||||||
|
tvdb_last_attempted TEXT,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
sort_title TEXT,
|
||||||
|
year INTEGER,
|
||||||
|
overview TEXT,
|
||||||
|
status TEXT, -- continuing | ended | upcoming
|
||||||
|
network TEXT,
|
||||||
|
airs_time TEXT, -- TVDB show air time, e.g. "21:00" (network local)
|
||||||
|
runtime_minutes INTEGER,
|
||||||
|
content_rating TEXT,
|
||||||
|
tagline TEXT,
|
||||||
|
rating REAL, -- TMDB audience score (0-10)
|
||||||
|
imdb_rating REAL, -- IMDb (0-10, via OMDb)
|
||||||
|
rt_rating INTEGER, -- Rotten Tomatoes (0-100)
|
||||||
|
metacritic INTEGER, -- Metacritic (0-100)
|
||||||
|
ratings_synced INTEGER NOT NULL DEFAULT 0, -- OMDb ratings fetched?
|
||||||
|
first_air_date TEXT,
|
||||||
|
last_air_date TEXT,
|
||||||
|
poster_url TEXT,
|
||||||
|
backdrop_url TEXT,
|
||||||
|
logo_url TEXT, -- transparent title logo (clearlogo)
|
||||||
|
episodes_synced INTEGER NOT NULL DEFAULT 0, -- full episode list pulled from metadata?
|
||||||
|
monitored INTEGER NOT NULL DEFAULT 1, -- "following" (watchlist)
|
||||||
|
quality_profile_id INTEGER REFERENCES quality_profiles(id) ON DELETE SET NULL,
|
||||||
|
root_folder_id INTEGER REFERENCES root_folders(id) ON DELETE SET NULL,
|
||||||
|
path TEXT,
|
||||||
|
added_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_shows_tvdb ON shows(tvdb_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_shows_monitored ON shows(monitored);
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS ux_shows_server ON shows(server_source, server_id);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS seasons (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
show_id INTEGER NOT NULL REFERENCES shows(id) ON DELETE CASCADE,
|
||||||
|
server_id TEXT, -- media server native id (for reference/refresh)
|
||||||
|
season_number INTEGER NOT NULL,
|
||||||
|
title TEXT,
|
||||||
|
overview TEXT,
|
||||||
|
poster_url TEXT,
|
||||||
|
monitored INTEGER NOT NULL DEFAULT 1,
|
||||||
|
UNIQUE (show_id, season_number)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_seasons_show ON seasons(show_id);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS episodes (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
show_id INTEGER NOT NULL REFERENCES shows(id) ON DELETE CASCADE,
|
||||||
|
season_id INTEGER NOT NULL REFERENCES seasons(id) ON DELETE CASCADE,
|
||||||
|
server_source TEXT, -- 'plex' | 'jellyfin'
|
||||||
|
server_id TEXT, -- media server native id
|
||||||
|
season_number INTEGER NOT NULL,
|
||||||
|
episode_number INTEGER NOT NULL,
|
||||||
|
title TEXT,
|
||||||
|
overview TEXT,
|
||||||
|
air_date TEXT, -- ISO date — drives the Calendar
|
||||||
|
runtime_minutes INTEGER,
|
||||||
|
still_url TEXT, -- per-episode thumbnail (server image path)
|
||||||
|
rating REAL, -- audience score (0-10)
|
||||||
|
tvdb_id INTEGER,
|
||||||
|
monitored INTEGER NOT NULL DEFAULT 1,
|
||||||
|
has_file INTEGER NOT NULL DEFAULT 0,
|
||||||
|
UNIQUE (show_id, season_number, episode_number)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_episodes_show ON episodes(show_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_episodes_air ON episodes(air_date);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_episodes_wanted ON episodes(monitored, has_file);
|
||||||
|
|
||||||
|
-- ── Genres (normalised many-to-many; no comma-blob) ─────────────────────────
|
||||||
|
CREATE TABLE IF NOT EXISTS genres (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
name TEXT NOT NULL UNIQUE COLLATE NOCASE
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS movie_genres (
|
||||||
|
movie_id INTEGER NOT NULL REFERENCES movies(id) ON DELETE CASCADE,
|
||||||
|
genre_id INTEGER NOT NULL REFERENCES genres(id) ON DELETE CASCADE,
|
||||||
|
PRIMARY KEY (movie_id, genre_id)
|
||||||
|
);
|
||||||
|
CREATE TABLE IF NOT EXISTS show_genres (
|
||||||
|
show_id INTEGER NOT NULL REFERENCES shows(id) ON DELETE CASCADE,
|
||||||
|
genre_id INTEGER NOT NULL REFERENCES genres(id) ON DELETE CASCADE,
|
||||||
|
PRIMARY KEY (show_id, genre_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_movie_genres_genre ON movie_genres(genre_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_show_genres_genre ON show_genres(genre_id);
|
||||||
|
|
||||||
|
-- ── People + credits (cast & crew; normalised, no blob) ─────────────────────
|
||||||
|
-- A person appears in many titles; deduped by their provider id. Each credit
|
||||||
|
-- belongs to exactly one movie OR show (separate nullable FKs + CHECK, no
|
||||||
|
-- polymorphic id), mirroring the media_files/downloads pattern.
|
||||||
|
CREATE TABLE IF NOT EXISTS people (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
tmdb_id INTEGER UNIQUE,
|
||||||
|
photo_url TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_people_name ON people(name);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS credits (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
person_id INTEGER NOT NULL REFERENCES people(id) ON DELETE CASCADE,
|
||||||
|
movie_id INTEGER REFERENCES movies(id) ON DELETE CASCADE,
|
||||||
|
show_id INTEGER REFERENCES shows(id) ON DELETE CASCADE,
|
||||||
|
department TEXT NOT NULL, -- 'cast' | 'crew'
|
||||||
|
job TEXT, -- Director | Writer | Creator (crew); 'Actor' (cast)
|
||||||
|
character TEXT, -- the role played (cast)
|
||||||
|
sort_order INTEGER NOT NULL DEFAULT 0,
|
||||||
|
CHECK ((movie_id IS NOT NULL) + (show_id IS NOT NULL) = 1)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_credits_movie ON credits(movie_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_credits_show ON credits(show_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_credits_person ON credits(person_id);
|
||||||
|
|
||||||
|
-- ── Discover: "Not interested" / ignore list ────────────────────────────────
|
||||||
|
-- Titles the user has hidden from Discover (movie/show level only — episodes can't be
|
||||||
|
-- individually ignored). Keyed by (kind, tmdb_id); a denormalised title/poster snapshot so
|
||||||
|
-- the manage-modal renders without a TMDB round-trip.
|
||||||
|
CREATE TABLE IF NOT EXISTS video_ignored (
|
||||||
|
kind TEXT NOT NULL, -- 'movie' | 'show'
|
||||||
|
tmdb_id INTEGER NOT NULL,
|
||||||
|
title TEXT,
|
||||||
|
year INTEGER,
|
||||||
|
poster_url TEXT,
|
||||||
|
added_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
PRIMARY KEY (kind, tmdb_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ── Content: YouTube (channels → videos) ────────────────────────────────────
|
||||||
|
CREATE TABLE IF NOT EXISTS channels (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
youtube_id TEXT NOT NULL UNIQUE, -- channel id
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
handle TEXT, -- @handle
|
||||||
|
description TEXT,
|
||||||
|
avatar_url TEXT,
|
||||||
|
banner_url TEXT,
|
||||||
|
monitored INTEGER NOT NULL DEFAULT 1, -- "subscribed" (watchlist)
|
||||||
|
quality_profile_id INTEGER REFERENCES quality_profiles(id) ON DELETE SET NULL,
|
||||||
|
root_folder_id INTEGER REFERENCES root_folders(id) ON DELETE SET NULL,
|
||||||
|
path TEXT,
|
||||||
|
added_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_channels_monitored ON channels(monitored);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS channel_videos (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
channel_id INTEGER NOT NULL REFERENCES channels(id) ON DELETE CASCADE,
|
||||||
|
youtube_id TEXT NOT NULL UNIQUE, -- video id
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
published_at TEXT, -- ISO datetime — drives feed/Calendar
|
||||||
|
duration_seconds INTEGER,
|
||||||
|
thumbnail_url TEXT,
|
||||||
|
monitored INTEGER NOT NULL DEFAULT 1,
|
||||||
|
has_file INTEGER NOT NULL DEFAULT 0
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_channel_videos_channel ON channel_videos(channel_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_channel_videos_published ON channel_videos(published_at);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_channel_videos_wanted ON channel_videos(monitored, has_file);
|
||||||
|
|
||||||
|
-- Cheap persistent cache of YouTube video upload dates (the flat listing omits
|
||||||
|
-- them). Filled from the channel RSS feed + any per-video metadata fetch, so the
|
||||||
|
-- channel page's year-seasons fill in over time without re-fetching. Standalone
|
||||||
|
-- (no channels FK) since the bridge stores channels in video_watchlist.
|
||||||
|
CREATE TABLE IF NOT EXISTS youtube_video_dates (
|
||||||
|
youtube_id TEXT PRIMARY KEY,
|
||||||
|
published_at TEXT,
|
||||||
|
cached_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Tracks which followed channels have had their full upload dates fetched (by the
|
||||||
|
-- background enricher) so we don't re-sweep them constantly.
|
||||||
|
CREATE TABLE IF NOT EXISTS youtube_channel_enrichment (
|
||||||
|
channel_id TEXT PRIMARY KEY,
|
||||||
|
enriched_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
date_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
method TEXT -- 'innertube' | 'fallback'; NULL = legacy → re-enrich once
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Remembered per-channel catalog so re-opening a channel (especially a watchlisted
|
||||||
|
-- one) is instant: served cache-first, then a background re-stream refreshes it.
|
||||||
|
-- Upload dates stay in youtube_video_dates (merged on read); this holds the list.
|
||||||
|
CREATE TABLE IF NOT EXISTS youtube_channel_videos (
|
||||||
|
channel_id TEXT NOT NULL,
|
||||||
|
youtube_id TEXT NOT NULL,
|
||||||
|
title TEXT,
|
||||||
|
thumbnail_url TEXT,
|
||||||
|
duration TEXT, -- overlay badge, e.g. "12:34"
|
||||||
|
view_count INTEGER, -- approximate (parsed from "2.6M views")
|
||||||
|
cached_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
PRIMARY KEY (channel_id, youtube_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_ycv_channel ON youtube_channel_videos(channel_id);
|
||||||
|
|
||||||
|
-- Remembered channel metadata (avatar/subs/tags/banner) so the header renders
|
||||||
|
-- instantly on re-open without a yt-dlp re-fetch.
|
||||||
|
CREATE TABLE IF NOT EXISTS youtube_channel_meta (
|
||||||
|
channel_id TEXT PRIMARY KEY,
|
||||||
|
title TEXT,
|
||||||
|
handle TEXT,
|
||||||
|
description TEXT,
|
||||||
|
avatar_url TEXT,
|
||||||
|
banner_url TEXT,
|
||||||
|
subscriber_count INTEGER,
|
||||||
|
view_count INTEGER,
|
||||||
|
tags TEXT, -- JSON array
|
||||||
|
cached_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Per-video supplementary stats from the no-key YouTube enrichers (keyed by
|
||||||
|
-- youtube_id, NOT by channel, so a video shared across playlists is enriched
|
||||||
|
-- once). Merged onto the cached catalog on read.
|
||||||
|
-- ryd_* Return YouTube Dislike -> like/dislike estimates
|
||||||
|
-- sb_* SponsorBlock -> crowd-sourced segments (in youtube_video_segments)
|
||||||
|
-- status columns: NULL = pending, 'ok' | 'not_found' | 'error'.
|
||||||
|
CREATE TABLE IF NOT EXISTS youtube_video_stats (
|
||||||
|
youtube_id TEXT PRIMARY KEY,
|
||||||
|
like_count INTEGER,
|
||||||
|
dislike_count INTEGER,
|
||||||
|
ryd_status TEXT,
|
||||||
|
ryd_attempted TEXT,
|
||||||
|
sb_status TEXT,
|
||||||
|
sb_attempted TEXT,
|
||||||
|
dearrow_title TEXT, -- DeArrow crowd-sourced better title
|
||||||
|
dearrow_status TEXT,
|
||||||
|
dearrow_attempted TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
-- SponsorBlock crowd segments (sponsor/intro/outro/selfpromo/…) for a video.
|
||||||
|
CREATE TABLE IF NOT EXISTS youtube_video_segments (
|
||||||
|
youtube_id TEXT NOT NULL,
|
||||||
|
category TEXT NOT NULL, -- sponsor | intro | outro | selfpromo | interaction | music_offtopic | preview | filler | poi_highlight | chapter
|
||||||
|
start_sec REAL NOT NULL,
|
||||||
|
end_sec REAL NOT NULL,
|
||||||
|
votes INTEGER,
|
||||||
|
uuid TEXT NOT NULL,
|
||||||
|
PRIMARY KEY (youtube_id, uuid)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_yvseg_video ON youtube_video_segments(youtube_id);
|
||||||
|
|
||||||
|
-- ── Owned media files (the Library = content that has a file) ────────────────
|
||||||
|
-- Exactly one owner FK is set (no polymorphic id). 1 row per physical file;
|
||||||
|
-- usually 1:1 with its content, but the table allows history/extras.
|
||||||
|
CREATE TABLE IF NOT EXISTS media_files (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
movie_id INTEGER REFERENCES movies(id) ON DELETE CASCADE,
|
||||||
|
episode_id INTEGER REFERENCES episodes(id) ON DELETE CASCADE,
|
||||||
|
video_id INTEGER REFERENCES channel_videos(id) ON DELETE CASCADE,
|
||||||
|
relative_path TEXT NOT NULL,
|
||||||
|
size_bytes INTEGER,
|
||||||
|
resolution TEXT, -- 480p | 720p | 1080p | 2160p
|
||||||
|
video_codec TEXT,
|
||||||
|
audio_codec TEXT,
|
||||||
|
release_source TEXT, -- bluray | web-dl | webrip | hdtv | youtube
|
||||||
|
quality TEXT, -- resolved quality name
|
||||||
|
runtime_seconds INTEGER,
|
||||||
|
added_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
CHECK ((movie_id IS NOT NULL) + (episode_id IS NOT NULL) + (video_id IS NOT NULL) = 1)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_media_files_movie ON media_files(movie_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_media_files_episode ON media_files(episode_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_media_files_video ON media_files(video_id);
|
||||||
|
|
||||||
|
-- ── Downloads (active queue + history) ──────────────────────────────────────
|
||||||
|
-- One target per row: a movie, a single episode, a whole season (pack), or a
|
||||||
|
-- youtube video. Exactly one FK set (CHECK), no polymorphic id.
|
||||||
|
CREATE TABLE IF NOT EXISTS downloads (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
movie_id INTEGER REFERENCES movies(id) ON DELETE SET NULL,
|
||||||
|
episode_id INTEGER REFERENCES episodes(id) ON DELETE SET NULL,
|
||||||
|
season_id INTEGER REFERENCES seasons(id) ON DELETE SET NULL,
|
||||||
|
video_id INTEGER REFERENCES channel_videos(id) ON DELETE SET NULL,
|
||||||
|
title TEXT NOT NULL, -- display label
|
||||||
|
release_title TEXT, -- actual release / nzb / torrent name
|
||||||
|
source TEXT, -- torrent | usenet | youtube
|
||||||
|
client TEXT, -- qbittorrent | sabnzbd | yt-dlp ...
|
||||||
|
client_download_id TEXT, -- hash / nzo_id to poll the client
|
||||||
|
indexer TEXT,
|
||||||
|
status TEXT NOT NULL DEFAULT 'queued'
|
||||||
|
CHECK (status IN ('queued','downloading','importing',
|
||||||
|
'completed','failed','paused')),
|
||||||
|
quality TEXT,
|
||||||
|
size_bytes INTEGER,
|
||||||
|
downloaded_bytes INTEGER NOT NULL DEFAULT 0,
|
||||||
|
progress REAL NOT NULL DEFAULT 0, -- 0..100
|
||||||
|
download_speed_bps INTEGER NOT NULL DEFAULT 0,
|
||||||
|
eta_seconds INTEGER,
|
||||||
|
error_message TEXT,
|
||||||
|
added_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
started_at TEXT,
|
||||||
|
completed_at TEXT,
|
||||||
|
CHECK ((movie_id IS NOT NULL) + (episode_id IS NOT NULL)
|
||||||
|
+ (season_id IS NOT NULL) + (video_id IS NOT NULL) = 1)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_downloads_status ON downloads(status);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_downloads_completed ON downloads(completed_at);
|
||||||
|
|
||||||
|
-- ── Activity feed (dashboard "Recent Activity") ─────────────────────────────
|
||||||
|
CREATE TABLE IF NOT EXISTS activity (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
event_type TEXT NOT NULL, -- added | grabbed | imported | failed | renamed
|
||||||
|
message TEXT NOT NULL,
|
||||||
|
movie_id INTEGER REFERENCES movies(id) ON DELETE SET NULL,
|
||||||
|
episode_id INTEGER REFERENCES episodes(id) ON DELETE SET NULL,
|
||||||
|
video_id INTEGER REFERENCES channel_videos(id) ON DELETE SET NULL,
|
||||||
|
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_activity_created ON activity(created_at);
|
||||||
|
|
||||||
|
-- ── User watchlist (curated follow-list: shows + people) ────────────────────
|
||||||
|
-- DISTINCT from the library-derived v_watchlist below: this is the user's
|
||||||
|
-- explicit follow-list and may include shows/people that are NOT in the library
|
||||||
|
-- yet (the whole point of following someone). Keyed on the stable cross-context
|
||||||
|
-- tmdb_id that both shows and people carry. The monitoring/discovery engine is a
|
||||||
|
-- later phase — this table just records membership + enough to render + link.
|
||||||
|
CREATE TABLE IF NOT EXISTS video_watchlist (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
kind TEXT NOT NULL, -- 'show' | 'person' | 'channel' (youtube)
|
||||||
|
tmdb_id INTEGER NOT NULL, -- tmdb id; for non-tmdb sources a stable surrogate of source_id
|
||||||
|
title TEXT NOT NULL, -- show title / person name / channel title
|
||||||
|
poster_url TEXT, -- poster (show) / photo (person) / avatar (channel)
|
||||||
|
library_id INTEGER, -- shows.id when owned (else NULL)
|
||||||
|
-- generic source bridge: 'tmdb' (default) or 'youtube'; source_id = native id
|
||||||
|
-- (channel youtube id) for non-tmdb rows. One table, both worlds.
|
||||||
|
source TEXT NOT NULL DEFAULT 'tmdb',
|
||||||
|
source_id TEXT,
|
||||||
|
-- 'follow' = explicit user follow. 'mute' = a TOMBSTONE: the user
|
||||||
|
-- un-followed something that is on the watchlist by default (an actively
|
||||||
|
-- airing library show), so the default must not re-add it. Library shows
|
||||||
|
-- that are still airing are watched by default WITHOUT a row here.
|
||||||
|
state TEXT NOT NULL DEFAULT 'follow',
|
||||||
|
date_added TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
UNIQUE(kind, tmdb_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_video_watchlist_kind ON video_watchlist(kind);
|
||||||
|
|
||||||
|
-- WISHLIST (curated 'get this') — atomic units are MOVIES and EPISODES. Adding a
|
||||||
|
-- whole show or a season just expands into episode rows. Upcoming (un-aired)
|
||||||
|
-- episodes do NOT live here; the watchlist/calendar promote them once they air,
|
||||||
|
-- so the wishlist only ever holds things you can actually acquire right now.
|
||||||
|
CREATE TABLE IF NOT EXISTS video_wishlist (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
kind TEXT NOT NULL, -- 'movie' | 'episode' | 'video' (youtube)
|
||||||
|
tmdb_id INTEGER NOT NULL, -- movie's tmdb id | the SHOW's tmdb id (episode) | channel surrogate (video)
|
||||||
|
title TEXT NOT NULL, -- movie title | show title | channel title (video rows)
|
||||||
|
poster_url TEXT, -- movie/show poster | channel avatar (video rows)
|
||||||
|
year INTEGER, -- movie year (movie rows)
|
||||||
|
season_number INTEGER, -- episode rows
|
||||||
|
episode_number INTEGER, -- episode rows
|
||||||
|
episode_title TEXT, -- episode rows | video title (video rows)
|
||||||
|
still_url TEXT, -- episode still | video thumbnail (video rows)
|
||||||
|
episode_overview TEXT, -- episode synopsis | video description (video rows)
|
||||||
|
season_poster_url TEXT, -- the episode's SEASON poster (episode rows)
|
||||||
|
air_date TEXT, -- episode air date | video published_at (video rows)
|
||||||
|
status TEXT NOT NULL DEFAULT 'wanted', -- wanted|searching|downloading|downloaded|failed
|
||||||
|
library_id INTEGER, -- owned movies.id/shows.id when re-downloading
|
||||||
|
server_source TEXT, -- server context that added it (informational)
|
||||||
|
-- generic source bridge (mirrors video_watchlist). For 'video' rows:
|
||||||
|
-- source='youtube', source_id=video youtube id, parent_source_id=channel youtube id.
|
||||||
|
source TEXT NOT NULL DEFAULT 'tmdb',
|
||||||
|
source_id TEXT,
|
||||||
|
parent_source_id TEXT, -- owning channel's youtube id (video rows)
|
||||||
|
date_added TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
-- one row per movie, one per (show, season, episode), one per youtube video —
|
||||||
|
-- partial uniques so the shapes don't collide and re-adding is an idempotent upsert.
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_video_wishlist_movie
|
||||||
|
ON video_wishlist(tmdb_id) WHERE kind = 'movie';
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_video_wishlist_episode
|
||||||
|
ON video_wishlist(tmdb_id, season_number, episode_number) WHERE kind = 'episode';
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_video_wishlist_show ON video_wishlist(tmdb_id) WHERE kind = 'episode';
|
||||||
|
-- NOTE: the source_id / parent_source_id partial indexes are created in code
|
||||||
|
-- (VideoDatabase._ensure_indexes) AFTER the column migrations run — they can't
|
||||||
|
-- live here because this script runs via executescript() BEFORE the ALTERs, so
|
||||||
|
-- on an upgraded DB the columns wouldn't exist yet.
|
||||||
|
|
||||||
|
-- ── Derived views: Watchlist / Wishlist / Calendar ──────────────────────────
|
||||||
|
-- WATCHLIST = things you follow for NEW content: monitored shows + channels.
|
||||||
|
CREATE VIEW IF NOT EXISTS v_watchlist AS
|
||||||
|
SELECT 'show' AS kind, id, title, status, poster_url, monitored
|
||||||
|
FROM shows WHERE monitored = 1
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'channel' AS kind, id, title, NULL AS status, avatar_url AS poster_url, monitored
|
||||||
|
FROM channels WHERE monitored = 1;
|
||||||
|
|
||||||
|
-- WISHLIST = wanted-but-missing: monitored movies without a file + monitored
|
||||||
|
-- episodes that have aired but aren't owned.
|
||||||
|
CREATE VIEW IF NOT EXISTS v_wishlist AS
|
||||||
|
SELECT 'movie' AS kind, m.id AS ref_id, m.title AS title,
|
||||||
|
NULL AS parent_title, m.release_date AS due_date
|
||||||
|
FROM movies m
|
||||||
|
WHERE m.monitored = 1 AND m.has_file = 0
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'episode' AS kind, e.id AS ref_id,
|
||||||
|
e.title AS title, s.title AS parent_title, e.air_date AS due_date
|
||||||
|
FROM episodes e
|
||||||
|
JOIN shows s ON s.id = e.show_id
|
||||||
|
WHERE e.monitored = 1 AND e.has_file = 0
|
||||||
|
AND e.air_date IS NOT NULL AND e.air_date <= date('now');
|
||||||
|
|
||||||
|
-- CALENDAR = dated items (episode air dates, movie releases, channel uploads).
|
||||||
|
CREATE VIEW IF NOT EXISTS v_calendar AS
|
||||||
|
SELECT 'episode' AS kind, e.id AS ref_id, e.air_date AS date,
|
||||||
|
e.title AS title, s.title AS parent_title
|
||||||
|
FROM episodes e JOIN shows s ON s.id = e.show_id
|
||||||
|
WHERE e.air_date IS NOT NULL
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'movie' AS kind, m.id AS ref_id, m.release_date AS date,
|
||||||
|
m.title AS title, NULL AS parent_title
|
||||||
|
FROM movies m WHERE m.release_date IS NOT NULL
|
||||||
|
UNION ALL
|
||||||
|
SELECT 'video' AS kind, v.id AS ref_id, v.published_at AS date,
|
||||||
|
v.title AS title, c.title AS parent_title
|
||||||
|
FROM channel_videos v JOIN channels c ON c.id = v.channel_id
|
||||||
|
WHERE v.published_at IS NOT NULL;
|
||||||
|
|
||||||
|
-- DOWNLOADS — every grab initiated from the video side lands here (movies/tv/youtube).
|
||||||
|
-- The pipeline starts the download, watches it, and on completion moves the file to the
|
||||||
|
-- per-type library folder and marks it completed. Status: queued|downloading|completed|failed.
|
||||||
|
CREATE TABLE IF NOT EXISTS video_downloads (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
kind TEXT NOT NULL, -- movie | show | youtube
|
||||||
|
title TEXT, -- human title (e.g. the movie name)
|
||||||
|
release_title TEXT, -- the release/file being grabbed
|
||||||
|
source TEXT, -- soulseek | torrent | usenet
|
||||||
|
username TEXT, -- slskd uploader (for the grab + status)
|
||||||
|
filename TEXT, -- slskd remote filename (full path)
|
||||||
|
size_bytes INTEGER DEFAULT 0,
|
||||||
|
quality_label TEXT,
|
||||||
|
media_id TEXT, -- the movie/show id (for the detail-page link)
|
||||||
|
media_source TEXT, -- library | tmdb
|
||||||
|
year INTEGER,
|
||||||
|
poster_url TEXT, -- poster for the Downloads card
|
||||||
|
target_dir TEXT, -- destination library folder
|
||||||
|
dest_path TEXT, -- final moved path (set on completion)
|
||||||
|
status TEXT NOT NULL DEFAULT 'downloading',
|
||||||
|
progress REAL DEFAULT 0,
|
||||||
|
error TEXT,
|
||||||
|
candidates TEXT, -- JSON: remaining best-first hits to retry
|
||||||
|
search_ctx TEXT, -- JSON: {scope,title,year,season,episode}
|
||||||
|
tried_queries TEXT, -- JSON: slskd queries already searched
|
||||||
|
tried_files TEXT, -- JSON: release filenames already attempted
|
||||||
|
attempts INTEGER DEFAULT 0,
|
||||||
|
created_at TEXT DEFAULT (datetime('now')),
|
||||||
|
updated_at TEXT DEFAULT (datetime('now')),
|
||||||
|
completed_at TEXT
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_video_downloads_status ON video_downloads(status);
|
||||||
|
|
||||||
|
-- video_download_history — a PERMANENT record of every grab SoulSync completed
|
||||||
|
-- (movies + episodes). video_downloads is the transient working queue (cleaned when
|
||||||
|
-- finished); this is the archive that powers the Download History modal AND the
|
||||||
|
-- smart post-download scan (newest completed item per library → probe the server).
|
||||||
|
CREATE TABLE IF NOT EXISTS video_download_history (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
download_id INTEGER, -- original video_downloads.id (transient)
|
||||||
|
kind TEXT NOT NULL, -- movie | show
|
||||||
|
media_type TEXT, -- movie | show (normalized; the scan scope)
|
||||||
|
title TEXT, -- movie/show title
|
||||||
|
year INTEGER,
|
||||||
|
season_number INTEGER, -- episodes only
|
||||||
|
episode_number INTEGER,
|
||||||
|
episode_title TEXT,
|
||||||
|
release_title TEXT, -- the release/file grabbed
|
||||||
|
source TEXT, -- soulseek | torrent | usenet
|
||||||
|
username TEXT, -- uploader (soulseek)
|
||||||
|
filename TEXT, -- remote filename grabbed
|
||||||
|
dest_path TEXT, -- final placed path
|
||||||
|
size_bytes INTEGER DEFAULT 0,
|
||||||
|
quality_label TEXT, -- e.g. "1080p", "2160p HDR"
|
||||||
|
resolution TEXT, -- parsed from the release name, best-effort
|
||||||
|
video_codec TEXT,
|
||||||
|
media_id TEXT, -- movie/show id for the detail deep-link
|
||||||
|
media_source TEXT, -- library | tmdb
|
||||||
|
poster_url TEXT,
|
||||||
|
outcome TEXT NOT NULL DEFAULT 'completed', -- completed | import_failed | failed | cancelled
|
||||||
|
error TEXT, -- failure reason (non-completed)
|
||||||
|
grabbed_at TEXT, -- when the download started
|
||||||
|
completed_at TEXT, -- terminal time
|
||||||
|
created_at TEXT DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_vdl_history_kind ON video_download_history(kind);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_vdl_history_completed ON video_download_history(completed_at);
|
||||||
|
-- one history row per terminal download (idempotent re-persist / restart-safe)
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_vdl_history_dedup
|
||||||
|
ON video_download_history(download_id, outcome, dest_path);
|
||||||
|
|
@ -32,10 +32,20 @@ services:
|
||||||
# Persistent data volumes
|
# Persistent data volumes
|
||||||
- ./config:/app/config
|
- ./config:/app/config
|
||||||
- ./logs:/app/logs
|
- ./logs:/app/logs
|
||||||
|
# Shared INPUT folder — slskd / torrent / usenet drop downloads here, and BOTH
|
||||||
|
# the music and video sides read from it (Settings → Downloads uses one path).
|
||||||
- ./downloads:/app/downloads
|
- ./downloads:/app/downloads
|
||||||
- ./Staging:/app/Staging
|
- ./Staging:/app/Staging
|
||||||
- ./MusicVideos:/app/MusicVideos
|
- ./MusicVideos:/app/MusicVideos
|
||||||
- ./scripts:/app/scripts
|
- ./scripts:/app/scripts
|
||||||
|
# ─── Video libraries (OUTPUT) — one per content type ─────────────────
|
||||||
|
# The video Downloads tab sorts finished files into these by type. Point each
|
||||||
|
# at where Plex/Jellyfin reads from, then set the matching path on
|
||||||
|
# Settings → Downloads (the in-app placeholders match the in-container paths):
|
||||||
|
# Movies → /media/movies · TV → /media/tv · YouTube → /media/youtube
|
||||||
|
- ./Movies:/media/movies
|
||||||
|
- ./TV:/media/tv
|
||||||
|
- ./YouTube:/media/youtube
|
||||||
# Use named volume for database persistence (separate from host database)
|
# Use named volume for database persistence (separate from host database)
|
||||||
# NOTE: Changed from /app/database to /app/data to avoid overwriting Python package
|
# NOTE: Changed from /app/database to /app/data to avoid overwriting Python package
|
||||||
- soulsync_database:/app/data
|
- soulsync_database:/app/data
|
||||||
|
|
|
||||||
|
|
@ -1,116 +1,58 @@
|
||||||
# SoulSync 2.6.4 — Merge `dev` → `main`
|
# 2.7.2
|
||||||
|
|
||||||
Patch release on top of 2.6.3. Headline:
|
a big feature + fix pass on top of 2.7.1 — playlist-folder mirroring, a redesigned quality-upgrade finder, smarter artist enrichment, better tidal/youtube/soundcloud imports, M3U export, and a stack of issue fixes.
|
||||||
|
|
||||||
- **#721 — Usenet album bundles stuck on "downloading release" when SAB History flips before storage lands.** Reported by @IamGroot60 against 2.6.3, validated on the `fix/usenet-bundle-save-path-handoff` branch, merged via PR #723.
|
## organize playlists into folders
|
||||||
|
|
||||||
Everything from 2.6.3 also rolls in unchanged (was bumped on dev but never tagged / published to main / docker, so this is the first time these changes reach users).
|
soulsync can now mirror each playlist into its own folder on disk, so external players (plex / jellyfin / music assistant) see them as real folders:
|
||||||
|
|
||||||
---
|
- **symlink or copy** — symlink for no extra disk, copy if you want standalone files.
|
||||||
|
- **self-maintaining** — rebuilds after every sync and prunes tracks you removed; separate output root + a manual rebuild button in settings.
|
||||||
|
- album / single / playlist downloads all share the exact same post-processing, so the folder view always matches the library.
|
||||||
|
|
||||||
## 2.6.4 — the patch
|
## quality upgrade finder (replaces the old quality scanner)
|
||||||
|
|
||||||
### #721 — Usenet album bundle stuck on "downloading release" when SAB History flips before `storage` lands
|
the old Quality Scanner judged quality by file extension only, ignored the bitrate profile, and auto-dumped everything into the wishlist with no review. it's gone — replaced by a proper **Library Maintenance job** (off by default):
|
||||||
Follow-up to the 2.6.3 queue→history handoff fix (#706). 2.6.3 covered the gap where SAB removes a job from the queue before adding it to history. **2.6.4** covers a second-stage gap: SAB flips `status` to `Completed` in History a few seconds **before** its post-processing writes the final `storage` field.
|
|
||||||
|
|
||||||
Pre-fix: `poll_album_download` saw the first `Completed` read with `save_path=None` and bailed. The bundle plugin marked the batch failed, but the UI froze on the last `downloading progress=0.61` emit because the terminal `failed` emit never registered (renderer holds the last-known progress).
|
- **bitrate-aware** — judges each track by format *and* bitrate against your quality profile, so a 320 mp3 passes a flac+320+256 profile and a 128 mp3 doesn't (enabling mp3-320/256 finally counts).
|
||||||
|
- **findings, not auto-action** — it scans (watchlist or whole library), finds below-quality tracks, and creates reviewable findings. nothing hits the wishlist until you Apply.
|
||||||
|
- **exact matching** — resolves the better version by the most precise identity available: the source track-ID embedded in the file → ISRC → the album's tracklist (by stored album-id or album search) → name search. carries real album context, and the fuzzy steps reject wrong-length cuts (live/edit/remix).
|
||||||
|
- skips tracks it already proposed, so re-runs are cheap. transcode/fake-lossless detection stays with the existing Fake Lossless Detector job.
|
||||||
|
|
||||||
- **`poll_album_download`**: separate transient counter for "completed but no save_path." Tolerates up to `transient_miss_threshold` (default 5) consecutive reads in that state — gives SAB ~10s to land the path. When it arrives, return normally. When it doesn't, fail loudly with an explicit error pointing at the missing field.
|
## smarter artist enrichment (#868)
|
||||||
- **Sticky save_path**: earlier `downloading` reads with a non-empty `save_path` (qBit / Transmission set this from the start) remain cached. So torrent flows aren't affected by the retry path.
|
|
||||||
- **SAB adapter (`_parse_history_slot`)**: widened the save_path fallback chain — `storage` → `path` → `download_path` → `dirname`. Covers SAB version differences (older builds populated `path`) and forks that expose `download_path` or `dirname`. Whitespace-only values skipped. `incomplete_path` intentionally NOT in the chain — it'd bypass the retry window and point at the in-progress staging dir.
|
|
||||||
- **Diagnostic**: loud debug log when none of the known fields land, dumping the slot keys so we can grow `_HISTORY_SAVE_PATH_KEYS` if a fork ships a novel field name.
|
|
||||||
|
|
||||||
**9 new tests**:
|
enrichment matched artists by name only — so for a common name (there are ~5 "Rone"s) it grabbed whichever the source ranked first, often the wrong one, which then drove a wrong/sparse library discography. now when multiple same-name artists clear the name gate, it picks the one whose catalog **overlaps the albums you actually own**. wired into Spotify (+ no-auth), iTunes, Deezer, and MusicBrainz. "click to re-match" now actually re-resolves (it used to re-confirm the wrong id).
|
||||||
- `test_album_bundle.py` (3): late-save_path arrival recovers; threshold-exhausted fails cleanly; sticky save_path keeps torrent flows working.
|
|
||||||
- `test_usenet_client_adapters.py` (6): each fallback field tier, whitespace-only skip, all-empty returns None, `incomplete_path` ignored.
|
|
||||||
|
|
||||||
132 album-bundle + usenet tests pass. Strictly additive — zero impact on users whose SAB returns `storage` on the first Completed read.
|
## tidal playlist discovery (#867)
|
||||||
|
|
||||||
---
|
- discovery used to show only a subset of a playlist's tracks (a 59-track playlist surfaced ~21) — now it shows them all, driven by the authoritative backend results, and `get_playlist` chunks its track-ID fetch to Tidal's page cap so nothing's dropped.
|
||||||
|
- the discovery modal opens **instantly** instead of hanging ~10s on a blocking pre-fetch, and it's no longer interactable while it's still loading.
|
||||||
|
|
||||||
## Everything else from 2.6.3 (carried forward)
|
## export server playlists as M3U
|
||||||
|
|
||||||
### Fixes
|
one-click **Export M3U** button in the Server Playlists compare/editor toolbar — writes a standard `.m3u` and downloads it to your browser (great for music assistant). resolved via one bulk db read so it doesn't hang under active enrichment/scan writes.
|
||||||
|
|
||||||
**#715 — Soulseek album downloads stuck on "failed" after slskd finished the release.**
|
## better youtube & soundcloud imports
|
||||||
`core/soulseek_client._resolve_downloaded_album_file` probed 3 hard-coded candidate paths. On the common slskd config `directories.downloads.username = true`, files land at `<download_dir>/<username>/<filename>` — none of the 3 candidates carried a username segment, so every file looked locally missing and the bundle poll silently spun for ~30 minutes before marking the batch failed.
|
|
||||||
|
|
||||||
- Lifted the per-track flow's recursive walk-by-basename helper into `core/downloads/file_finder.py` (`find_completed_audio_file`). Bundle resolver now delegates to it. Default-slskd users see zero behavior change (3-candidate fast path preserved).
|
- **#863** — youtube / youtube-music playlists that imported as "Unknown Artist" now recover the real artist from the track's music metadata, the "Artist - Title" pattern, or the uploading channel (recovery runs in the async discovery worker so the parse stays fast).
|
||||||
- Bundle poll detects "slskd reports Completed but local file can't be resolved past a 45s grace window" → exits early with explicit log line pointing at the likely `soulseek.download_path` mismatch.
|
- **#865** — paste a soundcloud track link, including unlisted / private share urls, into manual search to download it directly.
|
||||||
- Misleading `"(0 tracks, quality=)"` log on the preflight-reuse path fixed.
|
|
||||||
- **17 new tests** pin every slskd layout (flat, username-prefixed, full-tree-preserved, deep nested, dedup-suffix, quarantine-skip, YouTube/Tidal encoded, transfer-dir fallback, fuzzy variants).
|
|
||||||
|
|
||||||
**Auto-Sync ListenBrainz pipelines stuck on `Refreshing:` for 5+ minutes.**
|
## watchlist & playlists
|
||||||
Refresh path ran `_maybe_discover` inline AND Phase 2 ran the same matching engine via `run_playlist_discovery_worker`. LB tracks discovered twice; refresh-side run blocked with zero progress emission. Also: LB manager only exposed `update_all_playlists` (refreshing one playlist re-pulled all 12+ cached playlists). Also: LB adapter had a silent `except Exception: pass` masking real API failures.
|
|
||||||
|
|
||||||
- Pipeline sets `skip_discovery=True` on refresh config; Phase 2 handles discovery with proper progress emits.
|
- **follow-only watchlist** — per-artist "auto-download" toggle (on by default). off = scans still discover/surface new releases, they just don't auto-add to the wishlist.
|
||||||
- New `LBManager.refresh_playlist(mbid)` targeted refresh.
|
- **rename mirrored playlists** — a custom name that changes the display + sync name, survives upstream refreshes, and still tracks the same server playlist.
|
||||||
- LB adapter logs exceptions with traceback at warning level + returns `None`.
|
|
||||||
- **12 new tests**.
|
|
||||||
|
|
||||||
**Wishlist: harden Spotify backfill — poisoned `tn=1` can't mask a lean album.**
|
## more requested features
|
||||||
Spotify-API backfill that hydrates `release_date` / `total_tracks` was coupled to the "track_number missing" branch, so a poisoned default-1 track_number short-circuited it. Lifted to `core/downloads/track_metadata_backfill.py` with split concerns — track-number resolution keeps its precedence chain; album hydration runs whenever `release_date` / `total_tracks` missing, independent of track_number. Single API call still serves both. Also `core/wishlist/routes.py:_build_track_data` no longer defaults `track_number=1` / `disc_number=1` / `total_tracks=1` / `release_date=''`. **24 new tests**.
|
|
||||||
|
|
||||||
**Wishlist: fix three regressions causing all imports to land as track 01.**
|
- **export your roster** — one button + scope selector dumps the watchlist and/or whole library roster to JSON / CSV / text.
|
||||||
Track→dict conversion in payload helpers dropped everything except `album.name`; Deezer-sourced discovery matches saved without `track_number`/`disc_number`; import pipeline only consulted `album_info.track_number` before falling to the filename. Track_number resolution chain lifted into `core/imports/track_number.py:resolve_track_number` with 18 unit tests.
|
- **ReplayGain Filler (#437)** and **Empty Folder Cleaner** library-maintenance jobs.
|
||||||
|
- **#857** — custom in-container completed-download path for Torrent / Usenet so finished grabs in a category subfolder are found.
|
||||||
|
- **HiFi instances** — Restore Defaults button, bigger tap targets, and a confirmed-working instance auto-pushed to existing installs (thanks Sokhi).
|
||||||
|
- **Aria2** added to the torrent client list; artist detail **"DB Record"** inspector.
|
||||||
|
|
||||||
**Wishlist: only engage album-bundle when several tracks from the same album are missing.**
|
## bug fixes
|
||||||
New `core/wishlist/album_grouping.py`. Bundle path only engages when an album has ≥2 missing tracks; single-track items take the cheaper per-track path. Configurable via `wishlist.album_bundle_min_tracks`.
|
|
||||||
|
|
||||||
**Wishlist: distinguish Queued from Analyzing batches in the UI.**
|
- **#859** — a hung database update self-heals now instead of wedging on "Starting..." forever.
|
||||||
|
- **#862** — Library Reorganize finally works on media-server libraries (falls back to tag mode when an album has no source ID).
|
||||||
**Album-bundle staging: clean Soulseek copies + sweep orphans at startup.**
|
- **spotify (no-auth)** — shows as connected and the dashboard test reports it correctly, instead of claiming a deezer fallback.
|
||||||
Cleanup gate extended to include `soulseek` (was torrent/usenet only). New `sweep_orphan_album_bundle_staging` runs once at server boot. **12 new tests**.
|
- **navidrome** reconnects itself instead of latching "disconnected"; the orphan detector hard-bails on a mass-orphan flood; plus more #852 lock-screen hardening and login-password management in Manage Profiles.
|
||||||
|
|
||||||
**Usenet album poll: tolerate SAB queue→history handoff (#706).**
|
|
||||||
|
|
||||||
**Discogs: strip artist disambiguation suffixes everywhere (#634).**
|
|
||||||
|
|
||||||
**Library: Enhanced / Standard view toggle persists per browser.**
|
|
||||||
|
|
||||||
**Fix popup: manual matches survive Playlist Pipeline runs.**
|
|
||||||
|
|
||||||
**Fix popup: artist + track fields no longer surface unrelated covers.**
|
|
||||||
|
|
||||||
### UX overhauls
|
|
||||||
|
|
||||||
**Dashboard enrichment panel — equalizer-bar redesign.** 11 speedometer tiles → 11 vertical VU-meter equalizer bars in one symmetric flex row. Brand-logo avatar disc above each bar (Spotify/Apple Music/Deezer/Last.fm/Genius/MusicBrainz/AudioDB/Tidal/Qobuz/Discogs/Amazon with initial-letter CDN-fail fallback); peak-flash on cpm step-up; rolling counter; glass-surface reflection puddle. Last.fm circle-clipped; Tidal/Qobuz/Discogs/Amazon inverted to white silhouettes.
|
|
||||||
|
|
||||||
**Auto-Sync manager — full visual overhaul.** Selector-based override layer (zero JS/HTML changes). Every surface inside the modal restyled to match the dashboard's glassy / accent-radial aesthetic.
|
|
||||||
|
|
||||||
**Auto-Sync — weekly board cards now match the hourly board.** Same Run-now button, unschedule X, next-run countdown, health badge. Weekly cards now draggable between day columns.
|
|
||||||
|
|
||||||
**Auto-Sync sidebar — brand logo on each source-group header.**
|
|
||||||
|
|
||||||
**Sync page tabs — brand-logo chips with active label pill.** 14 tabs collapsed from cramped labeled pills to circular brand-logo chips; active tab swells into a pill with its label inline. `Link` variants (Spotify Link / Deezer Link / iTunes Link) carry a small chain-link badge bottom-right.
|
|
||||||
|
|
||||||
### Architectural lifts
|
|
||||||
|
|
||||||
**Unified Playlist Sources layer.** `PlaylistSource` ABC + registry in `core/playlists/sources/`. Refresh handler dropped from ~190 lines of if/elif to ~80 lines. ListenBrainz / Last.fm / SoulSync Discovery are now Sync-page tabs.
|
|
||||||
|
|
||||||
**Auto-Sync schedule types — weekday + time.** New Weekly Board tab on the Auto-Sync manager.
|
|
||||||
|
|
||||||
**iTunes / Apple Music link import.** New iTunes Link tab on the Sync page.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Test plan
|
|
||||||
|
|
||||||
- [x] 132 album-bundle + usenet tests pass (the new #721 path)
|
|
||||||
- [x] 488 downloads tests pass (full suite)
|
|
||||||
- [x] ~90 new unit tests across the cycle, including 9 new for #721
|
|
||||||
- [x] Smoke: dashboard equalizer renders w/ brand logos, peak-flash on cpm increase
|
|
||||||
- [x] Smoke: Auto-Sync manager renders glass overhaul, hourly + weekly cards both have action rows
|
|
||||||
- [x] Smoke: Sync page tab strip renders as logo chips; active expands; Link variants show chain-link badge
|
|
||||||
- [ ] Live: @IamGroot60 to re-test Forty Licks usenet bundle on dev (build with the #721 fix applied)
|
|
||||||
- [ ] Live: Soulseek album download on a username-subdir slskd config completes cleanly (#715, user-validated post-merge)
|
|
||||||
- [ ] Live: bundle staging dir cleaned on completion (user-validated post-merge)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Post-merge checklist
|
|
||||||
|
|
||||||
- [ ] Tag `v2.6.4` on `main`
|
|
||||||
- [ ] Trigger `docker-publish.yml` with `version_tag: 2.6.4` to push `boulderbadgedad/soulsync:2.6.4` + `ghcr.io/nezreka/soulsync:2.6.4` (default already updated)
|
|
||||||
- [ ] Discord release announcement (auto-fired by the workflow)
|
|
||||||
- [ ] Reply on #721 with the 2.6.4 release link
|
|
||||||
|
|
|
||||||
|
|
@ -81,3 +81,66 @@ def test_event_triggers_with_conditions_have_condition_fields():
|
||||||
assert 'condition_fields' in t, f"{t['type']} marked has_conditions but no condition_fields"
|
assert 'condition_fields' in t, f"{t['type']} marked has_conditions but no condition_fields"
|
||||||
assert isinstance(t['condition_fields'], list)
|
assert isinstance(t['condition_fields'], list)
|
||||||
assert len(t['condition_fields']) > 0
|
assert len(t['condition_fields']) > 0
|
||||||
|
|
||||||
|
|
||||||
|
# ── scope filtering (music vs the isolated video builder) ────────────────
|
||||||
|
|
||||||
|
def test_video_only_block_hidden_from_music_builder():
|
||||||
|
"""The video action must never appear on the music builder."""
|
||||||
|
music = blocks.blocks_for_scope('music')
|
||||||
|
assert 'video_scan_library' not in {a['type'] for a in music['actions']}
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_builder_gets_video_block_plus_generics():
|
||||||
|
video = blocks.blocks_for_scope('video')
|
||||||
|
action_types = {a['type'] for a in video['actions']}
|
||||||
|
# its own action…
|
||||||
|
assert 'video_scan_library' in action_types
|
||||||
|
# …plus the generic (scope='both') ones it shares with music…
|
||||||
|
assert 'notify_only' in action_types
|
||||||
|
assert 'run_script' in action_types
|
||||||
|
# …but NOT music-only actions.
|
||||||
|
assert 'process_wishlist' not in action_types
|
||||||
|
assert 'scan_library' not in action_types
|
||||||
|
|
||||||
|
|
||||||
|
def test_generic_blocks_appear_on_both_sides():
|
||||||
|
"""Every scope='both' block shows on music AND video."""
|
||||||
|
music = blocks.blocks_for_scope('music')
|
||||||
|
video = blocks.blocks_for_scope('video')
|
||||||
|
for key in ('triggers', 'actions', 'notifications'):
|
||||||
|
both = {b['type'] for b in getattr(blocks, key.upper()) if b.get('scope') == 'both'}
|
||||||
|
assert both, f"expected at least one scope='both' {key}"
|
||||||
|
assert both <= {b['type'] for b in music[key]}, f"music missing a 'both' {key}"
|
||||||
|
assert both <= {b['type'] for b in video[key]}, f"video missing a 'both' {key}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_music_scope_matches_legacy_full_lists_minus_video():
|
||||||
|
"""scope='music' must reproduce the pre-scope behaviour: everything that
|
||||||
|
isn't explicitly video-only. Guards against accidentally hiding a music
|
||||||
|
block when new scope tags are added."""
|
||||||
|
music = blocks.blocks_for_scope('music')
|
||||||
|
for key in ('triggers', 'actions', 'notifications'):
|
||||||
|
expected = {b['type'] for b in getattr(blocks, key.upper()) if b.get('scope') != 'video'}
|
||||||
|
assert {b['type'] for b in music[key]} == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_post_download_chain_blocks_are_video_scoped():
|
||||||
|
music = blocks.blocks_for_scope('music')
|
||||||
|
video = blocks.blocks_for_scope('video')
|
||||||
|
trig = {t['type'] for t in video['triggers']}
|
||||||
|
act = {a['type'] for a in video['actions']}
|
||||||
|
assert {'video_batch_complete', 'video_library_scan_completed'} <= trig
|
||||||
|
assert {'video_scan_server', 'video_update_database'} <= act
|
||||||
|
mtrig = {t['type'] for t in music['triggers']}
|
||||||
|
mact = {a['type'] for a in music['actions']}
|
||||||
|
assert not ({'video_batch_complete', 'video_library_scan_completed'} & mtrig)
|
||||||
|
assert not ({'video_scan_server', 'video_update_database'} & mact)
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_scan_library_block_shape():
|
||||||
|
action = next(a for a in blocks.ACTIONS if a['type'] == 'video_scan_library')
|
||||||
|
assert action['scope'] == 'video'
|
||||||
|
mode = next(f for f in action['config_fields'] if f['key'] == 'mode')
|
||||||
|
assert {o['value'] for o in mode['options']} == {'full', 'incremental', 'deep'}
|
||||||
|
assert mode['default'] == 'full'
|
||||||
|
|
|
||||||
|
|
@ -403,3 +403,164 @@ def test_end_to_end_monthly_schedule_produces_valid_db_string(engine_with_db):
|
||||||
assert parsed.day == 15
|
assert parsed.day == 15
|
||||||
assert parsed.hour == 9
|
assert parsed.hour == 9
|
||||||
assert parsed.minute == 0
|
assert parsed.minute == 0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# System-automation seeding — owned_by / action_config (video side).
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_ensure_system_automations_seeds_video_with_owned_by_and_mode():
|
||||||
|
"""The video post-download chain twins must seed with owned_by='video' so they
|
||||||
|
stay off the music page; the standalone 'Scan Video Library' is NO LONGER seeded
|
||||||
|
(superseded by the chain). Music system automations keep owned_by=None."""
|
||||||
|
db = MagicMock()
|
||||||
|
db.get_system_automation_by_action.return_value = None # nothing seeded yet
|
||||||
|
created = {}
|
||||||
|
|
||||||
|
def _create(**kw):
|
||||||
|
created[kw['action_type']] = kw
|
||||||
|
return 'id-' + kw['action_type']
|
||||||
|
|
||||||
|
db.create_automation.side_effect = _create
|
||||||
|
|
||||||
|
engine = AutomationEngine(db)
|
||||||
|
engine.ensure_system_automations()
|
||||||
|
|
||||||
|
# The standalone scheduled scan is gone; the chain twins are seeded, video-tagged.
|
||||||
|
assert 'video_scan_library' not in created
|
||||||
|
assert created['video_scan_server']['owned_by'] == 'video'
|
||||||
|
assert created['video_update_database']['owned_by'] == 'video'
|
||||||
|
assert json.loads(created['video_update_database']['action_config']) == {'mode': 'incremental'}
|
||||||
|
|
||||||
|
# Music automations stay owned_by=None (shown on the music page).
|
||||||
|
assert created['scan_library']['owned_by'] is None
|
||||||
|
assert json.loads(created['scan_library']['action_config']) == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_scan_library_system_automation_is_cleaned_up():
|
||||||
|
"""The obsolete standalone 'Scan Video Library' system automation is deleted when
|
||||||
|
present (superseded by the post-download chain). No stuck flag — it just keys off
|
||||||
|
the lookup, so once the row is gone it no-ops."""
|
||||||
|
# present → deleted (matches only the is_system-seeded row)
|
||||||
|
db = MagicMock()
|
||||||
|
db.get_system_automation_by_action.return_value = {'id': 99, 'is_system': 1}
|
||||||
|
AutomationEngine(db)._fix_video_scan_default()
|
||||||
|
db.delete_automation.assert_any_call(99)
|
||||||
|
|
||||||
|
# absent (already gone) → idempotent no-op, never errors or deletes
|
||||||
|
db2 = MagicMock()
|
||||||
|
db2.get_system_automation_by_action.return_value = None
|
||||||
|
AutomationEngine(db2)._fix_video_scan_default()
|
||||||
|
db2.delete_automation.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_airing_automation_migrates_24h_interval_to_daily_1am():
|
||||||
|
"""The old rolling-24h 'Auto-Wishlist Episodes Airing Today' row is rewritten to a
|
||||||
|
fixed daily 01:00 (server-local) so it stops drifting with restarts."""
|
||||||
|
db = MagicMock()
|
||||||
|
db.get_system_automation_by_action.return_value = {
|
||||||
|
'id': 42, 'is_system': 1, 'trigger_type': 'schedule',
|
||||||
|
'trigger_config': json.dumps({'interval': 24, 'unit': 'hours'})}
|
||||||
|
eng = AutomationEngine(db)
|
||||||
|
eng._default_tz = 'UTC'
|
||||||
|
eng._fix_airing_automation_schedule()
|
||||||
|
|
||||||
|
db.update_automation.assert_called_once()
|
||||||
|
args, kwargs = db.update_automation.call_args
|
||||||
|
assert args[0] == 42
|
||||||
|
assert kwargs['trigger_type'] == 'daily_time'
|
||||||
|
assert json.loads(kwargs['trigger_config']) == {'time': '01:00'}
|
||||||
|
# next_run is armed for 01:00 UTC (the next occurrence)
|
||||||
|
assert kwargs['next_run'].endswith(' 01:00:00')
|
||||||
|
|
||||||
|
|
||||||
|
def test_airing_automation_migration_is_idempotent():
|
||||||
|
"""Once the row is already daily_time, re-running the migration is a no-op (it
|
||||||
|
must not rewrite a user's edited time or re-arm next_run every startup)."""
|
||||||
|
db = MagicMock()
|
||||||
|
db.get_system_automation_by_action.return_value = {
|
||||||
|
'id': 42, 'is_system': 1, 'trigger_type': 'daily_time',
|
||||||
|
'trigger_config': json.dumps({'time': '06:30'})}
|
||||||
|
AutomationEngine(db)._fix_airing_automation_schedule()
|
||||||
|
db.update_automation.assert_not_called()
|
||||||
|
|
||||||
|
# absent (fresh install, created straight as daily_time) → no-op, never errors
|
||||||
|
db2 = MagicMock()
|
||||||
|
db2.get_system_automation_by_action.return_value = None
|
||||||
|
AutomationEngine(db2)._fix_airing_automation_schedule()
|
||||||
|
db2.update_automation.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_deep_scans_migrate_to_fixed_weekly_times():
|
||||||
|
"""The two video deep scans move from a rolling 7-day interval to fixed weekly
|
||||||
|
times — TV Mondays 02:00, Movies Tuesdays 02:00."""
|
||||||
|
rows = {
|
||||||
|
'video_deep_scan_tv': {'id': 7, 'is_system': 1, 'trigger_type': 'schedule',
|
||||||
|
'trigger_config': json.dumps({'interval': 7, 'unit': 'days'})},
|
||||||
|
'video_deep_scan_movies': {'id': 8, 'is_system': 1, 'trigger_type': 'schedule',
|
||||||
|
'trigger_config': json.dumps({'interval': 7, 'unit': 'days'})},
|
||||||
|
}
|
||||||
|
db = MagicMock()
|
||||||
|
db.get_system_automation_by_action.side_effect = lambda a: rows.get(a)
|
||||||
|
eng = AutomationEngine(db)
|
||||||
|
eng._default_tz = 'UTC'
|
||||||
|
eng._fix_deep_scan_schedules()
|
||||||
|
|
||||||
|
by_id = {c.args[0]: c.kwargs for c in db.update_automation.call_args_list}
|
||||||
|
assert json.loads(by_id[7]['trigger_config']) == {'time': '02:00', 'days': ['mon']} # TV → Mon
|
||||||
|
assert json.loads(by_id[8]['trigger_config']) == {'time': '02:00', 'days': ['tue']} # Movies → Tue
|
||||||
|
assert by_id[7]['trigger_type'] == 'weekly_time' and by_id[8]['trigger_type'] == 'weekly_time'
|
||||||
|
assert by_id[7]['next_run'].endswith(' 02:00:00') and by_id[8]['next_run'].endswith(' 02:00:00')
|
||||||
|
|
||||||
|
|
||||||
|
def test_deep_scan_migration_is_idempotent_and_safe_when_absent():
|
||||||
|
# already weekly_time → no-op (a hand-tuned day/time isn't reverted)
|
||||||
|
db = MagicMock()
|
||||||
|
db.get_system_automation_by_action.side_effect = lambda a: {
|
||||||
|
'id': 7, 'is_system': 1, 'trigger_type': 'weekly_time',
|
||||||
|
'trigger_config': json.dumps({'time': '05:00', 'days': ['sat']})}
|
||||||
|
AutomationEngine(db)._fix_deep_scan_schedules()
|
||||||
|
db.update_automation.assert_not_called()
|
||||||
|
|
||||||
|
# absent (not seeded yet) → no-op, never errors
|
||||||
|
db2 = MagicMock()
|
||||||
|
db2.get_system_automation_by_action.return_value = None
|
||||||
|
AutomationEngine(db2)._fix_deep_scan_schedules()
|
||||||
|
db2.update_automation.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_wishlist_processor_rename_removes_orphans_and_renames_youtube():
|
||||||
|
"""The Download→Process rename: a DB seeded under the old names has orphaned
|
||||||
|
'Auto-Download Movie/Episode Wishlist' rows (dead action_type) — they're deleted (after
|
||||||
|
lifting the is_system delete-guard); the YouTube row (action_type unchanged) is renamed
|
||||||
|
in place, never deleted."""
|
||||||
|
rows = {
|
||||||
|
'video_download_movie_wishlist': {'id': 10, 'is_system': 1, 'name': 'Auto-Download Movie Wishlist'},
|
||||||
|
'video_download_episode_wishlist': {'id': 11, 'is_system': 1, 'name': 'Auto-Download Episode Wishlist'},
|
||||||
|
'video_process_youtube_wishlist': {'id': 12, 'is_system': 1, 'name': 'Auto-Download YouTube Wishlist'},
|
||||||
|
}
|
||||||
|
db = MagicMock()
|
||||||
|
db.get_system_automation_by_action.side_effect = lambda a: rows.get(a)
|
||||||
|
AutomationEngine(db)._fix_wishlist_processor_rename()
|
||||||
|
|
||||||
|
db.delete_automation.assert_any_call(10)
|
||||||
|
db.delete_automation.assert_any_call(11)
|
||||||
|
# is_system=0 lifted on each orphan before deleting
|
||||||
|
cleared = {c.args[0] for c in db.update_automation.call_args_list if c.kwargs.get('is_system') == 0}
|
||||||
|
assert cleared == {10, 11}
|
||||||
|
# youtube renamed (in place), NOT deleted
|
||||||
|
renamed = [c for c in db.update_automation.call_args_list if c.kwargs.get('name')]
|
||||||
|
assert renamed and renamed[0].args[0] == 12
|
||||||
|
assert renamed[0].kwargs['name'] == 'Auto-Process YouTube Wishlist'
|
||||||
|
assert all(c.args[0] != 12 for c in db.delete_automation.call_args_list)
|
||||||
|
|
||||||
|
|
||||||
|
def test_wishlist_processor_rename_is_idempotent_when_clean():
|
||||||
|
# orphans already gone + youtube already renamed → no deletes, no rename
|
||||||
|
rows = {'video_process_youtube_wishlist': {'id': 12, 'is_system': 1, 'name': 'Auto-Process YouTube Wishlist'}}
|
||||||
|
db = MagicMock()
|
||||||
|
db.get_system_automation_by_action.side_effect = lambda a: rows.get(a)
|
||||||
|
AutomationEngine(db)._fix_wishlist_processor_rename()
|
||||||
|
db.delete_automation.assert_not_called()
|
||||||
|
assert not any(c.kwargs.get('name') for c in db.update_automation.call_args_list)
|
||||||
|
|
|
||||||
|
|
@ -53,6 +53,23 @@ EXPECTED_ACTION_NAMES = frozenset({
|
||||||
'full_cleanup',
|
'full_cleanup',
|
||||||
'run_script',
|
'run_script',
|
||||||
'search_and_download',
|
'search_and_download',
|
||||||
|
# Video side (isolated app, shared engine).
|
||||||
|
'video_scan_library',
|
||||||
|
'video_deep_scan_movies',
|
||||||
|
'video_deep_scan_tv',
|
||||||
|
'video_scan_server',
|
||||||
|
'video_update_database',
|
||||||
|
'video_add_airing_episodes',
|
||||||
|
'video_scan_watchlist_people',
|
||||||
|
'video_scan_watchlist_channels',
|
||||||
|
'video_scan_watchlist_playlists',
|
||||||
|
'video_process_movie_wishlist',
|
||||||
|
'video_process_episode_wishlist',
|
||||||
|
'video_process_youtube_wishlist',
|
||||||
|
'video_clean_search_history',
|
||||||
|
'video_clean_completed_downloads',
|
||||||
|
'video_full_cleanup',
|
||||||
|
'video_backup_database',
|
||||||
})
|
})
|
||||||
|
|
||||||
# Action names that MUST register a guard (duplicate-run prevention).
|
# Action names that MUST register a guard (duplicate-run prevention).
|
||||||
|
|
@ -66,6 +83,8 @@ EXPECTED_GUARDED_ACTIONS = frozenset({
|
||||||
'deep_scan_library',
|
'deep_scan_library',
|
||||||
'run_duplicate_cleaner',
|
'run_duplicate_cleaner',
|
||||||
'start_quality_scan',
|
'start_quality_scan',
|
||||||
|
'video_process_movie_wishlist',
|
||||||
|
'video_process_episode_wishlist',
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -174,6 +193,25 @@ class TestActionHandlerRegistration:
|
||||||
assert not missing, f"register_all dropped: {missing}"
|
assert not missing, f"register_all dropped: {missing}"
|
||||||
assert not extra, f"register_all added unexpected: {extra}"
|
assert not extra, f"register_all added unexpected: {extra}"
|
||||||
|
|
||||||
|
def test_deep_scans_route_to_readonly_update_in_deep_mode(self, monkeypatch):
|
||||||
|
"""A deep scan is a full READ + reconcile (music's full-refresh equivalent) —
|
||||||
|
it must NOT nudge Plex to rescan its disk. So the deep-scan action types route
|
||||||
|
to the read-only update-database handler in 'deep' mode, scoped per library,
|
||||||
|
never to the nudge+read scan-library handler."""
|
||||||
|
import core.automation.handlers.registration as reg
|
||||||
|
seen = []
|
||||||
|
monkeypatch.setattr(reg, 'auto_video_update_database',
|
||||||
|
lambda config, deps: seen.append(config) or {'status': 'completed'})
|
||||||
|
monkeypatch.setattr(reg, 'auto_video_scan_library',
|
||||||
|
lambda *a, **k: pytest.fail('deep scan must not nudge the server'))
|
||||||
|
engine = _RecordingEngine()
|
||||||
|
register_all(_build_deps(engine))
|
||||||
|
|
||||||
|
engine.action_handlers['video_deep_scan_tv']['handler']({'_automation_id': 'x'})
|
||||||
|
engine.action_handlers['video_deep_scan_movies']['handler']({})
|
||||||
|
assert seen[0]['media_type'] == 'show' and seen[0]['mode'] == 'deep'
|
||||||
|
assert seen[1]['media_type'] == 'movie' and seen[1]['mode'] == 'deep'
|
||||||
|
|
||||||
def test_guarded_actions_have_a_guard(self):
|
def test_guarded_actions_have_a_guard(self):
|
||||||
engine = _RecordingEngine()
|
engine = _RecordingEngine()
|
||||||
register_all(_build_deps(engine))
|
register_all(_build_deps(engine))
|
||||||
|
|
|
||||||
475
tests/automation/test_handler_video_scan_library.py
Normal file
475
tests/automation/test_handler_video_scan_library.py
Normal file
|
|
@ -0,0 +1,475 @@
|
||||||
|
"""Seam-level tests for the ``video_scan_library`` automation handler.
|
||||||
|
|
||||||
|
The handler is the VIDEO twin of ``scan_library``: it nudges the media
|
||||||
|
server to rescan the user's selected video sections, then reads the result
|
||||||
|
into video.db. Both side effects are injected (``server_refresh`` /
|
||||||
|
``run_video_scan``) so these tests exercise the real handler logic with
|
||||||
|
fakes — no Flask, no DB, no media server.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
from core.automation.handlers.video_scan_library import auto_video_scan_library
|
||||||
|
|
||||||
|
|
||||||
|
class _RecordingDeps:
|
||||||
|
"""Captures every update_progress call so tests can assert on the
|
||||||
|
streamed phase/log/status sequence."""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.calls: List[Dict[str, Any]] = []
|
||||||
|
|
||||||
|
def update_progress(self, automation_id: Any = None, **kw: Any) -> None:
|
||||||
|
kw['_id'] = automation_id
|
||||||
|
self.calls.append(kw)
|
||||||
|
|
||||||
|
# convenience accessors
|
||||||
|
def statuses(self) -> List[str]:
|
||||||
|
return [c['status'] for c in self.calls if 'status' in c]
|
||||||
|
|
||||||
|
def log_types(self) -> List[str]:
|
||||||
|
return [c['log_type'] for c in self.calls if 'log_type' in c]
|
||||||
|
|
||||||
|
|
||||||
|
def _refresh_ok(sections: int = 2):
|
||||||
|
return lambda media_type=None: {'ok': True, 'sections': sections}
|
||||||
|
|
||||||
|
|
||||||
|
def _scan_done(movies: int = 3, shows: int = 1, episodes: int = 9):
|
||||||
|
return lambda mode, media_type=None: {'state': 'done', 'movies': movies, 'shows': shows, 'episodes': episodes}
|
||||||
|
|
||||||
|
|
||||||
|
class TestHappyPath:
|
||||||
|
def test_returns_completed_with_counts(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
result = auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a', 'mode': 'full'}, deps,
|
||||||
|
server_refresh=_refresh_ok(), run_video_scan=_scan_done(3, 1, 9),
|
||||||
|
)
|
||||||
|
assert result == {
|
||||||
|
'status': 'completed', '_manages_own_progress': True,
|
||||||
|
'movies': 3, 'shows': 1, 'episodes': 9,
|
||||||
|
}
|
||||||
|
|
||||||
|
def test_finishes_at_100_percent(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a'}, deps,
|
||||||
|
server_refresh=_refresh_ok(), run_video_scan=_scan_done(),
|
||||||
|
)
|
||||||
|
# The final progress call drives the card to completion.
|
||||||
|
final = deps.calls[-1]
|
||||||
|
assert final.get('status') == 'finished'
|
||||||
|
assert final.get('progress') == 100
|
||||||
|
|
||||||
|
def test_passes_configured_mode_through_to_scan(self):
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def _scan(mode, media_type=None):
|
||||||
|
seen['mode'] = mode
|
||||||
|
return {'state': 'done'}
|
||||||
|
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a', 'mode': 'deep'}, deps,
|
||||||
|
server_refresh=_refresh_ok(), run_video_scan=_scan,
|
||||||
|
)
|
||||||
|
assert seen['mode'] == 'deep'
|
||||||
|
|
||||||
|
def test_defaults_mode_to_full(self):
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def _scan(mode, media_type=None):
|
||||||
|
seen['mode'] = mode
|
||||||
|
return {'state': 'done'}
|
||||||
|
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a'}, deps,
|
||||||
|
server_refresh=_refresh_ok(), run_video_scan=_scan,
|
||||||
|
)
|
||||||
|
assert seen['mode'] == 'full'
|
||||||
|
|
||||||
|
|
||||||
|
class TestMediaTypeScope:
|
||||||
|
"""The Movie and TV deep scans run the same handler scoped via media_type."""
|
||||||
|
|
||||||
|
def test_passes_media_type_through_to_scan(self):
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def _scan(mode, media_type=None):
|
||||||
|
seen['media_type'] = media_type
|
||||||
|
return {'state': 'done'}
|
||||||
|
|
||||||
|
auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a', 'mode': 'deep', 'media_type': 'show'}, _RecordingDeps(),
|
||||||
|
server_refresh=_refresh_ok(), run_video_scan=_scan)
|
||||||
|
assert seen['media_type'] == 'show'
|
||||||
|
|
||||||
|
def test_defaults_media_type_to_all(self):
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def _scan(mode, media_type=None):
|
||||||
|
seen['media_type'] = media_type
|
||||||
|
return {'state': 'done'}
|
||||||
|
|
||||||
|
auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a'}, _RecordingDeps(),
|
||||||
|
server_refresh=_refresh_ok(), run_video_scan=_scan)
|
||||||
|
assert seen['media_type'] == 'all'
|
||||||
|
|
||||||
|
def test_movie_scan_summary_names_only_movies(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a', 'media_type': 'movie'}, deps,
|
||||||
|
server_refresh=_refresh_ok(), run_video_scan=_scan_done(7, 0, 0))
|
||||||
|
summary = deps.calls[-1].get('log_line', '')
|
||||||
|
assert 'Movie library scanned: 7 movies' == summary # no "0 shows"
|
||||||
|
|
||||||
|
def test_tv_scan_summary_names_only_tv(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a', 'media_type': 'show'}, deps,
|
||||||
|
server_refresh=_refresh_ok(), run_video_scan=_scan_done(0, 4, 22))
|
||||||
|
summary = deps.calls[-1].get('log_line', '')
|
||||||
|
assert summary == 'TV library scanned: 4 shows, 22 episodes'
|
||||||
|
|
||||||
|
def test_busy_scanner_skips_cleanly(self):
|
||||||
|
# The singleton scanner reports another run in progress → skip, don't error.
|
||||||
|
res = auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a', 'media_type': 'movie'}, _RecordingDeps(),
|
||||||
|
server_refresh=_refresh_ok(),
|
||||||
|
run_video_scan=lambda mode, media_type=None: {'state': 'in_progress'})
|
||||||
|
assert res['status'] == 'skipped'
|
||||||
|
|
||||||
|
|
||||||
|
class TestServerUnavailable:
|
||||||
|
def test_warns_but_still_reads_library(self):
|
||||||
|
"""A server that can't be triggered is a warning, not a failure —
|
||||||
|
the read still mirrors whatever the server currently reports."""
|
||||||
|
scanned = {}
|
||||||
|
|
||||||
|
def _scan(mode, media_type=None):
|
||||||
|
scanned['ran'] = True
|
||||||
|
return {'state': 'done', 'movies': 5}
|
||||||
|
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
result = auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a'}, deps,
|
||||||
|
server_refresh=lambda media_type=None: {'ok': False, 'error': 'No video server configured'},
|
||||||
|
run_video_scan=_scan,
|
||||||
|
)
|
||||||
|
assert scanned.get('ran') is True
|
||||||
|
assert result['status'] == 'completed'
|
||||||
|
assert 'warning' in deps.log_types()
|
||||||
|
|
||||||
|
def test_none_refresh_result_is_tolerated(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
result = auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a'}, deps,
|
||||||
|
server_refresh=lambda media_type=None: None, run_video_scan=_scan_done(),
|
||||||
|
)
|
||||||
|
assert result['status'] == 'completed'
|
||||||
|
|
||||||
|
|
||||||
|
class TestScanFailure:
|
||||||
|
def test_scan_error_state_returns_error(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
result = auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a'}, deps,
|
||||||
|
server_refresh=_refresh_ok(),
|
||||||
|
run_video_scan=lambda mode, media_type=None: {'state': 'error', 'error': 'no connected server'},
|
||||||
|
)
|
||||||
|
assert result['status'] == 'error'
|
||||||
|
assert result['error'] == 'no connected server'
|
||||||
|
assert result['_manages_own_progress'] is True
|
||||||
|
assert 'error' in deps.statuses()
|
||||||
|
|
||||||
|
def test_none_scan_result_does_not_crash(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
result = auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a'}, deps,
|
||||||
|
server_refresh=_refresh_ok(), run_video_scan=lambda mode, media_type=None: None,
|
||||||
|
)
|
||||||
|
# No state -> treated as a (zero-count) completion, never raises.
|
||||||
|
assert result['status'] == 'completed'
|
||||||
|
assert result['movies'] == 0
|
||||||
|
|
||||||
|
|
||||||
|
class TestHandlerNeverRaises:
|
||||||
|
def test_swallows_refresh_exception(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
result = auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a'}, deps,
|
||||||
|
server_refresh=lambda media_type=None: (_ for _ in ()).throw(RuntimeError('boom')),
|
||||||
|
)
|
||||||
|
assert result['status'] == 'error'
|
||||||
|
assert result['error'] == 'boom'
|
||||||
|
assert result['_manages_own_progress'] is True
|
||||||
|
|
||||||
|
def test_swallows_scan_exception(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
result = auto_video_scan_library(
|
||||||
|
{'_automation_id': 'a'}, deps,
|
||||||
|
server_refresh=_refresh_ok(),
|
||||||
|
run_video_scan=lambda mode, media_type=None: (_ for _ in ()).throw(ValueError('kaboom')),
|
||||||
|
)
|
||||||
|
assert result['status'] == 'error'
|
||||||
|
assert result['error'] == 'kaboom'
|
||||||
|
|
||||||
|
def test_missing_automation_id_is_fine(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
result = auto_video_scan_library(
|
||||||
|
{}, deps, server_refresh=_refresh_ok(), run_video_scan=_scan_done(),
|
||||||
|
)
|
||||||
|
assert result['status'] == 'completed'
|
||||||
|
|
||||||
|
|
||||||
|
# ── post-download chain: video_scan_server (stage 1) ───────────────────────
|
||||||
|
from core.automation.handlers.video_scan_library import ( # noqa: E402
|
||||||
|
auto_video_scan_server, auto_video_update_database, wait_for_server_scan)
|
||||||
|
|
||||||
|
|
||||||
|
class TestWaitForServerScan:
|
||||||
|
"""Poll the server until its scan queue is idle; fixed wait only as a fallback."""
|
||||||
|
|
||||||
|
def test_returns_quickly_when_already_idle(self):
|
||||||
|
slept = []
|
||||||
|
waited = wait_for_server_scan(lambda: False, slept.append, grace_seconds=15)
|
||||||
|
assert waited == 15 and slept == [15] # just the grace, then idle
|
||||||
|
|
||||||
|
def test_polls_until_the_scan_finishes(self):
|
||||||
|
# scanning for three polls, then idle — handles a 10-20 min big-library scan
|
||||||
|
seq = iter([True, True, True, False])
|
||||||
|
slept = []
|
||||||
|
waited = wait_for_server_scan(lambda: next(seq), slept.append,
|
||||||
|
grace_seconds=15, interval_seconds=10)
|
||||||
|
assert waited == 15 + 30 # grace + 3 polls
|
||||||
|
assert slept == [15, 10, 10, 10]
|
||||||
|
|
||||||
|
def test_falls_back_to_fixed_wait_when_status_unknown(self):
|
||||||
|
slept = []
|
||||||
|
waited = wait_for_server_scan(lambda: None, slept.append,
|
||||||
|
grace_seconds=15, fallback_seconds=120)
|
||||||
|
assert waited == 120 and slept == [15, 105] # grace + (fallback - grace)
|
||||||
|
|
||||||
|
def test_stops_at_the_cap_on_a_runaway_scan(self):
|
||||||
|
slept = []
|
||||||
|
waited = wait_for_server_scan(lambda: True, slept.append,
|
||||||
|
grace_seconds=0, interval_seconds=10, cap_seconds=50)
|
||||||
|
assert waited == 50 # never hangs forever
|
||||||
|
|
||||||
|
def test_stops_if_status_becomes_unknown_mid_poll(self):
|
||||||
|
seq = iter([True, None])
|
||||||
|
slept = []
|
||||||
|
wait_for_server_scan(lambda: next(seq), slept.append, grace_seconds=0, interval_seconds=10)
|
||||||
|
assert slept == [10] # one poll, then bail — no hang
|
||||||
|
|
||||||
|
|
||||||
|
class TestScanServerStage:
|
||||||
|
def test_waits_for_idle_then_emits_scan_done(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
events = []
|
||||||
|
# scanning once, then idle
|
||||||
|
seq = iter([True, False])
|
||||||
|
r = auto_video_scan_server(
|
||||||
|
{'_automation_id': 'a'}, deps,
|
||||||
|
server_refresh=_refresh_ok(), sleep=lambda s: None,
|
||||||
|
latest_completed=lambda sc: None, scan_status=lambda mt: next(seq),
|
||||||
|
emit=lambda ev, data: events.append((ev, data)))
|
||||||
|
assert r['status'] == 'completed'
|
||||||
|
assert events == [('video_library_scan_completed', {'server': '', 'media_type': 'all'})]
|
||||||
|
|
||||||
|
def test_falls_back_to_fixed_wait_when_server_cant_report(self):
|
||||||
|
slept = []
|
||||||
|
auto_video_scan_server(
|
||||||
|
{'_automation_id': 'a', 'debounce_seconds': 90}, _RecordingDeps(),
|
||||||
|
server_refresh=_refresh_ok(), sleep=slept.append,
|
||||||
|
latest_completed=lambda sc: None, scan_status=lambda mt: None, emit=lambda ev, data: None)
|
||||||
|
assert sum(slept) == 90 # honoured the fallback wait
|
||||||
|
|
||||||
|
def test_server_unavailable_still_emits(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
events = []
|
||||||
|
auto_video_scan_server(
|
||||||
|
{'_automation_id': 'a'}, deps,
|
||||||
|
server_refresh=lambda media_type=None: {'ok': False, 'error': 'no server'},
|
||||||
|
sleep=lambda s: None, latest_completed=lambda sc: None, scan_status=lambda mt: False, emit=lambda ev, data: events.append(ev))
|
||||||
|
assert events == ['video_library_scan_completed'] # fires even if refresh failed
|
||||||
|
assert 'warning' in deps.log_types()
|
||||||
|
|
||||||
|
def test_never_raises(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
r = auto_video_scan_server(
|
||||||
|
{'_automation_id': 'a'}, deps,
|
||||||
|
server_refresh=lambda media_type=None: (_ for _ in ()).throw(RuntimeError('boom')),
|
||||||
|
sleep=lambda s: None, latest_completed=lambda sc: None, scan_status=lambda mt: False, emit=lambda *a: None)
|
||||||
|
assert r['status'] == 'error' and r['error'] == 'boom'
|
||||||
|
|
||||||
|
def test_scopes_refresh_and_status_and_carries_media_type_on_the_event(self):
|
||||||
|
seen = {}
|
||||||
|
events = []
|
||||||
|
|
||||||
|
def _refresh(media_type=None):
|
||||||
|
seen['refresh_mt'] = media_type
|
||||||
|
return {'ok': True}
|
||||||
|
|
||||||
|
auto_video_scan_server(
|
||||||
|
{'_automation_id': 'a', 'media_type': 'show'}, _RecordingDeps(),
|
||||||
|
server_refresh=_refresh, sleep=lambda s: None,
|
||||||
|
latest_completed=lambda sc: None, scan_status=lambda mt: seen.setdefault('status_mt', mt) and False,
|
||||||
|
emit=lambda ev, data: events.append((ev, data)))
|
||||||
|
assert seen['refresh_mt'] == 'show' # only TV sections nudged
|
||||||
|
assert seen['status_mt'] == 'show' # polled TV scan status
|
||||||
|
assert events[0][1]['media_type'] == 'show' # stage 2 inherits the scope
|
||||||
|
|
||||||
|
|
||||||
|
class TestSmartScanSkip:
|
||||||
|
"""Scanning is expensive — skip a library's crawl when the server already has the
|
||||||
|
newest grab (it auto-ingested). Always still emits so stage 2 reads."""
|
||||||
|
|
||||||
|
def _run(self, **kw):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
events, refreshed, polled = [], [], []
|
||||||
|
base = dict(
|
||||||
|
server_refresh=lambda mt=None: refreshed.append(mt) or {'ok': True},
|
||||||
|
sleep=lambda s: None, scan_status=lambda mt: polled.append(mt) or False,
|
||||||
|
emit=lambda ev, data: events.append((ev, data)))
|
||||||
|
base.update(kw)
|
||||||
|
res = auto_video_scan_server({'_automation_id': 'a'}, deps, **base)
|
||||||
|
return res, events, refreshed, polled, deps
|
||||||
|
|
||||||
|
def test_skips_entirely_when_server_has_both_newest(self):
|
||||||
|
res, events, refreshed, polled, deps = self._run(
|
||||||
|
latest_completed=lambda sc: {'title': 'X'}, # history has a newest for each
|
||||||
|
server_has_item=lambda sc, item: True) # …and the server already has it
|
||||||
|
assert refreshed == [] and polled == [] # no crawl, no poll — the win
|
||||||
|
assert events[0] == ('video_library_scan_completed', {'server': '', 'media_type': 'all'})
|
||||||
|
assert res['scanned'] == [] and set(res['skipped']) == {'movie', 'show'}
|
||||||
|
assert any('no scan needed' in (c.get('log_line') or '') for c in deps.calls)
|
||||||
|
|
||||||
|
def test_scans_only_the_library_the_server_is_missing(self):
|
||||||
|
# server has the newest MOVIE but not the newest EPISODE → scan TV only
|
||||||
|
res, events, refreshed, polled, _ = self._run(
|
||||||
|
latest_completed=lambda sc: {'title': 'X'},
|
||||||
|
server_has_item=lambda sc, item: sc == 'movie')
|
||||||
|
assert refreshed == ['show'] and polled == ['show'] # only the TV library crawled
|
||||||
|
assert res['scanned'] == ['show'] and res['skipped'] == ['movie']
|
||||||
|
|
||||||
|
def test_scans_when_there_is_no_history_to_probe(self):
|
||||||
|
res, events, refreshed, polled, _ = self._run(
|
||||||
|
latest_completed=lambda sc: None, # nothing grabbed yet
|
||||||
|
server_has_item=lambda sc, item: True) # (never consulted)
|
||||||
|
assert refreshed == ['all'] # both → scan all
|
||||||
|
assert set(res['scanned']) == {'movie', 'show'}
|
||||||
|
|
||||||
|
def test_toggle_off_always_scans(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
refreshed = []
|
||||||
|
auto_video_scan_server(
|
||||||
|
{'_automation_id': 'a', 'skip_if_present': False}, deps,
|
||||||
|
server_refresh=lambda mt=None: refreshed.append(mt) or {'ok': True},
|
||||||
|
sleep=lambda s: None, scan_status=lambda mt: False,
|
||||||
|
latest_completed=lambda sc: {'title': 'X'}, server_has_item=lambda sc, item: True,
|
||||||
|
emit=lambda ev, data: None)
|
||||||
|
assert refreshed == ['all'] # skip disabled → crawl anyway
|
||||||
|
|
||||||
|
def test_probe_error_is_treated_as_missing_so_we_scan(self):
|
||||||
|
def _boom(sc, item):
|
||||||
|
raise RuntimeError('plex down')
|
||||||
|
res, events, refreshed, polled, _ = self._run(
|
||||||
|
latest_completed=lambda sc: {'title': 'X'}, server_has_item=_boom)
|
||||||
|
assert refreshed == ['all'] # uncertainty → scan (safe)
|
||||||
|
|
||||||
|
def test_waits_for_the_servers_autoscan_then_skips(self):
|
||||||
|
# The file isn't on the server immediately (fresh drop) — it appears on the 3rd
|
||||||
|
# probe. We poll over the grace window and skip the crawl once it shows up.
|
||||||
|
calls = {'show': 0}
|
||||||
|
slept = []
|
||||||
|
|
||||||
|
def _has(sc, item):
|
||||||
|
calls[sc] += 1
|
||||||
|
return calls[sc] >= 3 # present only on the 3rd check
|
||||||
|
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
refreshed = []
|
||||||
|
res = auto_video_scan_server(
|
||||||
|
{'_automation_id': 'a', 'media_type': 'show', 'probe_grace_minutes': 5}, deps,
|
||||||
|
server_refresh=lambda mt=None: refreshed.append(mt) or {'ok': True},
|
||||||
|
sleep=slept.append, scan_status=lambda mt: False,
|
||||||
|
latest_completed=lambda sc: {'title': 'X'}, server_has_item=_has,
|
||||||
|
emit=lambda ev, data: None)
|
||||||
|
assert refreshed == [] # never crawled — auto-scan won
|
||||||
|
assert res['skipped'] == ['show'] and len(slept) == 2 # polled twice before it appeared
|
||||||
|
|
||||||
|
def test_probe_grace_zero_checks_once_immediately(self):
|
||||||
|
calls = {'show': 0}
|
||||||
|
|
||||||
|
def _has(sc, item):
|
||||||
|
calls[sc] += 1
|
||||||
|
return False
|
||||||
|
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
refreshed, slept = [], []
|
||||||
|
auto_video_scan_server(
|
||||||
|
{'_automation_id': 'a', 'media_type': 'show', 'probe_grace_minutes': 0}, deps,
|
||||||
|
server_refresh=lambda mt=None: refreshed.append(mt) or {'ok': True},
|
||||||
|
sleep=slept.append, scan_status=lambda mt: False,
|
||||||
|
latest_completed=lambda sc: {'title': 'X'}, server_has_item=_has,
|
||||||
|
emit=lambda ev, data: None)
|
||||||
|
assert calls['show'] == 1 and refreshed == ['show'] # one probe, no grace wait, then scan
|
||||||
|
|
||||||
|
|
||||||
|
# ── post-download chain: video_update_database (stage 2) ───────────────────
|
||||||
|
class TestUpdateDatabaseStage:
|
||||||
|
def test_incremental_read_returns_counts(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
r = auto_video_update_database({'_automation_id': 'a'}, deps, run_video_scan=_scan_done(2, 1, 5))
|
||||||
|
assert r == {'status': 'completed', '_manages_own_progress': True,
|
||||||
|
'movies': 2, 'shows': 1, 'episodes': 5}
|
||||||
|
|
||||||
|
def test_defaults_to_incremental_mode(self):
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def _scan(mode, media_type=None):
|
||||||
|
seen['mode'] = mode
|
||||||
|
return {'state': 'done'}
|
||||||
|
|
||||||
|
auto_video_update_database({'_automation_id': 'a'}, _RecordingDeps(), run_video_scan=_scan)
|
||||||
|
assert seen['mode'] == 'incremental'
|
||||||
|
|
||||||
|
def test_scan_error_propagates(self):
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
r = auto_video_update_database({'_automation_id': 'a'}, deps,
|
||||||
|
run_video_scan=lambda m, media_type=None: {'state': 'error', 'error': 'no server'})
|
||||||
|
assert r['status'] == 'error' and r['error'] == 'no server'
|
||||||
|
|
||||||
|
def test_inherits_media_type_from_the_scan_event(self):
|
||||||
|
# The post-download chain carries the scope: a TV-only rescan updates only TV.
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def _scan(mode, media_type=None):
|
||||||
|
seen['media_type'] = media_type
|
||||||
|
return {'state': 'done', 'shows': 3, 'episodes': 12}
|
||||||
|
|
||||||
|
deps = _RecordingDeps()
|
||||||
|
auto_video_update_database(
|
||||||
|
{'_automation_id': 'a', '_event_data': {'media_type': 'show'}}, deps, run_video_scan=_scan)
|
||||||
|
assert seen['media_type'] == 'show'
|
||||||
|
assert deps.calls[-1]['log_line'] == 'Video database updated: 3 shows, 12 episodes'
|
||||||
|
|
||||||
|
def test_explicit_media_type_beats_event(self):
|
||||||
|
seen = {}
|
||||||
|
auto_video_update_database(
|
||||||
|
{'_automation_id': 'a', 'media_type': 'movie', '_event_data': {'media_type': 'show'}},
|
||||||
|
_RecordingDeps(),
|
||||||
|
run_video_scan=lambda mode, media_type=None: seen.setdefault('mt', media_type) or {'state': 'done'})
|
||||||
|
assert seen['mt'] == 'movie'
|
||||||
|
|
||||||
|
def test_busy_scanner_skips_cleanly(self):
|
||||||
|
r = auto_video_update_database(
|
||||||
|
{'_automation_id': 'a'}, _RecordingDeps(),
|
||||||
|
run_video_scan=lambda mode, media_type=None: {'state': 'in_progress'})
|
||||||
|
assert r['status'] == 'skipped'
|
||||||
181
tests/automation/test_video_maintenance_twins.py
Normal file
181
tests/automation/test_video_maintenance_twins.py
Normal file
|
|
@ -0,0 +1,181 @@
|
||||||
|
"""Video twins of the music maintenance automations.
|
||||||
|
|
||||||
|
The music side has "Clean Search History" / "Clean Completed Downloads" / "Full
|
||||||
|
Cleanup" / "Auto-Backup Database". The video side gets its own copies so they
|
||||||
|
appear on the video Automations page too — distinct ``video_*`` action_type
|
||||||
|
(the system seeder keys on action_type, so a shared key would collide), the SAME
|
||||||
|
shared handler where the behaviour is identical, and ``owned_by='video'`` so the
|
||||||
|
music page is never disrupted.
|
||||||
|
|
||||||
|
These pin the registry + scope + seed contract, one twin per phase.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from core.automation.blocks import blocks_for_scope
|
||||||
|
from core.automation_engine import SYSTEM_AUTOMATIONS
|
||||||
|
|
||||||
|
|
||||||
|
def _action_types(scope):
|
||||||
|
return {a["type"] for a in blocks_for_scope(scope)["actions"]}
|
||||||
|
|
||||||
|
|
||||||
|
def _system_by_action(action_type):
|
||||||
|
return [s for s in SYSTEM_AUTOMATIONS if s.get("action_type") == action_type]
|
||||||
|
|
||||||
|
|
||||||
|
# ── Phase 2: Clean Search History ───────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_video_clean_search_history_is_video_scoped_only():
|
||||||
|
# Appears on the video builder, NOT the music builder (music block untouched).
|
||||||
|
assert "video_clean_search_history" in _action_types("video")
|
||||||
|
assert "video_clean_search_history" not in _action_types("music")
|
||||||
|
|
||||||
|
|
||||||
|
def test_music_clean_search_history_is_untouched():
|
||||||
|
# The original music action still exists and is still music-scoped — no disruption.
|
||||||
|
assert "clean_search_history" in _action_types("music")
|
||||||
|
assert "clean_search_history" not in _action_types("video")
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_clean_search_history_seeds_one_video_owned_system_row():
|
||||||
|
rows = _system_by_action("video_clean_search_history")
|
||||||
|
assert len(rows) == 1
|
||||||
|
row = rows[0]
|
||||||
|
assert row["owned_by"] == "video"
|
||||||
|
assert row["trigger_type"] == "schedule"
|
||||||
|
# mirrors the music cadence (every 1 hour)
|
||||||
|
assert row["trigger_config"] == {"interval": 1, "unit": "hours"}
|
||||||
|
|
||||||
|
|
||||||
|
def _registered_handlers():
|
||||||
|
from core.automation.handlers import register_all
|
||||||
|
|
||||||
|
class _Eng:
|
||||||
|
def __init__(self):
|
||||||
|
self.handlers = {}
|
||||||
|
def register_action_handler(self, t, fn, guard_fn=None):
|
||||||
|
self.handlers[t] = fn
|
||||||
|
def register_progress_callbacks(self, *a, **k):
|
||||||
|
pass
|
||||||
|
|
||||||
|
from tests.automation.test_handler_registration import _build_deps
|
||||||
|
eng = _Eng()
|
||||||
|
register_all(_build_deps(eng))
|
||||||
|
return eng.handlers
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_clean_search_history_reuses_the_music_handler():
|
||||||
|
# Registered to the SAME handler function as the music action (identical behaviour).
|
||||||
|
handlers = _registered_handlers()
|
||||||
|
assert "video_clean_search_history" in handlers
|
||||||
|
assert "clean_search_history" in handlers
|
||||||
|
|
||||||
|
|
||||||
|
# ── Phase 3: Clean Completed Downloads ──────────────────────────────────────
|
||||||
|
|
||||||
|
def test_video_clean_completed_downloads_is_video_scoped_only():
|
||||||
|
assert "video_clean_completed_downloads" in _action_types("video")
|
||||||
|
assert "video_clean_completed_downloads" not in _action_types("music")
|
||||||
|
# music original untouched
|
||||||
|
assert "clean_completed_downloads" in _action_types("music")
|
||||||
|
assert "clean_completed_downloads" not in _action_types("video")
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_clean_completed_downloads_seeds_one_video_owned_system_row():
|
||||||
|
rows = _system_by_action("video_clean_completed_downloads")
|
||||||
|
assert len(rows) == 1 and rows[0]["owned_by"] == "video"
|
||||||
|
assert rows[0]["trigger_config"] == {"interval": 5, "unit": "minutes"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_clean_completed_downloads_reuses_the_music_handler():
|
||||||
|
handlers = _registered_handlers()
|
||||||
|
assert "video_clean_completed_downloads" in handlers
|
||||||
|
assert "clean_completed_downloads" in handlers
|
||||||
|
|
||||||
|
|
||||||
|
# ── Phase 4: Full Cleanup ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_video_full_cleanup_is_video_scoped_only():
|
||||||
|
assert "video_full_cleanup" in _action_types("video")
|
||||||
|
assert "video_full_cleanup" not in _action_types("music")
|
||||||
|
assert "full_cleanup" in _action_types("music")
|
||||||
|
assert "full_cleanup" not in _action_types("video")
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_full_cleanup_seeds_one_video_owned_system_row():
|
||||||
|
rows = _system_by_action("video_full_cleanup")
|
||||||
|
assert len(rows) == 1 and rows[0]["owned_by"] == "video"
|
||||||
|
assert rows[0]["trigger_config"] == {"interval": 12, "unit": "hours"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_full_cleanup_reuses_the_music_handler():
|
||||||
|
handlers = _registered_handlers()
|
||||||
|
assert "video_full_cleanup" in handlers
|
||||||
|
assert "full_cleanup" in handlers
|
||||||
|
|
||||||
|
|
||||||
|
# ── Phase 5: Auto-Backup Database (video — CUSTOM, not a shared handler) ─────
|
||||||
|
|
||||||
|
def test_video_backup_database_is_video_scoped_only():
|
||||||
|
assert "video_backup_database" in _action_types("video")
|
||||||
|
assert "video_backup_database" not in _action_types("music")
|
||||||
|
assert "backup_database" in _action_types("music")
|
||||||
|
assert "backup_database" not in _action_types("video")
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_backup_database_seeds_one_video_owned_system_row():
|
||||||
|
rows = _system_by_action("video_backup_database")
|
||||||
|
assert len(rows) == 1 and rows[0]["owned_by"] == "video"
|
||||||
|
assert rows[0]["trigger_config"] == {"interval": 3, "unit": "days"}
|
||||||
|
|
||||||
|
|
||||||
|
def _mk_sqlite(path):
|
||||||
|
import sqlite3
|
||||||
|
con = sqlite3.connect(str(path))
|
||||||
|
con.execute("CREATE TABLE t (x)")
|
||||||
|
con.commit()
|
||||||
|
con.close()
|
||||||
|
|
||||||
|
|
||||||
|
class _Deps:
|
||||||
|
class _L:
|
||||||
|
def error(self, *a, **k):
|
||||||
|
pass
|
||||||
|
def debug(self, *a, **k):
|
||||||
|
pass
|
||||||
|
logger = _L()
|
||||||
|
def update_progress(self, *a, **k):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_backup_targets_video_db_and_music_backup_targets_music_db(tmp_path, monkeypatch):
|
||||||
|
# The whole reason this one can't be shared: each backs up ITS OWN db file.
|
||||||
|
import glob
|
||||||
|
from core.automation.handlers.maintenance import (
|
||||||
|
auto_backup_database, auto_backup_video_database)
|
||||||
|
music_db = tmp_path / "music_library.db"
|
||||||
|
video_db = tmp_path / "video_library.db"
|
||||||
|
_mk_sqlite(music_db)
|
||||||
|
_mk_sqlite(video_db)
|
||||||
|
monkeypatch.setenv("DATABASE_PATH", str(music_db))
|
||||||
|
monkeypatch.setenv("VIDEO_DATABASE_PATH", str(video_db))
|
||||||
|
deps = _Deps()
|
||||||
|
|
||||||
|
r_music = auto_backup_database({"_automation_id": "m"}, deps)
|
||||||
|
r_video = auto_backup_video_database({"_automation_id": "v"}, deps)
|
||||||
|
|
||||||
|
assert r_music["status"] == "completed" and r_video["status"] == "completed"
|
||||||
|
# each backup sits next to its OWN db — no cross-contamination
|
||||||
|
assert r_music["backup_path"].startswith(str(music_db))
|
||||||
|
assert r_video["backup_path"].startswith(str(video_db))
|
||||||
|
assert glob.glob(str(music_db) + ".backup_*")
|
||||||
|
assert glob.glob(str(video_db) + ".backup_*")
|
||||||
|
# the video backup did NOT touch the music db's backups and vice-versa
|
||||||
|
assert not glob.glob(str(music_db) + ".backup_*" )[0].startswith(str(video_db))
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_backup_database_has_its_own_handler():
|
||||||
|
handlers = _registered_handlers()
|
||||||
|
assert "video_backup_database" in handlers
|
||||||
|
assert "backup_database" in handlers
|
||||||
|
|
@ -16,6 +16,13 @@ endpoints and event handlers without importing the full web_server.py
|
||||||
# that call get_database() with no path → the real DB. Running those writers
|
# that call get_database() with no path → the real DB. Running those writers
|
||||||
# against the live DB over a WSL-mounted Windows drive corrupted a user's
|
# against the live DB over a WSL-mounted Windows drive corrupted a user's
|
||||||
# library. This guarantees it can't recur — tests get their own disposable DB.
|
# library. This guarantees it can't recur — tests get their own disposable DB.
|
||||||
|
#
|
||||||
|
# The VIDEO side has the SAME hazard: VideoDatabase()/get_video_db() with no
|
||||||
|
# path resolves from os.environ['VIDEO_DATABASE_PATH'] (see
|
||||||
|
# database/video_database.py) → the real database/video_library.db, and its
|
||||||
|
# enrichment threads WRITE. A blueprint/handler test that opened the default
|
||||||
|
# VideoDatabase corrupted the real video library once — same WSL/NTFS + WAL
|
||||||
|
# trap. So redirect VIDEO_DATABASE_PATH to /tmp here too, before any import.
|
||||||
import os as _os
|
import os as _os
|
||||||
import tempfile as _tempfile
|
import tempfile as _tempfile
|
||||||
import atexit as _atexit
|
import atexit as _atexit
|
||||||
|
|
@ -24,6 +31,7 @@ import shutil as _shutil
|
||||||
if not _os.environ.get('SOULSYNC_TEST_DB_READY'):
|
if not _os.environ.get('SOULSYNC_TEST_DB_READY'):
|
||||||
_TEST_DB_DIR = _tempfile.mkdtemp(prefix='soulsync-testdb-')
|
_TEST_DB_DIR = _tempfile.mkdtemp(prefix='soulsync-testdb-')
|
||||||
_os.environ['DATABASE_PATH'] = _os.path.join(_TEST_DB_DIR, 'test_music_library.db')
|
_os.environ['DATABASE_PATH'] = _os.path.join(_TEST_DB_DIR, 'test_music_library.db')
|
||||||
|
_os.environ['VIDEO_DATABASE_PATH'] = _os.path.join(_TEST_DB_DIR, 'test_video_library.db')
|
||||||
_os.environ['SOULSYNC_TEST_DB_READY'] = '1'
|
_os.environ['SOULSYNC_TEST_DB_READY'] = '1'
|
||||||
_atexit.register(lambda: _shutil.rmtree(_TEST_DB_DIR, ignore_errors=True))
|
_atexit.register(lambda: _shutil.rmtree(_TEST_DB_DIR, ignore_errors=True))
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,9 @@
|
||||||
"""Guard: the test suite must NEVER resolve the real database/music_library.db.
|
"""Guard: the test suite must NEVER resolve the real music OR video library DB.
|
||||||
|
|
||||||
Tests exercise modules that call get_database() with no path. If that resolves
|
Tests exercise modules that call get_database()/get_video_db() with no path. If
|
||||||
to the live DB, test writes can corrupt a real library (this happened, over a
|
that resolves to a live DB, test writes can corrupt a real library (this
|
||||||
WSL-mounted Windows drive). conftest.py sets DATABASE_PATH to a temp file before
|
happened to BOTH the music and the video library, over a WSL-mounted Windows
|
||||||
|
drive). conftest.py sets DATABASE_PATH + VIDEO_DATABASE_PATH to temp files before
|
||||||
anything imports; these tests prove it sticks for every default-path access.
|
anything imports; these tests prove it sticks for every default-path access.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
@ -30,3 +31,23 @@ def test_get_database_path_never_real():
|
||||||
from database.music_database import get_database
|
from database.music_database import get_database
|
||||||
resolved = str(Path(get_database().database_path).resolve())
|
resolved = str(Path(get_database().database_path).resolve())
|
||||||
assert 'soulsync-testdb-' in resolved, resolved
|
assert 'soulsync-testdb-' in resolved, resolved
|
||||||
|
|
||||||
|
|
||||||
|
# ── Video side — same hazard, same guarantee ──────────────────────────────
|
||||||
|
|
||||||
|
def test_video_database_path_env_is_isolated():
|
||||||
|
p = os.environ.get('VIDEO_DATABASE_PATH', '')
|
||||||
|
assert 'soulsync-testdb-' in p, f"VIDEO_DATABASE_PATH not isolated: {p!r}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_videodatabase_default_path_never_real():
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
resolved = str(Path(VideoDatabase().database_path).resolve())
|
||||||
|
assert 'soulsync-testdb-' in resolved, resolved
|
||||||
|
assert not resolved.replace('\\', '/').endswith('database/video_library.db'), resolved
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_video_db_path_never_real():
|
||||||
|
from api.video import get_video_db
|
||||||
|
resolved = str(Path(get_video_db().database_path).resolve())
|
||||||
|
assert 'soulsync-testdb-' in resolved, resolved
|
||||||
|
|
|
||||||
1192
tests/test_video_api.py
Normal file
1192
tests/test_video_api.py
Normal file
File diff suppressed because it is too large
Load diff
181
tests/test_video_auto_wishlist_airing.py
Normal file
181
tests/test_video_auto_wishlist_airing.py
Normal file
|
|
@ -0,0 +1,181 @@
|
||||||
|
"""Sonarr-style 'wishlist today's airings' automation handler — pure logic with the
|
||||||
|
calendar read + wishlist write injected, so it runs without a DB or media server."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from core.automation.handlers.video_auto_wishlist_airing import auto_video_add_airing_episodes
|
||||||
|
|
||||||
|
|
||||||
|
class _Deps:
|
||||||
|
def __init__(self):
|
||||||
|
self.progress = []
|
||||||
|
|
||||||
|
def update_progress(self, automation_id, **kw):
|
||||||
|
self.progress.append(kw)
|
||||||
|
|
||||||
|
|
||||||
|
def _row(tid, title, s, e, owned=False):
|
||||||
|
return {"show_tmdb_id": tid, "show_id": tid * 100, "show_title": title, "season_number": s,
|
||||||
|
"episode_number": e, "title": "Ep", "air_date": "2026-06-21", "has_file": owned}
|
||||||
|
|
||||||
|
|
||||||
|
def test_adds_unowned_airings_grouped_by_show():
|
||||||
|
rows = [
|
||||||
|
_row(1, "Widows Bay", 1, 1),
|
||||||
|
_row(1, "Widows Bay", 1, 2),
|
||||||
|
_row(2, "Another Show", 3, 5),
|
||||||
|
_row(1, "Widows Bay", 1, 3, owned=True), # already owned → skipped
|
||||||
|
{"show_title": "No id", "season_number": 1, "episode_number": 1}, # no tmdb id → skipped
|
||||||
|
]
|
||||||
|
added = []
|
||||||
|
|
||||||
|
def add(tid, title, eps, library_id=None, poster_url=None):
|
||||||
|
added.append((tid, title, len(eps), library_id, poster_url))
|
||||||
|
return len(eps)
|
||||||
|
|
||||||
|
res = auto_video_add_airing_episodes(
|
||||||
|
{"_automation_id": "a1", "prune_ended": False}, _Deps(),
|
||||||
|
fetch_airing=lambda today: rows, add_episodes=add, today_fn=lambda: "2026-06-21",
|
||||||
|
season_meta=lambda *a: None)
|
||||||
|
|
||||||
|
assert res["status"] == "completed"
|
||||||
|
assert res["episodes_added"] == 3 # 2 of Widows Bay + 1 of Another Show
|
||||||
|
assert res["shows"] == 2
|
||||||
|
# the show's library_id (show_id) + poster proxy are carried so the wishlist
|
||||||
|
# matches the show and the orb renders the show poster (like a manual add)
|
||||||
|
assert (1, "Widows Bay", 2, 100, "/api/video/poster/show/100") in added
|
||||||
|
assert (2, "Another Show", 1, 200, "/api/video/poster/show/200") in added
|
||||||
|
|
||||||
|
|
||||||
|
def test_uses_tmdb_season_metadata_like_a_manual_add():
|
||||||
|
# the SAME TMDB source the manual 'add to wishlist' uses — absolute still + overview
|
||||||
|
# + season poster — preferred over the patchy DB values.
|
||||||
|
rows = [{"show_tmdb_id": 5, "show_title": "Y", "season_number": 2, "episode_number": 3,
|
||||||
|
"title": "Ep", "air_date": "2026-06-21", "has_file": False,
|
||||||
|
"overview": "db overview", "still_url": "/db/still"}]
|
||||||
|
|
||||||
|
def season_meta(tid, sn):
|
||||||
|
assert (tid, sn) == (5, 2)
|
||||||
|
return {"poster_url": "https://img/tmdb/s2.jpg",
|
||||||
|
"episodes": [{"episode_number": 3, "overview": "TMDB overview",
|
||||||
|
"still_url": "https://img/tmdb/s2e3.jpg"}]}
|
||||||
|
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
def add(tid, title, eps, library_id=None, poster_url=None):
|
||||||
|
captured["eps"] = eps
|
||||||
|
return len(eps)
|
||||||
|
|
||||||
|
auto_video_add_airing_episodes({"_automation_id": "a", "prune_ended": False}, _Deps(),
|
||||||
|
fetch_airing=lambda t: rows, add_episodes=add,
|
||||||
|
today_fn=lambda: "2026-06-21", season_meta=season_meta)
|
||||||
|
ep = captured["eps"][0]
|
||||||
|
assert ep["overview"] == "TMDB overview" # TMDB preferred over DB
|
||||||
|
assert ep["still_url"] == "https://img/tmdb/s2e3.jpg"
|
||||||
|
assert ep["season_poster_url"] == "https://img/tmdb/s2.jpg"
|
||||||
|
|
||||||
|
|
||||||
|
def test_falls_back_to_db_values_when_tmdb_unavailable():
|
||||||
|
# if the TMDB fetch returns nothing, still carry the calendar/DB overview + still
|
||||||
|
rows = [{"show_tmdb_id": 1, "show_title": "X", "season_number": 1, "episode_number": 2,
|
||||||
|
"has_file": False, "overview": "db synopsis", "still_url": "/library/metadata/9/thumb/1"}]
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
def add(tid, title, eps, library_id=None, poster_url=None):
|
||||||
|
captured["eps"] = eps
|
||||||
|
return len(eps)
|
||||||
|
|
||||||
|
auto_video_add_airing_episodes({"_automation_id": "a", "prune_ended": False}, _Deps(),
|
||||||
|
fetch_airing=lambda t: rows, add_episodes=add,
|
||||||
|
today_fn=lambda: "2026-06-21", season_meta=lambda *a: None)
|
||||||
|
ep = captured["eps"][0]
|
||||||
|
assert ep["overview"] == "db synopsis"
|
||||||
|
assert ep["still_url"] == "/library/metadata/9/thumb/1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_queries_the_calendar_for_today():
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def fetch(today):
|
||||||
|
seen["today"] = today
|
||||||
|
return []
|
||||||
|
|
||||||
|
auto_video_add_airing_episodes({"_automation_id": "a", "prune_ended": False}, _Deps(),
|
||||||
|
fetch_airing=fetch, add_episodes=lambda *a: 0,
|
||||||
|
today_fn=lambda: "2026-06-21")
|
||||||
|
assert seen["today"] == "2026-06-21" # start == end == today
|
||||||
|
|
||||||
|
|
||||||
|
def test_nothing_airing_is_a_clean_noop():
|
||||||
|
res = auto_video_add_airing_episodes({"_automation_id": "a", "prune_ended": False}, _Deps(),
|
||||||
|
fetch_airing=lambda t: [], add_episodes=lambda *a: 0,
|
||||||
|
today_fn=lambda: "2026-06-21")
|
||||||
|
assert res["status"] == "completed" and res["episodes_added"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_error_is_caught_and_reported():
|
||||||
|
def boom(today):
|
||||||
|
raise RuntimeError("calendar down")
|
||||||
|
|
||||||
|
deps = _Deps()
|
||||||
|
res = auto_video_add_airing_episodes({"_automation_id": "a", "prune_ended": False}, deps,
|
||||||
|
fetch_airing=boom, add_episodes=lambda *a: 0)
|
||||||
|
assert res["status"] == "error" and "calendar down" in res["error"]
|
||||||
|
assert any(p.get("status") == "error" for p in deps.progress)
|
||||||
|
|
||||||
|
|
||||||
|
# ── watchlist hygiene: prune ended/canceled follows ─────────────────────────
|
||||||
|
from core.automation.handlers.video_auto_wishlist_airing import prune_ended_show_follows # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def test_prune_removes_ended_and_canceled_follows():
|
||||||
|
follows = [
|
||||||
|
{"tmdb_id": 1, "title": "Returning Show", "status": "Returning Series"},
|
||||||
|
{"tmdb_id": 2, "title": "Done Show", "status": "Ended"},
|
||||||
|
{"tmdb_id": 3, "title": "Axed Show", "status": "Canceled"},
|
||||||
|
{"tmdb_id": 4, "title": "Live Show", "status": "In Production"},
|
||||||
|
]
|
||||||
|
removed = []
|
||||||
|
n = prune_ended_show_follows(_Deps(), "a", fetch_follows=lambda: follows,
|
||||||
|
show_status=lambda t: None, remove_show=removed.append)
|
||||||
|
assert n == 2 and removed == [2, 3] # only Ended + Canceled
|
||||||
|
|
||||||
|
|
||||||
|
def test_prune_looks_up_status_for_tmdb_only_follows():
|
||||||
|
# no local status → fetch from TMDB; ended → prune
|
||||||
|
follows = [{"tmdb_id": 9, "title": "TMDB-only", "status": None}]
|
||||||
|
removed = []
|
||||||
|
prune_ended_show_follows(_Deps(), "a", fetch_follows=lambda: follows,
|
||||||
|
show_status=lambda t: "Ended", remove_show=removed.append)
|
||||||
|
assert removed == [9]
|
||||||
|
|
||||||
|
|
||||||
|
def test_prune_never_removes_on_unknown_status_or_lookup_error():
|
||||||
|
def _boom(t):
|
||||||
|
raise RuntimeError("tmdb down")
|
||||||
|
follows = [{"tmdb_id": 1, "title": "Unknown", "status": None}, # lookup returns None
|
||||||
|
{"tmdb_id": 2, "title": "Errored", "status": None}] # lookup raises
|
||||||
|
removed = []
|
||||||
|
n = prune_ended_show_follows(
|
||||||
|
_Deps(), "a", fetch_follows=lambda: follows,
|
||||||
|
show_status=lambda t: (_boom(t) if t == 2 else None), remove_show=removed.append)
|
||||||
|
assert n == 0 and removed == [] # uncertainty → keep
|
||||||
|
|
||||||
|
|
||||||
|
def test_airing_handler_runs_the_prune_pass():
|
||||||
|
removed = []
|
||||||
|
res = auto_video_add_airing_episodes(
|
||||||
|
{"_automation_id": "a"}, _Deps(),
|
||||||
|
fetch_airing=lambda t: [], add_episodes=lambda *a, **k: 0, today_fn=lambda: "2026-06-21",
|
||||||
|
prune_follows=lambda: [{"tmdb_id": 7, "title": "Old", "status": "Ended"}],
|
||||||
|
show_status=lambda t: None, remove_show=removed.append)
|
||||||
|
assert res["status"] == "completed" and res["shows_pruned"] == 1 and removed == [7]
|
||||||
|
|
||||||
|
|
||||||
|
def test_airing_handler_can_disable_the_prune():
|
||||||
|
called = {"n": 0}
|
||||||
|
auto_video_add_airing_episodes(
|
||||||
|
{"_automation_id": "a", "prune_ended": False}, _Deps(),
|
||||||
|
fetch_airing=lambda t: [], add_episodes=lambda *a, **k: 0, today_fn=lambda: "2026-06-21",
|
||||||
|
prune_follows=lambda: called.update(n=called["n"] + 1) or [])
|
||||||
|
assert called["n"] == 0 # prune skipped entirely
|
||||||
142
tests/test_video_automations_builder.py
Normal file
142
tests/test_video_automations_builder.py
Normal file
|
|
@ -0,0 +1,142 @@
|
||||||
|
"""Video automation BUILDER wiring — the isolated video page must be able to
|
||||||
|
create/edit its own automations without hijacking the music page's builder.
|
||||||
|
|
||||||
|
These pin the wiring contract that makes that work:
|
||||||
|
- the video automations subpage hosts its OWN builder DOM (vauto- prefixed ids)
|
||||||
|
+ a "New Automation" button, swapping list/builder like the music page;
|
||||||
|
- the shared builder (stats-automations.js) is context-aware: a video context
|
||||||
|
points at the video ids + the video-scoped blocks endpoint + owned_by='video',
|
||||||
|
and the card cog routes to the video builder when the video side is active;
|
||||||
|
- a save tags the row owned_by='video' so it stays off the music page;
|
||||||
|
- a generic config_fields renderer drives video block config (so the video
|
||||||
|
action's fields show up) and is gated to the video context (music untouched).
|
||||||
|
|
||||||
|
String-contract level (like tests/test_video_side_shell.py) so a refactor that
|
||||||
|
silently breaks the coupling fails here.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
_INDEX = (_ROOT / "webui" / "index.html").read_text(encoding="utf-8", errors="replace")
|
||||||
|
_STATS = (_ROOT / "webui" / "static" / "stats-automations.js").read_text(encoding="utf-8")
|
||||||
|
_VAUTO = (_ROOT / "webui" / "static" / "video" / "video-automations.js").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
# --- the video subpage builder DOM ----------------------------------------
|
||||||
|
|
||||||
|
def test_video_subpage_has_new_automation_button():
|
||||||
|
# Header button + empty-state button both open the VIDEO builder.
|
||||||
|
assert _INDEX.count('onclick="showVideoAutomationBuilder()"') >= 1
|
||||||
|
assert 'auto-new-btn' in _INDEX
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_subpage_has_list_and_builder_views():
|
||||||
|
for tok in ('id="vauto-list-view"', 'id="vauto-builder-view"'):
|
||||||
|
assert tok in _INDEX, tok
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_builder_has_prefixed_ids():
|
||||||
|
for tok in (
|
||||||
|
'id="vauto-builder-name"', 'id="vauto-builder-group-name"',
|
||||||
|
'id="vauto-builder-group-list"', 'id="vauto-builder-sidebar"',
|
||||||
|
'id="vauto-builder-canvas"',
|
||||||
|
):
|
||||||
|
assert tok in _INDEX, tok
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_builder_buttons_use_shared_handlers():
|
||||||
|
# Save / Cancel / Back reuse the SAME shared functions (ctx-driven).
|
||||||
|
assert 'onclick="saveAutomation()"' in _INDEX
|
||||||
|
assert 'onclick="hideAutomationBuilder()"' in _INDEX
|
||||||
|
|
||||||
|
|
||||||
|
# --- the shared builder is context-aware ----------------------------------
|
||||||
|
|
||||||
|
def test_shared_builder_has_video_entry_point():
|
||||||
|
assert 'function showVideoAutomationBuilder(' in _STATS
|
||||||
|
assert 'function showAutomationBuilder(' in _STATS
|
||||||
|
assert 'function _openAutomationBuilder(' in _STATS
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_context_targets_video_ids_and_endpoint():
|
||||||
|
assert "'/api/video/automations/blocks'" in _STATS
|
||||||
|
assert "ownedBy: 'video'" in _STATS
|
||||||
|
# The video context must reference the vauto- prefixed element ids.
|
||||||
|
for tok in ('vauto-builder-name', 'vauto-builder-sidebar', 'vauto-builder-canvas',
|
||||||
|
'vauto-list-view', 'vauto-builder-view'):
|
||||||
|
assert tok in _STATS, tok
|
||||||
|
|
||||||
|
|
||||||
|
def test_card_cog_routes_by_active_side():
|
||||||
|
# Cards call editAutomation (not showAutomationBuilder directly) so a video
|
||||||
|
# card opens the video builder instead of the hidden music one.
|
||||||
|
assert 'editAutomation(${a.id})' in _STATS
|
||||||
|
assert 'function editAutomation(' in _STATS
|
||||||
|
assert "getAttribute('data-side') === 'video'" in _STATS
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_tags_owned_by_from_context():
|
||||||
|
assert 'body.owned_by = _autoBuilderCtx.ownedBy' in _STATS
|
||||||
|
|
||||||
|
|
||||||
|
def test_open_clears_both_builders_to_avoid_id_collision():
|
||||||
|
# cfg-* ids exist in both builders; opening clears both canvases/sidebars.
|
||||||
|
assert "'vauto-builder-sidebar', 'vauto-builder-canvas'" in _STATS
|
||||||
|
|
||||||
|
|
||||||
|
def test_generic_config_renderer_is_video_gated():
|
||||||
|
# The generic config_fields renderer/reader must only run in the video
|
||||||
|
# context so the music side keeps its bespoke renderers (byte-identical).
|
||||||
|
assert 'function _renderGenericConfigField(' in _STATS
|
||||||
|
assert 'function _readGenericConfigField(' in _STATS
|
||||||
|
assert 'if (_autoBuilderCtx.ownedBy) {' in _STATS
|
||||||
|
|
||||||
|
|
||||||
|
# --- the video page exposes its reload hook -------------------------------
|
||||||
|
|
||||||
|
def test_video_page_exposes_reload_hook():
|
||||||
|
assert 'window._reloadVideoAutomations = load' in _VAUTO
|
||||||
|
|
||||||
|
|
||||||
|
# --- the System list is shown in a logical pipeline order -----------------
|
||||||
|
|
||||||
|
def test_video_automations_poll_does_not_rebuild_when_unchanged():
|
||||||
|
# regression: the 8s poll used to replaceChild the whole section every time → blink +
|
||||||
|
# wiped live progress. It must skip the rebuild when nothing structural changed.
|
||||||
|
assert 'if (sig === _lastSig) return;' in _VAUTO
|
||||||
|
assert 'var sig = JSON.stringify(' in _VAUTO
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_video_automation_has_a_friendly_label_and_icon():
|
||||||
|
# regression: a video action missing from the label/icon maps falls back to the gear +
|
||||||
|
# raw action_type ("⚙️ video_clean_search_history") — inconsistent. Every seeded video
|
||||||
|
# automation must have an entry in BOTH maps in stats-automations.js.
|
||||||
|
import core.automation_engine as ae
|
||||||
|
types = {a.get("action_type") for a in ae.SYSTEM_AUTOMATIONS if a.get("owned_by") == "video"}
|
||||||
|
missing = [t for t in types if _STATS.count(t + ":") < 2] # icon map + label map
|
||||||
|
assert not missing, "video actions missing a label/icon: " + ", ".join(sorted(missing))
|
||||||
|
|
||||||
|
|
||||||
|
def test_hourly_db_update_is_a_scheduled_safety_net():
|
||||||
|
# second trigger (a schedule) for the same incremental DB read, so manual library adds
|
||||||
|
# show up within the hour instead of waiting for the weekly deep scan.
|
||||||
|
import core.automation_engine as ae
|
||||||
|
row = next((a for a in ae.SYSTEM_AUTOMATIONS if a.get("action_type") == "video_update_database_hourly"), None)
|
||||||
|
assert row is not None
|
||||||
|
assert row["trigger_type"] == "schedule" and row["trigger_config"]["unit"] == "hours"
|
||||||
|
assert row["action_config"]["mode"] == "incremental" # cheap read, not a deep scan
|
||||||
|
|
||||||
|
|
||||||
|
def test_video_system_automations_are_sorted_by_pipeline_order():
|
||||||
|
# The API returns newest-created-first (jumbled); the page re-sorts by an
|
||||||
|
# explicit order so it reads scans → processors → library → maintenance.
|
||||||
|
assert 'sortSystem(all.filter(isVideoAutomation))' in _VAUTO
|
||||||
|
assert '_SYS_ORDER' in _VAUTO
|
||||||
|
# the order must put the watchlist SCANS before the wishlist PROCESSORS
|
||||||
|
scan = _VAUTO.index("'video_scan_watchlist_people'")
|
||||||
|
proc = _VAUTO.index("'video_process_movie_wishlist'")
|
||||||
|
maint = _VAUTO.index("'video_backup_database'")
|
||||||
|
assert scan < proc < maint
|
||||||
383
tests/test_video_backfill.py
Normal file
383
tests/test_video_backfill.py
Normal file
|
|
@ -0,0 +1,383 @@
|
||||||
|
"""Seam tests for the video BACKFILL workers (artwork / subtitles / no-key
|
||||||
|
YouTube extras). The network fetch is stubbed so the loop/queue/record/status
|
||||||
|
logic is tested without hitting fanart.tv / OpenSubtitles / RYD / SponsorBlock.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
from core.video.enrichment.backfill import (
|
||||||
|
RydWorker, SponsorBlockWorker, FanartWorker, OpenSubtitlesWorker, TraktWorker, TVmazeWorker,
|
||||||
|
AniListWorker, DeArrowWorker, WikidataWorker, VideoBackfillWorker, _RateLimited, _Unauthorized,
|
||||||
|
build_backfill_workers,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def db(tmp_path):
|
||||||
|
return VideoDatabase(database_path=str(tmp_path / "video_library.db"))
|
||||||
|
|
||||||
|
|
||||||
|
# ── DB seams ──────────────────────────────────────────────────────────────────
|
||||||
|
def test_youtube_enrich_queue_apply_breakdown(db):
|
||||||
|
db.cache_channel_videos("UC1", [{"youtube_id": "a", "title": "A"},
|
||||||
|
{"youtube_id": "b", "title": "B"}])
|
||||||
|
assert db.youtube_enrich_breakdown("ryd_status")["video"]["pending"] == 2
|
||||||
|
nxt = db.youtube_enrich_next("ryd_status")
|
||||||
|
assert nxt["youtube_id"] in ("a", "b") and nxt["kind"] == "video"
|
||||||
|
|
||||||
|
db.apply_youtube_votes("a", 100, 5, "ok")
|
||||||
|
db.apply_youtube_votes("b", None, None, "not_found")
|
||||||
|
bd = db.youtube_enrich_breakdown("ryd_status")["video"]
|
||||||
|
assert bd == {"matched": 1, "not_found": 1, "errors": 0, "pending": 0}
|
||||||
|
# likes/dislikes merge onto the cached catalog on read
|
||||||
|
vids = {v["youtube_id"]: v for v in db.get_channel_videos("UC1")}
|
||||||
|
assert vids["a"]["like_count"] == 100 and vids["a"]["dislike_count"] == 5
|
||||||
|
|
||||||
|
|
||||||
|
def test_youtube_enrich_shared_video_counted_once(db):
|
||||||
|
# Same video cached under two channels/playlists → one enrichment unit.
|
||||||
|
db.cache_channel_videos("UC1", [{"youtube_id": "x", "title": "X"}])
|
||||||
|
db.cache_channel_videos("PLfoo", [{"youtube_id": "x", "title": "X"}])
|
||||||
|
assert db.youtube_enrich_breakdown("sb_status")["video"]["pending"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_youtube_segments_stored_and_read(db):
|
||||||
|
db.cache_channel_videos("UC1", [{"youtube_id": "v", "title": "V"}])
|
||||||
|
segs = [{"category": "sponsor", "start_sec": 10.0, "end_sec": 20.0, "votes": 3, "uuid": "u1"}]
|
||||||
|
db.apply_youtube_segments("v", segs, "ok")
|
||||||
|
got = db.youtube_video_segments("v")
|
||||||
|
assert got == [{"category": "sponsor", "start_sec": 10.0, "end_sec": 20.0, "votes": 3}]
|
||||||
|
|
||||||
|
|
||||||
|
def test_backfill_next_mark_breakdown(db):
|
||||||
|
mid = db.upsert_movie("plex", {"server_id": "m1", "title": "Fight Club", "year": 1999})
|
||||||
|
with db.connect() as c:
|
||||||
|
c.execute("UPDATE movies SET tmdb_id=550, imdb_id='tt0137523' WHERE id=?", (mid,))
|
||||||
|
c.commit()
|
||||||
|
nxt = db.backfill_next("fanart")
|
||||||
|
assert nxt["kind"] == "movie" and nxt["tmdb_id"] == 550
|
||||||
|
|
||||||
|
db.backfill_mark("fanart", "movie", mid, "ok",
|
||||||
|
columns={"logo_url": "http://x/l.png", "bogus_col": "nope"})
|
||||||
|
with db.connect() as c:
|
||||||
|
r = c.execute("SELECT logo_url, fanart_status FROM movies WHERE id=?", (mid,)).fetchone()
|
||||||
|
assert r["logo_url"] == "http://x/l.png" and r["fanart_status"] == "ok" # whitelist drops bogus_col
|
||||||
|
assert db.backfill_breakdown("fanart")["movie"]["matched"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_backfill_mark_never_clobbers(db):
|
||||||
|
mid = db.upsert_movie("plex", {"server_id": "m1", "title": "M"})
|
||||||
|
with db.connect() as c:
|
||||||
|
c.execute("UPDATE movies SET tmdb_id=1, logo_url='SERVER_LOGO' WHERE id=?", (mid,))
|
||||||
|
c.commit()
|
||||||
|
db.backfill_mark("fanart", "movie", mid, "ok", columns={"logo_url": "FANART_LOGO"})
|
||||||
|
with db.connect() as c:
|
||||||
|
r = c.execute("SELECT logo_url FROM movies WHERE id=?", (mid,)).fetchone()
|
||||||
|
assert r["logo_url"] == "SERVER_LOGO" # gap-fill only
|
||||||
|
|
||||||
|
|
||||||
|
def test_backfill_show_requires_tvdb_id(db):
|
||||||
|
with db.connect() as c:
|
||||||
|
c.execute("INSERT INTO shows (title, year) VALUES ('S', 2008)")
|
||||||
|
sid = c.execute("SELECT id FROM shows WHERE title='S'").fetchone()["id"]
|
||||||
|
c.commit()
|
||||||
|
# No tvdb_id → fanart has nothing to key on, so it's not queued.
|
||||||
|
assert db.backfill_next("fanart") is None
|
||||||
|
with db.connect() as c:
|
||||||
|
c.execute("UPDATE shows SET tvdb_id=81189 WHERE id=?", (sid,))
|
||||||
|
c.commit()
|
||||||
|
assert db.backfill_next("fanart")["kind"] == "show"
|
||||||
|
|
||||||
|
|
||||||
|
# ── worker loop seams (stubbed fetch) ─────────────────────────────────────────
|
||||||
|
def _seed_video(db):
|
||||||
|
db.cache_channel_videos("UC1", [{"youtube_id": "v", "title": "V"}])
|
||||||
|
|
||||||
|
|
||||||
|
def test_ryd_worker_records_each_outcome(db):
|
||||||
|
_seed_video(db)
|
||||||
|
w = RydWorker(db)
|
||||||
|
w.fetch = lambda item: {"likes": 9, "dislikes": 2}
|
||||||
|
assert w.process_one() is True
|
||||||
|
assert db.youtube_enrich_breakdown("ryd_status")["video"]["matched"] == 1
|
||||||
|
assert w.stats["matched"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_ryd_worker_empty_marks_not_found(db):
|
||||||
|
_seed_video(db)
|
||||||
|
w = RydWorker(db)
|
||||||
|
w.fetch = lambda item: None
|
||||||
|
w.process_one()
|
||||||
|
assert db.youtube_enrich_breakdown("ryd_status")["video"]["not_found"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_call_error_records_error_not_notfound(db):
|
||||||
|
_seed_video(db)
|
||||||
|
w = SponsorBlockWorker(db)
|
||||||
|
|
||||||
|
def boom(item):
|
||||||
|
raise RuntimeError("network")
|
||||||
|
w.fetch = boom
|
||||||
|
w.process_one()
|
||||||
|
bd = db.youtube_enrich_breakdown("sb_status")["video"]
|
||||||
|
assert bd["errors"] == 1 and bd["not_found"] == 0
|
||||||
|
assert w.stats["errors"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_rate_limit_sets_cooldown_and_pauses_pending(db):
|
||||||
|
_seed_video(db)
|
||||||
|
w = SponsorBlockWorker(db)
|
||||||
|
|
||||||
|
def limited(item):
|
||||||
|
raise _RateLimited(30)
|
||||||
|
w.fetch = limited
|
||||||
|
assert w.process_one() is False # not consumed; will retry
|
||||||
|
assert w._cooldown_until > 0
|
||||||
|
assert w.get_stats()["cooldown"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_unauthorized_pauses_worker(db):
|
||||||
|
mid = db.upsert_movie("plex", {"server_id": "m1", "title": "M"})
|
||||||
|
with db.connect() as c:
|
||||||
|
c.execute("UPDATE movies SET tmdb_id=1 WHERE id=?", (mid,))
|
||||||
|
c.commit()
|
||||||
|
db.set_setting("fanart_api_key", "KEY")
|
||||||
|
w = FanartWorker(db)
|
||||||
|
|
||||||
|
def denied(item):
|
||||||
|
raise _Unauthorized()
|
||||||
|
w.fetch = denied
|
||||||
|
w.process_one()
|
||||||
|
assert w.paused is True and "key" in (w.note or "").lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_key_gated_workers_disabled_without_key(db):
|
||||||
|
assert FanartWorker(db).enabled is False
|
||||||
|
assert OpenSubtitlesWorker(db).enabled is False
|
||||||
|
db.set_setting("fanart_api_key", "KEY")
|
||||||
|
assert FanartWorker(db).enabled is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_nokey_workers_enabled_by_default_and_toggle(db):
|
||||||
|
assert RydWorker(db).enabled is True
|
||||||
|
db.set_setting("ryd_enabled", "0")
|
||||||
|
assert RydWorker(db).enabled is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_stats_shape_matches_matcher_worker(db):
|
||||||
|
_seed_video(db)
|
||||||
|
stats = RydWorker(db).get_stats()
|
||||||
|
assert set(stats) == {"enabled", "needs_key", "running", "paused", "idle", "current_item",
|
||||||
|
"note", "cooldown", "stats", "progress", "breakdown"}
|
||||||
|
assert set(stats["stats"]) == {"matched", "not_found", "errors", "pending"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_backfill_workers_set(db):
|
||||||
|
assert set(build_backfill_workers(db)) == {
|
||||||
|
"ryd", "sponsorblock", "fanart", "opensubtitles",
|
||||||
|
"trakt", "tvmaze", "anilist", "dearrow", "wikidata"}
|
||||||
|
|
||||||
|
|
||||||
|
# ── Wikidata (no-key, official-website lookup by imdb id) ─────────────────────
|
||||||
|
def test_wikidata_queue_keyed_on_imdb(db):
|
||||||
|
mid = db.upsert_movie("plex", {"server_id": "m1", "title": "Fight Club", "year": 1999})
|
||||||
|
assert db.backfill_next("wikidata") is None # no imdb id → not queued
|
||||||
|
with db.connect() as c:
|
||||||
|
c.execute("UPDATE movies SET imdb_id='tt0137523' WHERE id=?", (mid,))
|
||||||
|
c.commit()
|
||||||
|
assert db.backfill_next("wikidata")["kind"] == "movie"
|
||||||
|
|
||||||
|
|
||||||
|
def test_wikidata_worker_records_url(db):
|
||||||
|
mid = db.upsert_movie("plex", {"server_id": "m1", "title": "M"})
|
||||||
|
with db.connect() as c:
|
||||||
|
c.execute("UPDATE movies SET imdb_id='tt1' WHERE id=?", (mid,))
|
||||||
|
c.commit()
|
||||||
|
w = WikidataWorker(db)
|
||||||
|
assert w.enabled is True
|
||||||
|
w.fetch = lambda item: {"wikidata_url": "https://example.com"}
|
||||||
|
assert w.process_one() is True
|
||||||
|
with db.connect() as c:
|
||||||
|
r = c.execute("SELECT wikidata_url, wikidata_status FROM movies WHERE id=?", (mid,)).fetchone()
|
||||||
|
assert r["wikidata_url"] == "https://example.com" and r["wikidata_status"] == "ok"
|
||||||
|
|
||||||
|
|
||||||
|
def test_wikidata_fetch_two_step_lookup(db, monkeypatch):
|
||||||
|
import core.video.enrichment.backfill as bf
|
||||||
|
|
||||||
|
def fake(url, params=None, headers=None, timeout=12):
|
||||||
|
if (params or {}).get("action") == "query":
|
||||||
|
return {"query": {"search": [{"title": "Q190050"}]}}
|
||||||
|
return {"entities": {"Q190050": {"claims": {"P856": [
|
||||||
|
{"mainsnak": {"datavalue": {"value": "https://officialsite.example"}}}]}}}}
|
||||||
|
|
||||||
|
monkeypatch.setattr(bf, "_http_get_json", fake)
|
||||||
|
out = WikidataWorker(db).fetch({"kind": "movie", "imdb_id": "tt0137523"})
|
||||||
|
assert out == {"wikidata_url": "https://officialsite.example"}
|
||||||
|
|
||||||
|
|
||||||
|
# ── DeArrow (no-key, YouTube crowd titles) ────────────────────────────────────
|
||||||
|
def test_dearrow_queue_and_apply(db):
|
||||||
|
db.cache_channel_videos("UC1", [{"youtube_id": "v", "title": "clickbait TITLE!!!"}])
|
||||||
|
nxt = db.youtube_enrich_next("dearrow_status")
|
||||||
|
assert nxt and nxt["youtube_id"] == "v" and nxt["kind"] == "video"
|
||||||
|
db.apply_youtube_dearrow("v", "A calm, accurate title", "ok")
|
||||||
|
assert db.youtube_video_dearrow_title("v") == "A calm, accurate title"
|
||||||
|
assert db.youtube_enrich_breakdown("dearrow_status")["video"]["matched"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_dearrow_worker_records_title(db):
|
||||||
|
db.cache_channel_videos("UC1", [{"youtube_id": "v", "title": "X"}])
|
||||||
|
w = DeArrowWorker(db)
|
||||||
|
assert w.enabled is True
|
||||||
|
w.fetch = lambda item: {"title": "Better Crowd Title"}
|
||||||
|
assert w.process_one() is True
|
||||||
|
assert db.youtube_video_dearrow_title("v") == "Better Crowd Title"
|
||||||
|
|
||||||
|
|
||||||
|
def test_dearrow_fetch_parses_branding(db, monkeypatch):
|
||||||
|
import core.video.enrichment.backfill as bf
|
||||||
|
branding = {"titles": [
|
||||||
|
{"title": "Original", "original": True},
|
||||||
|
{"title": "Crowd Title", "original": False},
|
||||||
|
]}
|
||||||
|
monkeypatch.setattr(bf, "_http_get_json",
|
||||||
|
lambda url, params=None, headers=None, timeout=12: branding)
|
||||||
|
out = DeArrowWorker(db).fetch({"youtube_id": "v"})
|
||||||
|
assert out == {"title": "Crowd Title"}
|
||||||
|
|
||||||
|
|
||||||
|
# ── Trakt (community rating backfill, keyed on imdb id) ────────────────────────
|
||||||
|
def test_trakt_queue_keyed_on_imdb(db):
|
||||||
|
mid = db.upsert_movie("plex", {"server_id": "m1", "title": "Fight Club", "year": 1999})
|
||||||
|
assert db.backfill_next("trakt") is None # no imdb id → not queued
|
||||||
|
with db.connect() as c:
|
||||||
|
c.execute("UPDATE movies SET imdb_id='tt0137523' WHERE id=?", (mid,))
|
||||||
|
c.commit()
|
||||||
|
nxt = db.backfill_next("trakt")
|
||||||
|
assert nxt["kind"] == "movie" and nxt["imdb_id"] == "tt0137523"
|
||||||
|
|
||||||
|
|
||||||
|
def test_trakt_worker_records_rating(db):
|
||||||
|
mid = db.upsert_movie("plex", {"server_id": "m1", "title": "M"})
|
||||||
|
with db.connect() as c:
|
||||||
|
c.execute("UPDATE movies SET imdb_id='tt1' WHERE id=?", (mid,))
|
||||||
|
c.commit()
|
||||||
|
w = TraktWorker(db)
|
||||||
|
w.fetch = lambda item: {"trakt_rating": 8.2, "trakt_votes": 1234}
|
||||||
|
assert w.process_one() is True
|
||||||
|
with db.connect() as c:
|
||||||
|
r = c.execute("SELECT trakt_rating, trakt_votes, trakt_status FROM movies WHERE id=?", (mid,)).fetchone()
|
||||||
|
assert r["trakt_rating"] == 8.2 and r["trakt_votes"] == 1234 and r["trakt_status"] == "ok"
|
||||||
|
assert db.backfill_breakdown("trakt")["movie"]["matched"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_trakt_fetch_parses_summary(db, monkeypatch):
|
||||||
|
db.set_setting("trakt_api_key", "client-id")
|
||||||
|
import core.video.enrichment.backfill as bf
|
||||||
|
monkeypatch.setattr(bf, "_http_get_json",
|
||||||
|
lambda url, params=None, headers=None, timeout=12: {"rating": 8.234, "votes": 5000})
|
||||||
|
w = TraktWorker(db)
|
||||||
|
out = w.fetch({"kind": "movie", "imdb_id": "tt0137523"})
|
||||||
|
assert out == {"trakt_rating": 8.2, "trakt_votes": 5000} # rounded to 1dp
|
||||||
|
|
||||||
|
|
||||||
|
def test_trakt_fetch_needs_imdb_and_key(db, monkeypatch):
|
||||||
|
import core.video.enrichment.backfill as bf
|
||||||
|
monkeypatch.setattr(bf, "_http_get_json", lambda *a, **k: {"rating": 9})
|
||||||
|
w = TraktWorker(db)
|
||||||
|
assert w.fetch({"kind": "movie", "imdb_id": "tt1"}) is None # no key configured
|
||||||
|
db.set_setting("trakt_api_key", "k")
|
||||||
|
assert w.fetch({"kind": "movie", "imdb_id": "550"}) is None # not a tt-id
|
||||||
|
|
||||||
|
|
||||||
|
# ── TVmaze (no-key, TV-only community rating) ─────────────────────────────────
|
||||||
|
def test_tvmaze_is_show_only_and_enabled_by_default(db):
|
||||||
|
w = TVmazeWorker(db)
|
||||||
|
assert w.enabled is True # keyless → on by default
|
||||||
|
db.set_setting("tvmaze_enabled", "0")
|
||||||
|
assert w.enabled is False
|
||||||
|
# No movie entry in the backfill map → movies are never queued for tvmaze.
|
||||||
|
mid = db.upsert_movie("plex", {"server_id": "m1", "title": "M"})
|
||||||
|
with db.connect() as c:
|
||||||
|
c.execute("UPDATE movies SET imdb_id='tt1' WHERE id=?", (mid,))
|
||||||
|
c.commit()
|
||||||
|
assert db.backfill_next("tvmaze") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_tvmaze_worker_records_rating(db):
|
||||||
|
with db.connect() as c:
|
||||||
|
c.execute("INSERT INTO shows (title, year) VALUES ('S', 2008)")
|
||||||
|
sid = c.execute("SELECT id FROM shows WHERE title='S'").fetchone()["id"]
|
||||||
|
c.execute("UPDATE shows SET imdb_id='tt0903747' WHERE id=?", (sid,))
|
||||||
|
c.commit()
|
||||||
|
nxt = db.backfill_next("tvmaze")
|
||||||
|
assert nxt["kind"] == "show"
|
||||||
|
w = TVmazeWorker(db)
|
||||||
|
w.fetch = lambda item: {"tvmaze_rating": 9.3}
|
||||||
|
assert w.process_one() is True
|
||||||
|
with db.connect() as c:
|
||||||
|
r = c.execute("SELECT tvmaze_rating, tvmaze_status FROM shows WHERE id=?", (sid,)).fetchone()
|
||||||
|
assert r["tvmaze_rating"] == 9.3 and r["tvmaze_status"] == "ok"
|
||||||
|
assert db.backfill_breakdown("tvmaze")["show"]["matched"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_tvmaze_fetch_parses_lookup(db, monkeypatch):
|
||||||
|
import core.video.enrichment.backfill as bf
|
||||||
|
monkeypatch.setattr(bf, "_http_get_json",
|
||||||
|
lambda url, params=None, headers=None, timeout=12: {"rating": {"average": 9.34}})
|
||||||
|
out = TVmazeWorker(db).fetch({"kind": "show", "imdb_id": "tt0903747"})
|
||||||
|
assert out == {"tvmaze_rating": 9.3}
|
||||||
|
|
||||||
|
|
||||||
|
# ── AniList (no-key GraphQL, anime score, opt-in + title-match guard) ──────────
|
||||||
|
def test_anilist_off_by_default(db):
|
||||||
|
w = AniListWorker(db)
|
||||||
|
assert w.enabled is False # opt-in (anime-niche)
|
||||||
|
db.set_setting("anilist_enabled", "1")
|
||||||
|
assert w.enabled is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_anilist_fetch_matches_title_and_scores(db, monkeypatch):
|
||||||
|
import core.video.enrichment.backfill as bf
|
||||||
|
payload = {"data": {"Media": {"averageScore": 85,
|
||||||
|
"title": {"romaji": "Cowboy Bebop", "english": "Cowboy Bebop"}}}}
|
||||||
|
monkeypatch.setattr(bf, "_http_post_json", lambda url, body, headers=None, timeout=12: payload)
|
||||||
|
out = AniListWorker(db).fetch({"kind": "show", "title": "Cowboy Bebop"})
|
||||||
|
assert out == {"anilist_score": 85}
|
||||||
|
|
||||||
|
|
||||||
|
def test_anilist_rejects_title_mismatch(db, monkeypatch):
|
||||||
|
# AniList returns SOME anime for a non-anime title → the guard must reject it.
|
||||||
|
import core.video.enrichment.backfill as bf
|
||||||
|
payload = {"data": {"Media": {"averageScore": 90,
|
||||||
|
"title": {"romaji": "Naruto", "english": "Naruto"}}}}
|
||||||
|
monkeypatch.setattr(bf, "_http_post_json", lambda url, body, headers=None, timeout=12: payload)
|
||||||
|
assert AniListWorker(db).fetch({"kind": "show", "title": "The Office"}) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_anilist_worker_records_score(db):
|
||||||
|
with db.connect() as c:
|
||||||
|
c.execute("INSERT INTO shows (title, year) VALUES ('Bebop', 1998)")
|
||||||
|
sid = c.execute("SELECT id FROM shows WHERE title='Bebop'").fetchone()["id"]
|
||||||
|
c.commit()
|
||||||
|
w = AniListWorker(db)
|
||||||
|
w.fetch = lambda item: {"anilist_score": 87}
|
||||||
|
assert w.process_one() is True
|
||||||
|
with db.connect() as c:
|
||||||
|
r = c.execute("SELECT anilist_score, anilist_status FROM shows WHERE id=?", (sid,)).fetchone()
|
||||||
|
assert r["anilist_score"] == 87 and r["anilist_status"] == "ok"
|
||||||
|
|
||||||
|
|
||||||
|
def test_backfill_module_imports_nothing_from_music():
|
||||||
|
path = Path(__file__).resolve().parent.parent / "core" / "video" / "enrichment" / "backfill.py"
|
||||||
|
for line in path.read_text(encoding="utf-8").splitlines():
|
||||||
|
s = line.strip()
|
||||||
|
if s.startswith("import ") or s.startswith("from "):
|
||||||
|
assert "music" not in s.lower(), f"music import leaked: {s!r}"
|
||||||
61
tests/test_video_cache.py
Normal file
61
tests/test_video_cache.py
Normal file
|
|
@ -0,0 +1,61 @@
|
||||||
|
"""Seam tests for the video enrichment TTL+LRU cache (importable core/ module)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import threading
|
||||||
|
|
||||||
|
from core.video.enrichment.cache import TTLCache
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_miss_and_hit():
|
||||||
|
c = TTLCache(maxsize=4, ttl=100)
|
||||||
|
assert c.get("a") is None
|
||||||
|
c.put("a", 1)
|
||||||
|
assert c.get("a") == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_ttl_expiry_with_injected_clock():
|
||||||
|
t = {"now": 0.0}
|
||||||
|
c = TTLCache(maxsize=4, ttl=10, clock=lambda: t["now"])
|
||||||
|
c.put("a", 1)
|
||||||
|
t["now"] = 9.9
|
||||||
|
assert c.get("a") == 1 # still fresh
|
||||||
|
t["now"] = 10.1
|
||||||
|
assert c.get("a") is None # expired → evicted
|
||||||
|
assert len(c) == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_per_put_ttl_override():
|
||||||
|
t = {"now": 0.0}
|
||||||
|
c = TTLCache(ttl=1000, clock=lambda: t["now"])
|
||||||
|
c.put("a", 1, ttl=5)
|
||||||
|
t["now"] = 6
|
||||||
|
assert c.get("a") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_lru_eviction_not_wholesale():
|
||||||
|
c = TTLCache(maxsize=2, ttl=100)
|
||||||
|
c.put("a", 1)
|
||||||
|
c.put("b", 2)
|
||||||
|
assert c.get("a") == 1 # touch 'a' → most-recently-used
|
||||||
|
c.put("c", 3) # over capacity → evict LRU ('b'), NOT everything
|
||||||
|
assert c.get("a") == 1 # survived
|
||||||
|
assert c.get("c") == 3 # survived
|
||||||
|
assert c.get("b") is None # the only one evicted
|
||||||
|
assert len(c) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_thread_safe_under_concurrency():
|
||||||
|
c = TTLCache(maxsize=64, ttl=100)
|
||||||
|
|
||||||
|
def worker(n):
|
||||||
|
for i in range(500):
|
||||||
|
c.put(("k", (n * 500 + i) % 80), i)
|
||||||
|
c.get(("k", i % 80))
|
||||||
|
|
||||||
|
threads = [threading.Thread(target=worker, args=(n,)) for n in range(8)]
|
||||||
|
for t in threads:
|
||||||
|
t.start()
|
||||||
|
for t in threads:
|
||||||
|
t.join()
|
||||||
|
assert len(c) <= 64 # bound held, no crash/corruption
|
||||||
85
tests/test_video_calendar_scope.py
Normal file
85
tests/test_video_calendar_scope.py
Normal file
|
|
@ -0,0 +1,85 @@
|
||||||
|
"""Calendar source scope: watchlist (default) vs all-library.
|
||||||
|
|
||||||
|
watchlist_only restricts the feed to the EFFECTIVE watchlist — explicit show
|
||||||
|
follows ∪ airing library shows (not muted) — so the calendar tracks what you
|
||||||
|
follow, with an 'All library' escape hatch.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def db(tmp_path):
|
||||||
|
return VideoDatabase(database_path=str(tmp_path / "video_library.db"))
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_show(db, tmdb_id, title, server_id, air_date="2026-06-20"):
|
||||||
|
return db.upsert_show_tree("plex", {
|
||||||
|
"server_id": server_id, "title": title, "tmdb_id": tmdb_id,
|
||||||
|
"seasons": [{"season_number": 1, "episodes": [
|
||||||
|
{"episode_number": 1, "title": "E1", "air_date": air_date}]}]})
|
||||||
|
|
||||||
|
|
||||||
|
def _set_status(db, show_id, status):
|
||||||
|
conn = db._get_connection()
|
||||||
|
conn.execute("UPDATE shows SET status=? WHERE id=?", (status, show_id))
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _tmdbs(rows):
|
||||||
|
return {r["show_tmdb_id"] for r in rows}
|
||||||
|
|
||||||
|
|
||||||
|
def test_watchlist_scope_only_returns_followed_shows(db):
|
||||||
|
_seed_show(db, 11, "A", "sA")
|
||||||
|
_seed_show(db, 22, "B", "sB")
|
||||||
|
db.add_to_watchlist("show", 11, "A") # follow A only (B is not airing → not auto-included)
|
||||||
|
|
||||||
|
allp = db.calendar_upcoming("2026-06-01", "2026-06-30", server_source="plex", watchlist_only=False)
|
||||||
|
wl = db.calendar_upcoming("2026-06-01", "2026-06-30", server_source="plex", watchlist_only=True)
|
||||||
|
assert _tmdbs(allp) == {11, 22} # all-library sees both
|
||||||
|
assert _tmdbs(wl) == {11} # watchlist sees only the followed one
|
||||||
|
|
||||||
|
|
||||||
|
def test_watchlist_scope_includes_airing_default_and_respects_mute(db):
|
||||||
|
a = _seed_show(db, 33, "Airing", "sA")
|
||||||
|
_set_status(db, a, "Returning Series")
|
||||||
|
|
||||||
|
# airing show, not explicitly followed → still in the watchlist scope by default
|
||||||
|
wl = db.calendar_upcoming("2026-06-01", "2026-06-30", server_source="plex", watchlist_only=True)
|
||||||
|
assert _tmdbs(wl) == {33}
|
||||||
|
|
||||||
|
# mute it → drops out of the watchlist scope (but still in all-library)
|
||||||
|
db.remove_from_watchlist("show", 33) # stores a 'mute' tombstone
|
||||||
|
wl2 = db.calendar_upcoming("2026-06-01", "2026-06-30", server_source="plex", watchlist_only=True)
|
||||||
|
assert wl2 == []
|
||||||
|
allp = db.calendar_upcoming("2026-06-01", "2026-06-30", server_source="plex", watchlist_only=False)
|
||||||
|
assert _tmdbs(allp) == {33}
|
||||||
|
|
||||||
|
|
||||||
|
# ── frontend wiring (toggle defaults to watchlist, persists, refetches) ─────
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
_JS = (_ROOT / "webui" / "static" / "video" / "video-calendar.js").read_text(encoding="utf-8")
|
||||||
|
_INDEX = (_ROOT / "webui" / "index.html").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_calendar_has_scope_toggle_defaulting_to_watchlist():
|
||||||
|
assert 'data-video-cal-scope="watchlist"' in _INDEX
|
||||||
|
assert 'data-video-cal-scope="all"' in _INDEX
|
||||||
|
# the watchlist button is the one pre-selected (--on)
|
||||||
|
i = _INDEX.index('data-video-cal-scope="watchlist"')
|
||||||
|
assert 'vcal-filter-btn--on' in _INDEX[i - 80:i]
|
||||||
|
|
||||||
|
|
||||||
|
def test_calendar_js_defaults_watchlist_and_sends_scope():
|
||||||
|
assert "scope: 'watchlist'" in _JS # default state
|
||||||
|
assert "'&scope=' + (state.scope" in _JS # sent to the API
|
||||||
|
assert "function setScope(" in _JS # toggle refetches
|
||||||
|
assert "localStorage.setItem('vcalScope'" in _JS # remembers the choice
|
||||||
1249
tests/test_video_database.py
Normal file
1249
tests/test_video_database.py
Normal file
File diff suppressed because it is too large
Load diff
42
tests/test_video_detail_yt_reset.py
Normal file
42
tests/test_video_detail_yt_reset.py
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
"""Regression: a YouTube channel's playlists must not leak onto the next movie/show.
|
||||||
|
|
||||||
|
YouTube channels/playlists render into the SHOW-detail DOM (they reuse that page).
|
||||||
|
The playlist section is only in the show DOM, but a later movie/show load runs with
|
||||||
|
q() scoped to a different root — so the reset has to target the show subpage
|
||||||
|
directly AND run on every detail load (via resetExtras), or stale playlists show.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
_DETAIL = (_ROOT / "webui" / "static" / "video" / "video-detail.js").read_text(encoding="utf-8")
|
||||||
|
_INDEX = (_ROOT / "webui" / "index.html").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_playlist_section_lives_only_in_show_detail():
|
||||||
|
# The section markup is in the show subpage; the movie subpage must not have it.
|
||||||
|
show_start = _INDEX.index('data-video-detail="show"')
|
||||||
|
movie_start = _INDEX.index('data-video-detail="movie"')
|
||||||
|
show_block = _INDEX[show_start:movie_start]
|
||||||
|
movie_block = _INDEX[movie_start:movie_start + 4000]
|
||||||
|
assert 'data-vd-yt-pl-section' in show_block
|
||||||
|
assert 'data-vd-yt-pl-section' not in movie_block
|
||||||
|
|
||||||
|
|
||||||
|
def test_reset_targets_the_show_dom_directly():
|
||||||
|
# ytResetPlaylists must NOT rely on the kind-scoped q() (which points at the
|
||||||
|
# movie root during a movie load) — it queries the show subpage explicitly.
|
||||||
|
i = _DETAIL.index('function ytResetPlaylists(')
|
||||||
|
body = _DETAIL[i:i + 700]
|
||||||
|
assert "document.querySelector('[data-video-detail=\"show\"]')" in body
|
||||||
|
assert 'data-vd-yt-playlists' in body
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_detail_load_clears_stale_playlists():
|
||||||
|
# resetExtras() runs on loadMovie/loadShow/loadChannel/loadPlaylist, so wiring
|
||||||
|
# the clear there covers all navigations into a non-YouTube detail.
|
||||||
|
i = _DETAIL.index('function resetExtras(')
|
||||||
|
body = _DETAIL[i:i + 700]
|
||||||
|
assert 'ytResetPlaylists()' in body
|
||||||
106
tests/test_video_download_auto.py
Normal file
106
tests/test_video_download_auto.py
Normal file
|
|
@ -0,0 +1,106 @@
|
||||||
|
"""Video Download modal — per-source 'Auto' (search + auto-grab the best) wiring.
|
||||||
|
|
||||||
|
The download view already has a Manual search (you pick a release). This adds an
|
||||||
|
Auto button per source that runs the SAME search and then grabs the best release
|
||||||
|
for the quality profile. The "best" comes for free: the backend returns hits
|
||||||
|
sorted accepted→score→availability (see test_video_api.py
|
||||||
|
::test_downloads_search_endpoint_ranks_and_filters asserting results[0].accepted),
|
||||||
|
so Auto just takes the first accepted hit that has an uploader.
|
||||||
|
|
||||||
|
String-contract level (like tests/test_video_automations_builder.py) so a refactor
|
||||||
|
can't silently unwire the auto path or make manual + auto diverge.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
_VIEW = (_ROOT / "webui" / "static" / "video" / "video-download-view.js").read_text(encoding="utf-8")
|
||||||
|
_CSS = (_ROOT / "webui" / "static" / "video" / "video-side.css").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_each_source_has_manual_and_auto_buttons():
|
||||||
|
assert 'data-vdl-search="' in _VIEW # Manual (pick yourself)
|
||||||
|
assert 'data-vdl-auto="' in _VIEW # Auto (best pick)
|
||||||
|
# Flat/brutalist: plain UPPERCASE labels, no icon spans, no decorative glyphs.
|
||||||
|
assert '>MANUAL<' in _VIEW and '>AUTO<' in _VIEW
|
||||||
|
assert '⚡' not in _VIEW and '✦' not in _VIEW # lightning + sparkle redesigned out
|
||||||
|
assert 'vdl-btn-ic' not in _VIEW # icon spans are gone in the brutalist buttons
|
||||||
|
|
||||||
|
|
||||||
|
def test_header_has_single_auto_best_button():
|
||||||
|
# The old "Manual all" + "Auto all" pair is replaced by ONE header "Auto" that
|
||||||
|
# searches every source and grabs the single best release across all of them.
|
||||||
|
assert 'data-vdl-auto-best' in _VIEW
|
||||||
|
assert 'data-vdl-auto-all' not in _VIEW # the per-source-grab-all footgun is gone
|
||||||
|
assert 'data-vdl-search-all' not in _VIEW
|
||||||
|
assert 'Manual all' not in _VIEW and 'Auto all' not in _VIEW
|
||||||
|
|
||||||
|
|
||||||
|
def test_auto_best_picks_one_winner_across_all_sources():
|
||||||
|
assert 'function _autoBest(' in _VIEW
|
||||||
|
assert 'function _grabBestAcross(' in _VIEW
|
||||||
|
# it compares accepted+grabbable hits across every source by profile score…
|
||||||
|
assert "r.accepted && r.username" in _VIEW
|
||||||
|
assert '(r.score || 0) > (best.score || 0)' in _VIEW
|
||||||
|
# …and grabs exactly ONE (no per-source loop of grabs)
|
||||||
|
assert _VIEW.count('beginTracking(') >= 3 # def + doGrab + _autoPick (+ _grabBestAcross)
|
||||||
|
|
||||||
|
|
||||||
|
def test_click_handler_routes_auto_best_and_per_source():
|
||||||
|
assert "closest('[data-vdl-auto-best]')" in _VIEW
|
||||||
|
assert "closest('[data-vdl-auto]')" in _VIEW # per-source auto still works
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_threads_a_done_callback():
|
||||||
|
# Auto needs to act when the search SETTLES, not on the first tick.
|
||||||
|
assert 'function searchInto(container, resultsEl, params, triggerRows, onDone)' in _VIEW
|
||||||
|
assert 'function _pollSearch(resultsEl, params, id, triggerRows, pollMs, onDone)' in _VIEW
|
||||||
|
assert 'if (onDone) onDone();' in _VIEW
|
||||||
|
|
||||||
|
|
||||||
|
def test_autopick_takes_first_accepted_with_uploader():
|
||||||
|
assert 'function _autoPick(' in _VIEW
|
||||||
|
# picks the first accepted hit that has an uploader (best, since pre-sorted)
|
||||||
|
assert 'rows[i].accepted && rows[i].username' in _VIEW
|
||||||
|
|
||||||
|
|
||||||
|
def test_manual_and_auto_share_one_grab_path():
|
||||||
|
# Both go through buildGrabPayload + sendGrab so they can't diverge.
|
||||||
|
assert 'function buildGrabPayload(' in _VIEW
|
||||||
|
assert 'function sendGrab(' in _VIEW
|
||||||
|
assert _VIEW.count('sendGrab(buildGrabPayload(') >= 2 # doGrab + _autoPick
|
||||||
|
|
||||||
|
|
||||||
|
def test_auto_button_is_styled():
|
||||||
|
assert '.vdl-src-auto' in _CSS
|
||||||
|
assert '.vdl-res--auto' in _CSS # the chosen card gets a ring
|
||||||
|
assert '.vdl-src-actions' in _CSS
|
||||||
|
|
||||||
|
|
||||||
|
# --- Grab whole season (episode-level batch) ------------------------------
|
||||||
|
|
||||||
|
def test_episode_per_source_auto_is_wired():
|
||||||
|
# the show modal now routes the per-episode Auto button (was previously dead),
|
||||||
|
# reusing searchInto + _autoPick at episode scope
|
||||||
|
assert "closest('[data-vdl-auto]')" in _VIEW
|
||||||
|
assert "scope: 'episode'" in _VIEW
|
||||||
|
assert "_autoPick(resA, rowA)" in _VIEW
|
||||||
|
|
||||||
|
|
||||||
|
def test_grab_whole_season_button_and_batch():
|
||||||
|
assert 'data-vdl-season-grab="' in _VIEW # per-season button
|
||||||
|
assert '>Grab season<' in _VIEW
|
||||||
|
assert 'function grabSeason(' in _VIEW
|
||||||
|
assert 'function autoGrabEpisode(' in _VIEW
|
||||||
|
# episode-LEVEL: it loops missing episodes and auto-grabs each (no pack grab)
|
||||||
|
assert "st.epMeta[k].state === 'missing'" in _VIEW
|
||||||
|
assert ".vdl-season-grab" in _CSS
|
||||||
|
|
||||||
|
|
||||||
|
def test_season_grab_reuses_the_single_grab_path():
|
||||||
|
# each episode goes through the same searchInto + _autoPick as a manual Auto,
|
||||||
|
# so the batch can't diverge from the single path
|
||||||
|
assert 'autoGrabEpisode(container, st, sn, eps[idx++], src)' in _VIEW
|
||||||
|
assert 'searchInto(container, panel,' in _VIEW
|
||||||
68
tests/test_video_download_config.py
Normal file
68
tests/test_video_download_config.py
Normal file
|
|
@ -0,0 +1,68 @@
|
||||||
|
"""Video download source-config — pure normalize for mode + hybrid chain
|
||||||
|
(soulseek/torrent/usenet only), isolated from music."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
from core.video.download_config import (
|
||||||
|
MODES,
|
||||||
|
SOURCES,
|
||||||
|
load,
|
||||||
|
normalize_hybrid_order,
|
||||||
|
normalize_mode,
|
||||||
|
save,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_modes_are_video_only():
|
||||||
|
assert SOURCES == ("soulseek", "torrent", "usenet")
|
||||||
|
assert MODES == ("soulseek", "torrent", "usenet", "hybrid")
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_mode():
|
||||||
|
assert normalize_mode("torrent") == "torrent"
|
||||||
|
assert normalize_mode("HYBRID") == "hybrid"
|
||||||
|
assert normalize_mode("spotify") == "soulseek" # music sources rejected
|
||||||
|
assert normalize_mode(None) == "soulseek"
|
||||||
|
assert normalize_mode("") == "soulseek"
|
||||||
|
|
||||||
|
|
||||||
|
def test_normalize_hybrid_order_filters_dedupes_defaults():
|
||||||
|
assert normalize_hybrid_order(["torrent", "usenet"]) == ["torrent", "usenet"]
|
||||||
|
assert normalize_hybrid_order(["torrent", "torrent", "spotify"]) == ["torrent"]
|
||||||
|
assert normalize_hybrid_order([]) == ["soulseek"] # never empty
|
||||||
|
assert normalize_hybrid_order("garbage") == ["soulseek"]
|
||||||
|
# Accepts a JSON string (as stored in the KV table).
|
||||||
|
assert normalize_hybrid_order(json.dumps(["usenet", "soulseek"])) == ["usenet", "soulseek"]
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeDB:
|
||||||
|
def __init__(self):
|
||||||
|
self._kv = {}
|
||||||
|
|
||||||
|
def get_setting(self, key, default=None):
|
||||||
|
return self._kv.get(key, default)
|
||||||
|
|
||||||
|
def set_setting(self, key, value):
|
||||||
|
self._kv[key] = value
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_defaults():
|
||||||
|
assert load(_FakeDB()) == {"download_mode": "soulseek", "hybrid_order": ["soulseek"]}
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_validates_and_roundtrips():
|
||||||
|
db = _FakeDB()
|
||||||
|
out = save(db, {"download_mode": "hybrid", "hybrid_order": ["torrent", "bogus", "torrent", "usenet"]})
|
||||||
|
assert out == {"download_mode": "hybrid", "hybrid_order": ["torrent", "usenet"]}
|
||||||
|
assert load(db) == out # persisted + reloads identically
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_ignores_absent_keys():
|
||||||
|
db = _FakeDB()
|
||||||
|
save(db, {"download_mode": "usenet"})
|
||||||
|
assert load(db)["download_mode"] == "usenet"
|
||||||
|
save(db, {"hybrid_order": ["soulseek", "torrent"]}) # mode key absent → unchanged
|
||||||
|
assert load(db)["download_mode"] == "usenet"
|
||||||
|
assert load(db)["hybrid_order"] == ["soulseek", "torrent"]
|
||||||
79
tests/test_video_download_events.py
Normal file
79
tests/test_video_download_events.py
Normal file
|
|
@ -0,0 +1,79 @@
|
||||||
|
"""Video download → batch-complete event bridge + the monitor detection that fires it.
|
||||||
|
|
||||||
|
The isolated monitor (core/video) publishes 'batch complete' to a callback registry;
|
||||||
|
web_server bridges that to automation_engine.emit('video_batch_complete', …). These
|
||||||
|
pin the publish side: the registry, and the monitor firing exactly once when the last
|
||||||
|
download finishes (never while work is still in flight).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import core.video.download_monitor as mon
|
||||||
|
import core.video.download_events as events
|
||||||
|
|
||||||
|
|
||||||
|
def setup_function(_):
|
||||||
|
events._reset_for_tests()
|
||||||
|
|
||||||
|
|
||||||
|
# ── the event registry ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def test_register_and_notify_fires_callbacks():
|
||||||
|
got = []
|
||||||
|
events.register_batch_complete_callback(lambda d: got.append(d))
|
||||||
|
events.notify_batch_complete({"completed": 3})
|
||||||
|
assert got == [{"completed": 3}]
|
||||||
|
|
||||||
|
|
||||||
|
def test_register_is_idempotent_and_one_failure_is_isolated():
|
||||||
|
got = []
|
||||||
|
cb = lambda d: got.append(d)
|
||||||
|
events.register_batch_complete_callback(cb)
|
||||||
|
events.register_batch_complete_callback(cb) # dup → ignored
|
||||||
|
events.register_batch_complete_callback(lambda d: (_ for _ in ()).throw(RuntimeError("boom")))
|
||||||
|
events.notify_batch_complete({}) # must not raise despite the bad cb
|
||||||
|
assert got == [{}] # the good cb fired exactly once
|
||||||
|
|
||||||
|
|
||||||
|
# ── the monitor fires it on the last completion ────────────────────────────
|
||||||
|
|
||||||
|
class _FakeDb:
|
||||||
|
def __init__(self, active):
|
||||||
|
self._active = list(active)
|
||||||
|
|
||||||
|
def get_active_video_downloads(self):
|
||||||
|
return list(self._active)
|
||||||
|
|
||||||
|
def update_video_download(self, dl_id, **kw):
|
||||||
|
if kw.get("status") in ("completed", "failed", "cancelled"):
|
||||||
|
self._active = [d for d in self._active if d["id"] != dl_id]
|
||||||
|
|
||||||
|
|
||||||
|
def _patch(monkeypatch, result):
|
||||||
|
monkeypatch.setattr(mon, "list_downloads", lambda: [])
|
||||||
|
monkeypatch.setattr(mon, "process_download", lambda dl, *a, **k: dict(result))
|
||||||
|
|
||||||
|
|
||||||
|
def test_tick_fires_batch_complete_when_last_download_finishes(monkeypatch):
|
||||||
|
fired = []
|
||||||
|
events.register_batch_complete_callback(lambda d: fired.append(d))
|
||||||
|
_patch(monkeypatch, {"status": "completed", "progress": 100.0})
|
||||||
|
db = _FakeDb([{"id": 1, "status": "downloading", "filename": "a.mkv"}])
|
||||||
|
mon._tick(db)
|
||||||
|
assert fired == [{"completed": 1}] # one download done, none left → fired once
|
||||||
|
|
||||||
|
|
||||||
|
def test_tick_does_not_fire_while_a_download_is_still_active(monkeypatch):
|
||||||
|
fired = []
|
||||||
|
events.register_batch_complete_callback(lambda d: fired.append(d))
|
||||||
|
# one completes, one is still downloading → batch NOT done yet
|
||||||
|
monkeypatch.setattr(mon, "list_downloads", lambda: [])
|
||||||
|
|
||||||
|
def _proc(dl, *a, **k):
|
||||||
|
return {"status": "completed", "progress": 100.0} if dl["id"] == 1 else {"progress": 50.0}
|
||||||
|
|
||||||
|
monkeypatch.setattr(mon, "process_download", _proc)
|
||||||
|
db = _FakeDb([{"id": 1, "status": "downloading", "filename": "a.mkv"},
|
||||||
|
{"id": 2, "status": "downloading", "filename": "b.mkv"}])
|
||||||
|
mon._tick(db)
|
||||||
|
assert fired == [] # 2 still active → no batch-complete
|
||||||
99
tests/test_video_download_history.py
Normal file
99
tests/test_video_download_history.py
Normal file
|
|
@ -0,0 +1,99 @@
|
||||||
|
"""Permanent video download history — the archive that powers the History modal
|
||||||
|
and the smart post-download scan. video_downloads is the transient queue; this
|
||||||
|
table survives the cleanup, so it's snapshotted at terminal status."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def db(tmp_path):
|
||||||
|
return VideoDatabase(database_path=str(tmp_path / "video_library.db"))
|
||||||
|
|
||||||
|
|
||||||
|
def _movie(**over):
|
||||||
|
row = {"id": 1, "kind": "movie", "title": "Dune", "year": 2024, "status": "completed",
|
||||||
|
"release_title": "Dune.2024.2160p.UHD.BluRay.x265-GRP", "source": "soulseek",
|
||||||
|
"username": "bob", "filename": "Dune.2024.2160p.x265.mkv",
|
||||||
|
"dest_path": "/movies/Dune (2024)/Dune (2024).mkv", "size_bytes": 9_000_000_000,
|
||||||
|
"quality_label": "2160p", "media_id": "55", "media_source": "library",
|
||||||
|
"poster_url": "/p/dune.jpg", "created_at": "2026-06-20 10:00:00",
|
||||||
|
"completed_at": "2026-06-20 10:30:00"}
|
||||||
|
row.update(over)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def _episode(**over):
|
||||||
|
row = {"id": 2, "kind": "show", "title": "Severance", "year": 2025, "status": "completed",
|
||||||
|
"release_title": "Severance.S02E05.1080p.WEB.h264", "source": "soulseek",
|
||||||
|
"dest_path": "/tv/Severance/Season 02/Severance - S02E05.mkv", "size_bytes": 2_000_000_000,
|
||||||
|
"search_ctx": json.dumps({"scope": "episode", "title": "Severance", "season": 2, "episode": 5}),
|
||||||
|
"media_id": "9", "media_source": "library", "completed_at": "2026-06-21 02:00:00"}
|
||||||
|
row.update(over)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def test_records_a_completed_movie_with_parsed_quality(db):
|
||||||
|
hid = db.record_download_history(_movie())
|
||||||
|
assert hid > 0
|
||||||
|
d = db.download_history_detail(hid)
|
||||||
|
assert d["title"] == "Dune" and d["outcome"] == "completed" and d["media_type"] == "movie"
|
||||||
|
assert d["resolution"] == "2160p" and d["video_codec"] == "x265" # sniffed from the release name
|
||||||
|
assert d["size_bytes"] == 9_000_000_000 and d["dest_path"].endswith("Dune (2024).mkv")
|
||||||
|
|
||||||
|
|
||||||
|
def test_records_an_episode_with_season_episode_from_search_ctx(db):
|
||||||
|
hid = db.record_download_history(_episode())
|
||||||
|
d = db.download_history_detail(hid)
|
||||||
|
assert (d["kind"], d["media_type"]) == ("show", "show")
|
||||||
|
assert d["season_number"] == 2 and d["episode_number"] == 5
|
||||||
|
assert d["resolution"] == "1080p" and d["video_codec"] == "x264"
|
||||||
|
|
||||||
|
|
||||||
|
def test_history_is_idempotent_per_terminal_download(db):
|
||||||
|
first = db.record_download_history(_movie())
|
||||||
|
again = db.record_download_history(_movie()) # same download_id/outcome/dest_path
|
||||||
|
assert first > 0 and again == 0 # INSERT OR IGNORE → no dupe
|
||||||
|
assert db.download_history_counts()["movie"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_query_filters_by_kind_and_search(db):
|
||||||
|
db.record_download_history(_movie())
|
||||||
|
db.record_download_history(_episode())
|
||||||
|
assert db.query_download_history(kind="movie")["pagination"]["total_count"] == 1
|
||||||
|
assert db.query_download_history(kind="show")["items"][0]["title"] == "Severance"
|
||||||
|
hits = db.query_download_history(search="dune")["items"]
|
||||||
|
assert len(hits) == 1 and hits[0]["title"] == "Dune"
|
||||||
|
|
||||||
|
|
||||||
|
def test_counts_only_count_completed(db):
|
||||||
|
db.record_download_history(_movie())
|
||||||
|
db.record_download_history(_movie(id=3, status="failed", dest_path=None,
|
||||||
|
error="no release found"))
|
||||||
|
c = db.download_history_counts()
|
||||||
|
assert c == {"movie": 1, "show": 0, "total": 1} # the failed one isn't counted
|
||||||
|
|
||||||
|
|
||||||
|
def test_latest_completed_download_is_the_probe_target(db):
|
||||||
|
db.record_download_history(_movie(id=1, completed_at="2026-06-20 10:30:00"))
|
||||||
|
db.record_download_history(_movie(id=4, title="Wicked",
|
||||||
|
dest_path="/movies/Wicked (2024)/Wicked.mkv",
|
||||||
|
completed_at="2026-06-22 09:00:00"))
|
||||||
|
db.record_download_history(_episode())
|
||||||
|
assert db.latest_completed_download("movie")["title"] == "Wicked" # newest movie
|
||||||
|
assert db.latest_completed_download("show")["title"] == "Severance"
|
||||||
|
assert db.latest_completed_download("all")["title"] == "Wicked" # newest overall
|
||||||
|
|
||||||
|
|
||||||
|
def test_newest_first_ordering_in_the_feed(db):
|
||||||
|
db.record_download_history(_movie(id=1, title="Old", dest_path="/m/old.mkv",
|
||||||
|
completed_at="2026-01-01 00:00:00"))
|
||||||
|
db.record_download_history(_movie(id=2, title="New", dest_path="/m/new.mkv",
|
||||||
|
completed_at="2026-06-01 00:00:00"))
|
||||||
|
titles = [i["title"] for i in db.query_download_history()["items"]]
|
||||||
|
assert titles == ["New", "Old"]
|
||||||
62
tests/test_video_download_history_actions.py
Normal file
62
tests/test_video_download_history_actions.py
Normal file
|
|
@ -0,0 +1,62 @@
|
||||||
|
"""Download-history actions: forget one grab (the 'Re-download' button) + clear the whole
|
||||||
|
history. Removing a history row lets the scans re-add + re-grab the item."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def db(tmp_path):
|
||||||
|
d = VideoDatabase(database_path=str(tmp_path / "video_library.db"))
|
||||||
|
d.record_download_history({"id": 1, "kind": "youtube", "source": "youtube",
|
||||||
|
"media_id": "v1", "status": "completed", "dest_path": "/a.mp4"})
|
||||||
|
d.record_download_history({"id": 2, "kind": "youtube", "source": "youtube",
|
||||||
|
"media_id": "v2", "status": "completed", "dest_path": "/b.mp4"})
|
||||||
|
d.record_download_history({"id": 3, "kind": "movie", "source": "soulseek",
|
||||||
|
"media_id": "9", "status": "completed", "dest_path": "/m.mkv"})
|
||||||
|
return d
|
||||||
|
|
||||||
|
|
||||||
|
def _hist_ids(db):
|
||||||
|
return {r["media_id"] for r in db.query_download_history()["items"]}
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_one_history_entry_frees_it_to_redownload(db):
|
||||||
|
# the youtube dedup sees v1 + v2; forgetting v1's grab drops it from the dedup set
|
||||||
|
assert set(db.downloaded_youtube_video_ids()) == {"v1", "v2"}
|
||||||
|
target = next(r["id"] for r in db.query_download_history()["items"] if r["media_id"] == "v1")
|
||||||
|
assert db.delete_download_history(target) is True
|
||||||
|
assert set(db.downloaded_youtube_video_ids()) == {"v2"} # v1 will re-grab
|
||||||
|
assert db.delete_download_history(999999) is False # missing → no-op
|
||||||
|
|
||||||
|
|
||||||
|
def test_clear_history_all_and_by_kind(db):
|
||||||
|
assert db.clear_download_history(kind="youtube") == 2 # just the youtube grabs
|
||||||
|
assert _hist_ids(db) == {"9"} # movie survives
|
||||||
|
assert db.clear_download_history() == 1 # clear the rest
|
||||||
|
assert _hist_ids(db) == set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_history_action_endpoints(tmp_path):
|
||||||
|
from flask import Flask
|
||||||
|
import api.video as videoapi
|
||||||
|
db = VideoDatabase(database_path=str(tmp_path / "video_library.db"))
|
||||||
|
hid = db.record_download_history({"id": 1, "kind": "youtube", "source": "youtube",
|
||||||
|
"media_id": "v1", "status": "completed", "dest_path": "/a.mp4"})
|
||||||
|
db.record_download_history({"id": 2, "kind": "youtube", "source": "youtube",
|
||||||
|
"media_id": "v2", "status": "completed", "dest_path": "/b.mp4"})
|
||||||
|
videoapi._video_db = db
|
||||||
|
app = Flask(__name__)
|
||||||
|
app.register_blueprint(videoapi.create_video_blueprint(), url_prefix="/api/video")
|
||||||
|
client = app.test_client()
|
||||||
|
try:
|
||||||
|
assert client.delete("/api/video/downloads/history/%d" % hid).get_json()["success"] is True
|
||||||
|
assert set(db.downloaded_youtube_video_ids()) == {"v2"}
|
||||||
|
r = client.post("/api/video/downloads/history/clear", json={}).get_json()
|
||||||
|
assert r["success"] and r["removed"] == 1 # v2 left
|
||||||
|
assert db.downloaded_youtube_video_ids() == []
|
||||||
|
finally:
|
||||||
|
videoapi._video_db = None
|
||||||
210
tests/test_video_download_pipeline.py
Normal file
210
tests/test_video_download_pipeline.py
Normal file
|
|
@ -0,0 +1,210 @@
|
||||||
|
"""Video download pipeline — the pure seams: slskd state classification + flatten,
|
||||||
|
file location + destination resolution, and the video.db downloads CRUD. Isolated."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from core.video.download_pipeline import (
|
||||||
|
basename_of,
|
||||||
|
dest_path_for,
|
||||||
|
find_completed_file,
|
||||||
|
target_dir_for,
|
||||||
|
)
|
||||||
|
from core.video.slskd_download import (
|
||||||
|
classify_state,
|
||||||
|
find_transfer,
|
||||||
|
flatten_downloads,
|
||||||
|
progress_pct,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_classify_state():
|
||||||
|
assert classify_state("Completed, Succeeded") == "completed"
|
||||||
|
assert classify_state("InProgress") == "active"
|
||||||
|
assert classify_state("Queued, Remotely") == "active"
|
||||||
|
assert classify_state("Completed, Errored") == "failed"
|
||||||
|
assert classify_state("Completed, Cancelled") == "cancelled"
|
||||||
|
assert classify_state("Completed, TimedOut") == "failed"
|
||||||
|
assert classify_state("") == "active"
|
||||||
|
|
||||||
|
|
||||||
|
def test_flatten_and_find_transfer():
|
||||||
|
data = [{"username": "neo", "directories": [
|
||||||
|
{"files": [{"filename": r"@@a\Movie\movie.mkv", "id": "t1", "state": "InProgress",
|
||||||
|
"size": 100, "bytesTransferred": 25}]}]}]
|
||||||
|
flat = flatten_downloads(data)
|
||||||
|
assert len(flat) == 1 and flat[0]["username"] == "neo" and flat[0]["id"] == "t1"
|
||||||
|
assert progress_pct(flat[0]) == 25.0
|
||||||
|
assert find_transfer(flat, "neo", r"@@a\Movie\movie.mkv")["id"] == "t1"
|
||||||
|
assert find_transfer(flat, "neo", "other") == {}
|
||||||
|
assert flatten_downloads(None) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_progress_is_100_when_completed():
|
||||||
|
assert progress_pct({"state": "Completed, Succeeded", "size": 100, "transferred": 0}) == 100.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_basename_handles_both_separators():
|
||||||
|
assert basename_of(r"@@x\The.Wire.S02\ep.mkv") == "ep.mkv"
|
||||||
|
assert basename_of("a/b/c/movie.mp4") == "movie.mp4"
|
||||||
|
assert basename_of("") == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_completed_file_by_basename():
|
||||||
|
files = ["/dl/SomeFolder/movie.mkv", "/dl/Other/nope.mkv"]
|
||||||
|
assert find_completed_file("/dl", r"@@u\Remote\movie.mkv", lambda d: files) == "/dl/SomeFolder/movie.mkv"
|
||||||
|
assert find_completed_file("/dl", "missing.mkv", lambda d: files) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_dest_and_target_resolution():
|
||||||
|
assert dest_path_for("/media/movies", "/dl/x/movie.mkv") == "/media/movies/movie.mkv"
|
||||||
|
paths = {"movies_path": "/m", "tv_path": "/t", "youtube_path": "/y"}
|
||||||
|
assert target_dir_for("movie", paths) == "/m"
|
||||||
|
assert target_dir_for("show", paths) == "/t"
|
||||||
|
assert target_dir_for("season", paths) == "/t"
|
||||||
|
assert target_dir_for("youtube", paths) == "/y"
|
||||||
|
assert target_dir_for("weird", paths) == ""
|
||||||
|
|
||||||
|
|
||||||
|
# ── DB CRUD ───────────────────────────────────────────────────────────────────
|
||||||
|
def test_video_downloads_crud(tmp_path):
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
db = VideoDatabase(database_path=str(tmp_path / "video_library.db"))
|
||||||
|
dl_id = db.add_video_download({
|
||||||
|
"kind": "movie", "title": "The Matrix", "release_title": "The.Matrix.1999.1080p",
|
||||||
|
"source": "soulseek", "username": "neo", "filename": r"@@a\x\m.mkv",
|
||||||
|
"size_bytes": 8_000_000_000, "target_dir": "/media/movies", "status": "downloading",
|
||||||
|
})
|
||||||
|
assert dl_id > 0
|
||||||
|
active = db.get_active_video_downloads()
|
||||||
|
assert len(active) == 1 and active[0]["title"] == "The Matrix"
|
||||||
|
db.update_video_download(dl_id, status="completed", progress=100, dest_path="/media/movies/m.mkv")
|
||||||
|
assert db.get_active_video_downloads() == []
|
||||||
|
listed = db.list_video_downloads()
|
||||||
|
assert listed[0]["status"] == "completed" and listed[0]["dest_path"] == "/media/movies/m.mkv"
|
||||||
|
assert db.clear_finished_video_downloads() == 1
|
||||||
|
assert db.list_video_downloads() == []
|
||||||
|
|
||||||
|
|
||||||
|
# ── monitor decision (pure, injected fs) ──────────────────────────────────────
|
||||||
|
def _dl(**kw):
|
||||||
|
base = {"id": 1, "username": "neo", "filename": r"@@a\Folder\movie.mkv", "target_dir": "/media/movies"}
|
||||||
|
base.update(kw)
|
||||||
|
return base
|
||||||
|
|
||||||
|
|
||||||
|
def _xfer(state, **kw):
|
||||||
|
base = {"username": "neo", "filename": r"@@a\Folder\movie.mkv", "state": state, "size": 100, "transferred": 40}
|
||||||
|
base.update(kw)
|
||||||
|
return base
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_download_active_reports_progress():
|
||||||
|
from core.video.download_monitor import process_download
|
||||||
|
upd = process_download(_dl(), [_xfer("InProgress")], "/dl",
|
||||||
|
lister=lambda d: [], mover=lambda s, d: None)
|
||||||
|
assert upd == {"status": "downloading", "progress": 40.0}
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_download_failed():
|
||||||
|
from core.video.download_monitor import process_download
|
||||||
|
upd = process_download(_dl(), [_xfer("Completed, Errored")], "/dl",
|
||||||
|
lister=lambda d: [], mover=lambda s, d: None)
|
||||||
|
assert upd["status"] == "failed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_download_cancelled():
|
||||||
|
from core.video.download_monitor import process_download
|
||||||
|
upd = process_download(_dl(), [_xfer("Completed, Cancelled")], "/dl",
|
||||||
|
lister=lambda d: [], mover=lambda s, d: None)
|
||||||
|
assert upd["status"] == "cancelled"
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_download_missing_transfer_signals_missing():
|
||||||
|
from core.video.download_monitor import process_download
|
||||||
|
# slskd forgot it AND no file on disk → _missing (caller decides when to give up)
|
||||||
|
upd = process_download(_dl(), [], "/dl", lister=lambda d: [], mover=lambda s, d: None)
|
||||||
|
assert upd == {"_missing": True}
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_download_missing_but_file_present_completes():
|
||||||
|
from core.video.download_monitor import process_download
|
||||||
|
moved = {}
|
||||||
|
# slskd cleared the completed transfer (the music auto-clear) but the file is there
|
||||||
|
upd = process_download(_dl(), [], "/dl",
|
||||||
|
lister=lambda d: ["/dl/Folder/movie.mkv"],
|
||||||
|
mover=lambda s, d: moved.update(ok=True))
|
||||||
|
assert upd["status"] == "completed" and moved.get("ok") is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_download_completed_moves_file():
|
||||||
|
from core.video.download_monitor import process_download
|
||||||
|
moved = {}
|
||||||
|
upd = process_download(
|
||||||
|
_dl(), [_xfer("Completed, Succeeded")], "/dl",
|
||||||
|
lister=lambda d: ["/dl/Folder/movie.mkv"],
|
||||||
|
mover=lambda s, d: moved.update(src=s, dest=d))
|
||||||
|
assert upd == {"status": "completed", "progress": 100.0, "dest_path": "/media/movies/movie.mkv"}
|
||||||
|
assert moved == {"src": "/dl/Folder/movie.mkv", "dest": "/media/movies/movie.mkv"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_download_completed_but_file_not_settled():
|
||||||
|
from core.video.download_monitor import process_download
|
||||||
|
upd = process_download(_dl(), [_xfer("Completed, Succeeded")], "/dl",
|
||||||
|
lister=lambda d: [], mover=lambda s, d: None)
|
||||||
|
assert upd == {"progress": 100.0} # no status change — retries next tick
|
||||||
|
|
||||||
|
|
||||||
|
def test_fail_or_retry_starts_next_candidate(tmp_path, monkeypatch):
|
||||||
|
import json
|
||||||
|
import core.video.download_monitor as mon
|
||||||
|
import core.video.slskd_download as slskd
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
|
||||||
|
monkeypatch.setattr(slskd, "start_download", lambda *a, **k: {"ok": True})
|
||||||
|
monkeypatch.setattr(mon, "start_download", lambda *a, **k: {"ok": True})
|
||||||
|
|
||||||
|
db = VideoDatabase(database_path=str(tmp_path / "video_library.db"))
|
||||||
|
dl_id = db.add_video_download({
|
||||||
|
"kind": "movie", "title": "Dune", "release_title": "Dune.2021.1080p.x265-A",
|
||||||
|
"source": "soulseek", "username": "alice", "filename": r"@@a\A\dune.mkv",
|
||||||
|
"target_dir": "/m", "status": "downloading", "attempts": 0,
|
||||||
|
"tried_files": json.dumps([r"@@a\A\dune.mkv"]),
|
||||||
|
"candidates": json.dumps([{"username": "bob", "filename": r"@@b\B\dune.mkv",
|
||||||
|
"size_bytes": 9, "quality_label": "1080p", "release_title": "Dune.2021.1080p.x265-B"}]),
|
||||||
|
"search_ctx": json.dumps({"scope": "movie", "title": "Dune", "year": 2021}),
|
||||||
|
"tried_queries": json.dumps(["Dune 2021"]),
|
||||||
|
})
|
||||||
|
row = db.get_video_download(dl_id)
|
||||||
|
mon._fail_or_retry(db, row, "Soulseek transfer Errored")
|
||||||
|
after = db.get_video_download(dl_id)
|
||||||
|
# rolled onto the next candidate (bob's release), back to downloading, attempt counted
|
||||||
|
assert after["status"] == "downloading" and after["username"] == "bob"
|
||||||
|
assert after["release_title"] == "Dune.2021.1080p.x265-B" and after["attempts"] == 1
|
||||||
|
assert json.loads(after["candidates"]) == [] # pool consumed
|
||||||
|
|
||||||
|
|
||||||
|
def test_fail_or_retry_marks_failed_when_exhausted(tmp_path, monkeypatch):
|
||||||
|
import json
|
||||||
|
import core.video.download_monitor as mon
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
db = VideoDatabase(database_path=str(tmp_path / "video_library.db"))
|
||||||
|
dl_id = db.add_video_download({
|
||||||
|
"kind": "movie", "title": "Dune", "source": "soulseek", "username": "a",
|
||||||
|
"filename": "x.mkv", "target_dir": "/m", "status": "downloading", "attempts": 0,
|
||||||
|
"candidates": "[]", "tried_files": json.dumps(["x.mkv"]),
|
||||||
|
"search_ctx": json.dumps({"scope": "movie", "title": "Dune"}), # no year → only one query
|
||||||
|
"tried_queries": json.dumps(["Dune"]),
|
||||||
|
})
|
||||||
|
mon._fail_or_retry(db, db.get_video_download(dl_id), "boom")
|
||||||
|
assert db.get_video_download(dl_id)["status"] == "failed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_download_move_failure_marks_failed():
|
||||||
|
from core.video.download_monitor import process_download
|
||||||
|
|
||||||
|
def boom(s, d):
|
||||||
|
raise OSError("disk full")
|
||||||
|
|
||||||
|
upd = process_download(_dl(), [_xfer("Completed, Succeeded")], "/dl",
|
||||||
|
lister=lambda d: ["/dl/Folder/movie.mkv"], mover=boom)
|
||||||
|
assert upd["status"] == "failed" and "disk full" in upd["error"]
|
||||||
125
tests/test_video_download_tracking.py
Normal file
125
tests/test_video_download_tracking.py
Normal file
|
|
@ -0,0 +1,125 @@
|
||||||
|
"""Live download tracking wiring: after a grab the user must (a) see WHICH release
|
||||||
|
was selected, (b) get a live progress bar on it, and (c) get a button to the
|
||||||
|
Downloads page — and a movie's detail page shows live progress for an in-flight
|
||||||
|
download that jumps back to Downloads.
|
||||||
|
|
||||||
|
Backed by GET /api/video/downloads/status (tested in test_video_api.py
|
||||||
|
::test_downloads_status_lookup_by_id_and_media). These pin the frontend wiring so
|
||||||
|
a refactor can't quietly unhook the tracker, the detail chip, or the navigation.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
_VIEW = (_ROOT / "webui" / "static" / "video" / "video-download-view.js").read_text(encoding="utf-8")
|
||||||
|
_DETAIL = (_ROOT / "webui" / "static" / "video" / "video-detail.js").read_text(encoding="utf-8")
|
||||||
|
_SIDE = (_ROOT / "webui" / "static" / "video" / "video-side.js").read_text(encoding="utf-8")
|
||||||
|
_GETMODAL = (_ROOT / "webui" / "static" / "video" / "video-get-modal.js").read_text(encoding="utf-8")
|
||||||
|
_CSS = (_ROOT / "webui" / "static" / "video" / "video-side.css").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
# --- result card: redesign + selected/tracking state ----------------------
|
||||||
|
|
||||||
|
def test_result_card_redesigned_as_column_with_get_button():
|
||||||
|
assert 'vdl-res-main' in _VIEW # row wrapper so a tracker can dock below
|
||||||
|
assert 'data-vdl-card="' in _VIEW # addressable card (auto-pick targets it)
|
||||||
|
assert '[ GET ]' in _VIEW # brutalist grab button is a bracketed label
|
||||||
|
assert '.vdl-res-main' in _CSS
|
||||||
|
|
||||||
|
|
||||||
|
def test_grab_begins_live_tracking_on_the_card():
|
||||||
|
assert 'function beginTracking(' in _VIEW
|
||||||
|
assert 'vdl-res--grabbed' in _VIEW
|
||||||
|
assert 'data-vdl-track-fill' in _VIEW # the progress bar fill
|
||||||
|
assert "/api/video/downloads/status?id=" in _VIEW
|
||||||
|
# both the manual grab and the auto-pick start tracking the chosen card
|
||||||
|
assert _VIEW.count('beginTracking(') >= 3 # 1 def + doGrab + _autoPick
|
||||||
|
|
||||||
|
|
||||||
|
def test_track_button_goes_to_downloads_and_closes_modal():
|
||||||
|
assert 'Track on Downloads' in _VIEW
|
||||||
|
assert 'function gotoDownloads(' in _VIEW
|
||||||
|
assert 'VideoGet.close' in _VIEW
|
||||||
|
assert "'soulsync:video-navigate'" in _VIEW
|
||||||
|
assert '.vdl-res-track' in _CSS
|
||||||
|
|
||||||
|
|
||||||
|
def test_modal_exposes_close():
|
||||||
|
assert 'close: closeModal' in _GETMODAL
|
||||||
|
|
||||||
|
|
||||||
|
# --- movie detail page: live download chip --------------------------------
|
||||||
|
|
||||||
|
def test_detail_page_watches_movie_download():
|
||||||
|
assert 'function watchMovieDownload(' in _DETAIL
|
||||||
|
assert '/api/video/downloads/status?media_id=' in _DETAIL
|
||||||
|
assert 'data-vd-dlchip' in _DETAIL
|
||||||
|
# the chip jumps to the Downloads page
|
||||||
|
assert "'soulsync:video-navigate'" in _DETAIL
|
||||||
|
assert '.vd-dlchip' in _CSS
|
||||||
|
|
||||||
|
|
||||||
|
def test_detail_watch_started_for_library_movies():
|
||||||
|
assert 'watchMovieDownload(id)' in _DETAIL
|
||||||
|
assert 'stopMovieDownloadWatch()' in _DETAIL # cleared on (re)load / navigate away
|
||||||
|
|
||||||
|
|
||||||
|
# --- result cards: flat / brutalist redesign ------------------------------
|
||||||
|
|
||||||
|
def test_result_cards_are_flat_brutalist_three_line():
|
||||||
|
# 3 hard lines: [QUALITY] + title, an UPPERCASE spec line, then verdict + GET.
|
||||||
|
assert 'vdl-r-l1' in _VIEW and 'vdl-r-q' in _VIEW and 'vdl-r-title' in _VIEW
|
||||||
|
assert 'vdl-r-l2' in _VIEW and 'vdl-r-l3' in _VIEW and 'vdl-r-verdict' in _VIEW
|
||||||
|
assert '.vdl-r-q' in _CSS and '.vdl-r-l2' in _CSS and '.vdl-r-verdict' in _CSS
|
||||||
|
# the cinematic pill/badge language was redesigned out
|
||||||
|
assert 'vdl-info-tags' not in _VIEW and 'vdl-tag' not in _VIEW
|
||||||
|
assert 'vdl-q-res' not in _VIEW and 'vdl-flag' not in _VIEW
|
||||||
|
# sharp corners + monospace are the brutalist signature
|
||||||
|
assert 'border-radius: 0' in _CSS and 'monospace' in _CSS
|
||||||
|
|
||||||
|
|
||||||
|
# --- modal resumes an in-flight download on reopen ------------------------
|
||||||
|
|
||||||
|
def test_modal_resumes_active_download_on_reopen():
|
||||||
|
assert 'data-vdl-active' in _VIEW
|
||||||
|
assert 'function watchActiveDownload(' in _VIEW
|
||||||
|
assert '/api/video/downloads/status?media_id=' in _VIEW
|
||||||
|
# suppressed while a result card is already tracking inline (no double indicator)
|
||||||
|
assert "querySelector('[data-vdl-track]')" in _VIEW
|
||||||
|
assert '.vdl-active' in _CSS
|
||||||
|
|
||||||
|
|
||||||
|
# --- per-episode live tracking (TV parity with the movie inline tracker) ---
|
||||||
|
|
||||||
|
def test_episode_rows_get_live_download_status():
|
||||||
|
# each episode ROW shows its own Searching → Downloading % → Downloaded status
|
||||||
|
assert 'function epTrack(' in _VIEW and 'function epStatusRender(' in _VIEW
|
||||||
|
assert "/api/video/downloads/status?id=" in _VIEW
|
||||||
|
assert 'data-vdl-ep-status' in _VIEW
|
||||||
|
assert 'vdl-ep--dl-active' in _VIEW and 'vdl-ep--dl-done' in _VIEW and 'vdl-ep--dl-fail' in _VIEW
|
||||||
|
assert '.vdl-ep-dl' in _CSS and '.vdl-ep-dl-bar' in _CSS
|
||||||
|
|
||||||
|
|
||||||
|
def test_season_grab_lights_rows_and_resumes_on_reopen():
|
||||||
|
# season grab gives immediate per-row feedback (not headless), and reopening the
|
||||||
|
# modal resumes tracking in-flight episodes
|
||||||
|
assert 'function epSearching(' in _VIEW
|
||||||
|
assert 'eps.forEach(function (en) { epSearching(' in _VIEW # rows light up at once
|
||||||
|
assert 'function resumeEpisodeTracking(' in _VIEW
|
||||||
|
assert "/api/video/downloads/active" in _VIEW
|
||||||
|
assert 'resumeEpisodeTracking(container, st)' in _VIEW # called when rows build
|
||||||
|
|
||||||
|
|
||||||
|
def test_episode_grab_uses_the_shared_single_grab_path():
|
||||||
|
# the season batch grabs via the same payload as a manual grab (can't diverge)
|
||||||
|
assert 'function _pickAndGrab(' in _VIEW
|
||||||
|
assert 'sendGrab(buildGrabPayload(panel, best))' in _VIEW
|
||||||
|
|
||||||
|
|
||||||
|
# --- navigation plumbing --------------------------------------------------
|
||||||
|
|
||||||
|
def test_video_side_handles_navigate_event():
|
||||||
|
assert "'soulsync:video-navigate'" in _SIDE
|
||||||
|
assert 'navigate(pageId)' in _SIDE
|
||||||
80
tests/test_video_downloads_page.py
Normal file
80
tests/test_video_downloads_page.py
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
"""Downloads page UX overhaul — string-contract level (like test_video_side_shell.py), so a
|
||||||
|
refactor that silently drops the per-type theming, the sidebar count, or the new statuses
|
||||||
|
fails here."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
_JS = (_ROOT / "webui" / "static" / "video" / "video-downloads-page.js").read_text(encoding="utf-8")
|
||||||
|
_CSS = (_ROOT / "webui" / "static" / "video" / "video-side.css").read_text(encoding="utf-8")
|
||||||
|
_INDEX = (_ROOT / "webui" / "index.html").read_text(encoding="utf-8", errors="replace")
|
||||||
|
|
||||||
|
|
||||||
|
def test_cards_carry_a_type_for_cinema_theming():
|
||||||
|
assert "function dlType(" in _JS
|
||||||
|
assert "data-vtype" in _JS # set on each card
|
||||||
|
# the three Cinema palette colours are defined per type
|
||||||
|
for vt in ('[data-vtype="movie"]', '[data-vtype="tv"]', '[data-vtype="youtube"]'):
|
||||||
|
assert vt in _CSS, vt
|
||||||
|
assert "--vt: 79, 143, 247" in _CSS and "--vt: 240, 69, 75" in _CSS # azure + red
|
||||||
|
|
||||||
|
|
||||||
|
def test_importing_is_a_first_class_active_status():
|
||||||
|
assert "importing:" in _JS and "import_failed:" in _JS
|
||||||
|
assert "s === 'importing'" in _JS # counts as active
|
||||||
|
assert "vdpg-prog-indet" in _JS # indeterminate sweep for queued/searching/importing
|
||||||
|
|
||||||
|
|
||||||
|
def test_sidebar_has_a_live_downloads_count():
|
||||||
|
assert "data-video-downloads-badge" in _INDEX # the nav badge element
|
||||||
|
assert "function setDownloadsBadge(" in _JS
|
||||||
|
assert "function badgePoll(" in _JS # stays live off-page too
|
||||||
|
|
||||||
|
|
||||||
|
def test_cards_expand_into_a_detail_drawer():
|
||||||
|
assert "function drawerHTML(" in _JS and "function renderDrawer(" in _JS
|
||||||
|
assert "_expanded" in _JS # open state survives re-patches
|
||||||
|
assert "vdpg-dr-cast" in _JS and "vdpg-dr-syn" in _JS # cast + synopsis sections
|
||||||
|
assert "data-vdpg-copy" in _JS # copy-path action
|
||||||
|
# the lazy TMDB detail endpoint the drawer fetches synopsis/cast from
|
||||||
|
assert "/downloads/meta/" in _JS
|
||||||
|
|
||||||
|
|
||||||
|
def test_drawer_renders_the_rich_tmdb_fields():
|
||||||
|
# cast PHOTOS (the bug was the wrong field name) + logo header + trailer + providers
|
||||||
|
assert "c.photo" in _JS # correct TMDB cast-photo field
|
||||||
|
assert "vdpg-dr-logo" in _JS # title logo header
|
||||||
|
assert "vdpg-dr-trailer" in _JS and "vdpg-prov" in _JS # trailer + where-to-watch
|
||||||
|
assert "trailer_url" in _JS and "providers" in _JS
|
||||||
|
|
||||||
|
|
||||||
|
def test_youtube_open_button_opens_the_channel_page_not_a_show():
|
||||||
|
# regression: the open-show button ran parseInt(youtube_id) → opened a random library
|
||||||
|
# show (id 3 → '3rd Rock'). youtube cards must open the in-app CHANNEL page instead.
|
||||||
|
open_block = _JS.split("var openBtn")[1].split("var stateBtn")[0]
|
||||||
|
assert "dlType(d.kind) === 'youtube'" in open_block
|
||||||
|
assert "data-vdpg-open-channel" in open_block # opens the in-app channel page
|
||||||
|
assert "youtube.com/watch?v=" in open_block # fallback when the channel id is unknown
|
||||||
|
# clicking it dispatches an open-detail for the channel (reuses the show-detail page)
|
||||||
|
assert "kind: 'channel', source: 'youtube'" in _JS
|
||||||
|
|
||||||
|
|
||||||
|
def test_drawer_has_episode_youtube_and_availability_blocks():
|
||||||
|
assert "vdpg-dr-ytthumb" in _JS # youtube big thumbnail header
|
||||||
|
assert "vdpg-dr-ep" in _JS and "vdpg-dr-epstill" in _JS # episode still + block
|
||||||
|
assert "ctx.peer" in _JS # grab-time availability snapshot
|
||||||
|
assert "yt-meta" in _JS # youtube metadata fetch
|
||||||
|
assert "season=" in _JS and "episode=" in _JS # episode params on the meta fetch
|
||||||
|
|
||||||
|
|
||||||
|
def test_download_meta_routes_are_registered():
|
||||||
|
import api.video as videoapi
|
||||||
|
from flask import Flask
|
||||||
|
app = Flask(__name__)
|
||||||
|
app.register_blueprint(videoapi.create_video_blueprint(), url_prefix="/api/video")
|
||||||
|
rules = {r.rule for r in app.url_map.iter_rules()}
|
||||||
|
assert "/api/video/downloads/meta/<kind>/<int:tmdb_id>" in rules
|
||||||
|
assert "/api/video/downloads/yt-meta/<video_id>" in rules
|
||||||
|
|
||||||
1333
tests/test_video_enrichment.py
Normal file
1333
tests/test_video_enrichment.py
Normal file
File diff suppressed because it is too large
Load diff
59
tests/test_video_enrichment_manager.py
Normal file
59
tests/test_video_enrichment_manager.py
Normal file
|
|
@ -0,0 +1,59 @@
|
||||||
|
"""Wiring guards for the video "Manage Workers" modal (video-enrichment-manager.js).
|
||||||
|
|
||||||
|
Pins two behaviours the user asked for:
|
||||||
|
- clicking a coverage card just SWITCHES THE VIEW — it must not change the global
|
||||||
|
"process first" priority or silently re-queue failed items (those are the top
|
||||||
|
Movies/Shows/Auto tabs and the explicit "Retry all failed" button);
|
||||||
|
- "Retry all failed" re-queues EVERY coverage kind the worker handles, not just
|
||||||
|
the tab being viewed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
_JS = (_ROOT / "webui" / "static" / "video" / "video-enrichment-manager.js").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _func(name: str) -> str:
|
||||||
|
"""Return the body of `function name(` up to the next top-level `function `."""
|
||||||
|
i = _JS.index("function " + name + "(")
|
||||||
|
nxt = _JS.find("\n function ", i + 1)
|
||||||
|
return _JS[i:nxt if nxt != -1 else len(_JS)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_switch_kind_only_switches_view():
|
||||||
|
body = _func("switchKind")
|
||||||
|
assert "state.kind = kind" in body
|
||||||
|
# must NOT reach across to the global priority or auto-retry
|
||||||
|
assert "setPriority(" not in body
|
||||||
|
assert "requeueFailed(" not in body
|
||||||
|
|
||||||
|
|
||||||
|
def test_priority_changes_only_from_top_tabs():
|
||||||
|
# setPriority is still wired — but only to the data-em-priority (top tabs) click.
|
||||||
|
assert "function setPriority(" in _JS
|
||||||
|
assert "data-em-priority')) setPriority(" in _JS
|
||||||
|
# the dead per-coverage requeue helper is gone
|
||||||
|
assert "function requeueFailed(" not in _JS
|
||||||
|
|
||||||
|
|
||||||
|
def test_global_retry_all_failed_button_wired():
|
||||||
|
# a topbar button that re-queues failed items across ALL workers in one call
|
||||||
|
assert "data-em-retry-all-global" in _JS
|
||||||
|
assert "function retryAllFailedGlobal(" in _JS
|
||||||
|
assert "/api/video/enrichment/retry-all-failed" in _JS
|
||||||
|
# it's distinct from the per-worker retry (which targets one worker)
|
||||||
|
assert "retryAllFailedGlobal()" in _JS
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_all_failed_covers_every_kind_of_the_worker():
|
||||||
|
body = _func("retryAllFailed")
|
||||||
|
# iterate the worker's kinds (movie+show / show / video …), retry each
|
||||||
|
assert "workerDef(state.selected)" in body
|
||||||
|
assert "w.kinds" in body
|
||||||
|
assert re.search(r"kinds\.map\(", body)
|
||||||
|
# the button is wired to the multi-kind handler, not the single-kind retry
|
||||||
|
assert "data-em-retry-all')) retryAllFailed()" in _JS
|
||||||
35
tests/test_video_followed_shows.py
Normal file
35
tests/test_video_followed_shows.py
Normal file
|
|
@ -0,0 +1,35 @@
|
||||||
|
"""followed_shows(): explicit show follows + their library status — feeds the
|
||||||
|
watchlist-prune pass that drops ended/canceled shows."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def db(tmp_path):
|
||||||
|
return VideoDatabase(database_path=str(tmp_path / "video_library.db"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_lists_explicit_follows_with_library_status(db):
|
||||||
|
# a tmdb-only follow (no library row → status None)
|
||||||
|
db.add_to_watchlist("show", 100, "TMDB Only")
|
||||||
|
# a follow backed by a library show carrying a status
|
||||||
|
conn = db._get_connection()
|
||||||
|
conn.execute("INSERT INTO shows (id, server_source, title, tmdb_id, status) "
|
||||||
|
"VALUES (5, 'plex', 'Owned Show', 200, 'Ended')")
|
||||||
|
conn.commit(); conn.close()
|
||||||
|
db.add_to_watchlist("show", 200, "Owned Show", library_id=5)
|
||||||
|
|
||||||
|
rows = {r["tmdb_id"]: r for r in db.followed_shows()}
|
||||||
|
assert rows[100]["status"] is None # tmdb-only → no local status
|
||||||
|
assert rows[200]["status"] == "Ended" # owned → carries the status
|
||||||
|
|
||||||
|
|
||||||
|
def test_excludes_muted_and_people(db):
|
||||||
|
db.add_to_watchlist("show", 1, "A")
|
||||||
|
db.remove_from_watchlist("show", 1) # mute (tombstone)
|
||||||
|
db.add_to_watchlist("person", 2, "Someone")
|
||||||
|
assert db.followed_shows() == [] # neither muted shows nor people
|
||||||
61
tests/test_video_import_page.py
Normal file
61
tests/test_video_import_page.py
Normal file
|
|
@ -0,0 +1,61 @@
|
||||||
|
"""Video Import page — frontend wiring (string-contract, like the other video page
|
||||||
|
tests). Pins the page module, its container, nav registration, and the endpoints it
|
||||||
|
calls so a refactor can't quietly unhook the manual-import flow. The placement LOGIC
|
||||||
|
itself is covered by tests/test_video_importer.py + tests/test_video_manual_import.py.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_ROOT = Path(__file__).resolve().parent.parent
|
||||||
|
_JS = (_ROOT / "webui" / "static" / "video" / "video-import.js").read_text(encoding="utf-8")
|
||||||
|
_SIDE = (_ROOT / "webui" / "static" / "video" / "video-side.js").read_text(encoding="utf-8")
|
||||||
|
_INDEX = (_ROOT / "webui" / "index.html").read_text(encoding="utf-8")
|
||||||
|
_CSS = (_ROOT / "webui" / "static" / "video" / "video-side.css").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_module_is_an_isolated_iife():
|
||||||
|
s = _JS.strip()
|
||||||
|
assert s.startswith("/*") or s.startswith("(function")
|
||||||
|
assert "(function" in _JS and "})();" in _JS
|
||||||
|
# isolated: wrapped in an IIFE and never ASSIGNS a global (reads like
|
||||||
|
# window.confirm are fine). No `window.<name> =` and no `var X` at top level.
|
||||||
|
import re
|
||||||
|
assert not re.search(r"window\.\w+\s*=", _JS)
|
||||||
|
assert "PAGE_ID = 'video-import'" in _JS
|
||||||
|
|
||||||
|
|
||||||
|
def test_page_is_a_real_video_page_not_shared():
|
||||||
|
# the nav entry exists and is NO LONGER flagged shared (it's a true video page now)
|
||||||
|
assert 'data-video-page="video-import"' in _INDEX
|
||||||
|
assert "{ id: 'video-import', label: 'Import' }" in _SIDE
|
||||||
|
assert "'video-import', label: 'Import', shared" not in _SIDE
|
||||||
|
|
||||||
|
|
||||||
|
def test_subpage_container_and_script_present():
|
||||||
|
assert 'data-video-subpage="video-import"' in _INDEX
|
||||||
|
assert "data-vimp-grid" in _INDEX and "data-vimp-empty" in _INDEX
|
||||||
|
assert "video/video-import.js" in _INDEX # script include
|
||||||
|
assert ".vimp-card" in _CSS and ".vimp-modal" in _CSS
|
||||||
|
|
||||||
|
|
||||||
|
def test_loads_and_polls_the_failed_queue():
|
||||||
|
assert "/api/video/import/failed" in _JS
|
||||||
|
assert "soulsync:video-page-shown" in _JS
|
||||||
|
assert "setInterval(" in _JS # 5s poll while shown
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_flow_wired_to_place_and_search():
|
||||||
|
# the picker reuses the existing TMDB search, library-owned floated to the top
|
||||||
|
assert "/api/video/search?q=" in _JS
|
||||||
|
assert "library first" in _JS or "owned" in _JS
|
||||||
|
# movie vs episode placement + the place/dismiss endpoints
|
||||||
|
assert "scope: r.kind" in _JS
|
||||||
|
assert "/api/video/import/' + r.item.id + '/place'" in _JS
|
||||||
|
assert "/dismiss'" in _JS
|
||||||
|
|
||||||
|
|
||||||
|
def test_endpoints_registered_on_the_blueprint():
|
||||||
|
init = (_ROOT / "api" / "video" / "__init__.py").read_text(encoding="utf-8")
|
||||||
|
assert "reg_manual_import(bp)" in init
|
||||||
272
tests/test_video_importer.py
Normal file
272
tests/test_video_importer.py
Normal file
|
|
@ -0,0 +1,272 @@
|
||||||
|
"""Video post-process / import — the Radarr/Sonarr step that organises a finished
|
||||||
|
download into the library.
|
||||||
|
|
||||||
|
Covers the decision matrix (import / upgrade-replace / reject) and the orchestration
|
||||||
|
(copy in, carry subtitles, delete a worse existing file, reclaim the source unless
|
||||||
|
it's a torrent). The filesystem is injected so it's all unit-tested without disk.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
from core.video import importer
|
||||||
|
|
||||||
|
|
||||||
|
# ── a recording fake filesystem ──────────────────────────────────────────────
|
||||||
|
class FakeFS:
|
||||||
|
def __init__(self, dirs=None):
|
||||||
|
# dirs: {dirpath: [basename, ...]} — what list_dir returns
|
||||||
|
self.dirs = {k: list(v) for k, v in (dirs or {}).items()}
|
||||||
|
self.made = []
|
||||||
|
self.copied = [] # (src, dst)
|
||||||
|
self.moved = [] # (src, dst)
|
||||||
|
self.saved = [] # (url, dst)
|
||||||
|
self.removed = []
|
||||||
|
|
||||||
|
def list_dir(self, path):
|
||||||
|
return self.dirs.get(str(path), [])
|
||||||
|
|
||||||
|
def makedirs(self, path):
|
||||||
|
self.made.append(str(path))
|
||||||
|
|
||||||
|
def copy(self, src, dst):
|
||||||
|
self.copied.append((src, dst))
|
||||||
|
|
||||||
|
def move(self, src, dst):
|
||||||
|
self.moved.append((src, dst))
|
||||||
|
|
||||||
|
def save_url(self, url, dst):
|
||||||
|
self.saved.append((url, dst))
|
||||||
|
|
||||||
|
def remove(self, path):
|
||||||
|
self.removed.append(path)
|
||||||
|
|
||||||
|
|
||||||
|
def _movie_dl(release, source="soulseek", size=2_000_000_000, root="/lib/movies"):
|
||||||
|
return {
|
||||||
|
"kind": "movie", "title": "The Matrix", "year": 1999, "source": source,
|
||||||
|
"release_title": release, "size_bytes": size, "target_dir": root,
|
||||||
|
"search_ctx": json.dumps({"scope": "movie", "title": "The Matrix", "year": 1999}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _episode_dl(release, root="/lib/tv", season=1, episode=1):
|
||||||
|
return {
|
||||||
|
"kind": "show", "title": "Breaking Bad", "source": "soulseek",
|
||||||
|
"release_title": release, "size_bytes": 1_000_000_000, "target_dir": root,
|
||||||
|
"search_ctx": json.dumps({"scope": "episode", "title": "Breaking Bad",
|
||||||
|
"season": season, "episode": episode, "episode_title": "Pilot"}),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ── sanity gate ───────────────────────────────────────────────────────────────
|
||||||
|
def test_rejects_non_video_and_samples():
|
||||||
|
dl = _movie_dl("The Matrix 1999 1080p BluRay")
|
||||||
|
assert importer.plan_import(dl, "/dl/x/matrix.nfo", list_dir=lambda d: [])["action"] == "reject"
|
||||||
|
small = _movie_dl("The Matrix 1999 1080p BluRay", size=20_000_000)
|
||||||
|
p = importer.plan_import(small, "/dl/x/sample-matrix.mkv", list_dir=lambda d: [])
|
||||||
|
assert p["action"] == "reject" and "sample" in p["reason"].lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_rejects_wrong_episode():
|
||||||
|
dl = _episode_dl("Breaking Bad S01E02 1080p WEB-DL", season=1, episode=1)
|
||||||
|
p = importer.plan_import(dl, "/dl/x/bb.s01e02.mkv", list_dir=lambda d: [])
|
||||||
|
assert p["action"] == "reject" and "S01E02" in p["reason"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_rejects_season_pack_for_manual():
|
||||||
|
dl = _episode_dl("Breaking Bad S01 1080p WEB-DL", season=1, episode=1)
|
||||||
|
dl["search_ctx"] = json.dumps({"scope": "season", "title": "Breaking Bad", "season": 1})
|
||||||
|
p = importer.plan_import(dl, "/dl/x/bb.s01.mkv", list_dir=lambda d: [])
|
||||||
|
assert p["action"] == "reject" and "manual" in p["reason"].lower()
|
||||||
|
|
||||||
|
|
||||||
|
# ── import / upgrade / not-an-upgrade ─────────────────────────────────────────
|
||||||
|
def test_fresh_movie_import_path():
|
||||||
|
dl = _movie_dl("The Matrix 1999 1080p BluRay x265")
|
||||||
|
p = importer.plan_import(dl, "/dl/x/the.matrix.1999.1080p.bluray.x265.mkv", list_dir=lambda d: [])
|
||||||
|
assert p["action"] == "import"
|
||||||
|
assert p["dest"]["path"] == os.path.join("/lib/movies", "The Matrix (1999)",
|
||||||
|
"The Matrix (1999) Bluray-1080p.mkv")
|
||||||
|
|
||||||
|
|
||||||
|
def test_upgrade_replaces_worse_existing():
|
||||||
|
dl = _movie_dl("The Matrix 1999 2160p BluRay") # 4K incoming
|
||||||
|
folder = os.path.join("/lib/movies", "The Matrix (1999)")
|
||||||
|
fs_dirs = {folder: ["The Matrix (1999) Bluray-720p.mkv"]} # owns 720p
|
||||||
|
p = importer.plan_import(dl, "/dl/x/matrix.2160p.mkv", list_dir=lambda d: fs_dirs.get(d, []))
|
||||||
|
assert p["action"] == "upgrade"
|
||||||
|
assert p["replace_path"] == os.path.join(folder, "The Matrix (1999) Bluray-720p.mkv")
|
||||||
|
|
||||||
|
|
||||||
|
def test_not_an_upgrade_is_rejected():
|
||||||
|
dl = _movie_dl("The Matrix 1999 720p HDTV") # worse than owned
|
||||||
|
folder = os.path.join("/lib/movies", "The Matrix (1999)")
|
||||||
|
fs_dirs = {folder: ["The Matrix (1999) Bluray-1080p.mkv"]}
|
||||||
|
p = importer.plan_import(dl, "/dl/x/matrix.720p.mkv", list_dir=lambda d: fs_dirs.get(d, []))
|
||||||
|
assert p["action"] == "reject" and "upgrade" in p["reason"].lower()
|
||||||
|
|
||||||
|
|
||||||
|
# ── ffprobe verification ──────────────────────────────────────────────────────
|
||||||
|
def test_probe_true_resolution_overrides_lying_name():
|
||||||
|
# name claims 1080p; the file is really 720p → tag + folder use the truth
|
||||||
|
dl = _movie_dl("The Matrix 1999 1080p BluRay")
|
||||||
|
probe = {"ok": True, "resolution": "720p", "duration_sec": 8000, "video_codec": "x264"}
|
||||||
|
p = importer.plan_import(dl, "/dl/x/matrix.mkv", list_dir=lambda d: [], probe=probe)
|
||||||
|
assert p["action"] == "import"
|
||||||
|
assert p["dest"]["path"].endswith("The Matrix (1999) Bluray-720p.mkv")
|
||||||
|
|
||||||
|
|
||||||
|
def test_probe_rejects_corrupt_file():
|
||||||
|
dl = _movie_dl("The Matrix 1999 1080p BluRay")
|
||||||
|
probe = {"ok": False}
|
||||||
|
p = importer.plan_import(dl, "/dl/x/matrix.mkv", list_dir=lambda d: [], probe=probe)
|
||||||
|
assert p["action"] == "reject" and "corrupt" in p["reason"].lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_probe_rejects_short_runtime_movie_as_sample():
|
||||||
|
dl = _movie_dl("The Matrix 1999 1080p BluRay")
|
||||||
|
probe = {"ok": True, "resolution": "1080p", "duration_sec": 120} # 2 minutes
|
||||||
|
p = importer.plan_import(dl, "/dl/x/matrix.mkv", list_dir=lambda d: [], probe=probe)
|
||||||
|
assert p["action"] == "reject" and "sample" in p["reason"].lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_probe_resolution_drives_upgrade_decision():
|
||||||
|
# name says 1080p but the file is truly 2160p → beats an owned 1080p copy
|
||||||
|
dl = _movie_dl("The Matrix 1999 1080p BluRay")
|
||||||
|
folder = os.path.join("/lib/movies", "The Matrix (1999)")
|
||||||
|
fs_dirs = {folder: ["The Matrix (1999) Bluray-1080p.mkv"]}
|
||||||
|
probe = {"ok": True, "resolution": "2160p", "duration_sec": 8000}
|
||||||
|
p = importer.plan_import(dl, "/dl/x/matrix.mkv", list_dir=lambda d: fs_dirs.get(d, []), probe=probe)
|
||||||
|
assert p["action"] == "upgrade"
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_import_uses_prober_and_can_reject():
|
||||||
|
dl = _movie_dl("The Matrix 1999 1080p BluRay")
|
||||||
|
fs = FakeFS()
|
||||||
|
patch = importer.run_import(dl, "/dl/x/matrix.mkv", fs=fs,
|
||||||
|
prober=lambda p: {"ok": False})
|
||||||
|
assert patch["status"] == "import_failed" and not fs.copied
|
||||||
|
|
||||||
|
|
||||||
|
# ── orchestration via the fake fs ─────────────────────────────────────────────
|
||||||
|
def test_run_import_copies_and_reclaims_source():
|
||||||
|
dl = _movie_dl("The Matrix 1999 1080p BluRay")
|
||||||
|
fs = FakeFS()
|
||||||
|
patch = importer.run_import(dl, "/dl/x/matrix.mkv", fs=fs)
|
||||||
|
assert patch["status"] == "completed"
|
||||||
|
assert patch["dest_path"].endswith(os.path.join("The Matrix (1999)", "The Matrix (1999) Bluray-1080p.mkv"))
|
||||||
|
assert fs.copied and fs.copied[0][0] == "/dl/x/matrix.mkv"
|
||||||
|
assert "/dl/x/matrix.mkv" in fs.removed # soulseek source reclaimed
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_import_keeps_torrent_source():
|
||||||
|
dl = _movie_dl("The Matrix 1999 1080p BluRay", source="torrent")
|
||||||
|
fs = FakeFS()
|
||||||
|
importer.run_import(dl, "/dl/x/matrix.mkv", fs=fs)
|
||||||
|
assert "/dl/x/matrix.mkv" not in fs.removed # torrent left seeding
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_import_carries_subtitles_and_deletes_old_on_upgrade():
|
||||||
|
dl = _movie_dl("The Matrix 1999 2160p BluRay")
|
||||||
|
folder = os.path.join("/lib/movies", "The Matrix (1999)")
|
||||||
|
fs = FakeFS(dirs={
|
||||||
|
"/dl/x": ["matrix.mkv", "matrix.en.srt", "unrelated.srt"],
|
||||||
|
folder: ["The Matrix (1999) Bluray-720p.mkv"],
|
||||||
|
})
|
||||||
|
patch = importer.run_import(dl, "/dl/x/matrix.mkv", fs=fs)
|
||||||
|
assert patch["status"] == "completed"
|
||||||
|
# the matching sibling sub is carried + renamed to the dest stem, .en preserved
|
||||||
|
subs = [d for _s, d in fs.copied if d.endswith(".en.srt")]
|
||||||
|
assert subs and subs[0].endswith("The Matrix (1999) Bluray-2160p.en.srt")
|
||||||
|
# the worse 720p copy is deleted
|
||||||
|
assert os.path.join(folder, "The Matrix (1999) Bluray-720p.mkv") in fs.removed
|
||||||
|
|
||||||
|
|
||||||
|
# ── manual placement (force + override) ───────────────────────────────────────
|
||||||
|
def test_force_bypasses_sample_and_files_to_override_identity():
|
||||||
|
# a small "sample" file the auto path would reject — forced to a chosen movie
|
||||||
|
dl = _movie_dl("sample.mkv", size=20_000_000)
|
||||||
|
override = {"scope": "movie", "title": "Blade Runner", "year": 1982, "target_dir": "/lib/movies"}
|
||||||
|
p = importer.plan_import(dl, "/dl/x/sample.mkv", list_dir=lambda d: [],
|
||||||
|
force=True, override=override)
|
||||||
|
assert p["action"] == "import"
|
||||||
|
assert p["dest"]["path"] == os.path.join("/lib/movies", "Blade Runner (1982)",
|
||||||
|
"Blade Runner (1982).mkv")
|
||||||
|
|
||||||
|
|
||||||
|
def test_force_reidentifies_wrong_episode_to_the_right_one():
|
||||||
|
dl = _episode_dl("Show S01E02 1080p WEB-DL", season=1, episode=2)
|
||||||
|
override = {"scope": "episode", "title": "Show", "season": 3, "episode": 7,
|
||||||
|
"episode_title": "Reckoning", "target_dir": "/lib/tv"}
|
||||||
|
p = importer.plan_import(dl, "/dl/x/show.mkv", list_dir=lambda d: [],
|
||||||
|
force=True, override=override)
|
||||||
|
assert p["action"] == "import"
|
||||||
|
# re-identified to S03E07, but the file keeps its real quality tag from the release
|
||||||
|
assert p["dest"]["path"] == os.path.join("/lib/tv", "Show", "Season 03",
|
||||||
|
"Show - S03E07 - Reckoning WEBDL-1080p.mkv")
|
||||||
|
|
||||||
|
|
||||||
|
def test_force_replaces_existing_regardless_of_quality():
|
||||||
|
# forcing a 720p over an owned 1080p still places it (the user decided)
|
||||||
|
dl = _movie_dl("The Matrix 1999 720p HDTV")
|
||||||
|
folder = os.path.join("/lib/movies", "The Matrix (1999)")
|
||||||
|
fs_dirs = {folder: ["The Matrix (1999) Bluray-1080p.mkv"]}
|
||||||
|
override = {"scope": "movie", "title": "The Matrix", "year": 1999, "target_dir": "/lib/movies"}
|
||||||
|
p = importer.plan_import(dl, "/dl/x/m.mkv", list_dir=lambda d: fs_dirs.get(d, []),
|
||||||
|
force=True, override=override)
|
||||||
|
assert p["action"] == "upgrade"
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_import_reject_remembers_the_file_location():
|
||||||
|
dl = _movie_dl("The Matrix 1999 1080p BluRay")
|
||||||
|
fs = FakeFS()
|
||||||
|
patch = importer.run_import(dl, "/dl/x/readme.nfo", fs=fs) # non-video → reject
|
||||||
|
assert patch["status"] == "import_failed"
|
||||||
|
assert patch["dest_path"] == "/dl/x/readme.nfo" # so manual import can find it
|
||||||
|
|
||||||
|
|
||||||
|
# ── organisation settings drive behaviour ─────────────────────────────────────
|
||||||
|
def test_custom_template_changes_the_path():
|
||||||
|
dl = _movie_dl("The Matrix 1999 1080p BluRay")
|
||||||
|
settings = {"movie_template": "$year/$title/$title $resolution"}
|
||||||
|
p = importer.plan_import(dl, "/dl/x/matrix.mkv", list_dir=lambda d: [], settings=settings)
|
||||||
|
assert p["dest"]["path"] == os.path.join("/lib/movies", "1999", "The Matrix", "The Matrix 1080p.mkv")
|
||||||
|
|
||||||
|
|
||||||
|
def test_replace_disabled_rejects_instead_of_upgrading():
|
||||||
|
dl = _movie_dl("The Matrix 1999 2160p BluRay")
|
||||||
|
folder = os.path.join("/lib/movies", "The Matrix (1999)")
|
||||||
|
fs_dirs = {folder: ["The Matrix (1999) Bluray-720p.mkv"]}
|
||||||
|
p = importer.plan_import(dl, "/dl/x/m.mkv", list_dir=lambda d: fs_dirs.get(d, []),
|
||||||
|
settings={"replace_existing": False})
|
||||||
|
assert p["action"] == "reject" and "replace is turned off" in p["reason"].lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_move_mode_moves_and_does_not_reclaim():
|
||||||
|
dl = _movie_dl("The Matrix 1999 1080p BluRay")
|
||||||
|
fs = FakeFS()
|
||||||
|
importer.run_import(dl, "/dl/x/matrix.mkv", fs=fs, settings={"transfer_mode": "move"})
|
||||||
|
assert fs.moved and fs.moved[0][0] == "/dl/x/matrix.mkv"
|
||||||
|
assert not fs.copied and not fs.removed # moved, so no copy + no source reclaim
|
||||||
|
|
||||||
|
|
||||||
|
def test_carry_subtitles_toggle_off():
|
||||||
|
dl = _movie_dl("The Matrix 1999 1080p BluRay")
|
||||||
|
fs = FakeFS(dirs={"/dl/x": ["matrix.mkv", "matrix.en.srt"]})
|
||||||
|
importer.run_import(dl, "/dl/x/matrix.mkv", fs=fs, settings={"carry_subtitles": False})
|
||||||
|
assert not any(d.endswith(".srt") for _s, d in fs.copied)
|
||||||
|
|
||||||
|
|
||||||
|
# (artwork/NFO sidecars moved out of run_import into core/video/sidecars.py — see
|
||||||
|
# tests/test_video_sidecars.py. run_import is now purely the file mover.)
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_import_reject_leaves_file_and_flags_manual():
|
||||||
|
dl = _movie_dl("The Matrix 1999 1080p BluRay")
|
||||||
|
fs = FakeFS()
|
||||||
|
patch = importer.run_import(dl, "/dl/x/readme.nfo", fs=fs) # not a video file
|
||||||
|
assert patch["status"] == "import_failed" and patch["error"]
|
||||||
|
assert not fs.copied and not fs.removed # nothing touched on disk
|
||||||
69
tests/test_video_library_paths.py
Normal file
69
tests/test_video_library_paths.py
Normal file
|
|
@ -0,0 +1,69 @@
|
||||||
|
"""Video library organisation — the Radarr/Sonarr-standard naming seam.
|
||||||
|
|
||||||
|
A finished download must land at a canonical, server-identifiable path:
|
||||||
|
<root>/The Matrix (1999)/The Matrix (1999) Bluray-1080p.mkv
|
||||||
|
<root>/Breaking Bad/Season 01/Breaking Bad - S01E01 - Pilot WEBDL-1080p.mkv
|
||||||
|
Pure string logic — pinned here so a refactor can't quietly change the layout.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
from core.video.library_paths import (
|
||||||
|
episode_filename,
|
||||||
|
movie_filename,
|
||||||
|
movie_folder,
|
||||||
|
plan_path,
|
||||||
|
quality_full,
|
||||||
|
sanitize,
|
||||||
|
season_folder,
|
||||||
|
show_folder,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_sanitize_strips_illegal_and_trailing():
|
||||||
|
assert sanitize('A: Movie / "Title"?') == "A Movie Title"
|
||||||
|
assert sanitize("name with trailing dots... ") == "name with trailing dots"
|
||||||
|
assert sanitize(" spaced out ") == "spaced out"
|
||||||
|
assert sanitize(None) == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_quality_full_tag():
|
||||||
|
assert quality_full({"source": "bluray", "resolution": "1080p"}) == "Bluray-1080p"
|
||||||
|
assert quality_full({"source": "web-dl", "resolution": "1080p"}) == "WEBDL-1080p"
|
||||||
|
assert quality_full({"source": "remux", "resolution": "2160p"}) == "Remux-2160p"
|
||||||
|
# proper/repack is surfaced in the tag
|
||||||
|
assert quality_full({"source": "bluray", "resolution": "720p", "proper": True}) == "Bluray-720p Proper"
|
||||||
|
# partial / unknown
|
||||||
|
assert quality_full({"resolution": "1080p"}) == "1080p"
|
||||||
|
assert quality_full({}) == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_movie_naming():
|
||||||
|
assert movie_folder("The Matrix", 1999) == "The Matrix (1999)"
|
||||||
|
assert movie_folder("The Matrix", None) == "The Matrix" # no year → no suffix
|
||||||
|
assert movie_filename("The Matrix", 1999, "Bluray-1080p", ".mkv") == "The Matrix (1999) Bluray-1080p.mkv"
|
||||||
|
assert movie_filename("The Matrix", 1999, "", ".mkv") == "The Matrix (1999).mkv" # unknown quality omitted
|
||||||
|
|
||||||
|
|
||||||
|
def test_episode_naming():
|
||||||
|
assert show_folder("Breaking Bad") == "Breaking Bad"
|
||||||
|
assert season_folder(1) == "Season 01"
|
||||||
|
assert season_folder(0) == "Specials"
|
||||||
|
assert episode_filename("Breaking Bad", 1, 1, "Pilot", "WEBDL-1080p", ".mkv") == \
|
||||||
|
"Breaking Bad - S01E01 - Pilot WEBDL-1080p.mkv"
|
||||||
|
# missing episode title + missing quality both degrade gracefully
|
||||||
|
assert episode_filename("Breaking Bad", 2, 13, "", "", ".mkv") == "Breaking Bad - S02E13.mkv"
|
||||||
|
|
||||||
|
|
||||||
|
def test_plan_path_movie_and_episode():
|
||||||
|
movie = plan_path("movie", "/lib/movies", {"title": "The Matrix", "year": 1999}, "Bluray-1080p", ".mkv")
|
||||||
|
assert movie["dir"] == os.path.join("/lib/movies", "The Matrix (1999)")
|
||||||
|
assert movie["path"] == os.path.join("/lib/movies", "The Matrix (1999)", "The Matrix (1999) Bluray-1080p.mkv")
|
||||||
|
|
||||||
|
ep = plan_path("episode", "/lib/tv",
|
||||||
|
{"title": "Breaking Bad", "season": 1, "episode": 1, "episode_title": "Pilot"},
|
||||||
|
"WEBDL-1080p", ".mkv")
|
||||||
|
assert ep["dir"] == os.path.join("/lib/tv", "Breaking Bad", "Season 01")
|
||||||
|
assert ep["filename"] == "Breaking Bad - S01E01 - Pilot WEBDL-1080p.mkv"
|
||||||
120
tests/test_video_manual_import.py
Normal file
120
tests/test_video_manual_import.py
Normal file
|
|
@ -0,0 +1,120 @@
|
||||||
|
"""Manual / failed-import resolution endpoints — list the unplaced queue, force-place
|
||||||
|
a file to a chosen identity, and dismiss. Uses real temp files (the place flow runs
|
||||||
|
the real importer against disk)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from flask import Flask
|
||||||
|
|
||||||
|
import api.video as videoapi
|
||||||
|
from core.video import organization
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def env(tmp_path):
|
||||||
|
db = VideoDatabase(database_path=str(tmp_path / "video_library.db"))
|
||||||
|
movies = tmp_path / "Movies"
|
||||||
|
movies.mkdir()
|
||||||
|
db.set_setting("movies_path", str(movies))
|
||||||
|
db.set_setting("tv_path", str(tmp_path / "TV"))
|
||||||
|
organization.save(db, {"verify_with_ffprobe": False}) # don't depend on ffprobe in CI
|
||||||
|
|
||||||
|
dl_dir = tmp_path / "dl"
|
||||||
|
dl_dir.mkdir()
|
||||||
|
src = dl_dir / "the.matrix.1999.1080p.bluray.x265.mkv"
|
||||||
|
src.write_bytes(b"x" * 4096)
|
||||||
|
|
||||||
|
dl_id = db.add_video_download({
|
||||||
|
"kind": "movie", "title": "the matrix", "release_title": src.name,
|
||||||
|
"source": "soulseek", "username": "neo", "filename": src.name,
|
||||||
|
"size_bytes": 4096, "target_dir": str(movies), "status": "import_failed",
|
||||||
|
"search_ctx": "{}",
|
||||||
|
})
|
||||||
|
db.update_video_download(dl_id, dest_path=str(src), error="Looks like a sample, not the feature")
|
||||||
|
|
||||||
|
videoapi._video_db = db
|
||||||
|
app = Flask(__name__)
|
||||||
|
app.register_blueprint(videoapi.create_video_blueprint(), url_prefix="/api/video")
|
||||||
|
client = app.test_client()
|
||||||
|
try:
|
||||||
|
yield {"client": client, "db": db, "dl_id": dl_id, "src": src, "movies": movies}
|
||||||
|
finally:
|
||||||
|
videoapi._video_db = None
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_list_surfaces_unplaced_downloads(env):
|
||||||
|
items = env["client"].get("/api/video/import/failed").get_json()["items"]
|
||||||
|
assert len(items) == 1
|
||||||
|
it = items[0]
|
||||||
|
assert it["id"] == env["dl_id"]
|
||||||
|
assert it["file"] == str(env["src"]) # points at the unplaced file
|
||||||
|
assert "sample" in it["reason"].lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_place_force_imports_to_chosen_identity(env):
|
||||||
|
r = env["client"].post("/api/video/import/%d/place" % env["dl_id"],
|
||||||
|
json={"scope": "movie", "title": "The Matrix", "year": 1999}).get_json()
|
||||||
|
assert r["success"] and r["status"] == "completed"
|
||||||
|
final = env["movies"] / "The Matrix (1999)" / "The Matrix (1999) Bluray-1080p.mkv"
|
||||||
|
assert final.exists() # filed under the standard layout
|
||||||
|
assert not env["src"].exists() # source reclaimed (copy mode, non-torrent)
|
||||||
|
# the row is no longer in the failed queue
|
||||||
|
assert env["client"].get("/api/video/import/failed").get_json()["items"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_place_triggers_a_library_refresh(env):
|
||||||
|
# a successful manual place fires the same batch-complete refresh the auto path
|
||||||
|
# uses, so the title shows up without waiting for a scheduled scan.
|
||||||
|
from core.video import download_events
|
||||||
|
fired = []
|
||||||
|
download_events.register_batch_complete_callback(lambda d: fired.append(d))
|
||||||
|
try:
|
||||||
|
env["client"].post("/api/video/import/%d/place" % env["dl_id"],
|
||||||
|
json={"scope": "movie", "title": "The Matrix", "year": 1999})
|
||||||
|
assert fired and fired[-1].get("manual") is True
|
||||||
|
finally:
|
||||||
|
download_events._reset_for_tests()
|
||||||
|
|
||||||
|
|
||||||
|
def test_media_ids_resolves_tmdb_and_library_regrabs():
|
||||||
|
from core.video.download_monitor import _media_ids
|
||||||
|
# grabbed straight from TMDB → media_id is the tmdb id
|
||||||
|
assert _media_ids(None, {"media_source": "tmdb", "media_id": "603"}) == (603, None)
|
||||||
|
|
||||||
|
# owned re-grab → media_id is the LIBRARY id; resolve via the library row
|
||||||
|
class _DB:
|
||||||
|
def media_tmdb_id(self, kind, mid):
|
||||||
|
assert kind == "movie" and mid == "5107"
|
||||||
|
return (936075, "tt11378946")
|
||||||
|
assert _media_ids(_DB(), {"media_source": "library", "media_id": "5107", "kind": "movie"}) \
|
||||||
|
== (936075, "tt11378946")
|
||||||
|
|
||||||
|
assert _media_ids(None, {}) == (None, None) # unresolvable → no sidecars
|
||||||
|
|
||||||
|
|
||||||
|
def test_dismiss_does_not_trigger_a_refresh(env):
|
||||||
|
from core.video import download_events
|
||||||
|
fired = []
|
||||||
|
download_events.register_batch_complete_callback(lambda d: fired.append(d))
|
||||||
|
try:
|
||||||
|
env["client"].post("/api/video/import/%d/dismiss" % env["dl_id"], json={})
|
||||||
|
assert fired == [] # nothing landed → no scan
|
||||||
|
finally:
|
||||||
|
download_events._reset_for_tests()
|
||||||
|
|
||||||
|
|
||||||
|
def test_place_rejects_bad_scope(env):
|
||||||
|
r = env["client"].post("/api/video/import/%d/place" % env["dl_id"], json={"scope": "season"})
|
||||||
|
assert r.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
def test_dismiss_drops_row_and_can_delete_file(env):
|
||||||
|
r = env["client"].post("/api/video/import/%d/dismiss" % env["dl_id"],
|
||||||
|
json={"delete_file": True}).get_json()
|
||||||
|
assert r["success"]
|
||||||
|
assert not env["src"].exists() # file removed
|
||||||
|
assert env["client"].get("/api/video/import/failed").get_json()["items"] == []
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue