video downloads: monitor thread + grab/active/clear endpoints
Phase 2 — the engine: - core/video/download_monitor.py: a daemon thread polls slskd for active video downloads, updates progress, and on completion MOVES the file from the shared download folder into the per-type library folder + marks it completed. The per-download decision (process_download) is pure (fs + slskd injected) — 6 tests. - POST /downloads/grab: validates (Soulseek-only v1), resolves the target library by kind, starts the slskd download, records the row, lazily starts the monitor. - GET /downloads/active (list + ensure monitor running) · POST /downloads/clear (drop finished). 14 tests, isolation guard + ruff clean. Grab button + Downloads page next.
This commit is contained in:
parent
9f687c061a
commit
ceccb4ee65
4 changed files with 262 additions and 0 deletions
|
|
@ -177,6 +177,60 @@ def register_routes(bp):
|
||||||
r.pop("_avail", None)
|
r.pop("_avail", None)
|
||||||
return jsonify({"scope": scope, "results": results[:40], "live": live})
|
return jsonify({"scope": scope, "results": results[:40], "live": live})
|
||||||
|
|
||||||
|
@bp.route("/downloads/grab", methods=["POST"])
|
||||||
|
def video_downloads_grab():
|
||||||
|
"""Start a real download of a chosen release and track it. v1: Soulseek only.
|
||||||
|
Body: {kind, title, release_title, source, username, filename, size_bytes,
|
||||||
|
quality_label}."""
|
||||||
|
from . import get_video_db
|
||||||
|
from config.settings import config_manager
|
||||||
|
from core.video.download_monitor import ensure_started
|
||||||
|
from core.video.download_pipeline import target_dir_for
|
||||||
|
from core.video.slskd_download import start_download
|
||||||
|
|
||||||
|
body = request.get_json(silent=True) or {}
|
||||||
|
source = str(body.get("source") or "soulseek").lower()
|
||||||
|
if source != "soulseek":
|
||||||
|
return jsonify({"ok": False, "error": "Only Soulseek grabs are wired up so far."}), 400
|
||||||
|
username, filename = body.get("username"), body.get("filename")
|
||||||
|
if not username or not filename:
|
||||||
|
return jsonify({"ok": False, "error": "Missing the release's source info."}), 400
|
||||||
|
|
||||||
|
db = get_video_db()
|
||||||
|
paths = {k: db.get_setting(k) or "" for k in ("movies_path", "tv_path", "youtube_path")}
|
||||||
|
if not paths["movies_path"]:
|
||||||
|
paths["movies_path"] = db.get_setting("transfer_path") or ""
|
||||||
|
target = target_dir_for(body.get("kind"), paths)
|
||||||
|
if not target:
|
||||||
|
return jsonify({"ok": False, "error": "Set the library folder for this type on Settings → Downloads."}), 400
|
||||||
|
|
||||||
|
started = start_download(username, filename, body.get("size_bytes") or 0)
|
||||||
|
if not started.get("ok"):
|
||||||
|
return jsonify({"ok": False, "error": started.get("error") or "slskd refused the download."}), 502
|
||||||
|
|
||||||
|
dl_id = db.add_video_download({
|
||||||
|
"kind": str(body.get("kind") or "movie"), "title": body.get("title"),
|
||||||
|
"release_title": body.get("release_title") or body.get("filename"),
|
||||||
|
"source": "soulseek", "username": username, "filename": filename,
|
||||||
|
"size_bytes": int(body.get("size_bytes") or 0), "quality_label": body.get("quality_label"),
|
||||||
|
"target_dir": target, "status": "downloading",
|
||||||
|
})
|
||||||
|
ensure_started(get_video_db)
|
||||||
|
return jsonify({"ok": True, "id": dl_id})
|
||||||
|
|
||||||
|
@bp.route("/downloads/active", methods=["GET"])
|
||||||
|
def video_downloads_active():
|
||||||
|
from . import get_video_db
|
||||||
|
from core.video.download_monitor import ensure_started
|
||||||
|
db = get_video_db()
|
||||||
|
ensure_started(get_video_db) # also (re)start the monitor when the page is open
|
||||||
|
return jsonify({"downloads": db.list_video_downloads()})
|
||||||
|
|
||||||
|
@bp.route("/downloads/clear", methods=["POST"])
|
||||||
|
def video_downloads_clear():
|
||||||
|
from . import get_video_db
|
||||||
|
return jsonify({"cleared": get_video_db().clear_finished_video_downloads()})
|
||||||
|
|
||||||
@bp.route("/downloads/youtube-quality", methods=["GET"])
|
@bp.route("/downloads/youtube-quality", methods=["GET"])
|
||||||
def video_youtube_quality():
|
def video_youtube_quality():
|
||||||
# Separate, smaller profile — YouTube is yt-dlp, not scene/p2p releases.
|
# Separate, smaller profile — YouTube is yt-dlp, not scene/p2p releases.
|
||||||
|
|
|
||||||
109
core/video/download_monitor.py
Normal file
109
core/video/download_monitor.py
Normal file
|
|
@ -0,0 +1,109 @@
|
||||||
|
"""Background monitor that drives video downloads to completion.
|
||||||
|
|
||||||
|
A daemon thread polls slskd for the active video downloads, updates their progress,
|
||||||
|
and when one finishes MOVES the file from the shared download folder into the right
|
||||||
|
per-type library folder (Movies / TV / YouTube) and marks it completed. Simple v1:
|
||||||
|
slskd source only, flat move by basename.
|
||||||
|
|
||||||
|
The per-download decision (``process_download``) is pure — filesystem + slskd are
|
||||||
|
injected — so it's unit-tested; the thread loop is thin glue.
|
||||||
|
|
||||||
|
Isolated: stdlib + the sibling video modules + shared config_manager; no music imports.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
|
||||||
|
from utils.logging_config import get_logger
|
||||||
|
|
||||||
|
from core.video.download_pipeline import dest_path_for, find_completed_file
|
||||||
|
from core.video.slskd_download import classify_state, find_transfer, list_downloads, progress_pct
|
||||||
|
|
||||||
|
logger = get_logger("video.download_monitor")
|
||||||
|
|
||||||
|
_INTERVAL = 3 # seconds between polls
|
||||||
|
_started = False
|
||||||
|
_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def process_download(dl: dict, transfers: list, download_dir: str, *, lister, mover) -> dict | None:
|
||||||
|
"""Decide the next state for one active download given the current slskd transfers.
|
||||||
|
Returns a patch dict for the DB row (or None to leave it untouched this tick)."""
|
||||||
|
t = find_transfer(transfers, dl.get("username"), dl.get("filename"))
|
||||||
|
if not t:
|
||||||
|
return None # not registered yet (or cleared) — wait
|
||||||
|
state = classify_state(t.get("state"))
|
||||||
|
if state == "active":
|
||||||
|
return {"status": "downloading", "progress": progress_pct(t)}
|
||||||
|
if state == "failed":
|
||||||
|
return {"status": "failed", "error": "Soulseek transfer " + str(t.get("state") or "failed")}
|
||||||
|
# completed → locate the file on disk and move it into the library folder
|
||||||
|
src = find_completed_file(download_dir, dl.get("filename"), lister)
|
||||||
|
if not src:
|
||||||
|
return {"progress": 100.0} # slskd done; file still settling — retry
|
||||||
|
dest = dest_path_for(dl.get("target_dir"), src)
|
||||||
|
try:
|
||||||
|
mover(src, dest)
|
||||||
|
except Exception as e: # noqa: BLE001 - any move failure marks the download failed
|
||||||
|
return {"status": "failed", "error": "Move failed: " + str(e)}
|
||||||
|
return {"status": "completed", "progress": 100.0, "dest_path": dest}
|
||||||
|
|
||||||
|
|
||||||
|
def _move(src: str, dest: str) -> None:
|
||||||
|
os.makedirs(os.path.dirname(dest) or ".", exist_ok=True)
|
||||||
|
shutil.move(src, dest)
|
||||||
|
|
||||||
|
|
||||||
|
def _walk(root: str):
|
||||||
|
for dirpath, _dirs, files in os.walk(str(root or ".")):
|
||||||
|
for f in files:
|
||||||
|
yield os.path.join(dirpath, f)
|
||||||
|
|
||||||
|
|
||||||
|
def _tick(db) -> None:
|
||||||
|
active = db.get_active_video_downloads()
|
||||||
|
if not active:
|
||||||
|
return
|
||||||
|
from config.settings import config_manager
|
||||||
|
download_dir = str(config_manager.get("soulseek.download_path", "") or "")
|
||||||
|
transfers = list_downloads()
|
||||||
|
for dl in active:
|
||||||
|
upd = process_download(dl, transfers, download_dir, lister=_walk, mover=_move)
|
||||||
|
if not upd:
|
||||||
|
continue
|
||||||
|
if upd.get("status") in ("completed", "failed"):
|
||||||
|
upd.setdefault("completed_at", time.strftime("%Y-%m-%d %H:%M:%S"))
|
||||||
|
try:
|
||||||
|
db.update_video_download(dl["id"], **upd)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video download %s: failed to persist update", dl.get("id"))
|
||||||
|
|
||||||
|
|
||||||
|
def _run(db_provider) -> None:
|
||||||
|
logger.info("video download monitor started")
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
db = db_provider()
|
||||||
|
if db is not None:
|
||||||
|
_tick(db)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("video download monitor tick failed")
|
||||||
|
time.sleep(_INTERVAL)
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_started(db_provider) -> None:
|
||||||
|
"""Start the monitor thread once (idempotent). Called when the first grab happens."""
|
||||||
|
global _started
|
||||||
|
with _lock:
|
||||||
|
if _started:
|
||||||
|
return
|
||||||
|
_started = True
|
||||||
|
threading.Thread(target=_run, args=(db_provider,), daemon=True,
|
||||||
|
name="video-download-monitor").start()
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["process_download", "ensure_started"]
|
||||||
|
|
@ -503,6 +503,44 @@ def test_downloads_search_endpoint_ranks_and_filters(tmp_path):
|
||||||
videoapi._video_db = None
|
videoapi._video_db = None
|
||||||
|
|
||||||
|
|
||||||
|
def test_downloads_grab_and_active(tmp_path, monkeypatch):
|
||||||
|
import api.video as videoapi
|
||||||
|
import core.video.download_monitor as mon
|
||||||
|
import core.video.slskd_download as slskd
|
||||||
|
from database.video_database import VideoDatabase
|
||||||
|
|
||||||
|
monkeypatch.setattr(slskd, "start_download", lambda *a, **k: {"ok": True})
|
||||||
|
monkeypatch.setattr(mon, "ensure_started", lambda *a, **k: None) # don't spawn the thread
|
||||||
|
|
||||||
|
db = VideoDatabase(database_path=str(tmp_path / "video_library.db"))
|
||||||
|
db.set_setting("movies_path", "/media/movies")
|
||||||
|
videoapi._video_db = db
|
||||||
|
app = Flask(__name__)
|
||||||
|
app.register_blueprint(videoapi.create_video_blueprint(), url_prefix="/api/video")
|
||||||
|
client = app.test_client()
|
||||||
|
try:
|
||||||
|
r = client.post("/api/video/downloads/grab", json={
|
||||||
|
"kind": "movie", "title": "The Matrix", "source": "soulseek",
|
||||||
|
"username": "neo", "filename": r"@@a\x\m.mkv", "size_bytes": 8, "quality_label": "1080p"})
|
||||||
|
out = r.get_json()
|
||||||
|
assert out["ok"] is True and out["id"] > 0
|
||||||
|
# tracked + routed to the Movies library.
|
||||||
|
act = client.get("/api/video/downloads/active").get_json()["downloads"]
|
||||||
|
assert len(act) == 1 and act[0]["status"] == "downloading"
|
||||||
|
assert act[0]["target_dir"] == "/media/movies" and act[0]["kind"] == "movie"
|
||||||
|
# missing source info → 400.
|
||||||
|
assert client.post("/api/video/downloads/grab",
|
||||||
|
json={"kind": "movie", "source": "soulseek"}).status_code == 400
|
||||||
|
# non-soulseek not wired yet → 400.
|
||||||
|
assert client.post("/api/video/downloads/grab",
|
||||||
|
json={"kind": "movie", "source": "torrent", "username": "u",
|
||||||
|
"filename": "f"}).status_code == 400
|
||||||
|
# clear removes only finished (none yet).
|
||||||
|
assert client.post("/api/video/downloads/clear").get_json()["cleared"] == 0
|
||||||
|
finally:
|
||||||
|
videoapi._video_db = None
|
||||||
|
|
||||||
|
|
||||||
def test_slskd_config_shared_via_config_manager(tmp_path, monkeypatch):
|
def test_slskd_config_shared_via_config_manager(tmp_path, monkeypatch):
|
||||||
import api.video as videoapi
|
import api.video as videoapi
|
||||||
import config.settings as cfg
|
import config.settings as cfg
|
||||||
|
|
|
||||||
|
|
@ -83,3 +83,64 @@ def test_video_downloads_crud(tmp_path):
|
||||||
assert listed[0]["status"] == "completed" and listed[0]["dest_path"] == "/media/movies/m.mkv"
|
assert listed[0]["status"] == "completed" and listed[0]["dest_path"] == "/media/movies/m.mkv"
|
||||||
assert db.clear_finished_video_downloads() == 1
|
assert db.clear_finished_video_downloads() == 1
|
||||||
assert db.list_video_downloads() == []
|
assert db.list_video_downloads() == []
|
||||||
|
|
||||||
|
|
||||||
|
# ── monitor decision (pure, injected fs) ──────────────────────────────────────
|
||||||
|
def _dl(**kw):
|
||||||
|
base = {"id": 1, "username": "neo", "filename": r"@@a\Folder\movie.mkv", "target_dir": "/media/movies"}
|
||||||
|
base.update(kw)
|
||||||
|
return base
|
||||||
|
|
||||||
|
|
||||||
|
def _xfer(state, **kw):
|
||||||
|
base = {"username": "neo", "filename": r"@@a\Folder\movie.mkv", "state": state, "size": 100, "transferred": 40}
|
||||||
|
base.update(kw)
|
||||||
|
return base
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_download_active_reports_progress():
|
||||||
|
from core.video.download_monitor import process_download
|
||||||
|
upd = process_download(_dl(), [_xfer("InProgress")], "/dl",
|
||||||
|
lister=lambda d: [], mover=lambda s, d: None)
|
||||||
|
assert upd == {"status": "downloading", "progress": 40.0}
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_download_failed():
|
||||||
|
from core.video.download_monitor import process_download
|
||||||
|
upd = process_download(_dl(), [_xfer("Completed, Errored")], "/dl",
|
||||||
|
lister=lambda d: [], mover=lambda s, d: None)
|
||||||
|
assert upd["status"] == "failed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_download_no_transfer_yet():
|
||||||
|
from core.video.download_monitor import process_download
|
||||||
|
assert process_download(_dl(), [], "/dl", lister=lambda d: [], mover=lambda s, d: None) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_download_completed_moves_file():
|
||||||
|
from core.video.download_monitor import process_download
|
||||||
|
moved = {}
|
||||||
|
upd = process_download(
|
||||||
|
_dl(), [_xfer("Completed, Succeeded")], "/dl",
|
||||||
|
lister=lambda d: ["/dl/Folder/movie.mkv"],
|
||||||
|
mover=lambda s, d: moved.update(src=s, dest=d))
|
||||||
|
assert upd == {"status": "completed", "progress": 100.0, "dest_path": "/media/movies/movie.mkv"}
|
||||||
|
assert moved == {"src": "/dl/Folder/movie.mkv", "dest": "/media/movies/movie.mkv"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_download_completed_but_file_not_settled():
|
||||||
|
from core.video.download_monitor import process_download
|
||||||
|
upd = process_download(_dl(), [_xfer("Completed, Succeeded")], "/dl",
|
||||||
|
lister=lambda d: [], mover=lambda s, d: None)
|
||||||
|
assert upd == {"progress": 100.0} # no status change — retries next tick
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_download_move_failure_marks_failed():
|
||||||
|
from core.video.download_monitor import process_download
|
||||||
|
|
||||||
|
def boom(s, d):
|
||||||
|
raise OSError("disk full")
|
||||||
|
|
||||||
|
upd = process_download(_dl(), [_xfer("Completed, Succeeded")], "/dl",
|
||||||
|
lister=lambda d: ["/dl/Folder/movie.mkv"], mover=boom)
|
||||||
|
assert upd["status"] == "failed" and "disk full" in upd["error"]
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue