video downloads pipeline: data + pure logic foundation

Phase 1 of the real grab→transfer pipeline:
- video.db video_downloads table (kind/title/release/source/username/filename/size/
  target_dir/dest_path/status/progress/error/timestamps) + CRUD (add/list/get_active/
  update/clear_finished). Schema v14.
- core/video/slskd_download.py: start_download (POST /transfers/downloads/{user}),
  list_downloads (GET → flattened), + pure classify_state (completed/failed/active),
  progress_pct, find_transfer.
- core/video/download_pipeline.py: pure find_completed_file (locate by basename in the
  download dir), dest_path_for, target_dir_for (kind → movies/tv/youtube library).

All filesystem/HTTP is injected or thin glue; 7 tests, isolation guard + ruff clean.
Monitor + grab endpoint + Downloads page next.
This commit is contained in:
BoulderBadgeDad 2026-06-19 14:50:27 -07:00
parent 84fac1f80a
commit 9f687c061a
5 changed files with 333 additions and 1 deletions

View file

@ -0,0 +1,54 @@
"""Pure helpers for the video download pipeline: locate a finished file in the
download dir and work out where it should move to.
slskd writes a completed download somewhere under the shared download folder (it
mirrors the remote folder structure), so we locate the file by basename. Kept pure
(filesystem access is injected) so it's unit-tested without touching disk.
Isolated: stdlib only; no music imports.
"""
from __future__ import annotations
import os
from typing import Any, Callable
def basename_of(path: Any) -> str:
"""Final path segment, handling both / and \\ separators (slskd uses Windows-style)."""
return str(path or "").replace("\\", "/").rstrip("/").rsplit("/", 1)[-1]
def find_completed_file(download_dir: str, filename: str, lister: Callable) -> str | None:
"""Find the downloaded file on disk by basename. ``lister(dir)`` yields candidate
full paths (injected: os.walk-based in the monitor, a list in tests). Returns the
largest match (the real video, not a stray same-named bit), or None."""
base = basename_of(filename)
if not base or not download_dir:
return None
matches = [p for p in lister(download_dir) if basename_of(p) == base]
if not matches:
return None
return matches[0] if len(matches) == 1 else max(matches, key=len)
def dest_path_for(target_dir: str, src_path: str) -> str:
"""Where a finished file moves to inside its library folder (flat, by basename)."""
return os.path.join(str(target_dir or ""), basename_of(src_path))
def target_dir_for(kind: str, paths: dict) -> str:
"""Pick the library folder for a download's kind. ``paths`` = the config dict
({movies_path, tv_path, youtube_path})."""
paths = paths or {}
k = str(kind or "").lower()
if k == "movie":
return paths.get("movies_path") or ""
if k in ("show", "tv", "episode", "season", "series"):
return paths.get("tv_path") or ""
if k == "youtube":
return paths.get("youtube_path") or ""
return ""
__all__ = ["basename_of", "find_completed_file", "dest_path_for", "target_dir_for"]

View file

@ -0,0 +1,104 @@
"""Start + track Soulseek (slskd) downloads for the video pipeline.
Thin wrapper over slskd's transfer API (the same shared instance the music side uses):
- ``start_download(username, filename, size)`` POST /transfers/downloads/{username}
- ``list_downloads()`` GET /transfers/downloads (flattened)
The flatten + state-classification helpers are pure (unit-tested); the HTTP is glue.
Isolated: stdlib + requests + shared ``config_manager``; no music imports.
"""
from __future__ import annotations
from typing import Any
from urllib.parse import quote
import requests
def _conn():
from config.settings import config_manager
base = str(config_manager.get("soulseek.slskd_url", "") or "").rstrip("/")
key = config_manager.get("soulseek.api_key", "") or ""
return base, ({"X-API-Key": key} if key else {})
def start_download(username: str, filename: str, size_bytes: int = 0) -> dict:
"""Ask slskd to download one file from a user. Returns {ok[, error]}."""
base, headers = _conn()
if not base:
return {"ok": False, "error": "slskd isn't configured"}
try:
r = requests.post(base + "/api/v0/transfers/downloads/" + quote(str(username or "")),
json=[{"filename": filename, "size": int(size_bytes or 0)}],
headers=headers, timeout=15)
r.raise_for_status()
return {"ok": True}
except Exception as e: # noqa: BLE001 - surface any slskd/network failure to the caller
return {"ok": False, "error": str(e)}
def list_downloads() -> list:
"""Current slskd downloads, flattened to one dict per file."""
base, headers = _conn()
if not base:
return []
try:
r = requests.get(base + "/api/v0/transfers/downloads", headers=headers, timeout=15)
if not r.ok:
return []
data = r.json()
except Exception: # noqa: BLE001, S110 - transient slskd error → no downloads this tick
return []
return flatten_downloads(data)
def flatten_downloads(data: Any) -> list:
"""slskd's nested users→directories→files → a flat list of file transfers. Pure."""
out = []
for user in (data if isinstance(data, list) else []):
if not isinstance(user, dict):
continue
un = user.get("username", "")
for d in (user.get("directories") or []):
for f in (d.get("files") or []):
out.append({
"username": un, "filename": f.get("filename", ""), "id": f.get("id", ""),
"state": f.get("state", ""), "size": f.get("size", 0) or 0,
"transferred": f.get("bytesTransferred", 0) or 0,
})
return out
def classify_state(state: Any) -> str:
"""slskd state string → 'completed' | 'failed' | 'active'. Pure."""
s = str(state or "").lower()
if "completed" in s and "succeed" in s:
return "completed"
if any(x in s for x in ("error", "cancel", "timed", "failed", "reject")):
return "failed"
if "completed" in s: # completed but not succeeded → treat as failed
return "failed"
return "active"
def progress_pct(transfer: dict) -> float:
"""0100 from bytesTransferred/size (100 once completed). Pure."""
transfer = transfer or {}
if classify_state(transfer.get("state")) == "completed":
return 100.0
size = transfer.get("size", 0) or 0
tr = transfer.get("transferred", 0) or 0
return round(min(99.0, (tr / size * 100.0) if size else 0.0), 1)
def find_transfer(transfers: list, username: str, filename: str) -> dict:
"""The transfer matching a started download (by username + filename). Pure."""
for t in (transfers or []):
if t.get("username") == username and t.get("filename") == filename:
return t
return {}
__all__ = ["start_download", "list_downloads", "flatten_downloads", "classify_state",
"progress_pct", "find_transfer"]

View file

@ -31,7 +31,7 @@ logger = get_logger("video_database")
# Bump when video_schema.sql changes in a way worth recording. Stored in
# PRAGMA user_version as a backstop indicator (nothing gates on it yet).
SCHEMA_VERSION = 13
SCHEMA_VERSION = 14
_DEFAULT_DB_PATH = "database/video_library.db"
_SCHEMA_FILE = Path(__file__).resolve().parent / "video_schema.sql"
@ -1126,6 +1126,71 @@ class VideoDatabase:
finally:
conn.close()
# ── video downloads (the grab → transfer pipeline) ────────────────────────
_DL_FIELDS = ("kind", "title", "release_title", "source", "username", "filename",
"size_bytes", "quality_label", "target_dir", "status")
def add_video_download(self, rec: dict) -> int:
"""Insert a download row (status defaults to 'downloading'); returns its id."""
rec = rec or {}
cols = [f for f in self._DL_FIELDS if f in rec]
conn = self._get_connection()
try:
cur = conn.execute(
"INSERT INTO video_downloads (" + ", ".join(cols) + ") VALUES (" +
", ".join("?" for _ in cols) + ")",
tuple(rec[c] for c in cols),
)
conn.commit()
return int(cur.lastrowid)
finally:
conn.close()
def list_video_downloads(self, limit: int = 100) -> list:
conn = self._get_connection()
try:
rows = conn.execute(
"SELECT * FROM video_downloads ORDER BY "
"CASE status WHEN 'downloading' THEN 0 WHEN 'queued' THEN 1 ELSE 2 END, "
"id DESC LIMIT ?", (int(limit),)
).fetchall()
return [dict(r) for r in rows]
finally:
conn.close()
def get_active_video_downloads(self) -> list:
conn = self._get_connection()
try:
rows = conn.execute(
"SELECT * FROM video_downloads WHERE status IN ('queued', 'downloading') ORDER BY id"
).fetchall()
return [dict(r) for r in rows]
finally:
conn.close()
def update_video_download(self, dl_id: int, **fields) -> None:
"""Patch a download row; ``updated_at`` is always bumped."""
if not fields:
return
keys = list(fields.keys())
sets = ", ".join(k + " = ?" for k in keys) + ", updated_at = datetime('now')"
conn = self._get_connection()
try:
conn.execute("UPDATE video_downloads SET " + sets + " WHERE id = ?",
tuple(fields[k] for k in keys) + (int(dl_id),))
conn.commit()
finally:
conn.close()
def clear_finished_video_downloads(self) -> int:
conn = self._get_connection()
try:
cur = conn.execute("DELETE FROM video_downloads WHERE status IN ('completed', 'failed')")
conn.commit()
return cur.rowcount
finally:
conn.close()
# ── library mapping (which server library is Movies / TV) ─────────────────
def get_library_selection(self, server: str) -> dict:
return {

View file

@ -516,3 +516,27 @@ CREATE VIEW IF NOT EXISTS v_calendar AS
v.title AS title, c.title AS parent_title
FROM channel_videos v JOIN channels c ON c.id = v.channel_id
WHERE v.published_at IS NOT NULL;
-- DOWNLOADS — every grab initiated from the video side lands here (movies/tv/youtube).
-- The pipeline starts the download, watches it, and on completion moves the file to the
-- per-type library folder and marks it completed. Status: queued|downloading|completed|failed.
CREATE TABLE IF NOT EXISTS video_downloads (
id INTEGER PRIMARY KEY AUTOINCREMENT,
kind TEXT NOT NULL, -- movie | show | youtube
title TEXT, -- human title (e.g. the movie name)
release_title TEXT, -- the release/file being grabbed
source TEXT, -- soulseek | torrent | usenet
username TEXT, -- slskd uploader (for the grab + status)
filename TEXT, -- slskd remote filename (full path)
size_bytes INTEGER DEFAULT 0,
quality_label TEXT,
target_dir TEXT, -- destination library folder
dest_path TEXT, -- final moved path (set on completion)
status TEXT NOT NULL DEFAULT 'downloading',
progress REAL DEFAULT 0,
error TEXT,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now')),
completed_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_video_downloads_status ON video_downloads(status);

View file

@ -0,0 +1,85 @@
"""Video download pipeline — the pure seams: slskd state classification + flatten,
file location + destination resolution, and the video.db downloads CRUD. Isolated."""
from __future__ import annotations
from core.video.download_pipeline import (
basename_of,
dest_path_for,
find_completed_file,
target_dir_for,
)
from core.video.slskd_download import (
classify_state,
find_transfer,
flatten_downloads,
progress_pct,
)
def test_classify_state():
assert classify_state("Completed, Succeeded") == "completed"
assert classify_state("InProgress") == "active"
assert classify_state("Queued, Remotely") == "active"
assert classify_state("Completed, Errored") == "failed"
assert classify_state("Completed, Cancelled") == "failed"
assert classify_state("Completed, TimedOut") == "failed"
assert classify_state("") == "active"
def test_flatten_and_find_transfer():
data = [{"username": "neo", "directories": [
{"files": [{"filename": r"@@a\Movie\movie.mkv", "id": "t1", "state": "InProgress",
"size": 100, "bytesTransferred": 25}]}]}]
flat = flatten_downloads(data)
assert len(flat) == 1 and flat[0]["username"] == "neo" and flat[0]["id"] == "t1"
assert progress_pct(flat[0]) == 25.0
assert find_transfer(flat, "neo", r"@@a\Movie\movie.mkv")["id"] == "t1"
assert find_transfer(flat, "neo", "other") == {}
assert flatten_downloads(None) == []
def test_progress_is_100_when_completed():
assert progress_pct({"state": "Completed, Succeeded", "size": 100, "transferred": 0}) == 100.0
def test_basename_handles_both_separators():
assert basename_of(r"@@x\The.Wire.S02\ep.mkv") == "ep.mkv"
assert basename_of("a/b/c/movie.mp4") == "movie.mp4"
assert basename_of("") == ""
def test_find_completed_file_by_basename():
files = ["/dl/SomeFolder/movie.mkv", "/dl/Other/nope.mkv"]
assert find_completed_file("/dl", r"@@u\Remote\movie.mkv", lambda d: files) == "/dl/SomeFolder/movie.mkv"
assert find_completed_file("/dl", "missing.mkv", lambda d: files) is None
def test_dest_and_target_resolution():
assert dest_path_for("/media/movies", "/dl/x/movie.mkv") == "/media/movies/movie.mkv"
paths = {"movies_path": "/m", "tv_path": "/t", "youtube_path": "/y"}
assert target_dir_for("movie", paths) == "/m"
assert target_dir_for("show", paths) == "/t"
assert target_dir_for("season", paths) == "/t"
assert target_dir_for("youtube", paths) == "/y"
assert target_dir_for("weird", paths) == ""
# ── DB CRUD ───────────────────────────────────────────────────────────────────
def test_video_downloads_crud(tmp_path):
from database.video_database import VideoDatabase
db = VideoDatabase(database_path=str(tmp_path / "video_library.db"))
dl_id = db.add_video_download({
"kind": "movie", "title": "The Matrix", "release_title": "The.Matrix.1999.1080p",
"source": "soulseek", "username": "neo", "filename": r"@@a\x\m.mkv",
"size_bytes": 8_000_000_000, "target_dir": "/media/movies", "status": "downloading",
})
assert dl_id > 0
active = db.get_active_video_downloads()
assert len(active) == 1 and active[0]["title"] == "The Matrix"
db.update_video_download(dl_id, status="completed", progress=100, dest_path="/media/movies/m.mkv")
assert db.get_active_video_downloads() == []
listed = db.list_video_downloads()
assert listed[0]["status"] == "completed" and listed[0]["dest_path"] == "/media/movies/m.mkv"
assert db.clear_finished_video_downloads() == 1
assert db.list_video_downloads() == []