924 lines
32 KiB
Python
924 lines
32 KiB
Python
"""Generic task handler driven by JSON definitions."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import fnmatch
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from collections.abc import Mapping
|
|
from typing import TYPE_CHECKING, Any
|
|
from urllib.parse import urljoin
|
|
|
|
import httpx
|
|
import jmespath
|
|
from parsel import Selector
|
|
|
|
from app.features.tasks.definitions.results import HandleTask, TaskFailure, TaskItem, TaskResult
|
|
from app.features.tasks.definitions.schemas import (
|
|
ExtractionRule,
|
|
TaskDefinition,
|
|
)
|
|
from app.features.ytdlp.extractor import fetch_info
|
|
from app.features.ytdlp.utils import get_archive_id
|
|
from app.library.cache import Cache
|
|
from app.library.config import Config
|
|
from app.library.httpx_client import Globals, build_request_headers, get_async_client, resolve_curl_transport
|
|
from app.library.log import get_logger
|
|
|
|
from ._base_handler import BaseHandler
|
|
|
|
if TYPE_CHECKING:
|
|
from pathlib import Path
|
|
|
|
from parsel.selector import SelectorList
|
|
|
|
LOG = get_logger()
|
|
CACHE: Cache = Cache()
|
|
|
|
|
|
class GenericTaskHandler(BaseHandler):
|
|
"""Handler that scrapes arbitrary web pages based on JSON task definitions."""
|
|
|
|
_definitions: list[TaskDefinition] = []
|
|
"""Cached loaded task definitions."""
|
|
|
|
_sources_mtime: dict[Path, float] = {}
|
|
"""Modification times of source files to detect changes."""
|
|
|
|
@classmethod
|
|
async def refresh_definitions(cls, force: bool = False) -> list[TaskDefinition]:
|
|
"""
|
|
Refresh the cached task definitions if source files have changed.
|
|
|
|
Args:
|
|
force (bool): If True, force reload even if no changes detected.
|
|
|
|
"""
|
|
if cls._definitions and not force:
|
|
return cls._definitions
|
|
|
|
try:
|
|
from app.features.tasks.definitions.repository import TaskDefinitionsRepository
|
|
from app.features.tasks.definitions.utils import model_to_schema
|
|
|
|
repo = TaskDefinitionsRepository.get_instance()
|
|
models = await repo.list()
|
|
|
|
cls._definitions = [model_to_schema(model) for model in models]
|
|
return cls._definitions
|
|
except Exception as exc:
|
|
LOG.exception(
|
|
"Failed to load generic task definitions.",
|
|
extra={"error": str(exc), "exception_type": type(exc).__name__},
|
|
)
|
|
return []
|
|
|
|
@classmethod
|
|
async def _find_definition(cls, url: str) -> TaskDefinition | None:
|
|
"""
|
|
Find a task definition that matches the given URL.
|
|
|
|
Args:
|
|
url (str): The URL to match.
|
|
|
|
Returns:
|
|
(TaskDefinition|None): A matching TaskDefinition if found, None otherwise.
|
|
|
|
"""
|
|
await cls.refresh_definitions()
|
|
|
|
for definition in cls._definitions:
|
|
if not definition.enabled:
|
|
continue
|
|
|
|
try:
|
|
for matcher in definition.match_url:
|
|
pattern_str: str | None = None
|
|
|
|
if matcher.startswith("/") and matcher.endswith("/") and len(matcher) > 2:
|
|
pattern_str = matcher[1:-1]
|
|
else:
|
|
pattern_str = fnmatch.translate(matcher)
|
|
|
|
if pattern_str and re.match(pattern_str, url):
|
|
return definition
|
|
except Exception as exc:
|
|
LOG.exception(
|
|
"Failed to match a generic task definition.",
|
|
extra={
|
|
"definition": definition.name,
|
|
"url": url,
|
|
"error": str(exc),
|
|
"exception_type": type(exc).__name__,
|
|
},
|
|
)
|
|
|
|
return None
|
|
|
|
@staticmethod
|
|
async def can_handle(task: HandleTask) -> bool:
|
|
"""
|
|
Determine if this handler can process the given task.
|
|
|
|
Args:
|
|
task (Task): The task to check.
|
|
|
|
Returns:
|
|
(bool): True if the handler can process the task, False otherwise.
|
|
|
|
"""
|
|
definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
|
|
if definition:
|
|
LOG.debug(
|
|
"Task '%s' matched a generic task definition.",
|
|
task.name,
|
|
extra={"task_name": task.name, "url": task.url, "definition": definition.name},
|
|
)
|
|
return True
|
|
|
|
return False
|
|
|
|
@staticmethod
|
|
async def extract(task: HandleTask, config: Config | None = None) -> TaskResult | TaskFailure: # noqa: ARG004
|
|
definition: TaskDefinition | None = await GenericTaskHandler._find_definition(task.url)
|
|
if not definition:
|
|
return TaskFailure(message="No generic task definition matched the provided URL.")
|
|
|
|
ytdlp_opts: dict[str, Any] = task.get_ytdlp_opts().get_all()
|
|
target_url: str = definition.definition.request.url or task.url
|
|
|
|
LOG.debug(
|
|
"Fetching content for task '%s'.",
|
|
task.name,
|
|
extra={
|
|
"task_name": task.name,
|
|
"definition": definition.name,
|
|
"url": target_url,
|
|
"engine": definition.definition.engine.type,
|
|
},
|
|
)
|
|
|
|
try:
|
|
body_text, json_data = await GenericTaskHandler._fetch_content(
|
|
url=target_url, definition=definition, ytdlp_opts=ytdlp_opts
|
|
)
|
|
except httpx.HTTPError as exc:
|
|
return TaskFailure(message="Failed to fetch target URL.", error=str(exc))
|
|
except Exception as exc:
|
|
LOG.exception(
|
|
"Failed to fetch content for task '%s'.",
|
|
task.name,
|
|
extra={
|
|
"task_name": task.name,
|
|
"definition": definition.name,
|
|
"url": target_url,
|
|
"exception_type": type(exc).__name__,
|
|
},
|
|
)
|
|
return TaskFailure(message="Failed to fetch target URL.", error=str(exc))
|
|
|
|
if "json" == definition.definition.response.type and json_data is None:
|
|
return TaskFailure(message="Expected JSON response but decoding failed.")
|
|
|
|
if "json" != definition.definition.response.type and not body_text:
|
|
return TaskFailure(message="Received empty response body.")
|
|
|
|
raw_items: list[dict[str, str]] = GenericTaskHandler._parse_items(
|
|
definition=definition, html=body_text or "", base_url=target_url, json_data=json_data
|
|
)
|
|
|
|
task_items: list[TaskItem] = []
|
|
|
|
def _generic_id(url):
|
|
import os
|
|
from urllib import parse
|
|
|
|
return parse.unquote(os.path.splitext(url.rstrip("/").split("/")[-1])[0])
|
|
|
|
for entry in raw_items:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
|
|
if not (url := entry.get("link") or entry.get("url")):
|
|
continue
|
|
|
|
id_dict: dict[str, str | None] = get_archive_id(url=url)
|
|
archive_id: str | None = id_dict.get("archive_id")
|
|
if not archive_id:
|
|
cache_key: str = hashlib.sha256(f"{task.name}-{url}".encode()).hexdigest()
|
|
if CACHE.has(cache_key):
|
|
archive_id = CACHE.get(cache_key)
|
|
if not archive_id:
|
|
continue
|
|
else:
|
|
LOG.warning(
|
|
"Task '%s' could not generate a static archive ID. Fetching it with yt-dlp.",
|
|
task.name,
|
|
extra={"definition": definition.name, "task_name": task.name, "url": url},
|
|
)
|
|
|
|
(info, _) = await fetch_info(
|
|
config=task.get_ytdlp_opts().get_all(),
|
|
url=url,
|
|
no_archive=True,
|
|
no_log=True,
|
|
budget_sleep=True,
|
|
)
|
|
|
|
if not info:
|
|
LOG.error(
|
|
"Task '%s' failed to extract info to generate an archive ID. Skipping item.",
|
|
task.name,
|
|
extra={"definition": definition.name, "task_name": task.name, "url": url},
|
|
)
|
|
CACHE.set(cache_key, None)
|
|
continue
|
|
|
|
if not info.get("id") or not info.get("extractor_key"):
|
|
LOG.error(
|
|
"Task '%s' returned incomplete info while generating an archive ID. Skipping item.",
|
|
task.name,
|
|
extra={"definition": definition.name, "task_name": task.name, "url": url},
|
|
)
|
|
CACHE.set(cache_key, None)
|
|
continue
|
|
|
|
archive_id = f"{str(info.get('extractor_key', '')).lower()} {info.get('id')}"
|
|
CACHE.set(cache_key, archive_id)
|
|
|
|
metadata: dict[str, str] = {
|
|
k: v for k, v in entry.items() if k not in {"link", "url", "title", "published", "archive_id"}
|
|
}
|
|
|
|
task_items.append(
|
|
TaskItem(
|
|
url=url,
|
|
title=entry.get("title"),
|
|
archive_id=archive_id,
|
|
metadata={"published": entry.get("published"), **metadata},
|
|
)
|
|
)
|
|
|
|
return TaskResult(
|
|
items=task_items,
|
|
metadata={
|
|
"definition": definition.name,
|
|
"response_format": definition.definition.response.type,
|
|
},
|
|
)
|
|
|
|
@staticmethod
|
|
async def _fetch_content(
|
|
url: str,
|
|
definition: TaskDefinition,
|
|
ytdlp_opts: dict[str, Any],
|
|
) -> tuple[str | None, Any | None]:
|
|
"""
|
|
Fetch the content of the given URL using the specified engine.
|
|
|
|
Args:
|
|
url (str): The URL to fetch.
|
|
definition (TaskDefinitionRuntimeSchema): The task definition specifying the engine and request details.
|
|
ytdlp_opts (dict[str, Any]): yt-dlp options that may influence fetching
|
|
|
|
Returns:
|
|
(str|None): The fetched HTML content if successful, None otherwise.
|
|
|
|
"""
|
|
if "selenium" == definition.definition.engine.type:
|
|
return await GenericTaskHandler._fetch_with_selenium(url=url, definition=definition)
|
|
|
|
return await GenericTaskHandler._fetch_with_httpx(url=url, definition=definition, ytdlp_opts=ytdlp_opts)
|
|
|
|
@staticmethod
|
|
async def _fetch_with_httpx(
|
|
url: str,
|
|
definition: TaskDefinition,
|
|
ytdlp_opts: dict[str, Any],
|
|
) -> tuple[str | None, Any | None]:
|
|
"""
|
|
Fetch the content using httpx.
|
|
|
|
Args:
|
|
url (str): The URL to fetch.
|
|
definition (TaskDefinitionRuntimeSchema): The task definition specifying the request details.
|
|
ytdlp_opts (dict[str, Any]): yt-dlp options that may influence fetching
|
|
|
|
Returns:
|
|
(str|None): The fetched HTML content if successful, None otherwise.
|
|
|
|
"""
|
|
headers: dict[str, str] = {**definition.definition.request.headers}
|
|
use_curl = resolve_curl_transport()
|
|
request_headers = build_request_headers(
|
|
base_headers=headers,
|
|
user_agent=Globals.get_random_agent(),
|
|
use_curl=use_curl,
|
|
)
|
|
|
|
timeout_value: float | Any = definition.definition.request.timeout or ytdlp_opts.get("socket_timeout", 120)
|
|
|
|
client = get_async_client(proxy=ytdlp_opts.get("proxy"), use_curl=use_curl)
|
|
response: httpx.Response = await client.request(
|
|
method=definition.definition.request.method.upper(),
|
|
url=url,
|
|
params=definition.definition.request.params or None,
|
|
data=definition.definition.request.data,
|
|
json=definition.definition.request.json_data,
|
|
timeout=timeout_value,
|
|
headers=request_headers,
|
|
)
|
|
response.raise_for_status()
|
|
|
|
if "json" == definition.definition.response.type:
|
|
try:
|
|
json_data: dict[str, Any] = response.json()
|
|
except Exception as exc:
|
|
LOG.exception(
|
|
"Task definition '%s' returned invalid JSON for '%s'.",
|
|
definition.name,
|
|
url,
|
|
extra={
|
|
"definition": definition.name,
|
|
"url": url,
|
|
"error": str(exc),
|
|
"exception_type": type(exc).__name__,
|
|
},
|
|
)
|
|
return response.text, None
|
|
|
|
return response.text, json_data
|
|
|
|
return response.text, None
|
|
|
|
@staticmethod
|
|
async def _fetch_with_selenium(
|
|
url: str,
|
|
definition: TaskDefinition,
|
|
) -> tuple[str | None, Any | None]:
|
|
"""
|
|
Fetch the content using a Selenium WebDriver.
|
|
|
|
Args:
|
|
url (str): The URL to fetch.
|
|
definition (TaskDefinitionRuntimeSchema): The task definition specifying the engine options.
|
|
|
|
Returns:
|
|
(str|None): The fetched HTML content if successful, None otherwise.
|
|
|
|
"""
|
|
try:
|
|
from selenium import webdriver
|
|
from selenium.webdriver.chrome.options import Options as ChromeOptions
|
|
from selenium.webdriver.common.by import By
|
|
from selenium.webdriver.support import expected_conditions as EC
|
|
from selenium.webdriver.support.ui import WebDriverWait
|
|
except ImportError as exc:
|
|
LOG.exception(
|
|
"Task definition '%s' requested Selenium, but Selenium is not installed.",
|
|
definition.name,
|
|
extra={"definition": definition.name, "error": str(exc), "exception_type": type(exc).__name__},
|
|
)
|
|
return (None, None)
|
|
|
|
options_map: dict[str, Any] = definition.definition.engine.options
|
|
command_executor_value = options_map.get("url")
|
|
command_executor: str = (
|
|
str(command_executor_value)
|
|
if isinstance(command_executor_value, str) and command_executor_value
|
|
else "http://localhost:4444/wd/hub"
|
|
)
|
|
browser: str = str(options_map.get("browser", "chrome")).lower()
|
|
|
|
if "chrome" != browser:
|
|
LOG.error(
|
|
"Task definition '%s' requested unsupported Selenium browser '%s'.",
|
|
definition.name,
|
|
browser,
|
|
extra={"definition": definition.name, "browser": browser},
|
|
)
|
|
return (None, None)
|
|
|
|
arguments: list[str] | str = options_map.get("arguments", ["--headless", "--disable-gpu"])
|
|
if isinstance(arguments, str):
|
|
arguments = [arguments]
|
|
|
|
wait_for: Mapping | None = (
|
|
options_map.get("wait_for") if isinstance(options_map.get("wait_for"), Mapping) else None
|
|
)
|
|
wait_timeout = float(options_map.get("wait_timeout") or 15)
|
|
page_load_timeout = float(options_map.get("page_load_timeout") or 60)
|
|
|
|
def load_page() -> str | None:
|
|
chrome_options = ChromeOptions()
|
|
for arg in arguments:
|
|
chrome_options.add_argument(str(arg))
|
|
|
|
driver = webdriver.Remote(command_executor=command_executor, options=chrome_options)
|
|
try:
|
|
driver.set_page_load_timeout(page_load_timeout)
|
|
driver.get(url)
|
|
|
|
if wait_for:
|
|
wait_type: str = str(wait_for.get("type", "css")).lower()
|
|
expression: str | None = wait_for.get("expression") or wait_for.get("value")
|
|
if isinstance(expression, str) and expression:
|
|
locator = None
|
|
locator = (By.XPATH, expression) if "xpath" == wait_type else (By.CSS_SELECTOR, expression)
|
|
WebDriverWait(driver, wait_timeout).until(EC.presence_of_element_located(locator))
|
|
|
|
return driver.page_source
|
|
finally:
|
|
driver.quit()
|
|
|
|
html: str | None = await asyncio.to_thread(load_page)
|
|
return html, None
|
|
|
|
@staticmethod
|
|
def _parse_items(
|
|
definition: TaskDefinition,
|
|
html: str,
|
|
base_url: str,
|
|
json_data: Any | None = None,
|
|
) -> list[dict[str, str]]:
|
|
"""
|
|
Parse the HTML content and extract items based on the definition.
|
|
|
|
Args:
|
|
definition (TaskDefinitionRuntimeSchema): The task definition specifying the parsers.
|
|
html (str): The HTML content to parse.
|
|
base_url (str): The base URL to resolve relative links.
|
|
json_data (Any|None): The JSON data to parse if applicable.
|
|
|
|
Returns:
|
|
(list[dict[str, str]]): A list of extracted items as dictionaries.
|
|
|
|
"""
|
|
if "json" == definition.definition.response.type:
|
|
return GenericTaskHandler._parse_json_items(definition, json_data, base_url)
|
|
|
|
selector = Selector(text=html)
|
|
|
|
if definition.definition.parse.get("items"):
|
|
return GenericTaskHandler._parse_with_container(
|
|
definition=definition,
|
|
selector=selector,
|
|
html=html,
|
|
base_url=base_url,
|
|
)
|
|
|
|
extracted: dict[str, list[str]] = {}
|
|
|
|
for _field, rule_data in definition.definition.parse.field_items():
|
|
if not isinstance(rule_data, dict):
|
|
continue
|
|
rule = ExtractionRule.model_validate(rule_data)
|
|
values: list[str] = GenericTaskHandler._execute_rule(field=_field, selector=selector, html=html, rule=rule)
|
|
extracted[_field] = values
|
|
|
|
link_values: list[str] = extracted.get("link", [])
|
|
if not link_values:
|
|
LOG.debug(
|
|
"Definition '%s' produced no link values.", definition.name, extra={"definition": definition.name}
|
|
)
|
|
return []
|
|
|
|
total_items: int = len(link_values)
|
|
items: list[dict[str, str]] = []
|
|
|
|
for index in range(total_items):
|
|
entry: dict[str, str] = {}
|
|
link_value: str = link_values[index]
|
|
if not link_value:
|
|
continue
|
|
|
|
entry["link"] = urljoin(base_url, link_value)
|
|
|
|
for _field, values in extracted.items():
|
|
if "link" == _field:
|
|
continue
|
|
|
|
value: str | None = values[index] if index < len(values) else None
|
|
if value is None:
|
|
continue
|
|
|
|
entry[_field] = value
|
|
|
|
items.append(entry)
|
|
|
|
return items
|
|
|
|
@staticmethod
|
|
def _parse_json_items(
|
|
definition: TaskDefinition,
|
|
json_data: Any | None,
|
|
base_url: str,
|
|
) -> list[dict[str, str]]:
|
|
if json_data is None:
|
|
LOG.debug(
|
|
"Definition '%s' expects JSON but no data was parsed.",
|
|
definition.name,
|
|
extra={"definition": definition.name},
|
|
)
|
|
return []
|
|
|
|
if definition.definition.parse.get("items"):
|
|
return GenericTaskHandler._parse_json_with_container(definition, json_data, base_url)
|
|
|
|
items: list[dict[str, str]] = []
|
|
entry: dict[str, str] = {}
|
|
|
|
for _field, rule_data in definition.definition.parse.field_items():
|
|
if not isinstance(rule_data, dict):
|
|
continue
|
|
rule = ExtractionRule.model_validate(rule_data)
|
|
values: list[str] = GenericTaskHandler._execute_json_rule(_field, json_data, rule)
|
|
if values:
|
|
if "link" == _field:
|
|
entry["link"] = urljoin(base_url, values[0])
|
|
else:
|
|
entry[_field] = values[0]
|
|
|
|
if "link" in entry:
|
|
items.append(entry)
|
|
|
|
return items
|
|
|
|
@staticmethod
|
|
def _parse_with_container(
|
|
definition: TaskDefinition,
|
|
selector: Selector,
|
|
html: str,
|
|
base_url: str,
|
|
) -> list[dict[str, str]]:
|
|
container: dict[str, Any] | None = definition.definition.parse.get("items")
|
|
if not container:
|
|
return []
|
|
|
|
container_type = container.get("type", "css")
|
|
container_selector = container.get("selector") or container.get("expression") or ""
|
|
if not container_selector:
|
|
LOG.error(
|
|
"Task definition '%s' is missing an item container selector.",
|
|
definition.name,
|
|
extra={"definition": definition.name},
|
|
)
|
|
return []
|
|
container_fields = container.get("fields", {})
|
|
|
|
selection: SelectorList[Selector] = (
|
|
selector.css(container_selector) if "css" == container_type else selector.xpath(container_selector)
|
|
)
|
|
|
|
items: list[dict[str, str]] = []
|
|
|
|
for node in selection:
|
|
node_html: Any | str = node.get() or html
|
|
entry: dict[str, str] = {}
|
|
|
|
for _field, rule_data in container_fields.items():
|
|
rule = ExtractionRule.model_validate(rule_data)
|
|
values: list[str] = GenericTaskHandler._execute_rule(
|
|
field=_field,
|
|
selector=node,
|
|
html=node_html,
|
|
rule=rule,
|
|
)
|
|
|
|
value: str | None = values[0] if values else None
|
|
if value is None:
|
|
continue
|
|
|
|
if "link" == _field:
|
|
entry["link"] = urljoin(base_url, value)
|
|
else:
|
|
entry[_field] = value
|
|
|
|
if "link" not in entry:
|
|
continue
|
|
|
|
items.append(entry)
|
|
|
|
return items
|
|
|
|
@staticmethod
|
|
def _parse_json_with_container(
|
|
definition: TaskDefinition,
|
|
json_data: Any,
|
|
base_url: str,
|
|
) -> list[dict[str, str]]:
|
|
container: dict[str, Any] | None = definition.definition.parse.get("items")
|
|
if not container:
|
|
return []
|
|
|
|
container_type = container.get("type", "css")
|
|
container_selector = container.get("selector") or container.get("expression", "")
|
|
container_fields = container.get("fields", {})
|
|
|
|
if "jsonpath" != container_type:
|
|
LOG.error(
|
|
"JSON response requires container selector type 'jsonpath'. Definition '%s'.",
|
|
definition.name,
|
|
extra={"definition": definition.name, "container_type": container_type},
|
|
)
|
|
return []
|
|
|
|
nodes: Any = GenericTaskHandler._json_search(json_data, container_selector)
|
|
if nodes is None:
|
|
return []
|
|
|
|
if not isinstance(nodes, list):
|
|
nodes = [nodes]
|
|
|
|
items: list[dict[str, str]] = []
|
|
|
|
for node in nodes:
|
|
entry: dict[str, str] = {}
|
|
|
|
for _field, rule_data in container_fields.items():
|
|
rule = ExtractionRule.model_validate(rule_data)
|
|
values: list[str] = GenericTaskHandler._execute_json_rule(_field, node, rule)
|
|
if not values:
|
|
continue
|
|
|
|
if "link" == _field:
|
|
entry["link"] = urljoin(base_url, values[0])
|
|
else:
|
|
entry[_field] = values[0]
|
|
|
|
if "link" not in entry:
|
|
continue
|
|
|
|
items.append(entry)
|
|
|
|
return items
|
|
|
|
@staticmethod
|
|
def _execute_json_rule(field: str, data: Any, rule: ExtractionRule) -> list[str]:
|
|
values: list[str] = []
|
|
|
|
if "jsonpath" == rule.type:
|
|
result: Any = GenericTaskHandler._json_search(data, rule.expression)
|
|
candidates: list | list[Any] = result if isinstance(result, list) else [result]
|
|
for candidate in candidates:
|
|
if candidate is None:
|
|
continue
|
|
|
|
text: str = GenericTaskHandler._coerce_to_string(candidate)
|
|
processed: str | None = GenericTaskHandler._apply_post_filter(text, rule)
|
|
if processed is not None:
|
|
values.append(processed)
|
|
|
|
return values
|
|
|
|
if "regex" == rule.type:
|
|
target: str = GenericTaskHandler._coerce_to_string(data)
|
|
try:
|
|
pattern: re.Pattern[str] = re.compile(rule.expression, re.MULTILINE | re.DOTALL)
|
|
except re.error as exc:
|
|
LOG.exception(
|
|
"Invalid regex expression '%s'.",
|
|
rule.expression,
|
|
extra={
|
|
"field": field,
|
|
"expression": rule.expression,
|
|
"error": str(exc),
|
|
"exception_type": type(exc).__name__,
|
|
},
|
|
)
|
|
return values
|
|
|
|
for match in pattern.finditer(target):
|
|
raw: str | None = GenericTaskHandler._regex_value(match=match, attribute=rule.attribute)
|
|
processed = GenericTaskHandler._apply_post_filter(raw, rule)
|
|
if processed is not None:
|
|
values.append(processed)
|
|
|
|
return values
|
|
|
|
LOG.error(
|
|
"Unsupported extraction type '%s' for JSON data in field '%s'.",
|
|
rule.type,
|
|
field,
|
|
extra={"field": field, "rule_type": rule.type},
|
|
)
|
|
return values
|
|
|
|
@staticmethod
|
|
def _json_search(data: Any, expression: str) -> Any:
|
|
try:
|
|
return jmespath.search(expression, data)
|
|
except Exception as exc:
|
|
LOG.exception(
|
|
"JSONPath search failed for expression '%s'.",
|
|
expression,
|
|
extra={"expression": expression, "error": str(exc), "exception_type": type(exc).__name__},
|
|
)
|
|
return None
|
|
|
|
@staticmethod
|
|
def _coerce_to_string(value: Any) -> str:
|
|
if isinstance(value, str):
|
|
return value
|
|
if isinstance(value, (int, float, bool)) or value is None:
|
|
return "" if value is None else str(value)
|
|
try:
|
|
return json.dumps(value, ensure_ascii=False)
|
|
except Exception:
|
|
return str(value)
|
|
|
|
@staticmethod
|
|
def _execute_rule(field: str, selector: Selector, html: str, rule: ExtractionRule) -> list[str]:
|
|
"""
|
|
Execute a single extraction rule and return the list of extracted values.
|
|
|
|
Args:
|
|
field (str): The name of the field being extracted.
|
|
selector (Selector): The parsel Selector for the HTML content.
|
|
html (str): The raw HTML content.
|
|
rule (ExtractionRuleSchema): The extraction rule to execute.
|
|
|
|
Returns:
|
|
(list[str]): A list of extracted values.
|
|
|
|
"""
|
|
values: list[str] = []
|
|
|
|
if "regex" == rule.type:
|
|
try:
|
|
pattern: re.Pattern[str] = re.compile(rule.expression, re.MULTILINE | re.DOTALL)
|
|
except re.error as exc:
|
|
LOG.exception(
|
|
"Invalid regex expression '%s'.",
|
|
rule.expression,
|
|
extra={
|
|
"field": field,
|
|
"expression": rule.expression,
|
|
"error": str(exc),
|
|
"exception_type": type(exc).__name__,
|
|
},
|
|
)
|
|
return values
|
|
|
|
for match in pattern.finditer(html):
|
|
raw: str | None = GenericTaskHandler._regex_value(match=match, attribute=rule.attribute)
|
|
processed: str | None = GenericTaskHandler._apply_post_filter(raw, rule)
|
|
if processed is not None:
|
|
values.append(processed)
|
|
|
|
return values
|
|
|
|
if "jsonpath" == rule.type:
|
|
LOG.error("Field '%s' uses 'jsonpath' on a non-JSON response.", field, extra={"field": field})
|
|
return values
|
|
|
|
selection: SelectorList[Selector] = (
|
|
selector.css(rule.expression) if "css" == rule.type else selector.xpath(rule.expression)
|
|
)
|
|
|
|
for sel in selection:
|
|
raw = GenericTaskHandler._selector_value(field, sel, rule.attribute)
|
|
processed = GenericTaskHandler._apply_post_filter(raw, rule)
|
|
if processed is not None:
|
|
values.append(processed)
|
|
|
|
return values
|
|
|
|
@staticmethod
|
|
def _regex_value(match: re.Match[str], attribute: str | None) -> str | None:
|
|
"""
|
|
Extract a value from a regex match based on the attribute.
|
|
|
|
Args:
|
|
match (re.Match[str]): The regex match object.
|
|
attribute (str|None): Optional group name or index to extract.
|
|
|
|
Returns:
|
|
(str|None): The extracted value if found, None otherwise.
|
|
|
|
"""
|
|
if attribute:
|
|
try:
|
|
return match.group(attribute)
|
|
except (IndexError, KeyError):
|
|
LOG.debug(
|
|
"Regex group '%s' not found in pattern '%s'.",
|
|
attribute,
|
|
match.re.pattern,
|
|
extra={"attribute": attribute, "pattern": match.re.pattern},
|
|
)
|
|
return None
|
|
|
|
if match.groupdict():
|
|
return next((value for value in match.groupdict().values() if value), None)
|
|
|
|
if match.groups():
|
|
return match.group(1)
|
|
|
|
return match.group(0)
|
|
|
|
@staticmethod
|
|
def _selector_value(field: str, sel: Selector, attribute: str | None) -> str | None:
|
|
"""
|
|
Extract a value from a parsel Selector based on the attribute.
|
|
|
|
Args:
|
|
field (str): The name of the field being extracted.
|
|
sel (Selector): The parsel Selector object.
|
|
attribute (str|None): Optional attribute to extract (e.g. 'href', 'src', 'text', etc.).
|
|
|
|
Returns:
|
|
(str|None): The extracted value if found, None otherwise.
|
|
|
|
"""
|
|
attr: str | None = attribute.lower() if isinstance(attribute, str) else None
|
|
|
|
if attr in {"text", "inner_text"}:
|
|
return sel.xpath("normalize-space()").get()
|
|
|
|
if attr in {"html", "outer_html"}:
|
|
value: Any = sel.get()
|
|
return value if value is not None else None
|
|
|
|
if attr and attr not in {"html", "outer_html", "text", "inner_text"}:
|
|
try:
|
|
attributes: dict[str, str] | None = sel.attrib
|
|
except AttributeError:
|
|
attributes = None
|
|
|
|
if attributes and attr in attributes:
|
|
return attributes.get(attr)
|
|
|
|
attr_value: str | None = sel.xpath(f"@{attr}").get()
|
|
if attr_value is not None:
|
|
return attr_value
|
|
|
|
if attr is None and "link" == field.lower():
|
|
href = None
|
|
try:
|
|
attributes: dict[str, str] | None = sel.attrib
|
|
except AttributeError:
|
|
attributes = None
|
|
|
|
if attributes and "href" in attributes:
|
|
href: str | None = attributes.get("href")
|
|
if not href:
|
|
href: str | None = sel.xpath("@href").get()
|
|
if href:
|
|
return href
|
|
|
|
if attr is None:
|
|
text_value: str | None = sel.xpath("normalize-space()").get()
|
|
if text_value:
|
|
return text_value
|
|
|
|
value = sel.get()
|
|
return value if value is not None else None
|
|
|
|
@staticmethod
|
|
def _apply_post_filter(value: str | None, rule: ExtractionRule) -> str | None:
|
|
"""
|
|
Apply the post-filter to the extracted value if defined.
|
|
|
|
Args:
|
|
value (str|None): The extracted value to filter.
|
|
rule (ExtractionRuleSchema): The extraction rule containing the post-filter.
|
|
|
|
Returns:
|
|
(str|None): The filtered value if applicable, None otherwise.
|
|
|
|
"""
|
|
if value is None:
|
|
return None
|
|
|
|
cleaned: str = value.strip()
|
|
if rule.post_filter:
|
|
# Apply post-filter inline (removed helper method)
|
|
try:
|
|
pattern = re.compile(rule.post_filter.filter)
|
|
match = pattern.search(cleaned)
|
|
if not match:
|
|
return None
|
|
|
|
if rule.post_filter.value:
|
|
try:
|
|
return match.group(rule.post_filter.value)
|
|
except (IndexError, KeyError):
|
|
return None
|
|
|
|
if match.groupdict():
|
|
# Prefer first named group when available
|
|
for group_value in match.groupdict().values():
|
|
if group_value is not None:
|
|
return group_value
|
|
|
|
if match.groups():
|
|
return match.group(1)
|
|
|
|
return match.group(0)
|
|
except re.error:
|
|
return None
|
|
|
|
return cleaned or None
|