"""文本站 HTML 抓取与正文抽取。"""

from __future__ import annotations

import html as html_lib
import re
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
from urllib.request import Request, urlopen

import yaml

from niannian.core.config import resolve_config_path
from niannian.soul.search_providers import url_allowed
from niannian.vault.store import _config_vault_path

_STRIP_TAGS = re.compile(r"<(script|style|noscript)[^>]*>.*?</\1>", re.I | re.S)
_TAG_RE = re.compile(r"<[^>]+>")
_TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.I | re.S)
_ARTICLE_RE = re.compile(r"<article[^>]*>(.*?)</article>", re.I | re.S)
_MAIN_RE = re.compile(r"<main[^>]*>(.*?)</main>", re.I | re.S)


def load_text_hosts_config() -> dict[str, Any]:
    cfg_path = resolve_config_path()
    data: dict[str, Any] = {}
    if cfg_path.is_file():
        with cfg_path.open(encoding="utf-8") as f:
            loaded = yaml.safe_load(f) or {}
        vault = loaded.get("vault") if isinstance(loaded, dict) else {}
        if isinstance(vault, dict) and vault.get("text_hosts_file"):
            custom = Path(str(vault["text_hosts_file"]))
            if not custom.is_absolute():
                custom = cfg_path.parent / custom
            if custom.is_file():
                with custom.open(encoding="utf-8") as f:
                    data = yaml.safe_load(f) or {}
    if not data:
        path = _config_vault_path("text_hosts.yaml")
        if path.is_file():
            with path.open(encoding="utf-8") as f:
                data = yaml.safe_load(f) or {}
    if not isinstance(data, dict):
        return {"hosts": [], "user_agent": "NianNianBot/1.0", "max_bytes": 2_097_152}
    data.setdefault("hosts", [])
    data.setdefault("user_agent", "NianNianBot/1.0")
    data.setdefault("max_bytes", 2_097_152)
    data.setdefault("path_rules", [])
    return data


def is_text_url_allowed(url: str, cfg: dict[str, Any] | None = None) -> bool:
    cfg = cfg or load_text_hosts_config()
    hosts = [str(h) for h in (cfg.get("hosts") or []) if h]
    if not url_allowed(url, hosts):
        return False
    parsed = urlparse(url)
    host = (parsed.hostname or "").lower()
    path = parsed.path or "/"
    for rule in cfg.get("path_rules") or []:
        if not isinstance(rule, dict):
            continue
        rule_host = str(rule.get("host") or "").lower()
        prefix = str(rule.get("path_prefix") or "")
        if not rule_host:
            continue
        if host == rule_host or host.endswith("." + rule_host):
            if prefix and not path.startswith(prefix):
                return False
    return True


def fetch_url_html(url: str, *, cfg: dict[str, Any] | None = None, opener=None) -> str:
    cfg = cfg or load_text_hosts_config()
    if not is_text_url_allowed(url, cfg):
        raise ValueError("host_not_allowed")
    ua = str(cfg.get("user_agent") or "NianNianBot/1.0")
    max_bytes = int(cfg.get("max_bytes") or 2_097_152)
    req = Request(url, headers={"User-Agent": ua})
    open_fn = opener or urlopen
    with open_fn(req, timeout=30) as resp:
        raw = resp.read(max_bytes + 1)
    if len(raw) > max_bytes:
        raise ValueError("response_too_large")
    for enc in ("utf-8", "gbk", "latin-1"):
        try:
            return raw.decode(enc)
        except UnicodeDecodeError:
            continue
    return raw.decode("utf-8", errors="replace")


def extract_text_from_html(html: str) -> tuple[str, str]:
    title_m = _TITLE_RE.search(html)
    title = _clean_text(title_m.group(1)) if title_m else "未命名网页"
    body_html = html
    for pat in (_ARTICLE_RE, _MAIN_RE):
        m = pat.search(html)
        if m:
            body_html = m.group(1)
            break
    body_html = _STRIP_TAGS.sub("", body_html)
    text = _clean_text(_TAG_RE.sub("\n", body_html))
    if not text:
        text = _clean_text(_TAG_RE.sub("\n", _STRIP_TAGS.sub("", html)))
    return title, text[:50_000]


def _clean_text(raw: str) -> str:
    unescaped = html_lib.unescape(raw)
    lines = [ln.strip() for ln in unescaped.splitlines()]
    collapsed: list[str] = []
    for ln in lines:
        if not ln:
            if collapsed and collapsed[-1] != "":
                collapsed.append("")
            continue
        collapsed.append(ln)
    return "\n".join(collapsed).strip()
