"""Douyin plugin. Supports public Douyin share, video, note, and modal links. Cookie support is optional and reads from runtime/douyin_cookies.json when present. """ from __future__ import annotations import html import json import re from dataclasses import dataclass from pathlib import Path from typing import Any from urllib.parse import parse_qs, unquote, urlparse import httpx from pixiv_app.core.cookie_import import cookies_to_header, parse_cookie_text from pixiv_app.core.paths import runtime_path from pixiv_app.core.plugin.base import BasePlugin, PluginDownloadError, PluginParseError, Resource _AWEME_ID_RE = re.compile( r"(?:aweme_id=|modal_id=|/video/|/note/|/share/video/|/share/note/)(\d{6,})", re.IGNORECASE, ) _JSON_SCRIPT_RE = re.compile( r']+id=["\'](?PRENDER_DATA|__UNIVERSAL_DATA_FOR_REHYDRATION__)["\'][^>]*>' r"(?P.*?)", re.IGNORECASE | re.DOTALL, ) _JSON_URL_RE = re.compile(r"https?:\\?/\\?/[^\"'<>\\]+") @dataclass class DouyinPluginConfig: timeout: float = 30.0 max_redirects: int = 8 cookie_file: str = "" class DouyinPlugin(BasePlugin): domain = "douyin.com" name = "Douyin" version = "1.0.0" def __init__(self) -> None: self.config = DouyinPluginConfig(cookie_file=str(runtime_path("douyin_cookies.json"))) def can_handle(self, url: str) -> bool: lower = (url or "").strip().lower() return any( token in lower for token in ( "douyin.com", "v.douyin.com", "iesdouyin.com", "amemv.com", ) ) def get_headers(self) -> dict[str, str]: headers = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" ), "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", "Referer": "https://www.douyin.com/", } cookie_header = self._cookie_header() if cookie_header: headers["Cookie"] = cookie_header return headers def parse(self, url: str) -> list[Resource]: target = _extract_first_douyin_url(url) if not target: raise PluginParseError("No valid Douyin URL found.") final_url, body = self._fetch_page(target) aweme_id = _extract_aweme_id(final_url) or _extract_aweme_id(body) or _extract_aweme_id(target) payloads = _extract_json_payloads(body) focus_nodes = _find_nodes_for_aweme(payloads, aweme_id) if aweme_id else [] media = _extract_media(focus_nodes or payloads) if not media: media = _extract_media_from_raw_html(body) files = _dedupe_files(media) if not files: raise PluginParseError( "No Douyin video or image media was found. Login, captcha, or page changes may be required." ) title = _first_text(focus_nodes or payloads, ("desc", "title", "caption")) or "Douyin" author = _first_text(focus_nodes or payloads, ("nickname", "author_name", "unique_id")) or "" author_id = _first_text(focus_nodes or payloads, ("uid", "sec_uid", "short_id", "unique_id")) or "" created_at = _first_text(focus_nodes or payloads, ("create_time", "createTime")) return [ Resource( id=aweme_id or str(abs(hash(final_url))), url=final_url, title=title, author=author, author_id=author_id, files=files, metadata={ "source_platform": "douyin", "aweme_id": aweme_id or "", "kind": "douyin", "original_url": target, }, thumbnail=next( (item.get("thumbnail") or item.get("url") for item in files if item.get("type") == "image"), None, ), created_at=str(created_at) if created_at else None, ) ] def download(self, resource: Resource, save_path: Path) -> list[Path]: save_path.mkdir(parents=True, exist_ok=True) downloaded: list[Path] = [] with httpx.Client(timeout=max(30.0, self.config.timeout), follow_redirects=True, headers=self.get_headers()) as client: for index, item in enumerate(resource.files, start=1): media_url = str(item.get("url") or "").strip() if not media_url: continue suffix = _guess_suffix(media_url, str(item.get("type") or "")) target = save_path / f"{_safe_filename(resource.id)}_{index:02d}{suffix}" if target.exists() and target.stat().st_size > 512: downloaded.append(target) continue try: response = client.get(media_url, headers={**self.get_headers(), "Referer": resource.url}) response.raise_for_status() except Exception as exc: raise PluginDownloadError(f"Douyin media download failed: {exc}") from exc content_type = response.headers.get("content-type", "").lower() if "text/html" in content_type: raise PluginDownloadError("Douyin media download returned HTML. Login or captcha may be required.") target.write_bytes(response.content) if target.stat().st_size <= 512: target.unlink(missing_ok=True) raise PluginDownloadError("Douyin media download was too small and likely failed.") downloaded.append(target) return downloaded def _fetch_page(self, url: str) -> tuple[str, str]: try: with httpx.Client( timeout=self.config.timeout, follow_redirects=True, max_redirects=self.config.max_redirects, headers=self.get_headers(), ) as client: response = client.get(url) response.raise_for_status() return str(response.url), response.text except Exception as exc: raise PluginParseError(f"Douyin page load failed: {exc}") from exc def _cookie_header(self) -> str: cookie_file = Path(self.config.cookie_file or "") if not cookie_file.is_file(): return "" try: cookies = parse_cookie_text(cookie_file.read_text(encoding="utf-8")) return cookies_to_header(cookies, ("douyin.com", "iesdouyin.com", "amemv.com")) except Exception: return "" def _extract_first_douyin_url(text: str) -> str: value = (text or "").strip() match = re.search(r"https?://[^\s\"'<>]*?(?:douyin|iesdouyin|amemv)\.com/[^\s\"'<>]*", value, re.I) if match: return match.group(0).rstrip(",.;)") if "douyin.com" in value.lower() and not value.startswith(("http://", "https://")): return "https://" + value.lstrip("/") return value if value.startswith(("http://", "https://")) else "" def _extract_aweme_id(text: str) -> str: match = _AWEME_ID_RE.search(text or "") if match: return match.group(1) try: query = parse_qs(urlparse(text).query) except Exception: return "" for key in ("aweme_id", "modal_id", "item_id"): values = query.get(key) if values and str(values[0]).isdigit(): return str(values[0]) return "" def _extract_json_payloads(page: str) -> list[Any]: payloads: list[Any] = [] for match in _JSON_SCRIPT_RE.finditer(page or ""): raw = html.unescape(match.group("body") or "").strip() if match.group("id") == "RENDER_DATA": raw = unquote(raw) try: payloads.append(json.loads(raw)) except Exception: continue for pattern in ( r"window\.__INITIAL_STATE__\s*=\s*({.*?})\s*", r"window\._ROUTER_DATA\s*=\s*({.*?})\s*", ): for match in re.finditer(pattern, page or "", re.DOTALL): try: payloads.append(json.loads(match.group(1))) except Exception: continue return payloads def _find_nodes_for_aweme(payloads: list[Any], aweme_id: str) -> list[Any]: if not aweme_id: return [] matches: list[Any] = [] def walk(node: Any) -> None: if isinstance(node, dict): values = { str(node.get(key) or "") for key in ("aweme_id", "awemeId", "id", "item_id", "itemId", "modal_id", "modalId") } if aweme_id in values: matches.append(node) for value in node.values(): walk(value) elif isinstance(node, list): for item in node: walk(item) for payload in payloads: walk(payload) return matches def _extract_media(payloads: list[Any]) -> list[dict[str, str]]: files: list[dict[str, str]] = [] def add_video(node: dict[str, Any]) -> None: for key in ("play_addr", "download_addr", "playApi", "video", "videoModel"): urls = _urls_from_node(node.get(key)) if urls: files.append({"type": "video", "url": _normalize_url(urls[0])}) return def add_images(node: dict[str, Any]) -> None: for key in ("images", "image_post_info", "image", "imageAlbum"): for image_url in _urls_from_node(node.get(key)): files.append({"type": "image", "url": _normalize_url(image_url)}) def walk(node: Any) -> None: if isinstance(node, dict): add_images(node) if any(key in node for key in ("play_addr", "download_addr", "video", "playApi", "videoModel")): add_video(node) for value in node.values(): walk(value) elif isinstance(node, list): for item in node: walk(item) for payload in payloads: walk(payload) return files def _extract_media_from_raw_html(page: str) -> list[dict[str, str]]: files: list[dict[str, str]] = [] for raw in _JSON_URL_RE.findall(page or ""): url = _normalize_url(raw) low = url.lower() if any(token in low for token in ("douyin", "byteimg", "pstatp", "douyinvod", "amemv", "douyinpic")): if any(token in low for token in (".mp4", "playwm", "play/?", "video_id=")): files.append({"type": "video", "url": url}) elif any(token in low for token in (".jpeg", ".jpg", ".png", ".webp", "image")): files.append({"type": "image", "url": url}) return files def _urls_from_node(node: Any) -> list[str]: if node is None: return [] if isinstance(node, str): value = _normalize_url(node) return [value] if value.startswith(("http://", "https://", "//")) else [] if isinstance(node, list): urls: list[str] = [] for item in node: urls.extend(_urls_from_node(item)) return urls if isinstance(node, dict): urls: list[str] = [] for key in ( "url_list", "urlList", "urls", "uri", "url", "src", "download_url", "downloadUrl", "play_addr", "playAddr", "origin_cover", "cover", ): urls.extend(_urls_from_node(node.get(key))) return urls return [] def _dedupe_files(files: list[dict[str, str]]) -> list[dict[str, str]]: video: dict[str, str] | None = None images: list[dict[str, str]] = [] seen: set[str] = set() for item in files: url = _normalize_url(item.get("url", "")) if not url or url in seen: continue seen.add(url) media_type = item.get("type", "image") row = {"type": media_type, "url": url} if media_type == "video": if video is None: video = row continue images.append(row) return ([video] if video else []) + images def _first_text(payloads: list[Any], keys: tuple[str, ...]) -> str: result = "" def walk(node: Any) -> None: nonlocal result if result: return if isinstance(node, dict): for key in keys: value = node.get(key) if isinstance(value, (str, int)) and str(value).strip(): result = str(value).strip() return for value in node.values(): walk(value) elif isinstance(node, list): for item in node: walk(item) for payload in payloads: walk(payload) return result def _normalize_url(url: str) -> str: value = html.unescape(str(url or "")).strip().replace("\\/", "/") if value.startswith("//"): value = "https:" + value return value def _guess_suffix(url: str, media_type: str) -> str: path = urlparse(url).path.lower() suffix = Path(path).suffix if suffix in {".jpg", ".jpeg", ".png", ".webp", ".gif", ".mp4", ".mov", ".m4v"}: return suffix return ".mp4" if media_type == "video" else ".jpg" def _safe_filename(value: str) -> str: return re.sub(r"[^A-Za-z0-9_.-]+", "_", value or "douyin").strip("._") or "douyin" plugin_class = DouyinPlugin