first commit

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
kunkunxiaomaoandClaude committed 2026-11-16 20:45:22 +08:00
commit 9c5fef1a4d
412 files changed
+94582

No files matched your search

+387
View File
@@ -0,0 +1,387 @@
"""Douyin plugin.
Supports public Douyin share, video, note, and modal links. Cookie support is
optional and reads from runtime/douyin_cookies.json when present.
"""
from __future__ import annotations
import html
import json
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, unquote, urlparse
import httpx
from pixiv_app.core.cookie_import import cookies_to_header, parse_cookie_text
from pixiv_app.core.paths import runtime_path
from pixiv_app.core.plugin.base import BasePlugin, PluginDownloadError, PluginParseError, Resource
_AWEME_ID_RE = re.compile(
r"(?:aweme_id=|modal_id=|/video/|/note/|/share/video/|/share/note/)(\d{6,})",
re.IGNORECASE,
)
_JSON_SCRIPT_RE = re.compile(
r'<script[^>]+id=["\'](?P<id>RENDER_DATA|__UNIVERSAL_DATA_FOR_REHYDRATION__)["\'][^>]*>'
r"(?P<body>.*?)</script>",
re.IGNORECASE | re.DOTALL,
)
_JSON_URL_RE = re.compile(r"https?:\\?/\\?/[^\"'<>\\]+")
@dataclass
class DouyinPluginConfig:
timeout: float = 30.0
max_redirects: int = 8
cookie_file: str = ""
class DouyinPlugin(BasePlugin):
domain = "douyin.com"
name = "Douyin"
version = "1.0.0"
def __init__(self) -> None:
self.config = DouyinPluginConfig(cookie_file=str(runtime_path("douyin_cookies.json")))
def can_handle(self, url: str) -> bool:
lower = (url or "").strip().lower()
return any(
token in lower
for token in (
"douyin.com",
"v.douyin.com",
"iesdouyin.com",
"amemv.com",
)
)
def get_headers(self) -> dict[str, str]:
headers = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Referer": "https://www.douyin.com/",
}
cookie_header = self._cookie_header()
if cookie_header:
headers["Cookie"] = cookie_header
return headers
def parse(self, url: str) -> list[Resource]:
target = _extract_first_douyin_url(url)
if not target:
raise PluginParseError("No valid Douyin URL found.")
final_url, body = self._fetch_page(target)
aweme_id = _extract_aweme_id(final_url) or _extract_aweme_id(body) or _extract_aweme_id(target)
payloads = _extract_json_payloads(body)
focus_nodes = _find_nodes_for_aweme(payloads, aweme_id) if aweme_id else []
media = _extract_media(focus_nodes or payloads)
if not media:
media = _extract_media_from_raw_html(body)
files = _dedupe_files(media)
if not files:
raise PluginParseError(
"No Douyin video or image media was found. Login, captcha, or page changes may be required."
)
title = _first_text(focus_nodes or payloads, ("desc", "title", "caption")) or "Douyin"
author = _first_text(focus_nodes or payloads, ("nickname", "author_name", "unique_id")) or ""
author_id = _first_text(focus_nodes or payloads, ("uid", "sec_uid", "short_id", "unique_id")) or ""
created_at = _first_text(focus_nodes or payloads, ("create_time", "createTime"))
return [
Resource(
id=aweme_id or str(abs(hash(final_url))),
url=final_url,
title=title,
author=author,
author_id=author_id,
files=files,
metadata={
"source_platform": "douyin",
"aweme_id": aweme_id or "",
"kind": "douyin",
"original_url": target,
},
thumbnail=next(
(item.get("thumbnail") or item.get("url") for item in files if item.get("type") == "image"),
None,
),
created_at=str(created_at) if created_at else None,
)
]
def download(self, resource: Resource, save_path: Path) -> list[Path]:
save_path.mkdir(parents=True, exist_ok=True)
downloaded: list[Path] = []
with httpx.Client(timeout=max(30.0, self.config.timeout), follow_redirects=True, headers=self.get_headers()) as client:
for index, item in enumerate(resource.files, start=1):
media_url = str(item.get("url") or "").strip()
if not media_url:
continue
suffix = _guess_suffix(media_url, str(item.get("type") or ""))
target = save_path / f"{_safe_filename(resource.id)}_{index:02d}{suffix}"
if target.exists() and target.stat().st_size > 512:
downloaded.append(target)
continue
try:
response = client.get(media_url, headers={**self.get_headers(), "Referer": resource.url})
response.raise_for_status()
except Exception as exc:
raise PluginDownloadError(f"Douyin media download failed: {exc}") from exc
content_type = response.headers.get("content-type", "").lower()
if "text/html" in content_type:
raise PluginDownloadError("Douyin media download returned HTML. Login or captcha may be required.")
target.write_bytes(response.content)
if target.stat().st_size <= 512:
target.unlink(missing_ok=True)
raise PluginDownloadError("Douyin media download was too small and likely failed.")
downloaded.append(target)
return downloaded
def _fetch_page(self, url: str) -> tuple[str, str]:
try:
with httpx.Client(
timeout=self.config.timeout,
follow_redirects=True,
max_redirects=self.config.max_redirects,
headers=self.get_headers(),
) as client:
response = client.get(url)
response.raise_for_status()
return str(response.url), response.text
except Exception as exc:
raise PluginParseError(f"Douyin page load failed: {exc}") from exc
def _cookie_header(self) -> str:
cookie_file = Path(self.config.cookie_file or "")
if not cookie_file.is_file():
return ""
try:
cookies = parse_cookie_text(cookie_file.read_text(encoding="utf-8"))
return cookies_to_header(cookies, ("douyin.com", "iesdouyin.com", "amemv.com"))
except Exception:
return ""
def _extract_first_douyin_url(text: str) -> str:
value = (text or "").strip()
match = re.search(r"https?://[^\s\"'<>]*?(?:douyin|iesdouyin|amemv)\.com/[^\s\"'<>]*", value, re.I)
if match:
return match.group(0).rstrip(",.;)")
if "douyin.com" in value.lower() and not value.startswith(("http://", "https://")):
return "https://" + value.lstrip("/")
return value if value.startswith(("http://", "https://")) else ""
def _extract_aweme_id(text: str) -> str:
match = _AWEME_ID_RE.search(text or "")
if match:
return match.group(1)
try:
query = parse_qs(urlparse(text).query)
except Exception:
return ""
for key in ("aweme_id", "modal_id", "item_id"):
values = query.get(key)
if values and str(values[0]).isdigit():
return str(values[0])
return ""
def _extract_json_payloads(page: str) -> list[Any]:
payloads: list[Any] = []
for match in _JSON_SCRIPT_RE.finditer(page or ""):
raw = html.unescape(match.group("body") or "").strip()
if match.group("id") == "RENDER_DATA":
raw = unquote(raw)
try:
payloads.append(json.loads(raw))
except Exception:
continue
for pattern in (
r"window\.__INITIAL_STATE__\s*=\s*({.*?})\s*</script>",
r"window\._ROUTER_DATA\s*=\s*({.*?})\s*</script>",
):
for match in re.finditer(pattern, page or "", re.DOTALL):
try:
payloads.append(json.loads(match.group(1)))
except Exception:
continue
return payloads
def _find_nodes_for_aweme(payloads: list[Any], aweme_id: str) -> list[Any]:
if not aweme_id:
return []
matches: list[Any] = []
def walk(node: Any) -> None:
if isinstance(node, dict):
values = {
str(node.get(key) or "")
for key in ("aweme_id", "awemeId", "id", "item_id", "itemId", "modal_id", "modalId")
}
if aweme_id in values:
matches.append(node)
for value in node.values():
walk(value)
elif isinstance(node, list):
for item in node:
walk(item)
for payload in payloads:
walk(payload)
return matches
def _extract_media(payloads: list[Any]) -> list[dict[str, str]]:
files: list[dict[str, str]] = []
def add_video(node: dict[str, Any]) -> None:
for key in ("play_addr", "download_addr", "playApi", "video", "videoModel"):
urls = _urls_from_node(node.get(key))
if urls:
files.append({"type": "video", "url": _normalize_url(urls[0])})
return
def add_images(node: dict[str, Any]) -> None:
for key in ("images", "image_post_info", "image", "imageAlbum"):
for image_url in _urls_from_node(node.get(key)):
files.append({"type": "image", "url": _normalize_url(image_url)})
def walk(node: Any) -> None:
if isinstance(node, dict):
add_images(node)
if any(key in node for key in ("play_addr", "download_addr", "video", "playApi", "videoModel")):
add_video(node)
for value in node.values():
walk(value)
elif isinstance(node, list):
for item in node:
walk(item)
for payload in payloads:
walk(payload)
return files
def _extract_media_from_raw_html(page: str) -> list[dict[str, str]]:
files: list[dict[str, str]] = []
for raw in _JSON_URL_RE.findall(page or ""):
url = _normalize_url(raw)
low = url.lower()
if any(token in low for token in ("douyin", "byteimg", "pstatp", "douyinvod", "amemv", "douyinpic")):
if any(token in low for token in (".mp4", "playwm", "play/?", "video_id=")):
files.append({"type": "video", "url": url})
elif any(token in low for token in (".jpeg", ".jpg", ".png", ".webp", "image")):
files.append({"type": "image", "url": url})
return files
def _urls_from_node(node: Any) -> list[str]:
if node is None:
return []
if isinstance(node, str):
value = _normalize_url(node)
return [value] if value.startswith(("http://", "https://", "//")) else []
if isinstance(node, list):
urls: list[str] = []
for item in node:
urls.extend(_urls_from_node(item))
return urls
if isinstance(node, dict):
urls: list[str] = []
for key in (
"url_list",
"urlList",
"urls",
"uri",
"url",
"src",
"download_url",
"downloadUrl",
"play_addr",
"playAddr",
"origin_cover",
"cover",
):
urls.extend(_urls_from_node(node.get(key)))
return urls
return []
def _dedupe_files(files: list[dict[str, str]]) -> list[dict[str, str]]:
video: dict[str, str] | None = None
images: list[dict[str, str]] = []
seen: set[str] = set()
for item in files:
url = _normalize_url(item.get("url", ""))
if not url or url in seen:
continue
seen.add(url)
media_type = item.get("type", "image")
row = {"type": media_type, "url": url}
if media_type == "video":
if video is None:
video = row
continue
images.append(row)
return ([video] if video else []) + images
def _first_text(payloads: list[Any], keys: tuple[str, ...]) -> str:
result = ""
def walk(node: Any) -> None:
nonlocal result
if result:
return
if isinstance(node, dict):
for key in keys:
value = node.get(key)
if isinstance(value, (str, int)) and str(value).strip():
result = str(value).strip()
return
for value in node.values():
walk(value)
elif isinstance(node, list):
for item in node:
walk(item)
for payload in payloads:
walk(payload)
return result
def _normalize_url(url: str) -> str:
value = html.unescape(str(url or "")).strip().replace("\\/", "/")
if value.startswith("//"):
value = "https:" + value
return value
def _guess_suffix(url: str, media_type: str) -> str:
path = urlparse(url).path.lower()
suffix = Path(path).suffix
if suffix in {".jpg", ".jpeg", ".png", ".webp", ".gif", ".mp4", ".mov", ".m4v"}:
return suffix
return ".mp4" if media_type == "video" else ".jpg"
def _safe_filename(value: str) -> str:
return re.sub(r"[^A-Za-z0-9_.-]+", "_", value or "douyin").strip("._") or "douyin"
plugin_class = DouyinPlugin