commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Douyin plugin package."""
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "Douyin",
|
||||
"version": "1.0.0",
|
||||
"domain": "douyin.com",
|
||||
"author": "Sakura Downloader",
|
||||
"description": "Douyin video and image-note parser/downloader for public share URLs.",
|
||||
"requires_auth": false,
|
||||
"tags": ["social", "video", "image"],
|
||||
"min_platform_version": "1.0.0",
|
||||
"dependencies": []
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
"""Douyin plugin.
|
||||
|
||||
Supports public Douyin share, video, note, and modal links. Cookie support is
|
||||
optional and reads from runtime/douyin_cookies.json when present.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from pixiv_app.core.cookie_import import cookies_to_header, parse_cookie_text
|
||||
from pixiv_app.core.paths import runtime_path
|
||||
from pixiv_app.core.plugin.base import BasePlugin, PluginDownloadError, PluginParseError, Resource
|
||||
|
||||
|
||||
_AWEME_ID_RE = re.compile(
|
||||
r"(?:aweme_id=|modal_id=|/video/|/note/|/share/video/|/share/note/)(\d{6,})",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_JSON_SCRIPT_RE = re.compile(
|
||||
r'<script[^>]+id=["\'](?P<id>RENDER_DATA|__UNIVERSAL_DATA_FOR_REHYDRATION__)["\'][^>]*>'
|
||||
r"(?P<body>.*?)</script>",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_JSON_URL_RE = re.compile(r"https?:\\?/\\?/[^\"'<>\\]+")
|
||||
|
||||
|
||||
@dataclass
|
||||
class DouyinPluginConfig:
|
||||
timeout: float = 30.0
|
||||
max_redirects: int = 8
|
||||
cookie_file: str = ""
|
||||
|
||||
|
||||
class DouyinPlugin(BasePlugin):
|
||||
domain = "douyin.com"
|
||||
name = "Douyin"
|
||||
version = "1.0.0"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.config = DouyinPluginConfig(cookie_file=str(runtime_path("douyin_cookies.json")))
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
lower = (url or "").strip().lower()
|
||||
return any(
|
||||
token in lower
|
||||
for token in (
|
||||
"douyin.com",
|
||||
"v.douyin.com",
|
||||
"iesdouyin.com",
|
||||
"amemv.com",
|
||||
)
|
||||
)
|
||||
|
||||
def get_headers(self) -> dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Referer": "https://www.douyin.com/",
|
||||
}
|
||||
cookie_header = self._cookie_header()
|
||||
if cookie_header:
|
||||
headers["Cookie"] = cookie_header
|
||||
return headers
|
||||
|
||||
def parse(self, url: str) -> list[Resource]:
|
||||
target = _extract_first_douyin_url(url)
|
||||
if not target:
|
||||
raise PluginParseError("No valid Douyin URL found.")
|
||||
|
||||
final_url, body = self._fetch_page(target)
|
||||
aweme_id = _extract_aweme_id(final_url) or _extract_aweme_id(body) or _extract_aweme_id(target)
|
||||
payloads = _extract_json_payloads(body)
|
||||
focus_nodes = _find_nodes_for_aweme(payloads, aweme_id) if aweme_id else []
|
||||
media = _extract_media(focus_nodes or payloads)
|
||||
if not media:
|
||||
media = _extract_media_from_raw_html(body)
|
||||
|
||||
files = _dedupe_files(media)
|
||||
if not files:
|
||||
raise PluginParseError(
|
||||
"No Douyin video or image media was found. Login, captcha, or page changes may be required."
|
||||
)
|
||||
|
||||
title = _first_text(focus_nodes or payloads, ("desc", "title", "caption")) or "Douyin"
|
||||
author = _first_text(focus_nodes or payloads, ("nickname", "author_name", "unique_id")) or ""
|
||||
author_id = _first_text(focus_nodes or payloads, ("uid", "sec_uid", "short_id", "unique_id")) or ""
|
||||
created_at = _first_text(focus_nodes or payloads, ("create_time", "createTime"))
|
||||
|
||||
return [
|
||||
Resource(
|
||||
id=aweme_id or str(abs(hash(final_url))),
|
||||
url=final_url,
|
||||
title=title,
|
||||
author=author,
|
||||
author_id=author_id,
|
||||
files=files,
|
||||
metadata={
|
||||
"source_platform": "douyin",
|
||||
"aweme_id": aweme_id or "",
|
||||
"kind": "douyin",
|
||||
"original_url": target,
|
||||
},
|
||||
thumbnail=next(
|
||||
(item.get("thumbnail") or item.get("url") for item in files if item.get("type") == "image"),
|
||||
None,
|
||||
),
|
||||
created_at=str(created_at) if created_at else None,
|
||||
)
|
||||
]
|
||||
|
||||
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||||
save_path.mkdir(parents=True, exist_ok=True)
|
||||
downloaded: list[Path] = []
|
||||
with httpx.Client(timeout=max(30.0, self.config.timeout), follow_redirects=True, headers=self.get_headers()) as client:
|
||||
for index, item in enumerate(resource.files, start=1):
|
||||
media_url = str(item.get("url") or "").strip()
|
||||
if not media_url:
|
||||
continue
|
||||
suffix = _guess_suffix(media_url, str(item.get("type") or ""))
|
||||
target = save_path / f"{_safe_filename(resource.id)}_{index:02d}{suffix}"
|
||||
if target.exists() and target.stat().st_size > 512:
|
||||
downloaded.append(target)
|
||||
continue
|
||||
try:
|
||||
response = client.get(media_url, headers={**self.get_headers(), "Referer": resource.url})
|
||||
response.raise_for_status()
|
||||
except Exception as exc:
|
||||
raise PluginDownloadError(f"Douyin media download failed: {exc}") from exc
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
if "text/html" in content_type:
|
||||
raise PluginDownloadError("Douyin media download returned HTML. Login or captcha may be required.")
|
||||
target.write_bytes(response.content)
|
||||
if target.stat().st_size <= 512:
|
||||
target.unlink(missing_ok=True)
|
||||
raise PluginDownloadError("Douyin media download was too small and likely failed.")
|
||||
downloaded.append(target)
|
||||
return downloaded
|
||||
|
||||
def _fetch_page(self, url: str) -> tuple[str, str]:
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=self.config.timeout,
|
||||
follow_redirects=True,
|
||||
max_redirects=self.config.max_redirects,
|
||||
headers=self.get_headers(),
|
||||
) as client:
|
||||
response = client.get(url)
|
||||
response.raise_for_status()
|
||||
return str(response.url), response.text
|
||||
except Exception as exc:
|
||||
raise PluginParseError(f"Douyin page load failed: {exc}") from exc
|
||||
|
||||
def _cookie_header(self) -> str:
|
||||
cookie_file = Path(self.config.cookie_file or "")
|
||||
if not cookie_file.is_file():
|
||||
return ""
|
||||
try:
|
||||
cookies = parse_cookie_text(cookie_file.read_text(encoding="utf-8"))
|
||||
return cookies_to_header(cookies, ("douyin.com", "iesdouyin.com", "amemv.com"))
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_first_douyin_url(text: str) -> str:
|
||||
value = (text or "").strip()
|
||||
match = re.search(r"https?://[^\s\"'<>]*?(?:douyin|iesdouyin|amemv)\.com/[^\s\"'<>]*", value, re.I)
|
||||
if match:
|
||||
return match.group(0).rstrip(",.;)")
|
||||
if "douyin.com" in value.lower() and not value.startswith(("http://", "https://")):
|
||||
return "https://" + value.lstrip("/")
|
||||
return value if value.startswith(("http://", "https://")) else ""
|
||||
|
||||
|
||||
def _extract_aweme_id(text: str) -> str:
|
||||
match = _AWEME_ID_RE.search(text or "")
|
||||
if match:
|
||||
return match.group(1)
|
||||
try:
|
||||
query = parse_qs(urlparse(text).query)
|
||||
except Exception:
|
||||
return ""
|
||||
for key in ("aweme_id", "modal_id", "item_id"):
|
||||
values = query.get(key)
|
||||
if values and str(values[0]).isdigit():
|
||||
return str(values[0])
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_json_payloads(page: str) -> list[Any]:
|
||||
payloads: list[Any] = []
|
||||
for match in _JSON_SCRIPT_RE.finditer(page or ""):
|
||||
raw = html.unescape(match.group("body") or "").strip()
|
||||
if match.group("id") == "RENDER_DATA":
|
||||
raw = unquote(raw)
|
||||
try:
|
||||
payloads.append(json.loads(raw))
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
for pattern in (
|
||||
r"window\.__INITIAL_STATE__\s*=\s*({.*?})\s*</script>",
|
||||
r"window\._ROUTER_DATA\s*=\s*({.*?})\s*</script>",
|
||||
):
|
||||
for match in re.finditer(pattern, page or "", re.DOTALL):
|
||||
try:
|
||||
payloads.append(json.loads(match.group(1)))
|
||||
except Exception:
|
||||
continue
|
||||
return payloads
|
||||
|
||||
|
||||
def _find_nodes_for_aweme(payloads: list[Any], aweme_id: str) -> list[Any]:
|
||||
if not aweme_id:
|
||||
return []
|
||||
matches: list[Any] = []
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict):
|
||||
values = {
|
||||
str(node.get(key) or "")
|
||||
for key in ("aweme_id", "awemeId", "id", "item_id", "itemId", "modal_id", "modalId")
|
||||
}
|
||||
if aweme_id in values:
|
||||
matches.append(node)
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
for payload in payloads:
|
||||
walk(payload)
|
||||
return matches
|
||||
|
||||
|
||||
def _extract_media(payloads: list[Any]) -> list[dict[str, str]]:
|
||||
files: list[dict[str, str]] = []
|
||||
|
||||
def add_video(node: dict[str, Any]) -> None:
|
||||
for key in ("play_addr", "download_addr", "playApi", "video", "videoModel"):
|
||||
urls = _urls_from_node(node.get(key))
|
||||
if urls:
|
||||
files.append({"type": "video", "url": _normalize_url(urls[0])})
|
||||
return
|
||||
|
||||
def add_images(node: dict[str, Any]) -> None:
|
||||
for key in ("images", "image_post_info", "image", "imageAlbum"):
|
||||
for image_url in _urls_from_node(node.get(key)):
|
||||
files.append({"type": "image", "url": _normalize_url(image_url)})
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict):
|
||||
add_images(node)
|
||||
if any(key in node for key in ("play_addr", "download_addr", "video", "playApi", "videoModel")):
|
||||
add_video(node)
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
for payload in payloads:
|
||||
walk(payload)
|
||||
return files
|
||||
|
||||
|
||||
def _extract_media_from_raw_html(page: str) -> list[dict[str, str]]:
|
||||
files: list[dict[str, str]] = []
|
||||
for raw in _JSON_URL_RE.findall(page or ""):
|
||||
url = _normalize_url(raw)
|
||||
low = url.lower()
|
||||
if any(token in low for token in ("douyin", "byteimg", "pstatp", "douyinvod", "amemv", "douyinpic")):
|
||||
if any(token in low for token in (".mp4", "playwm", "play/?", "video_id=")):
|
||||
files.append({"type": "video", "url": url})
|
||||
elif any(token in low for token in (".jpeg", ".jpg", ".png", ".webp", "image")):
|
||||
files.append({"type": "image", "url": url})
|
||||
return files
|
||||
|
||||
|
||||
def _urls_from_node(node: Any) -> list[str]:
|
||||
if node is None:
|
||||
return []
|
||||
if isinstance(node, str):
|
||||
value = _normalize_url(node)
|
||||
return [value] if value.startswith(("http://", "https://", "//")) else []
|
||||
if isinstance(node, list):
|
||||
urls: list[str] = []
|
||||
for item in node:
|
||||
urls.extend(_urls_from_node(item))
|
||||
return urls
|
||||
if isinstance(node, dict):
|
||||
urls: list[str] = []
|
||||
for key in (
|
||||
"url_list",
|
||||
"urlList",
|
||||
"urls",
|
||||
"uri",
|
||||
"url",
|
||||
"src",
|
||||
"download_url",
|
||||
"downloadUrl",
|
||||
"play_addr",
|
||||
"playAddr",
|
||||
"origin_cover",
|
||||
"cover",
|
||||
):
|
||||
urls.extend(_urls_from_node(node.get(key)))
|
||||
return urls
|
||||
return []
|
||||
|
||||
|
||||
def _dedupe_files(files: list[dict[str, str]]) -> list[dict[str, str]]:
|
||||
video: dict[str, str] | None = None
|
||||
images: list[dict[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
for item in files:
|
||||
url = _normalize_url(item.get("url", ""))
|
||||
if not url or url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
media_type = item.get("type", "image")
|
||||
row = {"type": media_type, "url": url}
|
||||
if media_type == "video":
|
||||
if video is None:
|
||||
video = row
|
||||
continue
|
||||
images.append(row)
|
||||
return ([video] if video else []) + images
|
||||
|
||||
|
||||
def _first_text(payloads: list[Any], keys: tuple[str, ...]) -> str:
|
||||
result = ""
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
nonlocal result
|
||||
if result:
|
||||
return
|
||||
if isinstance(node, dict):
|
||||
for key in keys:
|
||||
value = node.get(key)
|
||||
if isinstance(value, (str, int)) and str(value).strip():
|
||||
result = str(value).strip()
|
||||
return
|
||||
for value in node.values():
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
for payload in payloads:
|
||||
walk(payload)
|
||||
return result
|
||||
|
||||
|
||||
def _normalize_url(url: str) -> str:
|
||||
value = html.unescape(str(url or "")).strip().replace("\\/", "/")
|
||||
if value.startswith("//"):
|
||||
value = "https:" + value
|
||||
return value
|
||||
|
||||
|
||||
def _guess_suffix(url: str, media_type: str) -> str:
|
||||
path = urlparse(url).path.lower()
|
||||
suffix = Path(path).suffix
|
||||
if suffix in {".jpg", ".jpeg", ".png", ".webp", ".gif", ".mp4", ".mov", ".m4v"}:
|
||||
return suffix
|
||||
return ".mp4" if media_type == "video" else ".jpg"
|
||||
|
||||
|
||||
def _safe_filename(value: str) -> str:
|
||||
return re.sub(r"[^A-Za-z0-9_.-]+", "_", value or "douyin").strip("._") or "douyin"
|
||||
|
||||
|
||||
plugin_class = DouyinPlugin
|
||||
Reference in new issue
Block a user