commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1,400 @@
|
||||
"""Taobao and Tmall product parser/downloader.
|
||||
|
||||
The plugin parses public product pages and optionally reuses a locally saved
|
||||
Taobao cookie. Tests mock every HTTP request; no live site is contacted there.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
from pixiv_app.core.cookie_import import cookies_to_header, parse_cookie_text
|
||||
from pixiv_app.core.paths import runtime_path
|
||||
from pixiv_app.core.plugin.base import BasePlugin, PluginDownloadError, PluginParseError, Resource
|
||||
|
||||
|
||||
_ALLOWED_DOMAINS = ("taobao.com", "tmall.com", "tb.cn")
|
||||
_TITLE_KEYS = {"title", "itemtitle", "item_title", "subject", "goodsname", "goods_name"}
|
||||
_AUTHOR_KEYS = {"shopname", "shop_name", "sellername", "seller_name", "nick", "nickname"}
|
||||
_ID_KEYS = {"itemid", "item_id", "nid", "auctionid", "auction_id", "goodsid", "goods_id"}
|
||||
_PRICE_KEYS = {"price", "promotionprice", "promotion_price", "saleprice", "sale_price"}
|
||||
_IMAGE_KEYS = {
|
||||
"images",
|
||||
"image",
|
||||
"pics",
|
||||
"picurl",
|
||||
"pic_url",
|
||||
"imageurl",
|
||||
"image_url",
|
||||
"itemimages",
|
||||
"item_images",
|
||||
"detailimages",
|
||||
"detail_images",
|
||||
"mainimage",
|
||||
"main_image",
|
||||
}
|
||||
_VIDEO_KEYS = {"video", "videos", "videourl", "video_url", "videoid", "video_id"}
|
||||
_URL_RE = re.compile(r"(?:https?:)?(?:\\?/){2}[^\s\"'<>]+", re.IGNORECASE)
|
||||
_JSON_SCRIPT_RE = re.compile(
|
||||
r"<script[^>]*(?:type=[\"']application/(?:ld\+)?json[\"']|id=[\"']__NEXT_DATA__[\"'])[^>]*>"
|
||||
r"(?P<body>.*?)</script>",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_META_RE = re.compile(
|
||||
r"<meta[^>]+(?:property|name)=[\"'](?P<name>og:title|og:image|description|og:description)[\"']"
|
||||
r"[^>]+content=[\"'](?P<content>.*?)[\"'][^>]*>",
|
||||
re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_IMG_RE = re.compile(
|
||||
r"<img[^>]+(?:data-src|data-original|data-ks-lazyload|src)=[\"'](?P<url>[^\"']+)[\"']",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaobaoPluginConfig:
|
||||
timeout: float = 25.0
|
||||
max_redirects: int = 8
|
||||
cookie_file: str = ""
|
||||
max_media: int = 100
|
||||
|
||||
|
||||
class TaobaoPlugin(BasePlugin):
|
||||
domain = "taobao.com"
|
||||
name = "Taobao"
|
||||
version = "1.0.0"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.config = TaobaoPluginConfig(cookie_file=str(runtime_path("taobao_cookies.txt")))
|
||||
|
||||
def can_handle(self, url: str) -> bool:
|
||||
target = _extract_first_product_url(url)
|
||||
if not target:
|
||||
return False
|
||||
try:
|
||||
host = (urlparse(target).hostname or "").lower()
|
||||
except ValueError:
|
||||
return False
|
||||
return _allowed_host(host)
|
||||
|
||||
def get_headers(self) -> dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/html,application/xhtml+xml,application/json;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.7",
|
||||
"Referer": "https://www.taobao.com/",
|
||||
}
|
||||
cookie = self._cookie_header()
|
||||
if cookie:
|
||||
headers["Cookie"] = cookie
|
||||
return headers
|
||||
|
||||
def parse(self, url: str) -> list[Resource]:
|
||||
target = _extract_first_product_url(url)
|
||||
if not target or not self.can_handle(target):
|
||||
raise PluginParseError("未识别到有效的淘宝或天猫商品链接。")
|
||||
|
||||
final_url, page = self._fetch_page(target)
|
||||
if not self.can_handle(final_url):
|
||||
raise PluginParseError("淘宝短链接跳转到了不受支持的站点。")
|
||||
if _looks_like_challenge(page, final_url):
|
||||
raise PluginParseError("淘宝要求登录或安全验证,请导入有效 Cookie 后重试。")
|
||||
|
||||
payloads = _extract_json_payloads(page)
|
||||
extracted = _extract_structured_fields(payloads)
|
||||
meta = _extract_meta(page)
|
||||
product_id = _extract_item_id(final_url) or extracted["id"] or _extract_item_id(target)
|
||||
title = extracted["title"] or meta.get("og:title", "")
|
||||
title = _clean_title(title) or f"淘宝商品 {product_id or ''}".strip()
|
||||
author = extracted["author"]
|
||||
description = meta.get("og:description") or meta.get("description") or ""
|
||||
|
||||
files = list(extracted["files"])
|
||||
if meta.get("og:image"):
|
||||
files.insert(0, {"type": "image", "url": _normalize_url(meta["og:image"])})
|
||||
for match in _IMG_RE.finditer(page):
|
||||
files.append({"type": "image", "url": _normalize_url(match.group("url"))})
|
||||
files.extend(_extract_media_from_html(page))
|
||||
files = _dedupe_media(files, limit=self.config.max_media)
|
||||
if not files:
|
||||
raise PluginParseError("商品页面已打开,但没有解析到商品图片或视频,页面结构可能已更新。")
|
||||
|
||||
thumbnail = next((item["url"] for item in files if item["type"] == "image"), None)
|
||||
return [
|
||||
Resource(
|
||||
id=product_id or str(abs(hash(final_url))),
|
||||
url=final_url,
|
||||
title=title,
|
||||
author=author,
|
||||
author_id="",
|
||||
files=files,
|
||||
metadata={
|
||||
"source_platform": "taobao",
|
||||
"kind": "product",
|
||||
"product_id": product_id,
|
||||
"price": extracted["price"],
|
||||
"description": description,
|
||||
"original_url": target,
|
||||
},
|
||||
thumbnail=thumbnail,
|
||||
)
|
||||
]
|
||||
|
||||
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||||
save_path.mkdir(parents=True, exist_ok=True)
|
||||
downloaded: list[Path] = []
|
||||
headers = self.get_headers()
|
||||
with httpx.Client(timeout=max(30.0, self.config.timeout), follow_redirects=True, headers=headers) as client:
|
||||
for index, item in enumerate(resource.files, start=1):
|
||||
media_url = _normalize_url(str(item.get("url") or ""))
|
||||
if not media_url:
|
||||
continue
|
||||
media_type = str(item.get("type") or "image")
|
||||
suffix = _guess_suffix(media_url, media_type)
|
||||
target = save_path / f"{_safe_filename(resource.id)}_{index:03d}{suffix}"
|
||||
if target.is_file() and target.stat().st_size > 256:
|
||||
downloaded.append(target)
|
||||
continue
|
||||
try:
|
||||
response = client.get(media_url, headers={**headers, "Referer": resource.url})
|
||||
response.raise_for_status()
|
||||
except Exception as exc:
|
||||
raise PluginDownloadError(f"淘宝媒体下载失败: {exc}") from exc
|
||||
content_type = response.headers.get("content-type", "").lower()
|
||||
if "text/html" in content_type or "application/json" in content_type:
|
||||
raise PluginDownloadError("淘宝媒体地址返回了验证页面,请更新 Cookie 后重试。")
|
||||
target.write_bytes(response.content)
|
||||
if target.stat().st_size <= 256:
|
||||
target.unlink(missing_ok=True)
|
||||
raise PluginDownloadError("下载内容过小,可能是失效链接或占位图。")
|
||||
downloaded.append(target)
|
||||
return downloaded
|
||||
|
||||
def _fetch_page(self, url: str) -> tuple[str, str]:
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=self.config.timeout,
|
||||
follow_redirects=True,
|
||||
max_redirects=self.config.max_redirects,
|
||||
headers=self.get_headers(),
|
||||
) as client:
|
||||
response = client.get(url)
|
||||
response.raise_for_status()
|
||||
return str(response.url), response.text
|
||||
except Exception as exc:
|
||||
raise PluginParseError(f"淘宝商品页面加载失败: {exc}") from exc
|
||||
|
||||
def _cookie_header(self) -> str:
|
||||
path = Path(self.config.cookie_file or "")
|
||||
if not path.is_file():
|
||||
return ""
|
||||
try:
|
||||
cookies = parse_cookie_text(path.read_text(encoding="utf-8"))
|
||||
return cookies_to_header(cookies, _ALLOWED_DOMAINS)
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _allowed_host(host: str) -> bool:
|
||||
return any(host == domain or host.endswith(f".{domain}") for domain in _ALLOWED_DOMAINS)
|
||||
|
||||
|
||||
def _extract_first_product_url(text: str) -> str:
|
||||
value = html.unescape((text or "").strip())
|
||||
match = re.search(r"https?://[^\s\"'<>]+", value, re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(0).rstrip(",.;)")
|
||||
if any(domain in value.lower() for domain in _ALLOWED_DOMAINS):
|
||||
return "https://" + value.lstrip("/")
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_item_id(url: str) -> str:
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
query = parse_qs(parsed.query)
|
||||
except ValueError:
|
||||
return ""
|
||||
for key in ("id", "itemId", "item_id", "auctionId"):
|
||||
value = (query.get(key) or [""])[0]
|
||||
match = re.search(r"\d{5,}", str(value))
|
||||
if match:
|
||||
return match.group(0)
|
||||
for pattern in (r"/item/(\d{5,})", r"/i(\d{5,})\.htm", r"item_id[=/](\d{5,})"):
|
||||
match = re.search(pattern, parsed.path, re.IGNORECASE)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_json_payloads(page: str) -> list[Any]:
|
||||
payloads: list[Any] = []
|
||||
for match in _JSON_SCRIPT_RE.finditer(page or ""):
|
||||
raw = html.unescape(match.group("body") or "").strip()
|
||||
try:
|
||||
payloads.append(json.loads(raw))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
for pattern in (
|
||||
r"window\.__INITIAL_STATE__\s*=\s*({.*?})\s*;?\s*</script>",
|
||||
r"window\.__INIT_DATA__\s*=\s*({.*?})\s*;?\s*</script>",
|
||||
):
|
||||
for match in re.finditer(pattern, page or "", re.IGNORECASE | re.DOTALL):
|
||||
try:
|
||||
payloads.append(json.loads(match.group(1)))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return payloads
|
||||
|
||||
|
||||
def _extract_structured_fields(payloads: list[Any]) -> dict[str, Any]:
|
||||
titles: list[str] = []
|
||||
authors: list[str] = []
|
||||
ids: list[str] = []
|
||||
prices: list[str] = []
|
||||
files: list[dict[str, str]] = []
|
||||
|
||||
def walk(node: Any) -> None:
|
||||
if isinstance(node, dict):
|
||||
for key, value in node.items():
|
||||
normalized = str(key).replace("-", "_").lower()
|
||||
compact = normalized.replace("_", "")
|
||||
values = _text_values(value)
|
||||
if normalized in _TITLE_KEYS or compact in _TITLE_KEYS:
|
||||
titles.extend(values)
|
||||
if normalized in _AUTHOR_KEYS or compact in _AUTHOR_KEYS:
|
||||
authors.extend(values)
|
||||
if normalized in _ID_KEYS or compact in _ID_KEYS:
|
||||
ids.extend(values)
|
||||
if normalized in _PRICE_KEYS or compact in _PRICE_KEYS:
|
||||
prices.extend(values)
|
||||
if normalized in _IMAGE_KEYS or compact in _IMAGE_KEYS:
|
||||
files.extend({"type": "image", "url": url} for url in _urls_from_node(value))
|
||||
if normalized in _VIDEO_KEYS or compact in _VIDEO_KEYS:
|
||||
files.extend({"type": "video", "url": url} for url in _urls_from_node(value))
|
||||
walk(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
walk(item)
|
||||
|
||||
for payload in payloads:
|
||||
walk(payload)
|
||||
product_id = next((match.group(0) for value in ids if (match := re.search(r"\d{5,}", value))), "")
|
||||
return {
|
||||
"title": _best_text(titles),
|
||||
"author": _best_text(authors),
|
||||
"id": product_id,
|
||||
"price": next((value for value in prices if value.strip()), ""),
|
||||
"files": files,
|
||||
}
|
||||
|
||||
|
||||
def _extract_meta(page: str) -> dict[str, str]:
|
||||
values: dict[str, str] = {}
|
||||
for match in _META_RE.finditer(page or ""):
|
||||
values[match.group("name").lower()] = html.unescape(match.group("content")).strip()
|
||||
return values
|
||||
|
||||
|
||||
def _extract_media_from_html(page: str) -> list[dict[str, str]]:
|
||||
files: list[dict[str, str]] = []
|
||||
for raw in _URL_RE.findall(page or ""):
|
||||
url = _normalize_url(raw)
|
||||
lower = url.lower().split("?", 1)[0]
|
||||
if any(lower.endswith(ext) for ext in (".jpg", ".jpeg", ".png", ".webp", ".avif")):
|
||||
files.append({"type": "image", "url": url})
|
||||
elif any(lower.endswith(ext) for ext in (".mp4", ".m3u8", ".webm")):
|
||||
files.append({"type": "video", "url": url})
|
||||
return files
|
||||
|
||||
|
||||
def _urls_from_node(node: Any) -> list[str]:
|
||||
if isinstance(node, str):
|
||||
value = _normalize_url(node)
|
||||
return [value] if value.startswith(("http://", "https://")) else []
|
||||
if isinstance(node, list):
|
||||
return [url for item in node for url in _urls_from_node(item)]
|
||||
if isinstance(node, dict):
|
||||
urls: list[str] = []
|
||||
for key, value in node.items():
|
||||
if str(key).lower() in {"url", "src", "urls", "url_list", "url_list_v2", "image", "video"}:
|
||||
urls.extend(_urls_from_node(value))
|
||||
elif isinstance(value, (dict, list)):
|
||||
urls.extend(_urls_from_node(value))
|
||||
return urls
|
||||
return []
|
||||
|
||||
|
||||
def _text_values(node: Any) -> list[str]:
|
||||
if isinstance(node, (str, int, float)):
|
||||
return [str(node).strip()]
|
||||
if isinstance(node, dict):
|
||||
return [str(value).strip() for key, value in node.items() if str(key).lower() in {"text", "value", "name"}]
|
||||
return []
|
||||
|
||||
|
||||
def _best_text(values: list[str]) -> str:
|
||||
cleaned = [html.unescape(value).strip() for value in values if value and len(value.strip()) > 1]
|
||||
return max(cleaned, key=len, default="")
|
||||
|
||||
|
||||
def _clean_title(value: str) -> str:
|
||||
title = re.sub(r"\s*[-_–—]\s*(淘宝网|天猫|tmall|taobao).*$", "", value or "", flags=re.IGNORECASE)
|
||||
return re.sub(r"\s+", " ", title).strip()
|
||||
|
||||
|
||||
def _normalize_url(value: str) -> str:
|
||||
url = html.unescape((value or "").strip()).replace("\\u002F", "/").replace("\\/", "/")
|
||||
if url.startswith("//"):
|
||||
return "https:" + url
|
||||
return url
|
||||
|
||||
|
||||
def _dedupe_media(files: list[dict[str, str]], *, limit: int) -> list[dict[str, str]]:
|
||||
result: list[dict[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
for item in files:
|
||||
url = _normalize_url(str(item.get("url") or ""))
|
||||
if not url.startswith(("http://", "https://")) or url in seen:
|
||||
continue
|
||||
lower = url.lower()
|
||||
if any(token in lower for token in ("sprite", "icon", "logo", "avatar", "qrcode")):
|
||||
continue
|
||||
seen.add(url)
|
||||
result.append({"type": "video" if item.get("type") == "video" else "image", "url": url})
|
||||
if len(result) >= max(1, limit):
|
||||
break
|
||||
return result
|
||||
|
||||
|
||||
def _looks_like_challenge(page: str, url: str) -> bool:
|
||||
sample = f"{url}\n{page[:20000]}".lower()
|
||||
markers = ("login.taobao.com", "punish", "captcha", "滑动验证", "安全验证", "请登录")
|
||||
return any(marker in sample for marker in markers)
|
||||
|
||||
|
||||
def _guess_suffix(url: str, media_type: str) -> str:
|
||||
suffix = Path(urlparse(url).path).suffix.lower()
|
||||
allowed = {".jpg", ".jpeg", ".png", ".webp", ".avif", ".gif", ".mp4", ".webm", ".m3u8"}
|
||||
if suffix in allowed:
|
||||
return suffix
|
||||
return ".mp4" if media_type == "video" else ".jpg"
|
||||
|
||||
|
||||
def _safe_filename(value: str) -> str:
|
||||
return re.sub(r"[^0-9A-Za-z._-]+", "_", value or "taobao")[:80] or "taobao"
|
||||
|
||||
|
||||
plugin_class = TaobaoPlugin
|
||||
|
||||
Reference in new issue
Block a user