"""Taobao and Tmall product parser/downloader. The plugin parses public product pages and optionally reuses a locally saved Taobao cookie. Tests mock every HTTP request; no live site is contacted there. """ from __future__ import annotations import html import json import re from dataclasses import dataclass from pathlib import Path from typing import Any from urllib.parse import parse_qs, urlparse import httpx from pixiv_app.core.cookie_import import cookies_to_header, parse_cookie_text from pixiv_app.core.paths import runtime_path from pixiv_app.core.plugin.base import BasePlugin, PluginDownloadError, PluginParseError, Resource _ALLOWED_DOMAINS = ("taobao.com", "tmall.com", "tb.cn") _TITLE_KEYS = {"title", "itemtitle", "item_title", "subject", "goodsname", "goods_name"} _AUTHOR_KEYS = {"shopname", "shop_name", "sellername", "seller_name", "nick", "nickname"} _ID_KEYS = {"itemid", "item_id", "nid", "auctionid", "auction_id", "goodsid", "goods_id"} _PRICE_KEYS = {"price", "promotionprice", "promotion_price", "saleprice", "sale_price"} _IMAGE_KEYS = { "images", "image", "pics", "picurl", "pic_url", "imageurl", "image_url", "itemimages", "item_images", "detailimages", "detail_images", "mainimage", "main_image", } _VIDEO_KEYS = {"video", "videos", "videourl", "video_url", "videoid", "video_id"} _URL_RE = re.compile(r"(?:https?:)?(?:\\?/){2}[^\s\"'<>]+", re.IGNORECASE) _JSON_SCRIPT_RE = re.compile( r"]*(?:type=[\"']application/(?:ld\+)?json[\"']|id=[\"']__NEXT_DATA__[\"'])[^>]*>" r"(?P.*?)", re.IGNORECASE | re.DOTALL, ) _META_RE = re.compile( r"]+(?:property|name)=[\"'](?Pog:title|og:image|description|og:description)[\"']" r"[^>]+content=[\"'](?P.*?)[\"'][^>]*>", re.IGNORECASE | re.DOTALL, ) _IMG_RE = re.compile( r"]+(?:data-src|data-original|data-ks-lazyload|src)=[\"'](?P[^\"']+)[\"']", re.IGNORECASE, ) @dataclass class TaobaoPluginConfig: timeout: float = 25.0 max_redirects: int = 8 cookie_file: str = "" max_media: int = 100 class TaobaoPlugin(BasePlugin): domain = "taobao.com" name = "Taobao" version = "1.0.0" def __init__(self) -> None: self.config = TaobaoPluginConfig(cookie_file=str(runtime_path("taobao_cookies.txt"))) def can_handle(self, url: str) -> bool: target = _extract_first_product_url(url) if not target: return False try: host = (urlparse(target).hostname or "").lower() except ValueError: return False return _allowed_host(host) def get_headers(self) -> dict[str, str]: headers = { "User-Agent": ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" ), "Accept": "text/html,application/xhtml+xml,application/json;q=0.9,image/avif,image/webp,*/*;q=0.8", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.7", "Referer": "https://www.taobao.com/", } cookie = self._cookie_header() if cookie: headers["Cookie"] = cookie return headers def parse(self, url: str) -> list[Resource]: target = _extract_first_product_url(url) if not target or not self.can_handle(target): raise PluginParseError("未识别到有效的淘宝或天猫商品链接。") final_url, page = self._fetch_page(target) if not self.can_handle(final_url): raise PluginParseError("淘宝短链接跳转到了不受支持的站点。") if _looks_like_challenge(page, final_url): raise PluginParseError("淘宝要求登录或安全验证,请导入有效 Cookie 后重试。") payloads = _extract_json_payloads(page) extracted = _extract_structured_fields(payloads) meta = _extract_meta(page) product_id = _extract_item_id(final_url) or extracted["id"] or _extract_item_id(target) title = extracted["title"] or meta.get("og:title", "") title = _clean_title(title) or f"淘宝商品 {product_id or ''}".strip() author = extracted["author"] description = meta.get("og:description") or meta.get("description") or "" files = list(extracted["files"]) if meta.get("og:image"): files.insert(0, {"type": "image", "url": _normalize_url(meta["og:image"])}) for match in _IMG_RE.finditer(page): files.append({"type": "image", "url": _normalize_url(match.group("url"))}) files.extend(_extract_media_from_html(page)) files = _dedupe_media(files, limit=self.config.max_media) if not files: raise PluginParseError("商品页面已打开,但没有解析到商品图片或视频,页面结构可能已更新。") thumbnail = next((item["url"] for item in files if item["type"] == "image"), None) return [ Resource( id=product_id or str(abs(hash(final_url))), url=final_url, title=title, author=author, author_id="", files=files, metadata={ "source_platform": "taobao", "kind": "product", "product_id": product_id, "price": extracted["price"], "description": description, "original_url": target, }, thumbnail=thumbnail, ) ] def download(self, resource: Resource, save_path: Path) -> list[Path]: save_path.mkdir(parents=True, exist_ok=True) downloaded: list[Path] = [] headers = self.get_headers() with httpx.Client(timeout=max(30.0, self.config.timeout), follow_redirects=True, headers=headers) as client: for index, item in enumerate(resource.files, start=1): media_url = _normalize_url(str(item.get("url") or "")) if not media_url: continue media_type = str(item.get("type") or "image") suffix = _guess_suffix(media_url, media_type) target = save_path / f"{_safe_filename(resource.id)}_{index:03d}{suffix}" if target.is_file() and target.stat().st_size > 256: downloaded.append(target) continue try: response = client.get(media_url, headers={**headers, "Referer": resource.url}) response.raise_for_status() except Exception as exc: raise PluginDownloadError(f"淘宝媒体下载失败: {exc}") from exc content_type = response.headers.get("content-type", "").lower() if "text/html" in content_type or "application/json" in content_type: raise PluginDownloadError("淘宝媒体地址返回了验证页面,请更新 Cookie 后重试。") target.write_bytes(response.content) if target.stat().st_size <= 256: target.unlink(missing_ok=True) raise PluginDownloadError("下载内容过小,可能是失效链接或占位图。") downloaded.append(target) return downloaded def _fetch_page(self, url: str) -> tuple[str, str]: try: with httpx.Client( timeout=self.config.timeout, follow_redirects=True, max_redirects=self.config.max_redirects, headers=self.get_headers(), ) as client: response = client.get(url) response.raise_for_status() return str(response.url), response.text except Exception as exc: raise PluginParseError(f"淘宝商品页面加载失败: {exc}") from exc def _cookie_header(self) -> str: path = Path(self.config.cookie_file or "") if not path.is_file(): return "" try: cookies = parse_cookie_text(path.read_text(encoding="utf-8")) return cookies_to_header(cookies, _ALLOWED_DOMAINS) except Exception: return "" def _allowed_host(host: str) -> bool: return any(host == domain or host.endswith(f".{domain}") for domain in _ALLOWED_DOMAINS) def _extract_first_product_url(text: str) -> str: value = html.unescape((text or "").strip()) match = re.search(r"https?://[^\s\"'<>]+", value, re.IGNORECASE) if match: return match.group(0).rstrip(",.;)") if any(domain in value.lower() for domain in _ALLOWED_DOMAINS): return "https://" + value.lstrip("/") return "" def _extract_item_id(url: str) -> str: try: parsed = urlparse(url) query = parse_qs(parsed.query) except ValueError: return "" for key in ("id", "itemId", "item_id", "auctionId"): value = (query.get(key) or [""])[0] match = re.search(r"\d{5,}", str(value)) if match: return match.group(0) for pattern in (r"/item/(\d{5,})", r"/i(\d{5,})\.htm", r"item_id[=/](\d{5,})"): match = re.search(pattern, parsed.path, re.IGNORECASE) if match: return match.group(1) return "" def _extract_json_payloads(page: str) -> list[Any]: payloads: list[Any] = [] for match in _JSON_SCRIPT_RE.finditer(page or ""): raw = html.unescape(match.group("body") or "").strip() try: payloads.append(json.loads(raw)) except (TypeError, ValueError): continue for pattern in ( r"window\.__INITIAL_STATE__\s*=\s*({.*?})\s*;?\s*", r"window\.__INIT_DATA__\s*=\s*({.*?})\s*;?\s*", ): for match in re.finditer(pattern, page or "", re.IGNORECASE | re.DOTALL): try: payloads.append(json.loads(match.group(1))) except (TypeError, ValueError): continue return payloads def _extract_structured_fields(payloads: list[Any]) -> dict[str, Any]: titles: list[str] = [] authors: list[str] = [] ids: list[str] = [] prices: list[str] = [] files: list[dict[str, str]] = [] def walk(node: Any) -> None: if isinstance(node, dict): for key, value in node.items(): normalized = str(key).replace("-", "_").lower() compact = normalized.replace("_", "") values = _text_values(value) if normalized in _TITLE_KEYS or compact in _TITLE_KEYS: titles.extend(values) if normalized in _AUTHOR_KEYS or compact in _AUTHOR_KEYS: authors.extend(values) if normalized in _ID_KEYS or compact in _ID_KEYS: ids.extend(values) if normalized in _PRICE_KEYS or compact in _PRICE_KEYS: prices.extend(values) if normalized in _IMAGE_KEYS or compact in _IMAGE_KEYS: files.extend({"type": "image", "url": url} for url in _urls_from_node(value)) if normalized in _VIDEO_KEYS or compact in _VIDEO_KEYS: files.extend({"type": "video", "url": url} for url in _urls_from_node(value)) walk(value) elif isinstance(node, list): for item in node: walk(item) for payload in payloads: walk(payload) product_id = next((match.group(0) for value in ids if (match := re.search(r"\d{5,}", value))), "") return { "title": _best_text(titles), "author": _best_text(authors), "id": product_id, "price": next((value for value in prices if value.strip()), ""), "files": files, } def _extract_meta(page: str) -> dict[str, str]: values: dict[str, str] = {} for match in _META_RE.finditer(page or ""): values[match.group("name").lower()] = html.unescape(match.group("content")).strip() return values def _extract_media_from_html(page: str) -> list[dict[str, str]]: files: list[dict[str, str]] = [] for raw in _URL_RE.findall(page or ""): url = _normalize_url(raw) lower = url.lower().split("?", 1)[0] if any(lower.endswith(ext) for ext in (".jpg", ".jpeg", ".png", ".webp", ".avif")): files.append({"type": "image", "url": url}) elif any(lower.endswith(ext) for ext in (".mp4", ".m3u8", ".webm")): files.append({"type": "video", "url": url}) return files def _urls_from_node(node: Any) -> list[str]: if isinstance(node, str): value = _normalize_url(node) return [value] if value.startswith(("http://", "https://")) else [] if isinstance(node, list): return [url for item in node for url in _urls_from_node(item)] if isinstance(node, dict): urls: list[str] = [] for key, value in node.items(): if str(key).lower() in {"url", "src", "urls", "url_list", "url_list_v2", "image", "video"}: urls.extend(_urls_from_node(value)) elif isinstance(value, (dict, list)): urls.extend(_urls_from_node(value)) return urls return [] def _text_values(node: Any) -> list[str]: if isinstance(node, (str, int, float)): return [str(node).strip()] if isinstance(node, dict): return [str(value).strip() for key, value in node.items() if str(key).lower() in {"text", "value", "name"}] return [] def _best_text(values: list[str]) -> str: cleaned = [html.unescape(value).strip() for value in values if value and len(value.strip()) > 1] return max(cleaned, key=len, default="") def _clean_title(value: str) -> str: title = re.sub(r"\s*[-_–—]\s*(淘宝网|天猫|tmall|taobao).*$", "", value or "", flags=re.IGNORECASE) return re.sub(r"\s+", " ", title).strip() def _normalize_url(value: str) -> str: url = html.unescape((value or "").strip()).replace("\\u002F", "/").replace("\\/", "/") if url.startswith("//"): return "https:" + url return url def _dedupe_media(files: list[dict[str, str]], *, limit: int) -> list[dict[str, str]]: result: list[dict[str, str]] = [] seen: set[str] = set() for item in files: url = _normalize_url(str(item.get("url") or "")) if not url.startswith(("http://", "https://")) or url in seen: continue lower = url.lower() if any(token in lower for token in ("sprite", "icon", "logo", "avatar", "qrcode")): continue seen.add(url) result.append({"type": "video" if item.get("type") == "video" else "image", "url": url}) if len(result) >= max(1, limit): break return result def _looks_like_challenge(page: str, url: str) -> bool: sample = f"{url}\n{page[:20000]}".lower() markers = ("login.taobao.com", "punish", "captcha", "滑动验证", "安全验证", "请登录") return any(marker in sample for marker in markers) def _guess_suffix(url: str, media_type: str) -> str: suffix = Path(urlparse(url).path).suffix.lower() allowed = {".jpg", ".jpeg", ".png", ".webp", ".avif", ".gif", ".mp4", ".webm", ".m3u8"} if suffix in allowed: return suffix return ".mp4" if media_type == "video" else ".jpg" def _safe_filename(value: str) -> str: return re.sub(r"[^0-9A-Za-z._-]+", "_", value or "taobao")[:80] or "taobao" plugin_class = TaobaoPlugin