# plugins/xiaohongshu/plugin.py """ 小红书爬虫插件 - 核心功能 - Playwright 浏览器自动化 - 动态代理池轮换 - 浏览器指纹伪装 - 行为节奏模拟 """ from __future__ import annotations import asyncio import json import random import re import threading import time from dataclasses import dataclass, field from datetime import datetime from pathlib import Path from typing import Any, Callable, Optional from urllib.parse import parse_qs, quote, unquote, urlparse # 动态加载 playwright try: from playwright.async_api import BrowserContext, Page, Playwright, async_playwright PLAYWRIGHT_AVAILABLE = True except ImportError: PLAYWRIGHT_AVAILABLE = False from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource from pixiv_app.core.paths import runtime_path from pixiv_app.core.playwright_runtime import ( configure_playwright_runtime_env, explain_missing_browser, preferred_chromium_executable, ) try: from pixiv_app.core.fingerprint import BrowserProfile, SessionFingerprintManager except Exception: # pragma: no cover BrowserProfile = None SessionFingerprintManager = None def _extract_note_id_from_text(text: str) -> str: patterns = [ r"xiaohongshu\.com/explore/([a-zA-Z0-9]+)", r"/explore/([a-zA-Z0-9]+)", r"note_id[=:]([a-zA-Z0-9]+)", ] for pattern in patterns: match = re.search(pattern, text or "", re.IGNORECASE) if match: return match.group(1) return "" def _extract_redirect_path(url: str) -> str: try: query = parse_qs(urlparse(url).query) value = (query.get("redirectPath") or [""])[0] except Exception: value = "" value = unquote(value or "").strip() if not value: return "" if value.startswith("//"): return "https:" + value if value.startswith("/"): return "https://www.xiaohongshu.com" + value if "xiaohongshu.com" in value.lower() or "xhslink.com" in value.lower(): return value if "://" in value else "https://" + value.lstrip("/") return "" def _normalize_xhs_url(url: str) -> str: value = (url or "").strip() if not value: return "" if value.startswith("//"): return "https:" + value if value.startswith("/"): return "https://www.xiaohongshu.com" + value lower = value.lower() if ("xiaohongshu.com" in lower or "xhslink.com" in lower) and "://" not in value: return "https://" + value.lstrip("/") return value # ==================== 配置定义 ==================== @dataclass class ProxyConfig: """代理配置""" server: str # http://ip:port 或 socks5://ip:port username: str = "" password: str = "" failures: int = 0 successes: int = 0 latency_ms: float = 0.0 healthy: bool = True @property def url(self) -> str: return self.server @property def score(self) -> float: """计算代理评分(用于加权选择)""" if not self.healthy: return 0.0 total = self.successes + self.failures if total == 0: return 1.0 success_rate = self.successes / total latency_penalty = max(0.1, min(1.0, 1000.0 / max(self.latency_ms, 1.0))) return success_rate * latency_penalty @dataclass class FingerprintConfig: """浏览器指纹配置""" user_agent: str viewport: dict[str, int] locale: str = "zh-CN" timezone_id: str = "Asia/Shanghai" color_scheme: str = "light" reduced_motion: str = "no-preference" device_scale_factor: float = 1.0 is_mobile: bool = False has_touch: bool = False @classmethod def from_browser_profile(cls, profile: Any) -> "FingerprintConfig": return cls( user_agent=profile.user_agent, viewport=dict(profile.viewport), locale=profile.locale, timezone_id=profile.timezone_id, color_scheme=profile.color_scheme, reduced_motion=profile.reduced_motion, device_scale_factor=profile.device_scale_factor, is_mobile=profile.is_mobile, has_touch=profile.has_touch, ) @classmethod def desktop_chrome(cls) -> "FingerprintConfig": return cls( user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", viewport={"width": 1920, "height": 1080}, device_scale_factor=1.0, ) @classmethod def desktop_edge(cls) -> "FingerprintConfig": return cls( user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0", viewport={"width": 1536, "height": 864}, device_scale_factor=1.25, ) @classmethod def mac_chrome(cls) -> "FingerprintConfig": return cls( user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", viewport={"width": 1512, "height": 982}, device_scale_factor=2.0, ) @classmethod def mobile_ios(cls) -> "FingerprintConfig": return cls( user_agent="Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1", viewport={"width": 390, "height": 844}, device_scale_factor=3.0, is_mobile=True, has_touch=True, ) @dataclass class XiaohongshuConfig: """小红书插件配置""" # 代理配置 proxy_enabled: bool = False proxy_list: list[ProxyConfig] = field(default_factory=list) proxy_rotation: str = "round_robin" # round_robin, random, session_sticky # 指纹配置 fingerprint_list: list[FingerprintConfig] = field(default_factory=lambda: [ FingerprintConfig.desktop_chrome(), FingerprintConfig.desktop_edge(), FingerprintConfig.mac_chrome(), ]) # 行为控制 request_delay_min: float = 3.0 # 最小延迟(秒) request_delay_max: float = 8.0 # 最大延迟(秒) scroll_delay_min: float = 0.5 # 滚动延迟(秒) scroll_delay_max: float = 1.5 batch_pause_min: float = 5.0 # 批次暂停(秒) batch_pause_max: float = 12.0 max_scrolls: int = 15 # 最大滚动次数 # 采集限制 max_notes_per_session: int = 50 # 单会话最大笔记数 max_concurrent: int = 2 # 最大并发数 # 会话持久化 profile_dir: str = "runtime/xiaohongshu_profile" cookie_file: str = "runtime/xiaohongshu_cookies.json" # ==================== 代理池管理器 ==================== class ProxyPool: """代理池管理器 - 支持轮换和健康检查""" def __init__(self, config: XiaohongshuConfig): self.config = config self._proxies: list[ProxyConfig] = list(config.proxy_list) self._current_index = 0 self._lock = threading.Lock() def add_proxy(self, proxy: ProxyConfig) -> None: """添加代理""" with self._lock: # 去重 existing = {p.server for p in self._proxies} if proxy.server not in existing: self._proxies.append(proxy) def get_proxy(self) -> Optional[ProxyConfig]: """获取一个代理(按策略)""" if not self.config.proxy_enabled or not self._proxies: return None with self._lock: healthy = [p for p in self._proxies if p.healthy] if not healthy: return None if self.config.proxy_rotation == "random": return random.choice(healthy) # round_robin proxy = healthy[self._current_index % len(healthy)] self._current_index += 1 return proxy def mark_success(self, proxy: Optional[ProxyConfig], latency_ms: float = 0) -> None: """标记代理成功""" if proxy is None: return with self._lock: proxy.successes += 1 proxy.failures = 0 proxy.healthy = True if latency_ms > 0: proxy.latency_ms = latency_ms def mark_failed(self, proxy: Optional[ProxyConfig]) -> None: """标记代理失败""" if proxy is None: return with self._lock: proxy.failures += 1 if proxy.failures >= 3: proxy.healthy = False def healthy_count(self) -> int: """可用代理数量""" with self._lock: return len([p for p in self._proxies if p.healthy]) def total_count(self) -> int: return len(self._proxies) # ==================== 指纹管理器 ==================== class FingerprintManager: """浏览器指纹管理器""" def __init__(self, config: XiaohongshuConfig): self.config = config self._fingerprints = list(config.fingerprint_list) self._current_index = 0 self._shared = SessionFingerprintManager(strategy="sticky") if SessionFingerprintManager else None def get_fingerprint(self, session_id: str = "default") -> FingerprintConfig: """获取一个指纹""" if self._shared is not None: return FingerprintConfig.from_browser_profile(self._shared.get_fingerprint(f"xiaohongshu:{session_id}")) if not self._fingerprints: return FingerprintConfig.desktop_chrome() index = abs(hash(session_id)) % len(self._fingerprints) return self._fingerprints[index] def apply_to_context(self, context: BrowserContext, fingerprint: FingerprintConfig) -> None: """将指纹应用到浏览器上下文""" # 注意:viewport 在创建 context 时设置,这里只做额外配置 pass # ==================== 浏览器会话管理器 ==================== class XiaohongshuSession: """小红书浏览器会话 - 管理单个 IP+指纹 的会话""" def __init__( self, session_id: str, proxy_pool: ProxyPool, fingerprint_manager: FingerprintManager, config: XiaohongshuConfig, ): self.session_id = session_id self.proxy_pool = proxy_pool self.fingerprint_manager = fingerprint_manager self.config = config self._pw: Optional[Playwright] = None self._context: Optional[BrowserContext] = None self._page: Optional[Page] = None self._current_proxy: Optional[ProxyConfig] = None self._fingerprint: Optional[FingerprintConfig] = None self.request_count = 0 self.created_at = datetime.now() async def start(self) -> None: """启动会话(创建独立的浏览器上下文)""" configure_playwright_runtime_env() self._pw = await async_playwright().start() # 获取代理和指纹 self._current_proxy = self.proxy_pool.get_proxy() self._fingerprint = self.fingerprint_manager.get_fingerprint(self.session_id) # 构建 context 参数 context_options = { "viewport": self._fingerprint.viewport, "user_agent": self._fingerprint.user_agent, "locale": self._fingerprint.locale, "timezone_id": self._fingerprint.timezone_id, "color_scheme": self._fingerprint.color_scheme, "reduced_motion": self._fingerprint.reduced_motion, "device_scale_factor": self._fingerprint.device_scale_factor, "is_mobile": self._fingerprint.is_mobile, "has_touch": self._fingerprint.has_touch, "ignore_default_args": ["--enable-automation"], "args": [ "--disable-blink-features=AutomationControlled", ], } # 添加代理配置 if self._current_proxy: context_options["proxy"] = { "server": self._current_proxy.server, } if self._current_proxy.username: context_options["proxy"]["username"] = self._current_proxy.username context_options["proxy"]["password"] = self._current_proxy.password # 创建上下文 self._context = await self._pw.chromium.launch_persistent_context( user_data_dir=str(Path(self.config.profile_dir) / self.session_id), headless=False, # 小红书能检测无头模式,必须 False **context_options ) # 加载保存的 Cookie await self._load_cookies() self._page = self._context.pages[0] if self._context.pages else await self._context.new_page() async def _load_cookies(self) -> None: """加载保存的 Cookie""" cookie_path = Path(self.config.cookie_file) if cookie_path.exists(): try: cookies = json.loads(cookie_path.read_text(encoding="utf-8")) if isinstance(cookies, list): await self._context.add_cookies(cookies) except Exception: pass async def _save_cookies(self) -> None: """保存 Cookie 到文件""" if self._context: cookies = await self._context.cookies() Path(self.config.cookie_file).write_text( json.dumps(cookies, indent=2, ensure_ascii=False), encoding="utf-8" ) async def get_page(self) -> Page: """获取页面对象""" if self._page is None: await self.start() return self._page async def random_delay(self) -> None: """随机延迟(模拟真人节奏)""" delay = random.uniform(self.config.request_delay_min, self.config.request_delay_max) await asyncio.sleep(delay) self.request_count += 1 async def random_scroll(self) -> None: """随机滚动页面""" scroll_y = random.randint(220, 620) try: await self._page.mouse.move(random.randint(80, 900), random.randint(120, 700), steps=random.randint(8, 18)) await self._page.mouse.wheel(0, scroll_y) except Exception: await self._page.evaluate(f"window.scrollBy(0, {scroll_y})") await asyncio.sleep(random.uniform(self.config.scroll_delay_min, self.config.scroll_delay_max)) async def simulate_browsing(self, duration: float = 2.0) -> None: """模拟浏览行为(停留、思考)""" await asyncio.sleep(duration * random.uniform(0.7, 1.3)) async def close(self) -> None: """关闭会话""" if self._context: await self._save_cookies() await self._context.close() if self._pw: await self._pw.stop() self._page = None self._context = None self._pw = None @property def proxy_info(self) -> str: return self._current_proxy.server if self._current_proxy else "直连" # ==================== 小红书 API 封装 ==================== class XiaohongshuAPI: """小红书 API 封装(基于 Playwright)""" BASE_URL = "https://www.xiaohongshu.com" def __init__(self, session: XiaohongshuSession): self.session = session async def search(self, keyword: str, limit: int = 20) -> list[dict]: """搜索笔记""" page = await self.session.get_page() # 访问搜索页面 search_url = f"{self.BASE_URL}/search_result?keyword={quote(keyword)}" await page.goto(search_url, wait_until="domcontentloaded", timeout=45000) await self.session.random_delay() # 滚动加载 notes = [] last_count = 0 scrolls_without_new = 0 for scroll_idx in range(self.session.config.max_scrolls): # 等待笔记容器加载 try: await page.wait_for_selector(".note-item, .feeds-container .note", timeout=5000) except: pass # 提取当前页面的笔记 current_notes = await page.evaluate(""" () => { const items = document.querySelectorAll('.note-item, .feeds-container .note, section, a[href*="/explore/"]'); return Array.from(items).map(el => { const anchor = el.matches?.('a') ? el : ( el.querySelector('a[href*="/explore/"], a[href*="xhslink.com"], a') ); const href = anchor?.href || anchor?.getAttribute('href') || ''; const img = el.querySelector('img'); const id = el.getAttribute('data-note-id') || href.match(/\\/explore\\/([a-zA-Z0-9]+)/)?.[1] || href.match(/note_id[=:]([a-zA-Z0-9]+)/)?.[1] || ''; return { id, title: el.querySelector('.title, .note-title, [class*="title"]')?.innerText?.trim() || '', author: el.querySelector('.author .name, .nickname, [class*="nickname"]')?.innerText?.trim() || '', likes: el.querySelector('.like-count, .likes, [class*="like"]')?.innerText?.trim() || '0', coverUrl: img?.currentSrc || img?.src || img?.getAttribute('data-src') || '', url: href }; }); } """) # 去重合并 existing_ids = {n.get("id") for n in notes} for note in current_notes: note_url = _normalize_xhs_url(str(note.get("url") or "")) note_id = str(note.get("id") or _extract_note_id_from_text(note_url)) if note_id and note_id not in existing_ids: note["id"] = note_id note["url"] = note_url existing_ids.add(note_id) notes.append(note) if len(notes) >= limit: break if len(notes) >= limit: break # 滚动 if len(current_notes) == last_count: scrolls_without_new += 1 if scrolls_without_new >= 3: break else: scrolls_without_new = 0 last_count = len(current_notes) await self.session.random_scroll() await self.session.simulate_browsing(1.0) return notes[:limit] async def get_note_detail(self, note_id: str = "", note_url: str | None = None) -> dict: """获取笔记详情""" page = await self.session.get_page() url = note_url or f"{self.BASE_URL}/explore/{note_id}" url = _normalize_xhs_url(url) await page.goto(url, wait_until="domcontentloaded", timeout=45000) await self.session.random_delay() current_url = page.url if "/404" in urlparse(current_url).path: redirect_url = _extract_redirect_path(current_url) if redirect_url and redirect_url != current_url and redirect_url != url: await page.goto(redirect_url, wait_until="domcontentloaded", timeout=45000) await self.session.random_delay() current_url = page.url final_note_id = note_id or _extract_note_id_from_text(current_url) try: await page.wait_for_selector("img", timeout=10000) except Exception: pass # 提取详情 detail = await page.evaluate(""" () => { // 标题 const title = document.querySelector('.title, .note-title')?.innerText?.trim() || ''; // 作者 const author = document.querySelector('.author .name, .nickname')?.innerText?.trim() || ''; // 内容描述 const desc = document.querySelector('.desc, .note-content')?.innerText?.trim() || ''; // 点赞数 const likes = document.querySelector('.like-count, .likes')?.innerText?.trim() || '0'; // 收藏数 const collects = document.querySelector('.collect-count')?.innerText?.trim() || '0'; // 评论数 const comments = document.querySelector('.comment-count')?.innerText?.trim() || '0'; // 图片列表 const seen = new Set(); const images = Array.from(document.querySelectorAll('img')) .map(img => img.currentSrc || img.src || img.getAttribute('data-src') || '') .filter(src => /^https?:\\/\\//.test(src)) .filter(src => /xhscdn|sns-webpic|xiaohongshu/.test(src)) .filter(src => { if (seen.has(src)) return false; seen.add(src); return true; }); const seenVideos = new Set(); const videos = Array.from(document.querySelectorAll('video, video source, source')) .map(el => el.currentSrc || el.src || el.getAttribute('src') || '') .filter(src => /^https?:\\/\\//.test(src)) .filter(src => /xhscdn|sns-video|xiaohongshu|xhs/.test(src)) .filter(src => { if (seenVideos.has(src)) return false; seenVideos.add(src); return true; }); return { title, author, desc, likes, collects, comments, images, videos }; } """) if "/404" in urlparse(current_url).path and not detail.get("images") and not detail.get("videos"): raise PluginParseError( "小红书页面跳转到 404。请粘贴浏览器地址栏里的完整笔记链接," "尤其要保留 ?xsec_token=...&xsec_source=... 参数。" ) if not detail.get("images") and not detail.get("videos"): raise PluginParseError( "页面已打开,但没有解析到图片。请确认链接是图文笔记," "并优先粘贴带 xsec_token 的完整浏览器地址栏链接。" ) detail["id"] = final_note_id detail["url"] = current_url return detail async def get_note_images(self, note_id: str, note_url: str | None = None) -> list[str]: """获取笔记中的图片 URL""" detail = await self.get_note_detail(note_id, note_url=note_url) return detail.get("images", []) async def get_note_media(self, note_id: str, note_url: str | None = None) -> dict: detail = await self.get_note_detail(note_id, note_url=note_url) return {"images": detail.get("images", []), "videos": detail.get("videos", [])} async def download_image(self, url: str, save_path: Path) -> Optional[Path]: """下载图片""" return await self.download_media(url, save_path, media_type="image") async def download_media(self, url: str, save_path: Path, *, media_type: str = "image") -> Optional[Path]: import httpx page = await self.session.get_page() # 构造文件名 filename = url.split("/")[-1].split("?")[0] if media_type == "video" and not filename.endswith((".mp4", ".mov", ".webm", ".m4v")): filename = f"{filename}.mp4" elif media_type != "video" and not filename.endswith((".jpg", ".jpeg", ".png", ".webp")): filename = f"{filename}.jpg" target = save_path / filename # 检查是否已存在 if target.exists() and target.stat().st_size > 1024: return target # 使用浏览器上下文下载(保持会话) try: async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client: # 获取当前页面的 cookies cookies = await page.context.cookies() cookie_dict = {c["name"]: c["value"] for c in cookies} response = await client.get( url, headers={ "User-Agent": self.session._fingerprint.user_agent, "Referer": page.url if page and page.url != "about:blank" else self.BASE_URL, }, cookies=cookie_dict, ) if response.status_code == 200: target.write_bytes(response.content) return target except Exception: pass return None # ==================== 插件主类 ==================== class XiaohongshuPlugin(BasePlugin): """小红书爬虫插件""" name = "小红书" domain = "xiaohongshu.com" version = "1.0.0" def __init__(self): self.config = XiaohongshuConfig() self.config.profile_dir = str(runtime_path("xiaohongshu_profile")) self.config.cookie_file = str(runtime_path("xiaohongshu_cookies.json")) self.progress_callback: Optional[Callable[[str, float], None]] = None self._proxy_pool: Optional[ProxyPool] = None self._fingerprint_manager: Optional[FingerprintManager] = None self._sessions: dict[str, XiaohongshuSession] = {} self._session_lock = threading.Lock() def _progress(self, message: str, value: float) -> None: if self.progress_callback is None: return try: self.progress_callback(message, value) except Exception: pass def can_handle(self, url: str) -> bool: """判断是否能处理该 URL""" text = url.strip().lower() return "xiaohongshu.com" in text or "xhslink.com" in text or text.isdigit() def validate(self) -> bool: """检查依赖是否安装""" return PLAYWRIGHT_AVAILABLE def get_headers(self) -> dict[str, str]: """获取请求头""" return { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", "Accept": "application/json, text/plain, */*", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", } def set_proxies(self, proxy_list: list[dict]) -> None: """ 设置代理列表 proxy_list: [{"server": "http://ip:port", "username": "", "password": ""}, ...] """ self.config.proxy_enabled = True self.config.proxy_list = [ ProxyConfig( server=p.get("server"), username=p.get("username", ""), password=p.get("password", ""), ) for p in proxy_list ] self._proxy_pool = ProxyPool(self.config) def set_fingerprints(self, fingerprints: list[dict]) -> None: """设置指纹列表""" self.config.fingerprint_list = [ FingerprintConfig( user_agent=f.get("user_agent"), viewport=f.get("viewport", {"width": 1920, "height": 1080}), locale=f.get("locale", "zh-CN"), timezone_id=f.get("timezone_id", "Asia/Shanghai"), ) for f in fingerprints ] def _get_session(self, session_id: str = "default") -> XiaohongshuSession: """获取或创建会话(每个会话独立 IP+指纹)""" with self._session_lock: if session_id not in self._sessions: if self._proxy_pool is None: self._proxy_pool = ProxyPool(self.config) if self._fingerprint_manager is None: self._fingerprint_manager = FingerprintManager(self.config) session = XiaohongshuSession( session_id=session_id, proxy_pool=self._proxy_pool, fingerprint_manager=self._fingerprint_manager, config=self.config, ) self._sessions[session_id] = session return self._sessions[session_id] def _close_session(self, session_id: str) -> None: """关闭会话""" with self._session_lock: if session_id in self._sessions: # 异步关闭需要在 async 环境中处理,这里触发但不等待 asyncio.create_task(self._sessions[session_id].close()) del self._sessions[session_id] def parse(self, url: str) -> list[Resource]: """解析 URL 为资源列表""" if not self.validate(): raise PluginParseError("请先安装 Playwright: pip install playwright && playwright install chromium") try: return asyncio.run(self._parse_async(url)) except Exception as exc: raise PluginParseError(f"小红书解析失败: {exc}") from exc async def _parse_async(self, url: str) -> list[Resource]: """异步解析 URL""" self._progress("小红书:准备解析目标", 0.05) # 提取关键词或笔记 ID note_id = self._extract_note_id(url) note_url = self._extract_note_url(url, note_id=note_id) keyword = self._extract_keyword(url) session = self._get_session(f"parse_{int(time.time())}") try: self._progress("小红书:启动浏览器会话", 0.10) await session.start() api = XiaohongshuAPI(session) resources = [] if note_id or note_url: # 单篇笔记 self._progress("小红书:打开笔记页面", 0.22) detail = await api.get_note_detail(note_id or "", note_url=note_url) self._progress("小红书:提取笔记图片", 0.55) note_id = detail.get("id") or note_id or _extract_note_id_from_text(detail.get("url", "")) if not note_id: note_id = f"note_{int(time.time())}" images = detail.get("images", []) videos = detail.get("videos", []) resource = Resource( id=note_id, url=detail.get("url") or note_url or f"https://www.xiaohongshu.com/explore/{note_id}", title=detail.get("title", "")[:200], author=detail.get("author", ""), author_id="", files=[{"url": video, "type": "video"} for video in videos] + [{"url": img, "type": "image"} for img in images], metadata=detail, thumbnail=images[0] if images else None, created_at="", ) resources.append(resource) elif keyword: # 关键词搜索 self._progress("小红书:打开搜索结果页", 0.18) notes = await api.search(keyword, limit=min(self.config.max_notes_per_session, 30)) self._progress(f"小红书:整理搜索结果({len(notes)} 条)", 0.62) for note in notes: note_url = _normalize_xhs_url(note.get("url", "")) note_id = note.get("id") or _extract_note_id_from_text(note_url) if not note_id: continue # 只返回元数据,实际图片在 download 时下载 resource = Resource( id=note_id, url=note_url, title=note.get("title", "")[:200], author=note.get("author", ""), author_id="", files=[], # 延迟下载 metadata={ "kind": "search_result", "keyword": keyword, "note_id": note_id, "note_url": note_url, "likes": note.get("likes"), "cover_url": note.get("coverUrl"), }, thumbnail=note.get("coverUrl"), created_at="", ) resources.append(resource) self._progress(f"小红书:解析完成({len(resources)} 个资源)", 0.72) return resources finally: await session.close() def download(self, resource: Resource, save_path: Path) -> list[Path]: """下载资源""" try: return asyncio.run(self._download_async(resource, save_path)) except Exception as exc: raise PluginParseError(f"小红书下载失败: {exc}") from exc async def _download_async(self, resource: Resource, save_path: Path) -> list[Path]: """异步下载""" self._progress(f"小红书:准备下载 {resource.id}", 0.0) session = self._get_session(f"download_{resource.id}_{int(time.time())}") try: self._progress(f"小红书:启动下载会话 {resource.id}", 0.10) await session.start() api = XiaohongshuAPI(session) # 创建保存目录 subdir = resource.metadata.get("keyword", resource.id) target_dir = save_path / "xiaohongshu" / subdir target_dir.mkdir(parents=True, exist_ok=True) downloaded = [] # 如果有直接的文件列表 if resource.files: if resource.url: try: self._progress(f"小红书:打开笔记页 {resource.id}", 0.20) page = await session.get_page() await page.goto(_normalize_xhs_url(resource.url), wait_until="domcontentloaded", timeout=45000) await session.random_delay() except Exception: pass total_files = max(len(resource.files), 1) for item in resource.files: file_url = item.get("url") if file_url: self._progress(f"小红书:下载图片 {len(downloaded) + 1}/{total_files}", 0.35 + (len(downloaded) / total_files) * 0.55) media_type = "video" if item.get("type") == "video" else "image" path = await api.download_media(file_url, target_dir, media_type=media_type) if path: downloaded.append(path) await session.random_delay() # 如果是搜索结果,需要先获取详情再下载 elif resource.metadata.get("kind") == "search_result": note_id = resource.metadata.get("note_id") note_url = _normalize_xhs_url(str(resource.metadata.get("note_url") or resource.url or "")) if note_id or note_url: self._progress(f"小红书:进入笔记详情 {resource.id}", 0.20) media = await api.get_note_media(note_id, note_url=note_url) media_items = [{"url": video, "type": "video"} for video in media.get("videos", [])] + [ {"url": image, "type": "image"} for image in media.get("images", []) ] total_files = max(len(media_items), 1) for media_item in media_items: img_url = media_item["url"] self._progress(f"小红书:下载图片 {len(downloaded) + 1}/{total_files}", 0.35 + (len(downloaded) / total_files) * 0.55) path = await api.download_media(img_url, target_dir, media_type=media_item["type"]) if path: downloaded.append(path) await session.random_delay() self._progress(f"小红书:资源下载完成 {resource.id}", 0.96) return downloaded finally: await session.close() def _extract_note_id(self, url: str) -> Optional[str]: """提取笔记 ID""" # 标准链接: xiaohongshu.com/explore/xxxxx patterns = [ r"xiaohongshu\.com/explore/([a-zA-Z0-9]+)", r"note_id[=:]([a-zA-Z0-9]+)", ] for pattern in patterns: match = re.search(pattern, url, re.IGNORECASE) if match: return match.group(1) # 纯数字 ID if url.strip().isdigit(): return url.strip() return None def _extract_note_url(self, url: str, *, note_id: str | None = None) -> Optional[str]: """提取或还原单篇笔记访问 URL,尽量保留 xsec_token 等安全参数。""" text = url.strip() if not text: return None lower = text.lower() if "xiaohongshu.com/404" in lower: redirect_url = _extract_redirect_path(text) if redirect_url: return redirect_url if "xiaohongshu.com" in lower or "xhslink.com" in lower: if text.startswith("//"): return "https:" + text if "://" not in text: return "https://" + text.lstrip("/") return text if note_id: return f"https://www.xiaohongshu.com/explore/{note_id}" return None def _extract_keyword(self, url: str) -> Optional[str]: """提取搜索关键词""" # 搜索链接: xiaohongshu.com/search_result?keyword=xxx match = re.search(r"keyword=([^&]+)", url) if match: return unquote(match.group(1)) if url.strip() and "xiaohongshu.com" not in url.lower() and "xhslink.com" not in url.lower(): return url.strip() return None def get_proxy_stats(self) -> dict: """获取代理池统计""" if self._proxy_pool: return { "total": self._proxy_pool.total_count(), "healthy": self._proxy_pool.healthy_count(), } return {"total": 0, "healthy": 0} def rotate_session(self, session_id: str = "default") -> None: """强制轮换会话(换 IP + 换指纹)""" self._close_session(session_id) def run_interactive_login(self, proceed: Optional[threading.Event] = None) -> bool: """交互式登录""" if not self.validate(): return False async def _login(): session = XiaohongshuSession( session_id="login", proxy_pool=self._proxy_pool or ProxyPool(self.config), fingerprint_manager=self._fingerprint_manager or FingerprintManager(self.config), config=self.config, ) try: await session.start() page = await session.get_page() await page.goto("https://www.xiaohongshu.com", wait_until="networkidle") if proceed: proceed.wait() else: input("\n在浏览器中完成小红书登录后,按 Enter 继续...\n") await session._save_cookies() return True finally: await session.close() return asyncio.run(_login()) # 导出插件类 plugin_class = XiaohongshuPlugin