Files
2026-11-16 20:45:22 +08:00

1022 lines
39 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
# plugins/xiaohongshu/plugin.py
"""
小红书爬虫插件 - 核心功能
- Playwright 浏览器自动化
- 动态代理池轮换
- 浏览器指纹伪装
- 行为节奏模拟
"""
from __future__ import annotations
import asyncio
import json
import random
import re
import threading
import time
from dataclasses import dataclass, field
from datetime import datetime
from pathlib import Path
from typing import Any, Callable, Optional
from urllib.parse import parse_qs, quote, unquote, urlparse
# 动态加载 playwright
try:
from playwright.async_api import BrowserContext, Page, Playwright, async_playwright
PLAYWRIGHT_AVAILABLE = True
except ImportError:
PLAYWRIGHT_AVAILABLE = False
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
from pixiv_app.core.paths import runtime_path
from pixiv_app.core.playwright_runtime import (
configure_playwright_runtime_env,
explain_missing_browser,
preferred_chromium_executable,
)
try:
from pixiv_app.core.fingerprint import BrowserProfile, SessionFingerprintManager
except Exception: # pragma: no cover
BrowserProfile = None
SessionFingerprintManager = None
def _extract_note_id_from_text(text: str) -> str:
patterns = [
r"xiaohongshu\.com/explore/([a-zA-Z0-9]+)",
r"/explore/([a-zA-Z0-9]+)",
r"note_id[=:]([a-zA-Z0-9]+)",
]
for pattern in patterns:
match = re.search(pattern, text or "", re.IGNORECASE)
if match:
return match.group(1)
return ""
def _extract_redirect_path(url: str) -> str:
try:
query = parse_qs(urlparse(url).query)
value = (query.get("redirectPath") or [""])[0]
except Exception:
value = ""
value = unquote(value or "").strip()
if not value:
return ""
if value.startswith("//"):
return "https:" + value
if value.startswith("/"):
return "https://www.xiaohongshu.com" + value
if "xiaohongshu.com" in value.lower() or "xhslink.com" in value.lower():
return value if "://" in value else "https://" + value.lstrip("/")
return ""
def _normalize_xhs_url(url: str) -> str:
value = (url or "").strip()
if not value:
return ""
if value.startswith("//"):
return "https:" + value
if value.startswith("/"):
return "https://www.xiaohongshu.com" + value
lower = value.lower()
if ("xiaohongshu.com" in lower or "xhslink.com" in lower) and "://" not in value:
return "https://" + value.lstrip("/")
return value
# ==================== 配置定义 ====================
@dataclass
class ProxyConfig:
"""代理配置"""
server: str # http://ip:port 或 socks5://ip:port
username: str = ""
password: str = ""
failures: int = 0
successes: int = 0
latency_ms: float = 0.0
healthy: bool = True
@property
def url(self) -> str:
return self.server
@property
def score(self) -> float:
"""计算代理评分(用于加权选择)"""
if not self.healthy:
return 0.0
total = self.successes + self.failures
if total == 0:
return 1.0
success_rate = self.successes / total
latency_penalty = max(0.1, min(1.0, 1000.0 / max(self.latency_ms, 1.0)))
return success_rate * latency_penalty
@dataclass
class FingerprintConfig:
"""浏览器指纹配置"""
user_agent: str
viewport: dict[str, int]
locale: str = "zh-CN"
timezone_id: str = "Asia/Shanghai"
color_scheme: str = "light"
reduced_motion: str = "no-preference"
device_scale_factor: float = 1.0
is_mobile: bool = False
has_touch: bool = False
@classmethod
def from_browser_profile(cls, profile: Any) -> "FingerprintConfig":
return cls(
user_agent=profile.user_agent,
viewport=dict(profile.viewport),
locale=profile.locale,
timezone_id=profile.timezone_id,
color_scheme=profile.color_scheme,
reduced_motion=profile.reduced_motion,
device_scale_factor=profile.device_scale_factor,
is_mobile=profile.is_mobile,
has_touch=profile.has_touch,
)
@classmethod
def desktop_chrome(cls) -> "FingerprintConfig":
return cls(
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
viewport={"width": 1920, "height": 1080},
device_scale_factor=1.0,
)
@classmethod
def desktop_edge(cls) -> "FingerprintConfig":
return cls(
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
viewport={"width": 1536, "height": 864},
device_scale_factor=1.25,
)
@classmethod
def mac_chrome(cls) -> "FingerprintConfig":
return cls(
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
viewport={"width": 1512, "height": 982},
device_scale_factor=2.0,
)
@classmethod
def mobile_ios(cls) -> "FingerprintConfig":
return cls(
user_agent="Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",
viewport={"width": 390, "height": 844},
device_scale_factor=3.0,
is_mobile=True,
has_touch=True,
)
@dataclass
class XiaohongshuConfig:
"""小红书插件配置"""
# 代理配置
proxy_enabled: bool = False
proxy_list: list[ProxyConfig] = field(default_factory=list)
proxy_rotation: str = "round_robin" # round_robin, random, session_sticky
# 指纹配置
fingerprint_list: list[FingerprintConfig] = field(default_factory=lambda: [
FingerprintConfig.desktop_chrome(),
FingerprintConfig.desktop_edge(),
FingerprintConfig.mac_chrome(),
])
# 行为控制
request_delay_min: float = 3.0 # 最小延迟(秒)
request_delay_max: float = 8.0 # 最大延迟(秒)
scroll_delay_min: float = 0.5 # 滚动延迟(秒)
scroll_delay_max: float = 1.5
batch_pause_min: float = 5.0 # 批次暂停(秒)
batch_pause_max: float = 12.0
max_scrolls: int = 15 # 最大滚动次数
# 采集限制
max_notes_per_session: int = 50 # 单会话最大笔记数
max_concurrent: int = 2 # 最大并发数
# 会话持久化
profile_dir: str = "runtime/xiaohongshu_profile"
cookie_file: str = "runtime/xiaohongshu_cookies.json"
# ==================== 代理池管理器 ====================
class ProxyPool:
"""代理池管理器 - 支持轮换和健康检查"""
def __init__(self, config: XiaohongshuConfig):
self.config = config
self._proxies: list[ProxyConfig] = list(config.proxy_list)
self._current_index = 0
self._lock = threading.Lock()
def add_proxy(self, proxy: ProxyConfig) -> None:
"""添加代理"""
with self._lock:
# 去重
existing = {p.server for p in self._proxies}
if proxy.server not in existing:
self._proxies.append(proxy)
def get_proxy(self) -> Optional[ProxyConfig]:
"""获取一个代理(按策略)"""
if not self.config.proxy_enabled or not self._proxies:
return None
with self._lock:
healthy = [p for p in self._proxies if p.healthy]
if not healthy:
return None
if self.config.proxy_rotation == "random":
return random.choice(healthy)
# round_robin
proxy = healthy[self._current_index % len(healthy)]
self._current_index += 1
return proxy
def mark_success(self, proxy: Optional[ProxyConfig], latency_ms: float = 0) -> None:
"""标记代理成功"""
if proxy is None:
return
with self._lock:
proxy.successes += 1
proxy.failures = 0
proxy.healthy = True
if latency_ms > 0:
proxy.latency_ms = latency_ms
def mark_failed(self, proxy: Optional[ProxyConfig]) -> None:
"""标记代理失败"""
if proxy is None:
return
with self._lock:
proxy.failures += 1
if proxy.failures >= 3:
proxy.healthy = False
def healthy_count(self) -> int:
"""可用代理数量"""
with self._lock:
return len([p for p in self._proxies if p.healthy])
def total_count(self) -> int:
return len(self._proxies)
# ==================== 指纹管理器 ====================
class FingerprintManager:
"""浏览器指纹管理器"""
def __init__(self, config: XiaohongshuConfig):
self.config = config
self._fingerprints = list(config.fingerprint_list)
self._current_index = 0
self._shared = SessionFingerprintManager(strategy="sticky") if SessionFingerprintManager else None
def get_fingerprint(self, session_id: str = "default") -> FingerprintConfig:
"""获取一个指纹"""
if self._shared is not None:
return FingerprintConfig.from_browser_profile(self._shared.get_fingerprint(f"xiaohongshu:{session_id}"))
if not self._fingerprints:
return FingerprintConfig.desktop_chrome()
index = abs(hash(session_id)) % len(self._fingerprints)
return self._fingerprints[index]
def apply_to_context(self, context: BrowserContext, fingerprint: FingerprintConfig) -> None:
"""将指纹应用到浏览器上下文"""
# 注意:viewport 在创建 context 时设置,这里只做额外配置
pass
# ==================== 浏览器会话管理器 ====================
class XiaohongshuSession:
"""小红书浏览器会话 - 管理单个 IP+指纹 的会话"""
def __init__(
self,
session_id: str,
proxy_pool: ProxyPool,
fingerprint_manager: FingerprintManager,
config: XiaohongshuConfig,
):
self.session_id = session_id
self.proxy_pool = proxy_pool
self.fingerprint_manager = fingerprint_manager
self.config = config
self._pw: Optional[Playwright] = None
self._context: Optional[BrowserContext] = None
self._page: Optional[Page] = None
self._current_proxy: Optional[ProxyConfig] = None
self._fingerprint: Optional[FingerprintConfig] = None
self.request_count = 0
self.created_at = datetime.now()
async def start(self) -> None:
"""启动会话(创建独立的浏览器上下文)"""
configure_playwright_runtime_env()
self._pw = await async_playwright().start()
# 获取代理和指纹
self._current_proxy = self.proxy_pool.get_proxy()
self._fingerprint = self.fingerprint_manager.get_fingerprint(self.session_id)
# 构建 context 参数
context_options = {
"viewport": self._fingerprint.viewport,
"user_agent": self._fingerprint.user_agent,
"locale": self._fingerprint.locale,
"timezone_id": self._fingerprint.timezone_id,
"color_scheme": self._fingerprint.color_scheme,
"reduced_motion": self._fingerprint.reduced_motion,
"device_scale_factor": self._fingerprint.device_scale_factor,
"is_mobile": self._fingerprint.is_mobile,
"has_touch": self._fingerprint.has_touch,
"ignore_default_args": ["--enable-automation"],
"args": [
"--disable-blink-features=AutomationControlled",
],
}
# 添加代理配置
if self._current_proxy:
context_options["proxy"] = {
"server": self._current_proxy.server,
}
if self._current_proxy.username:
context_options["proxy"]["username"] = self._current_proxy.username
context_options["proxy"]["password"] = self._current_proxy.password
# 创建上下文
self._context = await self._pw.chromium.launch_persistent_context(
user_data_dir=str(Path(self.config.profile_dir) / self.session_id),
headless=False, # 小红书能检测无头模式,必须 False
**context_options
)
# 加载保存的 Cookie
await self._load_cookies()
self._page = self._context.pages[0] if self._context.pages else await self._context.new_page()
async def _load_cookies(self) -> None:
"""加载保存的 Cookie"""
cookie_path = Path(self.config.cookie_file)
if cookie_path.exists():
try:
cookies = json.loads(cookie_path.read_text(encoding="utf-8"))
if isinstance(cookies, list):
await self._context.add_cookies(cookies)
except Exception:
pass
async def _save_cookies(self) -> None:
"""保存 Cookie 到文件"""
if self._context:
cookies = await self._context.cookies()
Path(self.config.cookie_file).write_text(
json.dumps(cookies, indent=2, ensure_ascii=False),
encoding="utf-8"
)
async def get_page(self) -> Page:
"""获取页面对象"""
if self._page is None:
await self.start()
return self._page
async def random_delay(self) -> None:
"""随机延迟(模拟真人节奏)"""
delay = random.uniform(self.config.request_delay_min, self.config.request_delay_max)
await asyncio.sleep(delay)
self.request_count += 1
async def random_scroll(self) -> None:
"""随机滚动页面"""
scroll_y = random.randint(220, 620)
try:
await self._page.mouse.move(random.randint(80, 900), random.randint(120, 700), steps=random.randint(8, 18))
await self._page.mouse.wheel(0, scroll_y)
except Exception:
await self._page.evaluate(f"window.scrollBy(0, {scroll_y})")
await asyncio.sleep(random.uniform(self.config.scroll_delay_min, self.config.scroll_delay_max))
async def simulate_browsing(self, duration: float = 2.0) -> None:
"""模拟浏览行为(停留、思考)"""
await asyncio.sleep(duration * random.uniform(0.7, 1.3))
async def close(self) -> None:
"""关闭会话"""
if self._context:
await self._save_cookies()
await self._context.close()
if self._pw:
await self._pw.stop()
self._page = None
self._context = None
self._pw = None
@property
def proxy_info(self) -> str:
return self._current_proxy.server if self._current_proxy else "直连"
# ==================== 小红书 API 封装 ====================
class XiaohongshuAPI:
"""小红书 API 封装(基于 Playwright)"""
BASE_URL = "https://www.xiaohongshu.com"
def __init__(self, session: XiaohongshuSession):
self.session = session
async def search(self, keyword: str, limit: int = 20) -> list[dict]:
"""搜索笔记"""
page = await self.session.get_page()
# 访问搜索页面
search_url = f"{self.BASE_URL}/search_result?keyword={quote(keyword)}"
await page.goto(search_url, wait_until="domcontentloaded", timeout=45000)
await self.session.random_delay()
# 滚动加载
notes = []
last_count = 0
scrolls_without_new = 0
for scroll_idx in range(self.session.config.max_scrolls):
# 等待笔记容器加载
try:
await page.wait_for_selector(".note-item, .feeds-container .note", timeout=5000)
except:
pass
# 提取当前页面的笔记
current_notes = await page.evaluate("""
() => {
const items = document.querySelectorAll('.note-item, .feeds-container .note, section, a[href*="/explore/"]');
return Array.from(items).map(el => {
const anchor = el.matches?.('a') ? el : (
el.querySelector('a[href*="/explore/"], a[href*="xhslink.com"], a')
);
const href = anchor?.href || anchor?.getAttribute('href') || '';
const img = el.querySelector('img');
const id = el.getAttribute('data-note-id') ||
href.match(/\\/explore\\/([a-zA-Z0-9]+)/)?.[1] ||
href.match(/note_id[=:]([a-zA-Z0-9]+)/)?.[1] ||
'';
return {
id,
title: el.querySelector('.title, .note-title, [class*="title"]')?.innerText?.trim() || '',
author: el.querySelector('.author .name, .nickname, [class*="nickname"]')?.innerText?.trim() || '',
likes: el.querySelector('.like-count, .likes, [class*="like"]')?.innerText?.trim() || '0',
coverUrl: img?.currentSrc || img?.src || img?.getAttribute('data-src') || '',
url: href
};
});
}
""")
# 去重合并
existing_ids = {n.get("id") for n in notes}
for note in current_notes:
note_url = _normalize_xhs_url(str(note.get("url") or ""))
note_id = str(note.get("id") or _extract_note_id_from_text(note_url))
if note_id and note_id not in existing_ids:
note["id"] = note_id
note["url"] = note_url
existing_ids.add(note_id)
notes.append(note)
if len(notes) >= limit:
break
if len(notes) >= limit:
break
# 滚动
if len(current_notes) == last_count:
scrolls_without_new += 1
if scrolls_without_new >= 3:
break
else:
scrolls_without_new = 0
last_count = len(current_notes)
await self.session.random_scroll()
await self.session.simulate_browsing(1.0)
return notes[:limit]
async def get_note_detail(self, note_id: str = "", note_url: str | None = None) -> dict:
"""获取笔记详情"""
page = await self.session.get_page()
url = note_url or f"{self.BASE_URL}/explore/{note_id}"
url = _normalize_xhs_url(url)
await page.goto(url, wait_until="domcontentloaded", timeout=45000)
await self.session.random_delay()
current_url = page.url
if "/404" in urlparse(current_url).path:
redirect_url = _extract_redirect_path(current_url)
if redirect_url and redirect_url != current_url and redirect_url != url:
await page.goto(redirect_url, wait_until="domcontentloaded", timeout=45000)
await self.session.random_delay()
current_url = page.url
final_note_id = note_id or _extract_note_id_from_text(current_url)
try:
await page.wait_for_selector("img", timeout=10000)
except Exception:
pass
# 提取详情
detail = await page.evaluate("""
() => {
// 标题
const title = document.querySelector('.title, .note-title')?.innerText?.trim() || '';
// 作者
const author = document.querySelector('.author .name, .nickname')?.innerText?.trim() || '';
// 内容描述
const desc = document.querySelector('.desc, .note-content')?.innerText?.trim() || '';
// 点赞数
const likes = document.querySelector('.like-count, .likes')?.innerText?.trim() || '0';
// 收藏数
const collects = document.querySelector('.collect-count')?.innerText?.trim() || '0';
// 评论数
const comments = document.querySelector('.comment-count')?.innerText?.trim() || '0';
// 图片列表
const seen = new Set();
const images = Array.from(document.querySelectorAll('img'))
.map(img => img.currentSrc || img.src || img.getAttribute('data-src') || '')
.filter(src => /^https?:\\/\\//.test(src))
.filter(src => /xhscdn|sns-webpic|xiaohongshu/.test(src))
.filter(src => {
if (seen.has(src)) return false;
seen.add(src);
return true;
});
const seenVideos = new Set();
const videos = Array.from(document.querySelectorAll('video, video source, source'))
.map(el => el.currentSrc || el.src || el.getAttribute('src') || '')
.filter(src => /^https?:\\/\\//.test(src))
.filter(src => /xhscdn|sns-video|xiaohongshu|xhs/.test(src))
.filter(src => {
if (seenVideos.has(src)) return false;
seenVideos.add(src);
return true;
});
return { title, author, desc, likes, collects, comments, images, videos };
}
""")
if "/404" in urlparse(current_url).path and not detail.get("images") and not detail.get("videos"):
raise PluginParseError(
"小红书页面跳转到 404。请粘贴浏览器地址栏里的完整笔记链接,"
"尤其要保留 ?xsec_token=...&xsec_source=... 参数。"
)
if not detail.get("images") and not detail.get("videos"):
raise PluginParseError(
"页面已打开,但没有解析到图片。请确认链接是图文笔记,"
"并优先粘贴带 xsec_token 的完整浏览器地址栏链接。"
)
detail["id"] = final_note_id
detail["url"] = current_url
return detail
async def get_note_images(self, note_id: str, note_url: str | None = None) -> list[str]:
"""获取笔记中的图片 URL"""
detail = await self.get_note_detail(note_id, note_url=note_url)
return detail.get("images", [])
async def get_note_media(self, note_id: str, note_url: str | None = None) -> dict:
detail = await self.get_note_detail(note_id, note_url=note_url)
return {"images": detail.get("images", []), "videos": detail.get("videos", [])}
async def download_image(self, url: str, save_path: Path) -> Optional[Path]:
"""下载图片"""
return await self.download_media(url, save_path, media_type="image")
async def download_media(self, url: str, save_path: Path, *, media_type: str = "image") -> Optional[Path]:
import httpx
page = await self.session.get_page()
# 构造文件名
filename = url.split("/")[-1].split("?")[0]
if media_type == "video" and not filename.endswith((".mp4", ".mov", ".webm", ".m4v")):
filename = f"{filename}.mp4"
elif media_type != "video" and not filename.endswith((".jpg", ".jpeg", ".png", ".webp")):
filename = f"{filename}.jpg"
target = save_path / filename
# 检查是否已存在
if target.exists() and target.stat().st_size > 1024:
return target
# 使用浏览器上下文下载(保持会话)
try:
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
# 获取当前页面的 cookies
cookies = await page.context.cookies()
cookie_dict = {c["name"]: c["value"] for c in cookies}
response = await client.get(
url,
headers={
"User-Agent": self.session._fingerprint.user_agent,
"Referer": page.url if page and page.url != "about:blank" else self.BASE_URL,
},
cookies=cookie_dict,
)
if response.status_code == 200:
target.write_bytes(response.content)
return target
except Exception:
pass
return None
# ==================== 插件主类 ====================
class XiaohongshuPlugin(BasePlugin):
"""小红书爬虫插件"""
name = "小红书"
domain = "xiaohongshu.com"
version = "1.0.0"
def __init__(self):
self.config = XiaohongshuConfig()
self.config.profile_dir = str(runtime_path("xiaohongshu_profile"))
self.config.cookie_file = str(runtime_path("xiaohongshu_cookies.json"))
self.progress_callback: Optional[Callable[[str, float], None]] = None
self._proxy_pool: Optional[ProxyPool] = None
self._fingerprint_manager: Optional[FingerprintManager] = None
self._sessions: dict[str, XiaohongshuSession] = {}
self._session_lock = threading.Lock()
def _progress(self, message: str, value: float) -> None:
if self.progress_callback is None:
return
try:
self.progress_callback(message, value)
except Exception:
pass
def can_handle(self, url: str) -> bool:
"""判断是否能处理该 URL"""
text = url.strip().lower()
return "xiaohongshu.com" in text or "xhslink.com" in text or text.isdigit()
def validate(self) -> bool:
"""检查依赖是否安装"""
return PLAYWRIGHT_AVAILABLE
def get_headers(self) -> dict[str, str]:
"""获取请求头"""
return {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Accept": "application/json, text/plain, */*",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
def set_proxies(self, proxy_list: list[dict]) -> None:
"""
设置代理列表
proxy_list: [{"server": "http://ip:port", "username": "", "password": ""}, ...]
"""
self.config.proxy_enabled = True
self.config.proxy_list = [
ProxyConfig(
server=p.get("server"),
username=p.get("username", ""),
password=p.get("password", ""),
)
for p in proxy_list
]
self._proxy_pool = ProxyPool(self.config)
def set_fingerprints(self, fingerprints: list[dict]) -> None:
"""设置指纹列表"""
self.config.fingerprint_list = [
FingerprintConfig(
user_agent=f.get("user_agent"),
viewport=f.get("viewport", {"width": 1920, "height": 1080}),
locale=f.get("locale", "zh-CN"),
timezone_id=f.get("timezone_id", "Asia/Shanghai"),
)
for f in fingerprints
]
def _get_session(self, session_id: str = "default") -> XiaohongshuSession:
"""获取或创建会话(每个会话独立 IP+指纹)"""
with self._session_lock:
if session_id not in self._sessions:
if self._proxy_pool is None:
self._proxy_pool = ProxyPool(self.config)
if self._fingerprint_manager is None:
self._fingerprint_manager = FingerprintManager(self.config)
session = XiaohongshuSession(
session_id=session_id,
proxy_pool=self._proxy_pool,
fingerprint_manager=self._fingerprint_manager,
config=self.config,
)
self._sessions[session_id] = session
return self._sessions[session_id]
def _close_session(self, session_id: str) -> None:
"""关闭会话"""
with self._session_lock:
if session_id in self._sessions:
# 异步关闭需要在 async 环境中处理,这里触发但不等待
asyncio.create_task(self._sessions[session_id].close())
del self._sessions[session_id]
def parse(self, url: str) -> list[Resource]:
"""解析 URL 为资源列表"""
if not self.validate():
raise PluginParseError("请先安装 Playwright: pip install playwright && playwright install chromium")
try:
return asyncio.run(self._parse_async(url))
except Exception as exc:
raise PluginParseError(f"小红书解析失败: {exc}") from exc
async def _parse_async(self, url: str) -> list[Resource]:
"""异步解析 URL"""
self._progress("小红书:准备解析目标", 0.05)
# 提取关键词或笔记 ID
note_id = self._extract_note_id(url)
note_url = self._extract_note_url(url, note_id=note_id)
keyword = self._extract_keyword(url)
session = self._get_session(f"parse_{int(time.time())}")
try:
self._progress("小红书:启动浏览器会话", 0.10)
await session.start()
api = XiaohongshuAPI(session)
resources = []
if note_id or note_url:
# 单篇笔记
self._progress("小红书:打开笔记页面", 0.22)
detail = await api.get_note_detail(note_id or "", note_url=note_url)
self._progress("小红书:提取笔记图片", 0.55)
note_id = detail.get("id") or note_id or _extract_note_id_from_text(detail.get("url", ""))
if not note_id:
note_id = f"note_{int(time.time())}"
images = detail.get("images", [])
videos = detail.get("videos", [])
resource = Resource(
id=note_id,
url=detail.get("url") or note_url or f"https://www.xiaohongshu.com/explore/{note_id}",
title=detail.get("title", "")[:200],
author=detail.get("author", ""),
author_id="",
files=[{"url": video, "type": "video"} for video in videos]
+ [{"url": img, "type": "image"} for img in images],
metadata=detail,
thumbnail=images[0] if images else None,
created_at="",
)
resources.append(resource)
elif keyword:
# 关键词搜索
self._progress("小红书:打开搜索结果页", 0.18)
notes = await api.search(keyword, limit=min(self.config.max_notes_per_session, 30))
self._progress(f"小红书:整理搜索结果({len(notes)} 条)", 0.62)
for note in notes:
note_url = _normalize_xhs_url(note.get("url", ""))
note_id = note.get("id") or _extract_note_id_from_text(note_url)
if not note_id:
continue
# 只返回元数据,实际图片在 download 时下载
resource = Resource(
id=note_id,
url=note_url,
title=note.get("title", "")[:200],
author=note.get("author", ""),
author_id="",
files=[], # 延迟下载
metadata={
"kind": "search_result",
"keyword": keyword,
"note_id": note_id,
"note_url": note_url,
"likes": note.get("likes"),
"cover_url": note.get("coverUrl"),
},
thumbnail=note.get("coverUrl"),
created_at="",
)
resources.append(resource)
self._progress(f"小红书:解析完成({len(resources)} 个资源)", 0.72)
return resources
finally:
await session.close()
def download(self, resource: Resource, save_path: Path) -> list[Path]:
"""下载资源"""
try:
return asyncio.run(self._download_async(resource, save_path))
except Exception as exc:
raise PluginParseError(f"小红书下载失败: {exc}") from exc
async def _download_async(self, resource: Resource, save_path: Path) -> list[Path]:
"""异步下载"""
self._progress(f"小红书:准备下载 {resource.id}", 0.0)
session = self._get_session(f"download_{resource.id}_{int(time.time())}")
try:
self._progress(f"小红书:启动下载会话 {resource.id}", 0.10)
await session.start()
api = XiaohongshuAPI(session)
# 创建保存目录
subdir = resource.metadata.get("keyword", resource.id)
target_dir = save_path / "xiaohongshu" / subdir
target_dir.mkdir(parents=True, exist_ok=True)
downloaded = []
# 如果有直接的文件列表
if resource.files:
if resource.url:
try:
self._progress(f"小红书:打开笔记页 {resource.id}", 0.20)
page = await session.get_page()
await page.goto(_normalize_xhs_url(resource.url), wait_until="domcontentloaded", timeout=45000)
await session.random_delay()
except Exception:
pass
total_files = max(len(resource.files), 1)
for item in resource.files:
file_url = item.get("url")
if file_url:
self._progress(f"小红书:下载图片 {len(downloaded) + 1}/{total_files}", 0.35 + (len(downloaded) / total_files) * 0.55)
media_type = "video" if item.get("type") == "video" else "image"
path = await api.download_media(file_url, target_dir, media_type=media_type)
if path:
downloaded.append(path)
await session.random_delay()
# 如果是搜索结果,需要先获取详情再下载
elif resource.metadata.get("kind") == "search_result":
note_id = resource.metadata.get("note_id")
note_url = _normalize_xhs_url(str(resource.metadata.get("note_url") or resource.url or ""))
if note_id or note_url:
self._progress(f"小红书:进入笔记详情 {resource.id}", 0.20)
media = await api.get_note_media(note_id, note_url=note_url)
media_items = [{"url": video, "type": "video"} for video in media.get("videos", [])] + [
{"url": image, "type": "image"} for image in media.get("images", [])
]
total_files = max(len(media_items), 1)
for media_item in media_items:
img_url = media_item["url"]
self._progress(f"小红书:下载图片 {len(downloaded) + 1}/{total_files}", 0.35 + (len(downloaded) / total_files) * 0.55)
path = await api.download_media(img_url, target_dir, media_type=media_item["type"])
if path:
downloaded.append(path)
await session.random_delay()
self._progress(f"小红书:资源下载完成 {resource.id}", 0.96)
return downloaded
finally:
await session.close()
def _extract_note_id(self, url: str) -> Optional[str]:
"""提取笔记 ID"""
# 标准链接: xiaohongshu.com/explore/xxxxx
patterns = [
r"xiaohongshu\.com/explore/([a-zA-Z0-9]+)",
r"note_id[=:]([a-zA-Z0-9]+)",
]
for pattern in patterns:
match = re.search(pattern, url, re.IGNORECASE)
if match:
return match.group(1)
# 纯数字 ID
if url.strip().isdigit():
return url.strip()
return None
def _extract_note_url(self, url: str, *, note_id: str | None = None) -> Optional[str]:
"""提取或还原单篇笔记访问 URL,尽量保留 xsec_token 等安全参数。"""
text = url.strip()
if not text:
return None
lower = text.lower()
if "xiaohongshu.com/404" in lower:
redirect_url = _extract_redirect_path(text)
if redirect_url:
return redirect_url
if "xiaohongshu.com" in lower or "xhslink.com" in lower:
if text.startswith("//"):
return "https:" + text
if "://" not in text:
return "https://" + text.lstrip("/")
return text
if note_id:
return f"https://www.xiaohongshu.com/explore/{note_id}"
return None
def _extract_keyword(self, url: str) -> Optional[str]:
"""提取搜索关键词"""
# 搜索链接: xiaohongshu.com/search_result?keyword=xxx
match = re.search(r"keyword=([^&]+)", url)
if match:
return unquote(match.group(1))
if url.strip() and "xiaohongshu.com" not in url.lower() and "xhslink.com" not in url.lower():
return url.strip()
return None
def get_proxy_stats(self) -> dict:
"""获取代理池统计"""
if self._proxy_pool:
return {
"total": self._proxy_pool.total_count(),
"healthy": self._proxy_pool.healthy_count(),
}
return {"total": 0, "healthy": 0}
def rotate_session(self, session_id: str = "default") -> None:
"""强制轮换会话(换 IP + 换指纹)"""
self._close_session(session_id)
def run_interactive_login(self, proceed: Optional[threading.Event] = None) -> bool:
"""交互式登录"""
if not self.validate():
return False
async def _login():
session = XiaohongshuSession(
session_id="login",
proxy_pool=self._proxy_pool or ProxyPool(self.config),
fingerprint_manager=self._fingerprint_manager or FingerprintManager(self.config),
config=self.config,
)
try:
await session.start()
page = await session.get_page()
await page.goto("https://www.xiaohongshu.com", wait_until="networkidle")
if proceed:
proceed.wait()
else:
input("\n在浏览器中完成小红书登录后,按 Enter 继续...\n")
await session._save_cookies()
return True
finally:
await session.close()
return asyncio.run(_login())
# 导出插件类
plugin_class = XiaohongshuPlugin