1022 lines
39 KiB
Python
1022 lines
39 KiB
Python
# plugins/xiaohongshu/plugin.py
|
||
"""
|
||
小红书爬虫插件 - 核心功能
|
||
- Playwright 浏览器自动化
|
||
- 动态代理池轮换
|
||
- 浏览器指纹伪装
|
||
- 行为节奏模拟
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import random
|
||
import re
|
||
import threading
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any, Callable, Optional
|
||
from urllib.parse import parse_qs, quote, unquote, urlparse
|
||
|
||
# 动态加载 playwright
|
||
try:
|
||
from playwright.async_api import BrowserContext, Page, Playwright, async_playwright
|
||
|
||
PLAYWRIGHT_AVAILABLE = True
|
||
except ImportError:
|
||
PLAYWRIGHT_AVAILABLE = False
|
||
|
||
from pixiv_app.core.plugin.base import BasePlugin, PluginParseError, Resource
|
||
from pixiv_app.core.paths import runtime_path
|
||
from pixiv_app.core.playwright_runtime import (
|
||
configure_playwright_runtime_env,
|
||
explain_missing_browser,
|
||
preferred_chromium_executable,
|
||
)
|
||
|
||
try:
|
||
from pixiv_app.core.fingerprint import BrowserProfile, SessionFingerprintManager
|
||
except Exception: # pragma: no cover
|
||
BrowserProfile = None
|
||
SessionFingerprintManager = None
|
||
|
||
|
||
def _extract_note_id_from_text(text: str) -> str:
|
||
patterns = [
|
||
r"xiaohongshu\.com/explore/([a-zA-Z0-9]+)",
|
||
r"/explore/([a-zA-Z0-9]+)",
|
||
r"note_id[=:]([a-zA-Z0-9]+)",
|
||
]
|
||
for pattern in patterns:
|
||
match = re.search(pattern, text or "", re.IGNORECASE)
|
||
if match:
|
||
return match.group(1)
|
||
return ""
|
||
|
||
|
||
def _extract_redirect_path(url: str) -> str:
|
||
try:
|
||
query = parse_qs(urlparse(url).query)
|
||
value = (query.get("redirectPath") or [""])[0]
|
||
except Exception:
|
||
value = ""
|
||
value = unquote(value or "").strip()
|
||
if not value:
|
||
return ""
|
||
if value.startswith("//"):
|
||
return "https:" + value
|
||
if value.startswith("/"):
|
||
return "https://www.xiaohongshu.com" + value
|
||
if "xiaohongshu.com" in value.lower() or "xhslink.com" in value.lower():
|
||
return value if "://" in value else "https://" + value.lstrip("/")
|
||
return ""
|
||
|
||
|
||
def _normalize_xhs_url(url: str) -> str:
|
||
value = (url or "").strip()
|
||
if not value:
|
||
return ""
|
||
if value.startswith("//"):
|
||
return "https:" + value
|
||
if value.startswith("/"):
|
||
return "https://www.xiaohongshu.com" + value
|
||
lower = value.lower()
|
||
if ("xiaohongshu.com" in lower or "xhslink.com" in lower) and "://" not in value:
|
||
return "https://" + value.lstrip("/")
|
||
return value
|
||
|
||
|
||
# ==================== 配置定义 ====================
|
||
|
||
@dataclass
|
||
class ProxyConfig:
|
||
"""代理配置"""
|
||
server: str # http://ip:port 或 socks5://ip:port
|
||
username: str = ""
|
||
password: str = ""
|
||
failures: int = 0
|
||
successes: int = 0
|
||
latency_ms: float = 0.0
|
||
healthy: bool = True
|
||
|
||
@property
|
||
def url(self) -> str:
|
||
return self.server
|
||
|
||
@property
|
||
def score(self) -> float:
|
||
"""计算代理评分(用于加权选择)"""
|
||
if not self.healthy:
|
||
return 0.0
|
||
total = self.successes + self.failures
|
||
if total == 0:
|
||
return 1.0
|
||
success_rate = self.successes / total
|
||
latency_penalty = max(0.1, min(1.0, 1000.0 / max(self.latency_ms, 1.0)))
|
||
return success_rate * latency_penalty
|
||
|
||
|
||
@dataclass
|
||
class FingerprintConfig:
|
||
"""浏览器指纹配置"""
|
||
user_agent: str
|
||
viewport: dict[str, int]
|
||
locale: str = "zh-CN"
|
||
timezone_id: str = "Asia/Shanghai"
|
||
color_scheme: str = "light"
|
||
reduced_motion: str = "no-preference"
|
||
device_scale_factor: float = 1.0
|
||
is_mobile: bool = False
|
||
has_touch: bool = False
|
||
|
||
@classmethod
|
||
def from_browser_profile(cls, profile: Any) -> "FingerprintConfig":
|
||
return cls(
|
||
user_agent=profile.user_agent,
|
||
viewport=dict(profile.viewport),
|
||
locale=profile.locale,
|
||
timezone_id=profile.timezone_id,
|
||
color_scheme=profile.color_scheme,
|
||
reduced_motion=profile.reduced_motion,
|
||
device_scale_factor=profile.device_scale_factor,
|
||
is_mobile=profile.is_mobile,
|
||
has_touch=profile.has_touch,
|
||
)
|
||
|
||
@classmethod
|
||
def desktop_chrome(cls) -> "FingerprintConfig":
|
||
return cls(
|
||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||
viewport={"width": 1920, "height": 1080},
|
||
device_scale_factor=1.0,
|
||
)
|
||
|
||
@classmethod
|
||
def desktop_edge(cls) -> "FingerprintConfig":
|
||
return cls(
|
||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||
viewport={"width": 1536, "height": 864},
|
||
device_scale_factor=1.25,
|
||
)
|
||
|
||
@classmethod
|
||
def mac_chrome(cls) -> "FingerprintConfig":
|
||
return cls(
|
||
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||
viewport={"width": 1512, "height": 982},
|
||
device_scale_factor=2.0,
|
||
)
|
||
|
||
@classmethod
|
||
def mobile_ios(cls) -> "FingerprintConfig":
|
||
return cls(
|
||
user_agent="Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",
|
||
viewport={"width": 390, "height": 844},
|
||
device_scale_factor=3.0,
|
||
is_mobile=True,
|
||
has_touch=True,
|
||
)
|
||
|
||
|
||
@dataclass
|
||
class XiaohongshuConfig:
|
||
"""小红书插件配置"""
|
||
# 代理配置
|
||
proxy_enabled: bool = False
|
||
proxy_list: list[ProxyConfig] = field(default_factory=list)
|
||
proxy_rotation: str = "round_robin" # round_robin, random, session_sticky
|
||
|
||
# 指纹配置
|
||
fingerprint_list: list[FingerprintConfig] = field(default_factory=lambda: [
|
||
FingerprintConfig.desktop_chrome(),
|
||
FingerprintConfig.desktop_edge(),
|
||
FingerprintConfig.mac_chrome(),
|
||
])
|
||
|
||
# 行为控制
|
||
request_delay_min: float = 3.0 # 最小延迟(秒)
|
||
request_delay_max: float = 8.0 # 最大延迟(秒)
|
||
scroll_delay_min: float = 0.5 # 滚动延迟(秒)
|
||
scroll_delay_max: float = 1.5
|
||
batch_pause_min: float = 5.0 # 批次暂停(秒)
|
||
batch_pause_max: float = 12.0
|
||
max_scrolls: int = 15 # 最大滚动次数
|
||
|
||
# 采集限制
|
||
max_notes_per_session: int = 50 # 单会话最大笔记数
|
||
max_concurrent: int = 2 # 最大并发数
|
||
|
||
# 会话持久化
|
||
profile_dir: str = "runtime/xiaohongshu_profile"
|
||
cookie_file: str = "runtime/xiaohongshu_cookies.json"
|
||
|
||
|
||
# ==================== 代理池管理器 ====================
|
||
|
||
class ProxyPool:
|
||
"""代理池管理器 - 支持轮换和健康检查"""
|
||
|
||
def __init__(self, config: XiaohongshuConfig):
|
||
self.config = config
|
||
self._proxies: list[ProxyConfig] = list(config.proxy_list)
|
||
self._current_index = 0
|
||
self._lock = threading.Lock()
|
||
|
||
def add_proxy(self, proxy: ProxyConfig) -> None:
|
||
"""添加代理"""
|
||
with self._lock:
|
||
# 去重
|
||
existing = {p.server for p in self._proxies}
|
||
if proxy.server not in existing:
|
||
self._proxies.append(proxy)
|
||
|
||
def get_proxy(self) -> Optional[ProxyConfig]:
|
||
"""获取一个代理(按策略)"""
|
||
if not self.config.proxy_enabled or not self._proxies:
|
||
return None
|
||
|
||
with self._lock:
|
||
healthy = [p for p in self._proxies if p.healthy]
|
||
if not healthy:
|
||
return None
|
||
|
||
if self.config.proxy_rotation == "random":
|
||
return random.choice(healthy)
|
||
|
||
# round_robin
|
||
proxy = healthy[self._current_index % len(healthy)]
|
||
self._current_index += 1
|
||
return proxy
|
||
|
||
def mark_success(self, proxy: Optional[ProxyConfig], latency_ms: float = 0) -> None:
|
||
"""标记代理成功"""
|
||
if proxy is None:
|
||
return
|
||
with self._lock:
|
||
proxy.successes += 1
|
||
proxy.failures = 0
|
||
proxy.healthy = True
|
||
if latency_ms > 0:
|
||
proxy.latency_ms = latency_ms
|
||
|
||
def mark_failed(self, proxy: Optional[ProxyConfig]) -> None:
|
||
"""标记代理失败"""
|
||
if proxy is None:
|
||
return
|
||
with self._lock:
|
||
proxy.failures += 1
|
||
if proxy.failures >= 3:
|
||
proxy.healthy = False
|
||
|
||
def healthy_count(self) -> int:
|
||
"""可用代理数量"""
|
||
with self._lock:
|
||
return len([p for p in self._proxies if p.healthy])
|
||
|
||
def total_count(self) -> int:
|
||
return len(self._proxies)
|
||
|
||
|
||
# ==================== 指纹管理器 ====================
|
||
|
||
class FingerprintManager:
|
||
"""浏览器指纹管理器"""
|
||
|
||
def __init__(self, config: XiaohongshuConfig):
|
||
self.config = config
|
||
self._fingerprints = list(config.fingerprint_list)
|
||
self._current_index = 0
|
||
self._shared = SessionFingerprintManager(strategy="sticky") if SessionFingerprintManager else None
|
||
|
||
def get_fingerprint(self, session_id: str = "default") -> FingerprintConfig:
|
||
"""获取一个指纹"""
|
||
if self._shared is not None:
|
||
return FingerprintConfig.from_browser_profile(self._shared.get_fingerprint(f"xiaohongshu:{session_id}"))
|
||
if not self._fingerprints:
|
||
return FingerprintConfig.desktop_chrome()
|
||
index = abs(hash(session_id)) % len(self._fingerprints)
|
||
return self._fingerprints[index]
|
||
|
||
def apply_to_context(self, context: BrowserContext, fingerprint: FingerprintConfig) -> None:
|
||
"""将指纹应用到浏览器上下文"""
|
||
# 注意:viewport 在创建 context 时设置,这里只做额外配置
|
||
pass
|
||
|
||
|
||
# ==================== 浏览器会话管理器 ====================
|
||
|
||
class XiaohongshuSession:
|
||
"""小红书浏览器会话 - 管理单个 IP+指纹 的会话"""
|
||
|
||
def __init__(
|
||
self,
|
||
session_id: str,
|
||
proxy_pool: ProxyPool,
|
||
fingerprint_manager: FingerprintManager,
|
||
config: XiaohongshuConfig,
|
||
):
|
||
self.session_id = session_id
|
||
self.proxy_pool = proxy_pool
|
||
self.fingerprint_manager = fingerprint_manager
|
||
self.config = config
|
||
|
||
self._pw: Optional[Playwright] = None
|
||
self._context: Optional[BrowserContext] = None
|
||
self._page: Optional[Page] = None
|
||
self._current_proxy: Optional[ProxyConfig] = None
|
||
self._fingerprint: Optional[FingerprintConfig] = None
|
||
|
||
self.request_count = 0
|
||
self.created_at = datetime.now()
|
||
|
||
async def start(self) -> None:
|
||
"""启动会话(创建独立的浏览器上下文)"""
|
||
configure_playwright_runtime_env()
|
||
self._pw = await async_playwright().start()
|
||
|
||
# 获取代理和指纹
|
||
self._current_proxy = self.proxy_pool.get_proxy()
|
||
self._fingerprint = self.fingerprint_manager.get_fingerprint(self.session_id)
|
||
|
||
# 构建 context 参数
|
||
context_options = {
|
||
"viewport": self._fingerprint.viewport,
|
||
"user_agent": self._fingerprint.user_agent,
|
||
"locale": self._fingerprint.locale,
|
||
"timezone_id": self._fingerprint.timezone_id,
|
||
"color_scheme": self._fingerprint.color_scheme,
|
||
"reduced_motion": self._fingerprint.reduced_motion,
|
||
"device_scale_factor": self._fingerprint.device_scale_factor,
|
||
"is_mobile": self._fingerprint.is_mobile,
|
||
"has_touch": self._fingerprint.has_touch,
|
||
"ignore_default_args": ["--enable-automation"],
|
||
"args": [
|
||
"--disable-blink-features=AutomationControlled",
|
||
],
|
||
}
|
||
|
||
# 添加代理配置
|
||
if self._current_proxy:
|
||
context_options["proxy"] = {
|
||
"server": self._current_proxy.server,
|
||
}
|
||
if self._current_proxy.username:
|
||
context_options["proxy"]["username"] = self._current_proxy.username
|
||
context_options["proxy"]["password"] = self._current_proxy.password
|
||
|
||
# 创建上下文
|
||
self._context = await self._pw.chromium.launch_persistent_context(
|
||
user_data_dir=str(Path(self.config.profile_dir) / self.session_id),
|
||
headless=False, # 小红书能检测无头模式,必须 False
|
||
**context_options
|
||
)
|
||
|
||
# 加载保存的 Cookie
|
||
await self._load_cookies()
|
||
|
||
self._page = self._context.pages[0] if self._context.pages else await self._context.new_page()
|
||
|
||
async def _load_cookies(self) -> None:
|
||
"""加载保存的 Cookie"""
|
||
cookie_path = Path(self.config.cookie_file)
|
||
if cookie_path.exists():
|
||
try:
|
||
cookies = json.loads(cookie_path.read_text(encoding="utf-8"))
|
||
if isinstance(cookies, list):
|
||
await self._context.add_cookies(cookies)
|
||
except Exception:
|
||
pass
|
||
|
||
async def _save_cookies(self) -> None:
|
||
"""保存 Cookie 到文件"""
|
||
if self._context:
|
||
cookies = await self._context.cookies()
|
||
Path(self.config.cookie_file).write_text(
|
||
json.dumps(cookies, indent=2, ensure_ascii=False),
|
||
encoding="utf-8"
|
||
)
|
||
|
||
async def get_page(self) -> Page:
|
||
"""获取页面对象"""
|
||
if self._page is None:
|
||
await self.start()
|
||
return self._page
|
||
|
||
async def random_delay(self) -> None:
|
||
"""随机延迟(模拟真人节奏)"""
|
||
delay = random.uniform(self.config.request_delay_min, self.config.request_delay_max)
|
||
await asyncio.sleep(delay)
|
||
self.request_count += 1
|
||
|
||
async def random_scroll(self) -> None:
|
||
"""随机滚动页面"""
|
||
scroll_y = random.randint(220, 620)
|
||
try:
|
||
await self._page.mouse.move(random.randint(80, 900), random.randint(120, 700), steps=random.randint(8, 18))
|
||
await self._page.mouse.wheel(0, scroll_y)
|
||
except Exception:
|
||
await self._page.evaluate(f"window.scrollBy(0, {scroll_y})")
|
||
await asyncio.sleep(random.uniform(self.config.scroll_delay_min, self.config.scroll_delay_max))
|
||
|
||
async def simulate_browsing(self, duration: float = 2.0) -> None:
|
||
"""模拟浏览行为(停留、思考)"""
|
||
await asyncio.sleep(duration * random.uniform(0.7, 1.3))
|
||
|
||
async def close(self) -> None:
|
||
"""关闭会话"""
|
||
if self._context:
|
||
await self._save_cookies()
|
||
await self._context.close()
|
||
if self._pw:
|
||
await self._pw.stop()
|
||
self._page = None
|
||
self._context = None
|
||
self._pw = None
|
||
|
||
@property
|
||
def proxy_info(self) -> str:
|
||
return self._current_proxy.server if self._current_proxy else "直连"
|
||
|
||
|
||
# ==================== 小红书 API 封装 ====================
|
||
|
||
class XiaohongshuAPI:
|
||
"""小红书 API 封装(基于 Playwright)"""
|
||
|
||
BASE_URL = "https://www.xiaohongshu.com"
|
||
|
||
def __init__(self, session: XiaohongshuSession):
|
||
self.session = session
|
||
|
||
async def search(self, keyword: str, limit: int = 20) -> list[dict]:
|
||
"""搜索笔记"""
|
||
page = await self.session.get_page()
|
||
|
||
# 访问搜索页面
|
||
search_url = f"{self.BASE_URL}/search_result?keyword={quote(keyword)}"
|
||
await page.goto(search_url, wait_until="domcontentloaded", timeout=45000)
|
||
await self.session.random_delay()
|
||
|
||
# 滚动加载
|
||
notes = []
|
||
last_count = 0
|
||
scrolls_without_new = 0
|
||
|
||
for scroll_idx in range(self.session.config.max_scrolls):
|
||
# 等待笔记容器加载
|
||
try:
|
||
await page.wait_for_selector(".note-item, .feeds-container .note", timeout=5000)
|
||
except:
|
||
pass
|
||
|
||
# 提取当前页面的笔记
|
||
current_notes = await page.evaluate("""
|
||
() => {
|
||
const items = document.querySelectorAll('.note-item, .feeds-container .note, section, a[href*="/explore/"]');
|
||
return Array.from(items).map(el => {
|
||
const anchor = el.matches?.('a') ? el : (
|
||
el.querySelector('a[href*="/explore/"], a[href*="xhslink.com"], a')
|
||
);
|
||
const href = anchor?.href || anchor?.getAttribute('href') || '';
|
||
const img = el.querySelector('img');
|
||
const id = el.getAttribute('data-note-id') ||
|
||
href.match(/\\/explore\\/([a-zA-Z0-9]+)/)?.[1] ||
|
||
href.match(/note_id[=:]([a-zA-Z0-9]+)/)?.[1] ||
|
||
'';
|
||
return {
|
||
id,
|
||
title: el.querySelector('.title, .note-title, [class*="title"]')?.innerText?.trim() || '',
|
||
author: el.querySelector('.author .name, .nickname, [class*="nickname"]')?.innerText?.trim() || '',
|
||
likes: el.querySelector('.like-count, .likes, [class*="like"]')?.innerText?.trim() || '0',
|
||
coverUrl: img?.currentSrc || img?.src || img?.getAttribute('data-src') || '',
|
||
url: href
|
||
};
|
||
});
|
||
}
|
||
""")
|
||
|
||
# 去重合并
|
||
existing_ids = {n.get("id") for n in notes}
|
||
for note in current_notes:
|
||
note_url = _normalize_xhs_url(str(note.get("url") or ""))
|
||
note_id = str(note.get("id") or _extract_note_id_from_text(note_url))
|
||
if note_id and note_id not in existing_ids:
|
||
note["id"] = note_id
|
||
note["url"] = note_url
|
||
existing_ids.add(note_id)
|
||
notes.append(note)
|
||
if len(notes) >= limit:
|
||
break
|
||
|
||
if len(notes) >= limit:
|
||
break
|
||
|
||
# 滚动
|
||
if len(current_notes) == last_count:
|
||
scrolls_without_new += 1
|
||
if scrolls_without_new >= 3:
|
||
break
|
||
else:
|
||
scrolls_without_new = 0
|
||
last_count = len(current_notes)
|
||
|
||
await self.session.random_scroll()
|
||
await self.session.simulate_browsing(1.0)
|
||
|
||
return notes[:limit]
|
||
|
||
async def get_note_detail(self, note_id: str = "", note_url: str | None = None) -> dict:
|
||
"""获取笔记详情"""
|
||
page = await self.session.get_page()
|
||
|
||
url = note_url or f"{self.BASE_URL}/explore/{note_id}"
|
||
url = _normalize_xhs_url(url)
|
||
await page.goto(url, wait_until="domcontentloaded", timeout=45000)
|
||
await self.session.random_delay()
|
||
|
||
current_url = page.url
|
||
if "/404" in urlparse(current_url).path:
|
||
redirect_url = _extract_redirect_path(current_url)
|
||
if redirect_url and redirect_url != current_url and redirect_url != url:
|
||
await page.goto(redirect_url, wait_until="domcontentloaded", timeout=45000)
|
||
await self.session.random_delay()
|
||
current_url = page.url
|
||
|
||
final_note_id = note_id or _extract_note_id_from_text(current_url)
|
||
|
||
try:
|
||
await page.wait_for_selector("img", timeout=10000)
|
||
except Exception:
|
||
pass
|
||
|
||
# 提取详情
|
||
detail = await page.evaluate("""
|
||
() => {
|
||
// 标题
|
||
const title = document.querySelector('.title, .note-title')?.innerText?.trim() || '';
|
||
// 作者
|
||
const author = document.querySelector('.author .name, .nickname')?.innerText?.trim() || '';
|
||
// 内容描述
|
||
const desc = document.querySelector('.desc, .note-content')?.innerText?.trim() || '';
|
||
// 点赞数
|
||
const likes = document.querySelector('.like-count, .likes')?.innerText?.trim() || '0';
|
||
// 收藏数
|
||
const collects = document.querySelector('.collect-count')?.innerText?.trim() || '0';
|
||
// 评论数
|
||
const comments = document.querySelector('.comment-count')?.innerText?.trim() || '0';
|
||
// 图片列表
|
||
const seen = new Set();
|
||
const images = Array.from(document.querySelectorAll('img'))
|
||
.map(img => img.currentSrc || img.src || img.getAttribute('data-src') || '')
|
||
.filter(src => /^https?:\\/\\//.test(src))
|
||
.filter(src => /xhscdn|sns-webpic|xiaohongshu/.test(src))
|
||
.filter(src => {
|
||
if (seen.has(src)) return false;
|
||
seen.add(src);
|
||
return true;
|
||
});
|
||
|
||
const seenVideos = new Set();
|
||
const videos = Array.from(document.querySelectorAll('video, video source, source'))
|
||
.map(el => el.currentSrc || el.src || el.getAttribute('src') || '')
|
||
.filter(src => /^https?:\\/\\//.test(src))
|
||
.filter(src => /xhscdn|sns-video|xiaohongshu|xhs/.test(src))
|
||
.filter(src => {
|
||
if (seenVideos.has(src)) return false;
|
||
seenVideos.add(src);
|
||
return true;
|
||
});
|
||
|
||
return { title, author, desc, likes, collects, comments, images, videos };
|
||
}
|
||
""")
|
||
|
||
if "/404" in urlparse(current_url).path and not detail.get("images") and not detail.get("videos"):
|
||
raise PluginParseError(
|
||
"小红书页面跳转到 404。请粘贴浏览器地址栏里的完整笔记链接,"
|
||
"尤其要保留 ?xsec_token=...&xsec_source=... 参数。"
|
||
)
|
||
|
||
if not detail.get("images") and not detail.get("videos"):
|
||
raise PluginParseError(
|
||
"页面已打开,但没有解析到图片。请确认链接是图文笔记,"
|
||
"并优先粘贴带 xsec_token 的完整浏览器地址栏链接。"
|
||
)
|
||
|
||
detail["id"] = final_note_id
|
||
detail["url"] = current_url
|
||
|
||
return detail
|
||
|
||
async def get_note_images(self, note_id: str, note_url: str | None = None) -> list[str]:
|
||
"""获取笔记中的图片 URL"""
|
||
detail = await self.get_note_detail(note_id, note_url=note_url)
|
||
return detail.get("images", [])
|
||
|
||
async def get_note_media(self, note_id: str, note_url: str | None = None) -> dict:
|
||
detail = await self.get_note_detail(note_id, note_url=note_url)
|
||
return {"images": detail.get("images", []), "videos": detail.get("videos", [])}
|
||
|
||
async def download_image(self, url: str, save_path: Path) -> Optional[Path]:
|
||
"""下载图片"""
|
||
return await self.download_media(url, save_path, media_type="image")
|
||
|
||
async def download_media(self, url: str, save_path: Path, *, media_type: str = "image") -> Optional[Path]:
|
||
import httpx
|
||
|
||
page = await self.session.get_page()
|
||
|
||
# 构造文件名
|
||
filename = url.split("/")[-1].split("?")[0]
|
||
if media_type == "video" and not filename.endswith((".mp4", ".mov", ".webm", ".m4v")):
|
||
filename = f"{filename}.mp4"
|
||
elif media_type != "video" and not filename.endswith((".jpg", ".jpeg", ".png", ".webp")):
|
||
filename = f"{filename}.jpg"
|
||
|
||
target = save_path / filename
|
||
|
||
# 检查是否已存在
|
||
if target.exists() and target.stat().st_size > 1024:
|
||
return target
|
||
|
||
# 使用浏览器上下文下载(保持会话)
|
||
try:
|
||
async with httpx.AsyncClient(timeout=60.0, follow_redirects=True) as client:
|
||
# 获取当前页面的 cookies
|
||
cookies = await page.context.cookies()
|
||
cookie_dict = {c["name"]: c["value"] for c in cookies}
|
||
|
||
response = await client.get(
|
||
url,
|
||
headers={
|
||
"User-Agent": self.session._fingerprint.user_agent,
|
||
"Referer": page.url if page and page.url != "about:blank" else self.BASE_URL,
|
||
},
|
||
cookies=cookie_dict,
|
||
)
|
||
|
||
if response.status_code == 200:
|
||
target.write_bytes(response.content)
|
||
return target
|
||
except Exception:
|
||
pass
|
||
|
||
return None
|
||
|
||
|
||
# ==================== 插件主类 ====================
|
||
|
||
class XiaohongshuPlugin(BasePlugin):
|
||
"""小红书爬虫插件"""
|
||
|
||
name = "小红书"
|
||
domain = "xiaohongshu.com"
|
||
version = "1.0.0"
|
||
|
||
def __init__(self):
|
||
self.config = XiaohongshuConfig()
|
||
self.config.profile_dir = str(runtime_path("xiaohongshu_profile"))
|
||
self.config.cookie_file = str(runtime_path("xiaohongshu_cookies.json"))
|
||
self.progress_callback: Optional[Callable[[str, float], None]] = None
|
||
self._proxy_pool: Optional[ProxyPool] = None
|
||
self._fingerprint_manager: Optional[FingerprintManager] = None
|
||
self._sessions: dict[str, XiaohongshuSession] = {}
|
||
self._session_lock = threading.Lock()
|
||
|
||
def _progress(self, message: str, value: float) -> None:
|
||
if self.progress_callback is None:
|
||
return
|
||
try:
|
||
self.progress_callback(message, value)
|
||
except Exception:
|
||
pass
|
||
|
||
def can_handle(self, url: str) -> bool:
|
||
"""判断是否能处理该 URL"""
|
||
text = url.strip().lower()
|
||
return "xiaohongshu.com" in text or "xhslink.com" in text or text.isdigit()
|
||
|
||
def validate(self) -> bool:
|
||
"""检查依赖是否安装"""
|
||
return PLAYWRIGHT_AVAILABLE
|
||
|
||
def get_headers(self) -> dict[str, str]:
|
||
"""获取请求头"""
|
||
return {
|
||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
||
"Accept": "application/json, text/plain, */*",
|
||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||
}
|
||
|
||
def set_proxies(self, proxy_list: list[dict]) -> None:
|
||
"""
|
||
设置代理列表
|
||
proxy_list: [{"server": "http://ip:port", "username": "", "password": ""}, ...]
|
||
"""
|
||
self.config.proxy_enabled = True
|
||
self.config.proxy_list = [
|
||
ProxyConfig(
|
||
server=p.get("server"),
|
||
username=p.get("username", ""),
|
||
password=p.get("password", ""),
|
||
)
|
||
for p in proxy_list
|
||
]
|
||
self._proxy_pool = ProxyPool(self.config)
|
||
|
||
def set_fingerprints(self, fingerprints: list[dict]) -> None:
|
||
"""设置指纹列表"""
|
||
self.config.fingerprint_list = [
|
||
FingerprintConfig(
|
||
user_agent=f.get("user_agent"),
|
||
viewport=f.get("viewport", {"width": 1920, "height": 1080}),
|
||
locale=f.get("locale", "zh-CN"),
|
||
timezone_id=f.get("timezone_id", "Asia/Shanghai"),
|
||
)
|
||
for f in fingerprints
|
||
]
|
||
|
||
def _get_session(self, session_id: str = "default") -> XiaohongshuSession:
|
||
"""获取或创建会话(每个会话独立 IP+指纹)"""
|
||
with self._session_lock:
|
||
if session_id not in self._sessions:
|
||
if self._proxy_pool is None:
|
||
self._proxy_pool = ProxyPool(self.config)
|
||
if self._fingerprint_manager is None:
|
||
self._fingerprint_manager = FingerprintManager(self.config)
|
||
|
||
session = XiaohongshuSession(
|
||
session_id=session_id,
|
||
proxy_pool=self._proxy_pool,
|
||
fingerprint_manager=self._fingerprint_manager,
|
||
config=self.config,
|
||
)
|
||
self._sessions[session_id] = session
|
||
|
||
return self._sessions[session_id]
|
||
|
||
def _close_session(self, session_id: str) -> None:
|
||
"""关闭会话"""
|
||
with self._session_lock:
|
||
if session_id in self._sessions:
|
||
# 异步关闭需要在 async 环境中处理,这里触发但不等待
|
||
asyncio.create_task(self._sessions[session_id].close())
|
||
del self._sessions[session_id]
|
||
|
||
def parse(self, url: str) -> list[Resource]:
|
||
"""解析 URL 为资源列表"""
|
||
if not self.validate():
|
||
raise PluginParseError("请先安装 Playwright: pip install playwright && playwright install chromium")
|
||
|
||
try:
|
||
return asyncio.run(self._parse_async(url))
|
||
except Exception as exc:
|
||
raise PluginParseError(f"小红书解析失败: {exc}") from exc
|
||
|
||
async def _parse_async(self, url: str) -> list[Resource]:
|
||
"""异步解析 URL"""
|
||
self._progress("小红书:准备解析目标", 0.05)
|
||
# 提取关键词或笔记 ID
|
||
note_id = self._extract_note_id(url)
|
||
note_url = self._extract_note_url(url, note_id=note_id)
|
||
keyword = self._extract_keyword(url)
|
||
|
||
session = self._get_session(f"parse_{int(time.time())}")
|
||
|
||
try:
|
||
self._progress("小红书:启动浏览器会话", 0.10)
|
||
await session.start()
|
||
api = XiaohongshuAPI(session)
|
||
|
||
resources = []
|
||
|
||
if note_id or note_url:
|
||
# 单篇笔记
|
||
self._progress("小红书:打开笔记页面", 0.22)
|
||
detail = await api.get_note_detail(note_id or "", note_url=note_url)
|
||
self._progress("小红书:提取笔记图片", 0.55)
|
||
note_id = detail.get("id") or note_id or _extract_note_id_from_text(detail.get("url", ""))
|
||
if not note_id:
|
||
note_id = f"note_{int(time.time())}"
|
||
images = detail.get("images", [])
|
||
videos = detail.get("videos", [])
|
||
|
||
resource = Resource(
|
||
id=note_id,
|
||
url=detail.get("url") or note_url or f"https://www.xiaohongshu.com/explore/{note_id}",
|
||
title=detail.get("title", "")[:200],
|
||
author=detail.get("author", ""),
|
||
author_id="",
|
||
files=[{"url": video, "type": "video"} for video in videos]
|
||
+ [{"url": img, "type": "image"} for img in images],
|
||
metadata=detail,
|
||
thumbnail=images[0] if images else None,
|
||
created_at="",
|
||
)
|
||
resources.append(resource)
|
||
|
||
elif keyword:
|
||
# 关键词搜索
|
||
self._progress("小红书:打开搜索结果页", 0.18)
|
||
notes = await api.search(keyword, limit=min(self.config.max_notes_per_session, 30))
|
||
self._progress(f"小红书:整理搜索结果({len(notes)} 条)", 0.62)
|
||
|
||
for note in notes:
|
||
note_url = _normalize_xhs_url(note.get("url", ""))
|
||
note_id = note.get("id") or _extract_note_id_from_text(note_url)
|
||
if not note_id:
|
||
continue
|
||
# 只返回元数据,实际图片在 download 时下载
|
||
resource = Resource(
|
||
id=note_id,
|
||
url=note_url,
|
||
title=note.get("title", "")[:200],
|
||
author=note.get("author", ""),
|
||
author_id="",
|
||
files=[], # 延迟下载
|
||
metadata={
|
||
"kind": "search_result",
|
||
"keyword": keyword,
|
||
"note_id": note_id,
|
||
"note_url": note_url,
|
||
"likes": note.get("likes"),
|
||
"cover_url": note.get("coverUrl"),
|
||
},
|
||
thumbnail=note.get("coverUrl"),
|
||
created_at="",
|
||
)
|
||
resources.append(resource)
|
||
|
||
self._progress(f"小红书:解析完成({len(resources)} 个资源)", 0.72)
|
||
return resources
|
||
|
||
finally:
|
||
await session.close()
|
||
|
||
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||
"""下载资源"""
|
||
try:
|
||
return asyncio.run(self._download_async(resource, save_path))
|
||
except Exception as exc:
|
||
raise PluginParseError(f"小红书下载失败: {exc}") from exc
|
||
|
||
async def _download_async(self, resource: Resource, save_path: Path) -> list[Path]:
|
||
"""异步下载"""
|
||
self._progress(f"小红书:准备下载 {resource.id}", 0.0)
|
||
session = self._get_session(f"download_{resource.id}_{int(time.time())}")
|
||
|
||
try:
|
||
self._progress(f"小红书:启动下载会话 {resource.id}", 0.10)
|
||
await session.start()
|
||
api = XiaohongshuAPI(session)
|
||
|
||
# 创建保存目录
|
||
subdir = resource.metadata.get("keyword", resource.id)
|
||
target_dir = save_path / "xiaohongshu" / subdir
|
||
target_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
downloaded = []
|
||
|
||
# 如果有直接的文件列表
|
||
if resource.files:
|
||
if resource.url:
|
||
try:
|
||
self._progress(f"小红书:打开笔记页 {resource.id}", 0.20)
|
||
page = await session.get_page()
|
||
await page.goto(_normalize_xhs_url(resource.url), wait_until="domcontentloaded", timeout=45000)
|
||
await session.random_delay()
|
||
except Exception:
|
||
pass
|
||
total_files = max(len(resource.files), 1)
|
||
for item in resource.files:
|
||
file_url = item.get("url")
|
||
if file_url:
|
||
self._progress(f"小红书:下载图片 {len(downloaded) + 1}/{total_files}", 0.35 + (len(downloaded) / total_files) * 0.55)
|
||
media_type = "video" if item.get("type") == "video" else "image"
|
||
path = await api.download_media(file_url, target_dir, media_type=media_type)
|
||
if path:
|
||
downloaded.append(path)
|
||
await session.random_delay()
|
||
|
||
# 如果是搜索结果,需要先获取详情再下载
|
||
elif resource.metadata.get("kind") == "search_result":
|
||
note_id = resource.metadata.get("note_id")
|
||
note_url = _normalize_xhs_url(str(resource.metadata.get("note_url") or resource.url or ""))
|
||
if note_id or note_url:
|
||
self._progress(f"小红书:进入笔记详情 {resource.id}", 0.20)
|
||
media = await api.get_note_media(note_id, note_url=note_url)
|
||
media_items = [{"url": video, "type": "video"} for video in media.get("videos", [])] + [
|
||
{"url": image, "type": "image"} for image in media.get("images", [])
|
||
]
|
||
total_files = max(len(media_items), 1)
|
||
for media_item in media_items:
|
||
img_url = media_item["url"]
|
||
self._progress(f"小红书:下载图片 {len(downloaded) + 1}/{total_files}", 0.35 + (len(downloaded) / total_files) * 0.55)
|
||
path = await api.download_media(img_url, target_dir, media_type=media_item["type"])
|
||
if path:
|
||
downloaded.append(path)
|
||
await session.random_delay()
|
||
|
||
self._progress(f"小红书:资源下载完成 {resource.id}", 0.96)
|
||
return downloaded
|
||
|
||
finally:
|
||
await session.close()
|
||
|
||
def _extract_note_id(self, url: str) -> Optional[str]:
|
||
"""提取笔记 ID"""
|
||
# 标准链接: xiaohongshu.com/explore/xxxxx
|
||
patterns = [
|
||
r"xiaohongshu\.com/explore/([a-zA-Z0-9]+)",
|
||
r"note_id[=:]([a-zA-Z0-9]+)",
|
||
]
|
||
for pattern in patterns:
|
||
match = re.search(pattern, url, re.IGNORECASE)
|
||
if match:
|
||
return match.group(1)
|
||
|
||
# 纯数字 ID
|
||
if url.strip().isdigit():
|
||
return url.strip()
|
||
|
||
return None
|
||
|
||
def _extract_note_url(self, url: str, *, note_id: str | None = None) -> Optional[str]:
|
||
"""提取或还原单篇笔记访问 URL,尽量保留 xsec_token 等安全参数。"""
|
||
text = url.strip()
|
||
if not text:
|
||
return None
|
||
lower = text.lower()
|
||
if "xiaohongshu.com/404" in lower:
|
||
redirect_url = _extract_redirect_path(text)
|
||
if redirect_url:
|
||
return redirect_url
|
||
if "xiaohongshu.com" in lower or "xhslink.com" in lower:
|
||
if text.startswith("//"):
|
||
return "https:" + text
|
||
if "://" not in text:
|
||
return "https://" + text.lstrip("/")
|
||
return text
|
||
if note_id:
|
||
return f"https://www.xiaohongshu.com/explore/{note_id}"
|
||
return None
|
||
|
||
def _extract_keyword(self, url: str) -> Optional[str]:
|
||
"""提取搜索关键词"""
|
||
# 搜索链接: xiaohongshu.com/search_result?keyword=xxx
|
||
match = re.search(r"keyword=([^&]+)", url)
|
||
if match:
|
||
return unquote(match.group(1))
|
||
if url.strip() and "xiaohongshu.com" not in url.lower() and "xhslink.com" not in url.lower():
|
||
return url.strip()
|
||
return None
|
||
|
||
def get_proxy_stats(self) -> dict:
|
||
"""获取代理池统计"""
|
||
if self._proxy_pool:
|
||
return {
|
||
"total": self._proxy_pool.total_count(),
|
||
"healthy": self._proxy_pool.healthy_count(),
|
||
}
|
||
return {"total": 0, "healthy": 0}
|
||
|
||
def rotate_session(self, session_id: str = "default") -> None:
|
||
"""强制轮换会话(换 IP + 换指纹)"""
|
||
self._close_session(session_id)
|
||
|
||
def run_interactive_login(self, proceed: Optional[threading.Event] = None) -> bool:
|
||
"""交互式登录"""
|
||
if not self.validate():
|
||
return False
|
||
|
||
async def _login():
|
||
session = XiaohongshuSession(
|
||
session_id="login",
|
||
proxy_pool=self._proxy_pool or ProxyPool(self.config),
|
||
fingerprint_manager=self._fingerprint_manager or FingerprintManager(self.config),
|
||
config=self.config,
|
||
)
|
||
try:
|
||
await session.start()
|
||
page = await session.get_page()
|
||
await page.goto("https://www.xiaohongshu.com", wait_until="networkidle")
|
||
|
||
if proceed:
|
||
proceed.wait()
|
||
else:
|
||
input("\n在浏览器中完成小红书登录后,按 Enter 继续...\n")
|
||
|
||
await session._save_cookies()
|
||
return True
|
||
finally:
|
||
await session.close()
|
||
|
||
return asyncio.run(_login())
|
||
|
||
|
||
# 导出插件类
|
||
plugin_class = XiaohongshuPlugin
|