Files
SakuraHub/pixiv_app/core/browser_proxy_pool.py
T

353 lines
12 KiB
Python
Raw Normal View History

2026-11-16 20:45:22 +08:00
"""
Playwright 浏览器代理池 - 支持动态换 IP
功能:
- 多代理轮换(轮询/随机/会话粘性)
- 健康检查(自动剔除失效代理)
- 延迟统计(加权选择最优代理)
- 线程安全
"""
import random
import threading
import time
from dataclasses import dataclass, field
from typing import Optional, List, Dict, Any
from enum import Enum
class RotationStrategy(Enum):
"""轮换策略"""
ROUND_ROBIN = "round_robin" # 轮询
RANDOM = "random" # 随机
WEIGHTED = "weighted" # 加权(按成功率+延迟)
SESSION_STICKY = "session_sticky" # 会话粘性(同会话同IP)
@dataclass
class ProxyInfo:
"""代理信息"""
server: str # http://ip:port 或 socks5://ip:port
username: str = ""
password: str = ""
# 统计数据
success_count: int = 0
fail_count: int = 0
total_latency_ms: float = 0.0
avg_latency_ms: float = 0.0
last_check_time: float = 0.0
last_fail_time: float = 0.0
# 状态
healthy: bool = True
cooldown_until: float = 0.0 # 冷却结束时间
@property
def proxy_url(self) -> str:
"""获取完整的代理URL(含认证)"""
if self.username and self.password:
# http://user:pass@ip:port
protocol = self.server.split("://")[0]
rest = self.server.split("://")[1]
return f"{protocol}://{self.username}:{self.password}@{rest}"
return self.server
def to_requests_proxies(self) -> dict[str, str]:
return {"http": self.proxy_url, "https": self.proxy_url}
def to_playwright_proxy(self) -> dict[str, str]:
data = {"server": self.server}
if self.username:
data["username"] = self.username
data["password"] = self.password
return data
@property
def success_rate(self) -> float:
"""成功率"""
total = self.success_count + self.fail_count
if total == 0:
return 1.0
return self.success_count / total
@property
def weight(self) -> float:
"""
计算代理权重(用于加权随机选择)
考虑因素:成功率、延迟、冷却状态
"""
if not self.healthy:
return 0.0
# 冷却中
if self.cooldown_until > time.time():
return 0.0
# 成功率权重
success_weight = self.success_rate
# 延迟权重(延迟越低权重越高)
if self.avg_latency_ms > 0:
latency_weight = max(0.1, 1000.0 / max(self.avg_latency_ms, 1))
else:
latency_weight = 1.0
# 失败惩罚
fail_penalty = 1.0 / (1.0 + self.fail_count * 0.5)
return success_weight * latency_weight * fail_penalty
class ProxyPool:
"""代理池 - 管理多个代理的获取和轮换"""
def __init__(
self,
strategy: RotationStrategy = RotationStrategy.WEIGHTED,
health_check_url: str = "https://httpbin.org/ip",
health_check_timeout: int = 8,
max_failures: int = 3,
cooldown_seconds: int = 60,
auto_health_check: bool = True,
):
"""
Args:
strategy: 轮换策略
health_check_url: 健康检查URL
health_check_timeout: 健康检查超时(秒)
max_failures: 最大失败次数,超过后标记为不健康
cooldown_seconds: 不健康代理的冷却时间(秒)
auto_health_check: 是否自动健康检查
"""
self.strategy = strategy
self.health_check_url = health_check_url
self.health_check_timeout = health_check_timeout
self.max_failures = max_failures
self.cooldown_seconds = cooldown_seconds
self.auto_health_check = auto_health_check
self._proxies: List[ProxyInfo] = []
self._session_proxy_map: Dict[str, ProxyInfo] = {} # 会话粘性用
self._round_robin_index = 0
self._lock = threading.Lock()
def add_proxy(self, server: str, username: str = "", password: str = "") -> None:
"""添加单个代理"""
with self._lock:
# 去重
existing = {p.server for p in self._proxies}
if server not in existing:
self._proxies.append(ProxyInfo(
server=server,
username=username,
password=password,
))
def add_proxy_url(self, value: str) -> None:
parsed = parse_proxy_url(value)
if parsed is not None:
self.add_proxy(parsed.server, parsed.username, parsed.password)
def add_proxies(self, proxies: List[dict]) -> None:
"""批量添加代理
Args:
proxies: [{"server": "http://1.2.3.4:8080", "username": "", "password": ""}, ...]
"""
for p in proxies:
self.add_proxy(
server=p["server"],
username=p.get("username", ""),
password=p.get("password", ""),
)
def add_proxy_text(self, text: str) -> int:
count = 0
for line in text.splitlines():
parsed = parse_proxy_url(line)
if parsed is None:
continue
self.add_proxy(parsed.server, parsed.username, parsed.password)
count += 1
return count
def remove_proxy(self, server: str) -> None:
"""移除代理"""
with self._lock:
self._proxies = [p for p in self._proxies if p.server != server]
def get_proxy(self, session_id: str = None) -> Optional[ProxyInfo]:
"""
获取一个代理
Args:
session_id: 会话ID,用于SESSION_STICKY策略
Returns:
代理信息,如果没有可用代理返回None
"""
with self._lock:
# 过滤出健康的代理
now = time.time()
healthy = [p for p in self._proxies if p.healthy and p.cooldown_until <= now]
if not healthy:
# 尝试从冷却中恢复
now = time.time()
recovered = []
for p in self._proxies:
if not p.healthy and p.cooldown_until <= now:
p.healthy = True
p.fail_count = 0
recovered.append(p)
if recovered:
healthy = recovered
if not healthy:
return None
# 根据策略选择代理
if self.strategy == RotationStrategy.RANDOM:
return random.choice(healthy)
elif self.strategy == RotationStrategy.WEIGHTED:
# 加权随机选择
weights = [p.weight for p in healthy]
total_weight = sum(weights)
if total_weight <= 0:
return random.choice(healthy)
r = random.uniform(0, total_weight)
cumsum = 0
for i, w in enumerate(weights):
cumsum += w
if r <= cumsum:
return healthy[i]
return healthy[0]
elif self.strategy == RotationStrategy.SESSION_STICKY:
if session_id and session_id in self._session_proxy_map:
proxy = self._session_proxy_map[session_id]
if proxy in healthy:
return proxy
# 新会话,分配一个代理
proxy = self._get_round_robin(healthy)
if session_id:
self._session_proxy_map[session_id] = proxy
return proxy
else: # ROUND_ROBIN
return self._get_round_robin(healthy)
def _get_round_robin(self, proxies: List[ProxyInfo]) -> ProxyInfo:
"""轮询获取代理"""
proxy = proxies[self._round_robin_index % len(proxies)]
self._round_robin_index += 1
return proxy
def mark_success(self, proxy: ProxyInfo, latency_ms: float = 0) -> None:
"""标记代理请求成功"""
if proxy is None:
return
with self._lock:
proxy.success_count += 1
proxy.fail_count = 0
proxy.healthy = True
if latency_ms > 0:
# 更新平均延迟(指数移动平均)
if proxy.avg_latency_ms == 0:
proxy.avg_latency_ms = latency_ms
else:
proxy.avg_latency_ms = proxy.avg_latency_ms * 0.7 + latency_ms * 0.3
proxy.last_check_time = time.time()
def mark_failed(self, proxy: ProxyInfo) -> None:
"""标记代理请求失败"""
if proxy is None:
return
with self._lock:
proxy.fail_count += 1
if proxy.fail_count >= self.max_failures:
proxy.healthy = False
proxy.cooldown_until = time.time() + self.cooldown_seconds
def health_check(self, proxy: ProxyInfo = None) -> bool:
"""
健康检查单个或所有代理
Returns:
如果检查通过返回True
"""
import requests
proxies_to_check = [proxy] if proxy else self._proxies
for p in proxies_to_check:
if p is None:
continue
try:
start = time.time()
resp = requests.get(
self.health_check_url,
proxies={"http": p.proxy_url, "https": p.proxy_url},
timeout=self.health_check_timeout,
)
if resp.status_code == 200:
latency_ms = (time.time() - start) * 1000
self.mark_success(p, latency_ms)
else:
self.mark_failed(p)
except Exception:
self.mark_failed(p)
return proxy.healthy if proxy else len(self.get_healthy_proxies()) > 0
def get_healthy_proxies(self) -> List[ProxyInfo]:
"""获取所有健康代理"""
with self._lock:
return [p for p in self._proxies if p.healthy]
def get_stats(self) -> dict:
"""获取代理池统计信息"""
with self._lock:
healthy = [p for p in self._proxies if p.healthy]
return {
"total": len(self._proxies),
"healthy": len(healthy),
"unhealthy": len(self._proxies) - len(healthy),
"proxies": [
{
"server": p.server,
"healthy": p.healthy,
"success_rate": p.success_rate,
"avg_latency_ms": round(p.avg_latency_ms, 2),
"fail_count": p.fail_count,
}
for p in self._proxies
]
}
def release_session(self, session_id: str) -> None:
"""释放会话绑定的代理"""
with self._lock:
self._session_proxy_map.pop(session_id, None)
def parse_proxy_url(value: str) -> Optional[ProxyInfo]:
text = value.strip()
if not text or text.startswith("#"):
return None
if "://" not in text:
text = "http://" + text
import re
match = re.match(r"^(https?|socks5)://(?:(?P<user>[^:@/]+):(?P<pwd>[^@/]+)@)?(?P<host>[\w.\-]+):(?P<port>\d{2,5})$", text, re.I)
if not match:
return None
scheme = text.split("://", 1)[0].lower()
host = match.group("host")
port = match.group("port")
user = match.group("user") or ""
pwd = match.group("pwd") or ""
return ProxyInfo(server=f"{scheme}://{host}:{port}", username=user, password=pwd)