Files
SakuraHub/pixiv_app/core/browser_proxy_pool.py
2026-11-16 20:45:22 +08:00

354 lines
12 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
Playwright 浏览器代理池 - 支持动态换 IP
功能:
- 多代理轮换(轮询/随机/会话粘性)
- 健康检查(自动剔除失效代理)
- 延迟统计(加权选择最优代理)
- 线程安全
"""
import random
import threading
import time
from dataclasses import dataclass, field
from typing import Optional, List, Dict, Any
from enum import Enum
class RotationStrategy(Enum):
"""轮换策略"""
ROUND_ROBIN = "round_robin" # 轮询
RANDOM = "random" # 随机
WEIGHTED = "weighted" # 加权(按成功率+延迟)
SESSION_STICKY = "session_sticky" # 会话粘性(同会话同IP)
@dataclass
class ProxyInfo:
"""代理信息"""
server: str # http://ip:port 或 socks5://ip:port
username: str = ""
password: str = ""
# 统计数据
success_count: int = 0
fail_count: int = 0
total_latency_ms: float = 0.0
avg_latency_ms: float = 0.0
last_check_time: float = 0.0
last_fail_time: float = 0.0
# 状态
healthy: bool = True
cooldown_until: float = 0.0 # 冷却结束时间
@property
def proxy_url(self) -> str:
"""获取完整的代理URL(含认证)"""
if self.username and self.password:
# http://user:pass@ip:port
protocol = self.server.split("://")[0]
rest = self.server.split("://")[1]
return f"{protocol}://{self.username}:{self.password}@{rest}"
return self.server
def to_requests_proxies(self) -> dict[str, str]:
return {"http": self.proxy_url, "https": self.proxy_url}
def to_playwright_proxy(self) -> dict[str, str]:
data = {"server": self.server}
if self.username:
data["username"] = self.username
data["password"] = self.password
return data
@property
def success_rate(self) -> float:
"""成功率"""
total = self.success_count + self.fail_count
if total == 0:
return 1.0
return self.success_count / total
@property
def weight(self) -> float:
"""
计算代理权重(用于加权随机选择)
考虑因素:成功率、延迟、冷却状态
"""
if not self.healthy:
return 0.0
# 冷却中
if self.cooldown_until > time.time():
return 0.0
# 成功率权重
success_weight = self.success_rate
# 延迟权重(延迟越低权重越高)
if self.avg_latency_ms > 0:
latency_weight = max(0.1, 1000.0 / max(self.avg_latency_ms, 1))
else:
latency_weight = 1.0
# 失败惩罚
fail_penalty = 1.0 / (1.0 + self.fail_count * 0.5)
return success_weight * latency_weight * fail_penalty
class ProxyPool:
"""代理池 - 管理多个代理的获取和轮换"""
def __init__(
self,
strategy: RotationStrategy = RotationStrategy.WEIGHTED,
health_check_url: str = "https://httpbin.org/ip",
health_check_timeout: int = 8,
max_failures: int = 3,
cooldown_seconds: int = 60,
auto_health_check: bool = True,
):
"""
Args:
strategy: 轮换策略
health_check_url: 健康检查URL
health_check_timeout: 健康检查超时(秒)
max_failures: 最大失败次数,超过后标记为不健康
cooldown_seconds: 不健康代理的冷却时间(秒)
auto_health_check: 是否自动健康检查
"""
self.strategy = strategy
self.health_check_url = health_check_url
self.health_check_timeout = health_check_timeout
self.max_failures = max_failures
self.cooldown_seconds = cooldown_seconds
self.auto_health_check = auto_health_check
self._proxies: List[ProxyInfo] = []
self._session_proxy_map: Dict[str, ProxyInfo] = {} # 会话粘性用
self._round_robin_index = 0
self._lock = threading.Lock()
def add_proxy(self, server: str, username: str = "", password: str = "") -> None:
"""添加单个代理"""
with self._lock:
# 去重
existing = {p.server for p in self._proxies}
if server not in existing:
self._proxies.append(ProxyInfo(
server=server,
username=username,
password=password,
))
def add_proxy_url(self, value: str) -> None:
parsed = parse_proxy_url(value)
if parsed is not None:
self.add_proxy(parsed.server, parsed.username, parsed.password)
def add_proxies(self, proxies: List[dict]) -> None:
"""批量添加代理
Args:
proxies: [{"server": "http://1.2.3.4:8080", "username": "", "password": ""}, ...]
"""
for p in proxies:
self.add_proxy(
server=p["server"],
username=p.get("username", ""),
password=p.get("password", ""),
)
def add_proxy_text(self, text: str) -> int:
count = 0
for line in text.splitlines():
parsed = parse_proxy_url(line)
if parsed is None:
continue
self.add_proxy(parsed.server, parsed.username, parsed.password)
count += 1
return count
def remove_proxy(self, server: str) -> None:
"""移除代理"""
with self._lock:
self._proxies = [p for p in self._proxies if p.server != server]
def get_proxy(self, session_id: str = None) -> Optional[ProxyInfo]:
"""
获取一个代理
Args:
session_id: 会话ID,用于SESSION_STICKY策略
Returns:
代理信息,如果没有可用代理返回None
"""
with self._lock:
# 过滤出健康的代理
now = time.time()
healthy = [p for p in self._proxies if p.healthy and p.cooldown_until <= now]
if not healthy:
# 尝试从冷却中恢复
now = time.time()
recovered = []
for p in self._proxies:
if not p.healthy and p.cooldown_until <= now:
p.healthy = True
p.fail_count = 0
recovered.append(p)
if recovered:
healthy = recovered
if not healthy:
return None
# 根据策略选择代理
if self.strategy == RotationStrategy.RANDOM:
return random.choice(healthy)
elif self.strategy == RotationStrategy.WEIGHTED:
# 加权随机选择
weights = [p.weight for p in healthy]
total_weight = sum(weights)
if total_weight <= 0:
return random.choice(healthy)
r = random.uniform(0, total_weight)
cumsum = 0
for i, w in enumerate(weights):
cumsum += w
if r <= cumsum:
return healthy[i]
return healthy[0]
elif self.strategy == RotationStrategy.SESSION_STICKY:
if session_id and session_id in self._session_proxy_map:
proxy = self._session_proxy_map[session_id]
if proxy in healthy:
return proxy
# 新会话,分配一个代理
proxy = self._get_round_robin(healthy)
if session_id:
self._session_proxy_map[session_id] = proxy
return proxy
else: # ROUND_ROBIN
return self._get_round_robin(healthy)
def _get_round_robin(self, proxies: List[ProxyInfo]) -> ProxyInfo:
"""轮询获取代理"""
proxy = proxies[self._round_robin_index % len(proxies)]
self._round_robin_index += 1
return proxy
def mark_success(self, proxy: ProxyInfo, latency_ms: float = 0) -> None:
"""标记代理请求成功"""
if proxy is None:
return
with self._lock:
proxy.success_count += 1
proxy.fail_count = 0
proxy.healthy = True
if latency_ms > 0:
# 更新平均延迟(指数移动平均)
if proxy.avg_latency_ms == 0:
proxy.avg_latency_ms = latency_ms
else:
proxy.avg_latency_ms = proxy.avg_latency_ms * 0.7 + latency_ms * 0.3
proxy.last_check_time = time.time()
def mark_failed(self, proxy: ProxyInfo) -> None:
"""标记代理请求失败"""
if proxy is None:
return
with self._lock:
proxy.fail_count += 1
if proxy.fail_count >= self.max_failures:
proxy.healthy = False
proxy.cooldown_until = time.time() + self.cooldown_seconds
def health_check(self, proxy: ProxyInfo = None) -> bool:
"""
健康检查单个或所有代理
Returns:
如果检查通过返回True
"""
import requests
proxies_to_check = [proxy] if proxy else self._proxies
for p in proxies_to_check:
if p is None:
continue
try:
start = time.time()
resp = requests.get(
self.health_check_url,
proxies={"http": p.proxy_url, "https": p.proxy_url},
timeout=self.health_check_timeout,
)
if resp.status_code == 200:
latency_ms = (time.time() - start) * 1000
self.mark_success(p, latency_ms)
else:
self.mark_failed(p)
except Exception:
self.mark_failed(p)
return proxy.healthy if proxy else len(self.get_healthy_proxies()) > 0
def get_healthy_proxies(self) -> List[ProxyInfo]:
"""获取所有健康代理"""
with self._lock:
return [p for p in self._proxies if p.healthy]
def get_stats(self) -> dict:
"""获取代理池统计信息"""
with self._lock:
healthy = [p for p in self._proxies if p.healthy]
return {
"total": len(self._proxies),
"healthy": len(healthy),
"unhealthy": len(self._proxies) - len(healthy),
"proxies": [
{
"server": p.server,
"healthy": p.healthy,
"success_rate": p.success_rate,
"avg_latency_ms": round(p.avg_latency_ms, 2),
"fail_count": p.fail_count,
}
for p in self._proxies
]
}
def release_session(self, session_id: str) -> None:
"""释放会话绑定的代理"""
with self._lock:
self._session_proxy_map.pop(session_id, None)
def parse_proxy_url(value: str) -> Optional[ProxyInfo]:
text = value.strip()
if not text or text.startswith("#"):
return None
if "://" not in text:
text = "http://" + text
import re
match = re.match(r"^(https?|socks5)://(?:(?P<user>[^:@/]+):(?P<pwd>[^@/]+)@)?(?P<host>[\w.\-]+):(?P<port>\d{2,5})$", text, re.I)
if not match:
return None
scheme = text.split("://", 1)[0].lower()
host = match.group("host")
port = match.group("port")
user = match.group("user") or ""
pwd = match.group("pwd") or ""
return ProxyInfo(server=f"{scheme}://{host}:{port}", username=user, password=pwd)