354 lines
12 KiB
Python
354 lines
12 KiB
Python
"""
|
||
Playwright 浏览器代理池 - 支持动态换 IP
|
||
功能:
|
||
- 多代理轮换(轮询/随机/会话粘性)
|
||
- 健康检查(自动剔除失效代理)
|
||
- 延迟统计(加权选择最优代理)
|
||
- 线程安全
|
||
"""
|
||
|
||
import random
|
||
import threading
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from typing import Optional, List, Dict, Any
|
||
from enum import Enum
|
||
|
||
|
||
class RotationStrategy(Enum):
|
||
"""轮换策略"""
|
||
ROUND_ROBIN = "round_robin" # 轮询
|
||
RANDOM = "random" # 随机
|
||
WEIGHTED = "weighted" # 加权(按成功率+延迟)
|
||
SESSION_STICKY = "session_sticky" # 会话粘性(同会话同IP)
|
||
|
||
|
||
@dataclass
|
||
class ProxyInfo:
|
||
"""代理信息"""
|
||
server: str # http://ip:port 或 socks5://ip:port
|
||
username: str = ""
|
||
password: str = ""
|
||
|
||
# 统计数据
|
||
success_count: int = 0
|
||
fail_count: int = 0
|
||
total_latency_ms: float = 0.0
|
||
avg_latency_ms: float = 0.0
|
||
last_check_time: float = 0.0
|
||
last_fail_time: float = 0.0
|
||
|
||
# 状态
|
||
healthy: bool = True
|
||
cooldown_until: float = 0.0 # 冷却结束时间
|
||
|
||
@property
|
||
def proxy_url(self) -> str:
|
||
"""获取完整的代理URL(含认证)"""
|
||
if self.username and self.password:
|
||
# http://user:pass@ip:port
|
||
protocol = self.server.split("://")[0]
|
||
rest = self.server.split("://")[1]
|
||
return f"{protocol}://{self.username}:{self.password}@{rest}"
|
||
return self.server
|
||
|
||
def to_requests_proxies(self) -> dict[str, str]:
|
||
return {"http": self.proxy_url, "https": self.proxy_url}
|
||
|
||
def to_playwright_proxy(self) -> dict[str, str]:
|
||
data = {"server": self.server}
|
||
if self.username:
|
||
data["username"] = self.username
|
||
data["password"] = self.password
|
||
return data
|
||
|
||
@property
|
||
def success_rate(self) -> float:
|
||
"""成功率"""
|
||
total = self.success_count + self.fail_count
|
||
if total == 0:
|
||
return 1.0
|
||
return self.success_count / total
|
||
|
||
@property
|
||
def weight(self) -> float:
|
||
"""
|
||
计算代理权重(用于加权随机选择)
|
||
考虑因素:成功率、延迟、冷却状态
|
||
"""
|
||
if not self.healthy:
|
||
return 0.0
|
||
|
||
# 冷却中
|
||
if self.cooldown_until > time.time():
|
||
return 0.0
|
||
|
||
# 成功率权重
|
||
success_weight = self.success_rate
|
||
|
||
# 延迟权重(延迟越低权重越高)
|
||
if self.avg_latency_ms > 0:
|
||
latency_weight = max(0.1, 1000.0 / max(self.avg_latency_ms, 1))
|
||
else:
|
||
latency_weight = 1.0
|
||
|
||
# 失败惩罚
|
||
fail_penalty = 1.0 / (1.0 + self.fail_count * 0.5)
|
||
|
||
return success_weight * latency_weight * fail_penalty
|
||
|
||
|
||
class ProxyPool:
|
||
"""代理池 - 管理多个代理的获取和轮换"""
|
||
|
||
def __init__(
|
||
self,
|
||
strategy: RotationStrategy = RotationStrategy.WEIGHTED,
|
||
health_check_url: str = "https://httpbin.org/ip",
|
||
health_check_timeout: int = 8,
|
||
max_failures: int = 3,
|
||
cooldown_seconds: int = 60,
|
||
auto_health_check: bool = True,
|
||
):
|
||
"""
|
||
Args:
|
||
strategy: 轮换策略
|
||
health_check_url: 健康检查URL
|
||
health_check_timeout: 健康检查超时(秒)
|
||
max_failures: 最大失败次数,超过后标记为不健康
|
||
cooldown_seconds: 不健康代理的冷却时间(秒)
|
||
auto_health_check: 是否自动健康检查
|
||
"""
|
||
self.strategy = strategy
|
||
self.health_check_url = health_check_url
|
||
self.health_check_timeout = health_check_timeout
|
||
self.max_failures = max_failures
|
||
self.cooldown_seconds = cooldown_seconds
|
||
self.auto_health_check = auto_health_check
|
||
|
||
self._proxies: List[ProxyInfo] = []
|
||
self._session_proxy_map: Dict[str, ProxyInfo] = {} # 会话粘性用
|
||
self._round_robin_index = 0
|
||
self._lock = threading.Lock()
|
||
|
||
def add_proxy(self, server: str, username: str = "", password: str = "") -> None:
|
||
"""添加单个代理"""
|
||
with self._lock:
|
||
# 去重
|
||
existing = {p.server for p in self._proxies}
|
||
if server not in existing:
|
||
self._proxies.append(ProxyInfo(
|
||
server=server,
|
||
username=username,
|
||
password=password,
|
||
))
|
||
|
||
def add_proxy_url(self, value: str) -> None:
|
||
parsed = parse_proxy_url(value)
|
||
if parsed is not None:
|
||
self.add_proxy(parsed.server, parsed.username, parsed.password)
|
||
|
||
def add_proxies(self, proxies: List[dict]) -> None:
|
||
"""批量添加代理
|
||
|
||
Args:
|
||
proxies: [{"server": "http://1.2.3.4:8080", "username": "", "password": ""}, ...]
|
||
"""
|
||
for p in proxies:
|
||
self.add_proxy(
|
||
server=p["server"],
|
||
username=p.get("username", ""),
|
||
password=p.get("password", ""),
|
||
)
|
||
|
||
def add_proxy_text(self, text: str) -> int:
|
||
count = 0
|
||
for line in text.splitlines():
|
||
parsed = parse_proxy_url(line)
|
||
if parsed is None:
|
||
continue
|
||
self.add_proxy(parsed.server, parsed.username, parsed.password)
|
||
count += 1
|
||
return count
|
||
|
||
def remove_proxy(self, server: str) -> None:
|
||
"""移除代理"""
|
||
with self._lock:
|
||
self._proxies = [p for p in self._proxies if p.server != server]
|
||
|
||
def get_proxy(self, session_id: str = None) -> Optional[ProxyInfo]:
|
||
"""
|
||
获取一个代理
|
||
|
||
Args:
|
||
session_id: 会话ID,用于SESSION_STICKY策略
|
||
|
||
Returns:
|
||
代理信息,如果没有可用代理返回None
|
||
"""
|
||
with self._lock:
|
||
# 过滤出健康的代理
|
||
now = time.time()
|
||
healthy = [p for p in self._proxies if p.healthy and p.cooldown_until <= now]
|
||
|
||
if not healthy:
|
||
# 尝试从冷却中恢复
|
||
now = time.time()
|
||
recovered = []
|
||
for p in self._proxies:
|
||
if not p.healthy and p.cooldown_until <= now:
|
||
p.healthy = True
|
||
p.fail_count = 0
|
||
recovered.append(p)
|
||
if recovered:
|
||
healthy = recovered
|
||
|
||
if not healthy:
|
||
return None
|
||
|
||
# 根据策略选择代理
|
||
if self.strategy == RotationStrategy.RANDOM:
|
||
return random.choice(healthy)
|
||
|
||
elif self.strategy == RotationStrategy.WEIGHTED:
|
||
# 加权随机选择
|
||
weights = [p.weight for p in healthy]
|
||
total_weight = sum(weights)
|
||
if total_weight <= 0:
|
||
return random.choice(healthy)
|
||
r = random.uniform(0, total_weight)
|
||
cumsum = 0
|
||
for i, w in enumerate(weights):
|
||
cumsum += w
|
||
if r <= cumsum:
|
||
return healthy[i]
|
||
return healthy[0]
|
||
|
||
elif self.strategy == RotationStrategy.SESSION_STICKY:
|
||
if session_id and session_id in self._session_proxy_map:
|
||
proxy = self._session_proxy_map[session_id]
|
||
if proxy in healthy:
|
||
return proxy
|
||
# 新会话,分配一个代理
|
||
proxy = self._get_round_robin(healthy)
|
||
if session_id:
|
||
self._session_proxy_map[session_id] = proxy
|
||
return proxy
|
||
|
||
else: # ROUND_ROBIN
|
||
return self._get_round_robin(healthy)
|
||
|
||
def _get_round_robin(self, proxies: List[ProxyInfo]) -> ProxyInfo:
|
||
"""轮询获取代理"""
|
||
proxy = proxies[self._round_robin_index % len(proxies)]
|
||
self._round_robin_index += 1
|
||
return proxy
|
||
|
||
def mark_success(self, proxy: ProxyInfo, latency_ms: float = 0) -> None:
|
||
"""标记代理请求成功"""
|
||
if proxy is None:
|
||
return
|
||
with self._lock:
|
||
proxy.success_count += 1
|
||
proxy.fail_count = 0
|
||
proxy.healthy = True
|
||
if latency_ms > 0:
|
||
# 更新平均延迟(指数移动平均)
|
||
if proxy.avg_latency_ms == 0:
|
||
proxy.avg_latency_ms = latency_ms
|
||
else:
|
||
proxy.avg_latency_ms = proxy.avg_latency_ms * 0.7 + latency_ms * 0.3
|
||
proxy.last_check_time = time.time()
|
||
|
||
def mark_failed(self, proxy: ProxyInfo) -> None:
|
||
"""标记代理请求失败"""
|
||
if proxy is None:
|
||
return
|
||
with self._lock:
|
||
proxy.fail_count += 1
|
||
if proxy.fail_count >= self.max_failures:
|
||
proxy.healthy = False
|
||
proxy.cooldown_until = time.time() + self.cooldown_seconds
|
||
|
||
def health_check(self, proxy: ProxyInfo = None) -> bool:
|
||
"""
|
||
健康检查单个或所有代理
|
||
|
||
Returns:
|
||
如果检查通过返回True
|
||
"""
|
||
import requests
|
||
|
||
proxies_to_check = [proxy] if proxy else self._proxies
|
||
|
||
for p in proxies_to_check:
|
||
if p is None:
|
||
continue
|
||
try:
|
||
start = time.time()
|
||
resp = requests.get(
|
||
self.health_check_url,
|
||
proxies={"http": p.proxy_url, "https": p.proxy_url},
|
||
timeout=self.health_check_timeout,
|
||
)
|
||
if resp.status_code == 200:
|
||
latency_ms = (time.time() - start) * 1000
|
||
self.mark_success(p, latency_ms)
|
||
else:
|
||
self.mark_failed(p)
|
||
except Exception:
|
||
self.mark_failed(p)
|
||
|
||
return proxy.healthy if proxy else len(self.get_healthy_proxies()) > 0
|
||
|
||
def get_healthy_proxies(self) -> List[ProxyInfo]:
|
||
"""获取所有健康代理"""
|
||
with self._lock:
|
||
return [p for p in self._proxies if p.healthy]
|
||
|
||
def get_stats(self) -> dict:
|
||
"""获取代理池统计信息"""
|
||
with self._lock:
|
||
healthy = [p for p in self._proxies if p.healthy]
|
||
return {
|
||
"total": len(self._proxies),
|
||
"healthy": len(healthy),
|
||
"unhealthy": len(self._proxies) - len(healthy),
|
||
"proxies": [
|
||
{
|
||
"server": p.server,
|
||
"healthy": p.healthy,
|
||
"success_rate": p.success_rate,
|
||
"avg_latency_ms": round(p.avg_latency_ms, 2),
|
||
"fail_count": p.fail_count,
|
||
}
|
||
for p in self._proxies
|
||
]
|
||
}
|
||
|
||
def release_session(self, session_id: str) -> None:
|
||
"""释放会话绑定的代理"""
|
||
with self._lock:
|
||
self._session_proxy_map.pop(session_id, None)
|
||
|
||
|
||
def parse_proxy_url(value: str) -> Optional[ProxyInfo]:
|
||
text = value.strip()
|
||
if not text or text.startswith("#"):
|
||
return None
|
||
if "://" not in text:
|
||
text = "http://" + text
|
||
import re
|
||
|
||
match = re.match(r"^(https?|socks5)://(?:(?P<user>[^:@/]+):(?P<pwd>[^@/]+)@)?(?P<host>[\w.\-]+):(?P<port>\d{2,5})$", text, re.I)
|
||
if not match:
|
||
return None
|
||
scheme = text.split("://", 1)[0].lower()
|
||
host = match.group("host")
|
||
port = match.group("port")
|
||
user = match.group("user") or ""
|
||
pwd = match.group("pwd") or ""
|
||
return ProxyInfo(server=f"{scheme}://{host}:{port}", username=user, password=pwd)
|
||
|
||
|