commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1,353 @@
|
||||
"""
|
||||
Playwright 浏览器代理池 - 支持动态换 IP
|
||||
功能:
|
||||
- 多代理轮换(轮询/随机/会话粘性)
|
||||
- 健康检查(自动剔除失效代理)
|
||||
- 延迟统计(加权选择最优代理)
|
||||
- 线程安全
|
||||
"""
|
||||
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional, List, Dict, Any
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class RotationStrategy(Enum):
|
||||
"""轮换策略"""
|
||||
ROUND_ROBIN = "round_robin" # 轮询
|
||||
RANDOM = "random" # 随机
|
||||
WEIGHTED = "weighted" # 加权(按成功率+延迟)
|
||||
SESSION_STICKY = "session_sticky" # 会话粘性(同会话同IP)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProxyInfo:
|
||||
"""代理信息"""
|
||||
server: str # http://ip:port 或 socks5://ip:port
|
||||
username: str = ""
|
||||
password: str = ""
|
||||
|
||||
# 统计数据
|
||||
success_count: int = 0
|
||||
fail_count: int = 0
|
||||
total_latency_ms: float = 0.0
|
||||
avg_latency_ms: float = 0.0
|
||||
last_check_time: float = 0.0
|
||||
last_fail_time: float = 0.0
|
||||
|
||||
# 状态
|
||||
healthy: bool = True
|
||||
cooldown_until: float = 0.0 # 冷却结束时间
|
||||
|
||||
@property
|
||||
def proxy_url(self) -> str:
|
||||
"""获取完整的代理URL(含认证)"""
|
||||
if self.username and self.password:
|
||||
# http://user:pass@ip:port
|
||||
protocol = self.server.split("://")[0]
|
||||
rest = self.server.split("://")[1]
|
||||
return f"{protocol}://{self.username}:{self.password}@{rest}"
|
||||
return self.server
|
||||
|
||||
def to_requests_proxies(self) -> dict[str, str]:
|
||||
return {"http": self.proxy_url, "https": self.proxy_url}
|
||||
|
||||
def to_playwright_proxy(self) -> dict[str, str]:
|
||||
data = {"server": self.server}
|
||||
if self.username:
|
||||
data["username"] = self.username
|
||||
data["password"] = self.password
|
||||
return data
|
||||
|
||||
@property
|
||||
def success_rate(self) -> float:
|
||||
"""成功率"""
|
||||
total = self.success_count + self.fail_count
|
||||
if total == 0:
|
||||
return 1.0
|
||||
return self.success_count / total
|
||||
|
||||
@property
|
||||
def weight(self) -> float:
|
||||
"""
|
||||
计算代理权重(用于加权随机选择)
|
||||
考虑因素:成功率、延迟、冷却状态
|
||||
"""
|
||||
if not self.healthy:
|
||||
return 0.0
|
||||
|
||||
# 冷却中
|
||||
if self.cooldown_until > time.time():
|
||||
return 0.0
|
||||
|
||||
# 成功率权重
|
||||
success_weight = self.success_rate
|
||||
|
||||
# 延迟权重(延迟越低权重越高)
|
||||
if self.avg_latency_ms > 0:
|
||||
latency_weight = max(0.1, 1000.0 / max(self.avg_latency_ms, 1))
|
||||
else:
|
||||
latency_weight = 1.0
|
||||
|
||||
# 失败惩罚
|
||||
fail_penalty = 1.0 / (1.0 + self.fail_count * 0.5)
|
||||
|
||||
return success_weight * latency_weight * fail_penalty
|
||||
|
||||
|
||||
class ProxyPool:
|
||||
"""代理池 - 管理多个代理的获取和轮换"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
strategy: RotationStrategy = RotationStrategy.WEIGHTED,
|
||||
health_check_url: str = "https://httpbin.org/ip",
|
||||
health_check_timeout: int = 8,
|
||||
max_failures: int = 3,
|
||||
cooldown_seconds: int = 60,
|
||||
auto_health_check: bool = True,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
strategy: 轮换策略
|
||||
health_check_url: 健康检查URL
|
||||
health_check_timeout: 健康检查超时(秒)
|
||||
max_failures: 最大失败次数,超过后标记为不健康
|
||||
cooldown_seconds: 不健康代理的冷却时间(秒)
|
||||
auto_health_check: 是否自动健康检查
|
||||
"""
|
||||
self.strategy = strategy
|
||||
self.health_check_url = health_check_url
|
||||
self.health_check_timeout = health_check_timeout
|
||||
self.max_failures = max_failures
|
||||
self.cooldown_seconds = cooldown_seconds
|
||||
self.auto_health_check = auto_health_check
|
||||
|
||||
self._proxies: List[ProxyInfo] = []
|
||||
self._session_proxy_map: Dict[str, ProxyInfo] = {} # 会话粘性用
|
||||
self._round_robin_index = 0
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def add_proxy(self, server: str, username: str = "", password: str = "") -> None:
|
||||
"""添加单个代理"""
|
||||
with self._lock:
|
||||
# 去重
|
||||
existing = {p.server for p in self._proxies}
|
||||
if server not in existing:
|
||||
self._proxies.append(ProxyInfo(
|
||||
server=server,
|
||||
username=username,
|
||||
password=password,
|
||||
))
|
||||
|
||||
def add_proxy_url(self, value: str) -> None:
|
||||
parsed = parse_proxy_url(value)
|
||||
if parsed is not None:
|
||||
self.add_proxy(parsed.server, parsed.username, parsed.password)
|
||||
|
||||
def add_proxies(self, proxies: List[dict]) -> None:
|
||||
"""批量添加代理
|
||||
|
||||
Args:
|
||||
proxies: [{"server": "http://1.2.3.4:8080", "username": "", "password": ""}, ...]
|
||||
"""
|
||||
for p in proxies:
|
||||
self.add_proxy(
|
||||
server=p["server"],
|
||||
username=p.get("username", ""),
|
||||
password=p.get("password", ""),
|
||||
)
|
||||
|
||||
def add_proxy_text(self, text: str) -> int:
|
||||
count = 0
|
||||
for line in text.splitlines():
|
||||
parsed = parse_proxy_url(line)
|
||||
if parsed is None:
|
||||
continue
|
||||
self.add_proxy(parsed.server, parsed.username, parsed.password)
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def remove_proxy(self, server: str) -> None:
|
||||
"""移除代理"""
|
||||
with self._lock:
|
||||
self._proxies = [p for p in self._proxies if p.server != server]
|
||||
|
||||
def get_proxy(self, session_id: str = None) -> Optional[ProxyInfo]:
|
||||
"""
|
||||
获取一个代理
|
||||
|
||||
Args:
|
||||
session_id: 会话ID,用于SESSION_STICKY策略
|
||||
|
||||
Returns:
|
||||
代理信息,如果没有可用代理返回None
|
||||
"""
|
||||
with self._lock:
|
||||
# 过滤出健康的代理
|
||||
now = time.time()
|
||||
healthy = [p for p in self._proxies if p.healthy and p.cooldown_until <= now]
|
||||
|
||||
if not healthy:
|
||||
# 尝试从冷却中恢复
|
||||
now = time.time()
|
||||
recovered = []
|
||||
for p in self._proxies:
|
||||
if not p.healthy and p.cooldown_until <= now:
|
||||
p.healthy = True
|
||||
p.fail_count = 0
|
||||
recovered.append(p)
|
||||
if recovered:
|
||||
healthy = recovered
|
||||
|
||||
if not healthy:
|
||||
return None
|
||||
|
||||
# 根据策略选择代理
|
||||
if self.strategy == RotationStrategy.RANDOM:
|
||||
return random.choice(healthy)
|
||||
|
||||
elif self.strategy == RotationStrategy.WEIGHTED:
|
||||
# 加权随机选择
|
||||
weights = [p.weight for p in healthy]
|
||||
total_weight = sum(weights)
|
||||
if total_weight <= 0:
|
||||
return random.choice(healthy)
|
||||
r = random.uniform(0, total_weight)
|
||||
cumsum = 0
|
||||
for i, w in enumerate(weights):
|
||||
cumsum += w
|
||||
if r <= cumsum:
|
||||
return healthy[i]
|
||||
return healthy[0]
|
||||
|
||||
elif self.strategy == RotationStrategy.SESSION_STICKY:
|
||||
if session_id and session_id in self._session_proxy_map:
|
||||
proxy = self._session_proxy_map[session_id]
|
||||
if proxy in healthy:
|
||||
return proxy
|
||||
# 新会话,分配一个代理
|
||||
proxy = self._get_round_robin(healthy)
|
||||
if session_id:
|
||||
self._session_proxy_map[session_id] = proxy
|
||||
return proxy
|
||||
|
||||
else: # ROUND_ROBIN
|
||||
return self._get_round_robin(healthy)
|
||||
|
||||
def _get_round_robin(self, proxies: List[ProxyInfo]) -> ProxyInfo:
|
||||
"""轮询获取代理"""
|
||||
proxy = proxies[self._round_robin_index % len(proxies)]
|
||||
self._round_robin_index += 1
|
||||
return proxy
|
||||
|
||||
def mark_success(self, proxy: ProxyInfo, latency_ms: float = 0) -> None:
|
||||
"""标记代理请求成功"""
|
||||
if proxy is None:
|
||||
return
|
||||
with self._lock:
|
||||
proxy.success_count += 1
|
||||
proxy.fail_count = 0
|
||||
proxy.healthy = True
|
||||
if latency_ms > 0:
|
||||
# 更新平均延迟(指数移动平均)
|
||||
if proxy.avg_latency_ms == 0:
|
||||
proxy.avg_latency_ms = latency_ms
|
||||
else:
|
||||
proxy.avg_latency_ms = proxy.avg_latency_ms * 0.7 + latency_ms * 0.3
|
||||
proxy.last_check_time = time.time()
|
||||
|
||||
def mark_failed(self, proxy: ProxyInfo) -> None:
|
||||
"""标记代理请求失败"""
|
||||
if proxy is None:
|
||||
return
|
||||
with self._lock:
|
||||
proxy.fail_count += 1
|
||||
if proxy.fail_count >= self.max_failures:
|
||||
proxy.healthy = False
|
||||
proxy.cooldown_until = time.time() + self.cooldown_seconds
|
||||
|
||||
def health_check(self, proxy: ProxyInfo = None) -> bool:
|
||||
"""
|
||||
健康检查单个或所有代理
|
||||
|
||||
Returns:
|
||||
如果检查通过返回True
|
||||
"""
|
||||
import requests
|
||||
|
||||
proxies_to_check = [proxy] if proxy else self._proxies
|
||||
|
||||
for p in proxies_to_check:
|
||||
if p is None:
|
||||
continue
|
||||
try:
|
||||
start = time.time()
|
||||
resp = requests.get(
|
||||
self.health_check_url,
|
||||
proxies={"http": p.proxy_url, "https": p.proxy_url},
|
||||
timeout=self.health_check_timeout,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
latency_ms = (time.time() - start) * 1000
|
||||
self.mark_success(p, latency_ms)
|
||||
else:
|
||||
self.mark_failed(p)
|
||||
except Exception:
|
||||
self.mark_failed(p)
|
||||
|
||||
return proxy.healthy if proxy else len(self.get_healthy_proxies()) > 0
|
||||
|
||||
def get_healthy_proxies(self) -> List[ProxyInfo]:
|
||||
"""获取所有健康代理"""
|
||||
with self._lock:
|
||||
return [p for p in self._proxies if p.healthy]
|
||||
|
||||
def get_stats(self) -> dict:
|
||||
"""获取代理池统计信息"""
|
||||
with self._lock:
|
||||
healthy = [p for p in self._proxies if p.healthy]
|
||||
return {
|
||||
"total": len(self._proxies),
|
||||
"healthy": len(healthy),
|
||||
"unhealthy": len(self._proxies) - len(healthy),
|
||||
"proxies": [
|
||||
{
|
||||
"server": p.server,
|
||||
"healthy": p.healthy,
|
||||
"success_rate": p.success_rate,
|
||||
"avg_latency_ms": round(p.avg_latency_ms, 2),
|
||||
"fail_count": p.fail_count,
|
||||
}
|
||||
for p in self._proxies
|
||||
]
|
||||
}
|
||||
|
||||
def release_session(self, session_id: str) -> None:
|
||||
"""释放会话绑定的代理"""
|
||||
with self._lock:
|
||||
self._session_proxy_map.pop(session_id, None)
|
||||
|
||||
|
||||
def parse_proxy_url(value: str) -> Optional[ProxyInfo]:
|
||||
text = value.strip()
|
||||
if not text or text.startswith("#"):
|
||||
return None
|
||||
if "://" not in text:
|
||||
text = "http://" + text
|
||||
import re
|
||||
|
||||
match = re.match(r"^(https?|socks5)://(?:(?P<user>[^:@/]+):(?P<pwd>[^@/]+)@)?(?P<host>[\w.\-]+):(?P<port>\d{2,5})$", text, re.I)
|
||||
if not match:
|
||||
return None
|
||||
scheme = text.split("://", 1)[0].lower()
|
||||
host = match.group("host")
|
||||
port = match.group("port")
|
||||
user = match.group("user") or ""
|
||||
pwd = match.group("pwd") or ""
|
||||
return ProxyInfo(server=f"{scheme}://{host}:{port}", username=user, password=pwd)
|
||||
|
||||
|
||||
Reference in new issue
Block a user