Files
SakuraHub/distributed_crawler/crawler/results.py
T

164 lines
6.1 KiB
Python
Raw Normal View History

2026-11-16 20:45:22 +08:00
from __future__ import annotations
import json
from abc import ABC, abstractmethod
from pathlib import Path
import asyncpg
from motor.motor_asyncio import AsyncIOMotorClient
from .config import StorageSettings
from .models import CrawlResult, make_url_hash
class ResultRepository(ABC):
@abstractmethod
async def setup(self) -> None: ...
@abstractmethod
async def store(self, result: CrawlResult) -> None: ...
@abstractmethod
async def close(self) -> None: ...
class JsonlResultRepository(ResultRepository):
def __init__(self, path: Path) -> None:
self.path = path
self.index_path = path.with_suffix(path.suffix + ".url_hashes")
self._seen_url_hashes: set[str] = set()
self.path.parent.mkdir(parents=True, exist_ok=True)
async def setup(self) -> None:
if self.index_path.exists():
self._seen_url_hashes = {
line.strip()
for line in self.index_path.read_text(encoding="utf-8").splitlines()
if line.strip()
}
return
if self.path.exists():
with self.path.open("r", encoding="utf-8") as file_handle:
for line in file_handle:
try:
payload = json.loads(line)
except json.JSONDecodeError:
continue
url_hash = payload.get("url_hash") or (make_url_hash(str(payload["url"])) if payload.get("url") else None)
if url_hash:
self._seen_url_hashes.add(str(url_hash))
if self._seen_url_hashes:
self.index_path.write_text("\n".join(sorted(self._seen_url_hashes)) + "\n", encoding="utf-8")
async def store(self, result: CrawlResult) -> None:
if result.url_hash in self._seen_url_hashes:
return
with self.path.open("a", encoding="utf-8") as file_handle:
file_handle.write(json.dumps(result.to_dict(), ensure_ascii=False) + "\n")
with self.index_path.open("a", encoding="utf-8") as file_handle:
file_handle.write(result.url_hash + "\n")
self._seen_url_hashes.add(result.url_hash)
async def close(self) -> None:
return None
class MongoResultRepository(ResultRepository):
def __init__(self, settings: StorageSettings) -> None:
self.client = AsyncIOMotorClient(settings.mongodb_dsn)
self.collection = self.client[settings.mongodb_database][settings.mongodb_collection]
async def setup(self) -> None:
await self.collection.create_index("task_id", unique=False)
await self.collection.create_index(
"url_hash",
unique=True,
partialFilterExpression={"url_hash": {"$exists": True}},
)
async def store(self, result: CrawlResult) -> None:
await self.collection.update_one(
{"url_hash": result.url_hash},
{"$set": result.to_dict()},
upsert=True,
)
async def close(self) -> None:
self.client.close()
class PostgresResultRepository(ResultRepository):
def __init__(self, settings: StorageSettings) -> None:
self.dsn = settings.postgres_dsn
self.table = settings.postgres_table
self.pool: asyncpg.Pool | None = None
async def setup(self) -> None:
self.pool = await asyncpg.create_pool(self.dsn)
async with self.pool.acquire() as connection:
await connection.execute(
f"""
CREATE TABLE IF NOT EXISTS {self.table} (
id BIGSERIAL PRIMARY KEY,
task_id TEXT NOT NULL,
worker_id TEXT NOT NULL,
status TEXT NOT NULL,
url TEXT NOT NULL,
url_hash TEXT NOT NULL,
spider TEXT NOT NULL,
http_status INTEGER,
payload JSONB NOT NULL,
error TEXT,
finished_at TIMESTAMPTZ NOT NULL
)
"""
)
await connection.execute(f"ALTER TABLE {self.table} ADD COLUMN IF NOT EXISTS url_hash TEXT")
await connection.execute(
f"CREATE UNIQUE INDEX IF NOT EXISTS {self.table}_url_hash_uidx ON {self.table} (url_hash)"
)
async def store(self, result: CrawlResult) -> None:
if self.pool is None:
raise RuntimeError("Postgres repository is not initialized")
async with self.pool.acquire() as connection:
await connection.execute(
f"""
INSERT INTO {self.table}
(task_id, worker_id, status, url, url_hash, spider, http_status, payload, error, finished_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb, $9, $10::timestamptz)
ON CONFLICT (url_hash) DO UPDATE SET
task_id = EXCLUDED.task_id,
worker_id = EXCLUDED.worker_id,
status = EXCLUDED.status,
url = EXCLUDED.url,
spider = EXCLUDED.spider,
http_status = EXCLUDED.http_status,
payload = EXCLUDED.payload,
error = EXCLUDED.error,
finished_at = EXCLUDED.finished_at
""",
result.task_id,
result.worker_id,
result.status.value,
result.url,
result.url_hash,
result.spider,
result.http_status,
json.dumps(result.data, ensure_ascii=False),
result.error,
result.finished_at,
)
async def close(self) -> None:
if self.pool is not None:
await self.pool.close()
def build_result_repository(settings: StorageSettings, jsonl_path: Path) -> ResultRepository:
backend = settings.backend.lower()
if backend == "mongodb":
return MongoResultRepository(settings)
if backend == "postgresql":
return PostgresResultRepository(settings)
return JsonlResultRepository(jsonl_path)