commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1,128 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from pixiv_app.core.library import DEFAULT_LIBRARY_DB, PixivLibrary
|
||||
|
||||
|
||||
OcrExtractor = Callable[[Path], tuple[str, float | None, str]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class LocalOcrResult:
|
||||
artwork_id: int
|
||||
status: str
|
||||
text_length: int = 0
|
||||
confidence: float | None = None
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class LocalOcrIndexer:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
db_path: str | Path = DEFAULT_LIBRARY_DB,
|
||||
enabled: bool | None = None,
|
||||
max_dimension: int | None = None,
|
||||
language: str | None = None,
|
||||
max_concurrency: int | None = None,
|
||||
) -> None:
|
||||
self.db_path = Path(db_path)
|
||||
self.enabled = enabled if enabled is not None else _env_bool("SAKURA_LOCAL_OCR_ENABLED", False)
|
||||
self.max_dimension = max_dimension or int(os.getenv("SAKURA_LOCAL_OCR_MAX_DIMENSION", "1800"))
|
||||
self.language = language or os.getenv("SAKURA_LOCAL_OCR_LANGUAGE", "ch")
|
||||
self._semaphore = threading.BoundedSemaphore(
|
||||
max_concurrency or int(os.getenv("SAKURA_LOCAL_OCR_MAX_CONCURRENCY", "1"))
|
||||
)
|
||||
|
||||
def index_artwork(self, artwork_id: int, *, extractor: OcrExtractor | None = None) -> LocalOcrResult:
|
||||
if not self.enabled and extractor is None:
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="disabled")
|
||||
if not self._semaphore.acquire(blocking=False):
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="busy")
|
||||
library = PixivLibrary(self.db_path)
|
||||
try:
|
||||
existing = library.get_ocr_text(artwork_id)
|
||||
if existing is not None and str(existing["status"]) == "done" and str(existing["ocr_text"]).strip():
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="done", text_length=len(str(existing["ocr_text"])))
|
||||
|
||||
row = library.connection.execute(
|
||||
"""
|
||||
SELECT file_path
|
||||
FROM artwork_files
|
||||
WHERE artwork_id = ?
|
||||
ORDER BY page_index ASC
|
||||
LIMIT 1
|
||||
""",
|
||||
(int(artwork_id),),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="failed", error="file_not_found")
|
||||
path = Path(str(row["file_path"]))
|
||||
if path.suffix.lower() not in {".jpg", ".jpeg", ".png", ".webp"}:
|
||||
library.upsert_ocr_text(artwork_id, text="", status="failed", attempts=1)
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="failed", error="unsupported_file_type")
|
||||
|
||||
text, confidence, language = (extractor or self._paddle_ocr_extract)(path)
|
||||
library.upsert_ocr_text(
|
||||
artwork_id,
|
||||
text=text.strip(),
|
||||
confidence=confidence,
|
||||
language=language or self.language,
|
||||
status="done",
|
||||
attempts=1,
|
||||
)
|
||||
logging.info("Local OCR done artwork=%s text_len=%s confidence=%s", artwork_id, len(text.strip()), confidence)
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="done", text_length=len(text.strip()), confidence=confidence)
|
||||
except Exception as exc:
|
||||
library.upsert_ocr_text(artwork_id, text="", status="failed", attempts=1)
|
||||
logging.warning("Local OCR failed artwork=%s error=%s", artwork_id, exc.__class__.__name__)
|
||||
return LocalOcrResult(artwork_id=artwork_id, status="failed", error=exc.__class__.__name__)
|
||||
finally:
|
||||
library.close()
|
||||
self._semaphore.release()
|
||||
|
||||
def _paddle_ocr_extract(self, path: Path) -> tuple[str, float | None, str]:
|
||||
from PIL import Image
|
||||
from paddleocr import PaddleOCR
|
||||
|
||||
work_path = path
|
||||
with Image.open(path) as image:
|
||||
original_size = image.size
|
||||
image.thumbnail((self.max_dimension, self.max_dimension))
|
||||
if image.size != original_size:
|
||||
tmp = path.with_suffix(".local-ocr-resized.jpg")
|
||||
image.convert("RGB").save(tmp, quality=90)
|
||||
work_path = tmp
|
||||
try:
|
||||
ocr = PaddleOCR(use_angle_cls=True, lang=self.language, use_gpu=False)
|
||||
result = ocr.ocr(str(work_path), cls=True)
|
||||
texts: list[str] = []
|
||||
scores: list[float] = []
|
||||
for page in result or []:
|
||||
for line in page or []:
|
||||
if len(line) >= 2 and isinstance(line[1], (list, tuple)):
|
||||
text = str(line[1][0]).strip()
|
||||
if text:
|
||||
texts.append(text)
|
||||
try:
|
||||
scores.append(float(line[1][1]))
|
||||
except (TypeError, ValueError, IndexError):
|
||||
pass
|
||||
confidence = sum(scores) / len(scores) if scores else None
|
||||
return "\n".join(texts), confidence, self.language
|
||||
finally:
|
||||
if work_path != path:
|
||||
work_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool) -> bool:
|
||||
value = os.getenv(name)
|
||||
if value is None:
|
||||
return default
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
Reference in new issue
Block a user