commit
9c5fef1a4d
412 files changed
+94582
No files matched your search
@@ -0,0 +1 @@
|
||||
"""Application services."""
|
||||
@@ -0,0 +1,45 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from sakura_cloud.models.admin import AdminAuditLog
|
||||
from sakura_cloud.models.user import User
|
||||
|
||||
|
||||
def write_admin_audit_log(
|
||||
db: Session,
|
||||
admin: User,
|
||||
*,
|
||||
action: str,
|
||||
target_type: str | None = None,
|
||||
target_id: str | None = None,
|
||||
request: Request | None = None,
|
||||
detail: dict | None = None,
|
||||
) -> AdminAuditLog:
|
||||
ip_address = request.client.host if request is not None and request.client is not None else None
|
||||
user_agent = request.headers.get("User-Agent") if request is not None else None
|
||||
row = AdminAuditLog(
|
||||
admin_user_id=admin.id,
|
||||
action=action,
|
||||
target_type=target_type,
|
||||
target_id=target_id,
|
||||
ip_address=ip_address,
|
||||
user_agent=user_agent,
|
||||
detail_json=_redact_detail(detail or {}),
|
||||
)
|
||||
db.add(row)
|
||||
db.commit()
|
||||
db.refresh(row)
|
||||
return row
|
||||
|
||||
|
||||
def _redact_detail(value: dict) -> dict:
|
||||
redacted: dict = {}
|
||||
for key, item in value.items():
|
||||
lowered = str(key).lower()
|
||||
if any(marker in lowered for marker in ("password", "token", "secret", "cookie", "key")):
|
||||
redacted[key] = "***"
|
||||
else:
|
||||
redacted[key] = item
|
||||
return redacted
|
||||
@@ -0,0 +1,21 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
|
||||
from sakura_cloud.core.config import settings
|
||||
from sakura_cloud.core.logging import logger
|
||||
|
||||
|
||||
def upgrade_database_to_head() -> None:
|
||||
cloud_dir = Path(__file__).resolve().parents[1]
|
||||
alembic_ini = cloud_dir / "alembic.ini"
|
||||
|
||||
config = Config(str(alembic_ini))
|
||||
config.set_main_option("script_location", str(cloud_dir / "migrations"))
|
||||
config.set_main_option("sqlalchemy.url", settings.database_url)
|
||||
|
||||
logger.info("Running database migrations to head")
|
||||
command.upgrade(config, "head")
|
||||
@@ -0,0 +1,163 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from sakura_cloud.core.config import settings
|
||||
from sakura_cloud.db.session import SessionLocal
|
||||
from sakura_cloud.models.artwork import Artwork
|
||||
from sakura_cloud.models.ocr import ArtworkOcrText
|
||||
|
||||
|
||||
OcrExtractor = Callable[[Path], tuple[str, float | None, str]]
|
||||
_semaphore = threading.BoundedSemaphore(settings.ocr_max_concurrency)
|
||||
|
||||
|
||||
def queue_ocr_for_artwork(db: Session, artwork: Artwork) -> ArtworkOcrText | None:
|
||||
if not settings.ocr_enabled:
|
||||
return None
|
||||
row = _get_or_create_row(db, artwork)
|
||||
if row.status in {"pending", "running", "done"}:
|
||||
return row
|
||||
row.status = "pending"
|
||||
row.error = None
|
||||
db.commit()
|
||||
return row
|
||||
|
||||
|
||||
def run_ocr_for_artwork_id(artwork_id: str, user_id: str) -> None:
|
||||
with SessionLocal() as db:
|
||||
artwork = db.get(Artwork, artwork_id)
|
||||
if artwork is None or artwork.user_id != user_id:
|
||||
return
|
||||
run_ocr_for_artwork(db, artwork)
|
||||
|
||||
|
||||
def run_ocr_for_artwork(
|
||||
db: Session,
|
||||
artwork: Artwork,
|
||||
*,
|
||||
extractor: OcrExtractor | None = None,
|
||||
) -> ArtworkOcrText:
|
||||
row = _get_or_create_row(db, artwork)
|
||||
if not settings.ocr_enabled and extractor is None:
|
||||
row.status = "failed"
|
||||
row.error = "ocr_disabled"
|
||||
db.commit()
|
||||
return row
|
||||
if not _is_supported_image(artwork):
|
||||
row.status = "failed"
|
||||
row.error = "unsupported_file_type"
|
||||
db.commit()
|
||||
logger.info("OCR skipped artwork={} mime={}", artwork.id, artwork.mime_type)
|
||||
return row
|
||||
if row.status == "running":
|
||||
return row
|
||||
if row.attempts >= settings.ocr_max_attempts and extractor is None:
|
||||
return row
|
||||
|
||||
acquired = _semaphore.acquire(blocking=False)
|
||||
if not acquired:
|
||||
row.status = "pending"
|
||||
row.error = "ocr_busy"
|
||||
db.commit()
|
||||
return row
|
||||
try:
|
||||
row.status = "running"
|
||||
row.attempts += 1
|
||||
row.error = None
|
||||
db.commit()
|
||||
|
||||
text, confidence, language = (extractor or _paddle_ocr_extract)(Path(artwork.file_path))
|
||||
row.ocr_text = text.strip()
|
||||
row.confidence = confidence
|
||||
row.language = language or "mixed"
|
||||
row.engine = "paddleocr"
|
||||
row.status = "done"
|
||||
row.error = None
|
||||
db.commit()
|
||||
logger.info("OCR done artwork={} text_len={} confidence={}", artwork.id, len(row.ocr_text), confidence)
|
||||
return row
|
||||
except Exception as exc:
|
||||
row.status = "failed"
|
||||
row.error = exc.__class__.__name__
|
||||
db.commit()
|
||||
logger.warning("OCR failed artwork={} attempt={} error={}", artwork.id, row.attempts, exc.__class__.__name__)
|
||||
return row
|
||||
finally:
|
||||
_semaphore.release()
|
||||
|
||||
|
||||
def _get_or_create_row(db: Session, artwork: Artwork) -> ArtworkOcrText:
|
||||
row = db.scalar(
|
||||
select(ArtworkOcrText).where(
|
||||
ArtworkOcrText.user_id == artwork.user_id,
|
||||
ArtworkOcrText.artwork_id == artwork.id,
|
||||
)
|
||||
)
|
||||
if row is None:
|
||||
row = ArtworkOcrText(
|
||||
user_id=artwork.user_id,
|
||||
artwork_id=artwork.id,
|
||||
file_id=artwork.file_id,
|
||||
source_platform=artwork.source_platform,
|
||||
source_id=artwork.source_id,
|
||||
status="pending",
|
||||
)
|
||||
db.add(row)
|
||||
db.commit()
|
||||
db.refresh(row)
|
||||
else:
|
||||
row.file_id = artwork.file_id
|
||||
row.source_platform = artwork.source_platform
|
||||
row.source_id = artwork.source_id
|
||||
return row
|
||||
|
||||
|
||||
def _is_supported_image(artwork: Artwork) -> bool:
|
||||
mime = (artwork.mime_type or "").lower()
|
||||
if mime in {"image/jpeg", "image/png", "image/webp"}:
|
||||
return True
|
||||
suffix = Path(artwork.file_path or "").suffix.lower()
|
||||
return suffix in {".jpg", ".jpeg", ".png", ".webp"} and mime != "image/gif"
|
||||
|
||||
|
||||
def _paddle_ocr_extract(path: Path) -> tuple[str, float | None, str]:
|
||||
from PIL import Image
|
||||
from paddleocr import PaddleOCR
|
||||
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError("image_not_found")
|
||||
|
||||
work_path = path
|
||||
with Image.open(path) as image:
|
||||
original_size = image.size
|
||||
image.thumbnail((settings.ocr_max_image_dimension, settings.ocr_max_image_dimension))
|
||||
if image.size != original_size:
|
||||
tmp = path.with_suffix(".ocr-resized.jpg")
|
||||
image.convert("RGB").save(tmp, quality=90)
|
||||
work_path = tmp
|
||||
|
||||
ocr = PaddleOCR(use_angle_cls=True, lang=settings.ocr_language, use_gpu=False)
|
||||
result = ocr.ocr(str(work_path), cls=True)
|
||||
texts: list[str] = []
|
||||
scores: list[float] = []
|
||||
for page in result or []:
|
||||
for line in page or []:
|
||||
if len(line) >= 2 and isinstance(line[1], (list, tuple)):
|
||||
text = str(line[1][0]).strip()
|
||||
if text:
|
||||
texts.append(text)
|
||||
try:
|
||||
scores.append(float(line[1][1]))
|
||||
except (TypeError, ValueError, IndexError):
|
||||
pass
|
||||
if work_path != path:
|
||||
work_path.unlink(missing_ok=True)
|
||||
confidence = sum(scores) / len(scores) if scores else None
|
||||
return "\n".join(texts), confidence, settings.ocr_language
|
||||
@@ -0,0 +1,537 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import re
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from html import unescape
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any
|
||||
from urllib.parse import quote, urljoin, urlparse
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from sakura_cloud.services import online_search_engine
|
||||
from sakura_cloud.services.online_search_engine import (
|
||||
OnlineSearchEngine,
|
||||
SearchEngineError,
|
||||
assert_public_url,
|
||||
build_search_url,
|
||||
parse_comic,
|
||||
parse_novel,
|
||||
resolve_resource_url,
|
||||
search_resource,
|
||||
source_type,
|
||||
)
|
||||
|
||||
|
||||
class OnlineSearchError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OnlineSearchItem:
|
||||
source_name: str
|
||||
source_url: str
|
||||
title: str
|
||||
author: str | None
|
||||
cover_url: str | None
|
||||
detail_url: str
|
||||
kind: str
|
||||
|
||||
|
||||
def parse_source_payload(payload: Any) -> list[dict[str, Any]]:
|
||||
if isinstance(payload, str):
|
||||
payload = json.loads(payload)
|
||||
if isinstance(payload, list):
|
||||
rows = payload
|
||||
elif isinstance(payload, dict):
|
||||
for key in ("sources", "bookSources", "data", "list", "items", "results"):
|
||||
value = payload.get(key)
|
||||
if isinstance(value, list):
|
||||
rows = value
|
||||
break
|
||||
else:
|
||||
rows = [payload]
|
||||
else:
|
||||
rows = []
|
||||
return [dict(item) for item in rows if isinstance(item, dict)]
|
||||
|
||||
|
||||
def search_sources(
|
||||
sources: list[dict[str, Any]],
|
||||
keyword: str,
|
||||
*,
|
||||
mode: str,
|
||||
limit_sources: int = 30,
|
||||
per_source_limit: int = 20,
|
||||
) -> list[OnlineSearchItem]:
|
||||
results: list[OnlineSearchItem] = []
|
||||
expected_type = 2 if mode == "manga" else 0
|
||||
engine = OnlineSearchEngine(timeout=10.0)
|
||||
for source in sources[:limit_sources]:
|
||||
if _source_type(source) != expected_type:
|
||||
continue
|
||||
try:
|
||||
online_search_engine.assert_public_url = _assert_public_url
|
||||
engine._request_text = ( # type: ignore[method-assign]
|
||||
lambda rule, url, data=None, _source=source: _fetch_text(url, rule.base_url, _source)
|
||||
)
|
||||
candidates = engine.search_legado_source(source, keyword, mode=mode, limit=per_source_limit)
|
||||
results.extend(
|
||||
OnlineSearchItem(
|
||||
source_name=item.source_name,
|
||||
source_url=item.source_url,
|
||||
title=item.title,
|
||||
author=item.author,
|
||||
cover_url=item.cover,
|
||||
detail_url=item.url,
|
||||
kind=mode,
|
||||
)
|
||||
for item in candidates
|
||||
)
|
||||
except SearchEngineError:
|
||||
continue
|
||||
except Exception:
|
||||
continue
|
||||
return results
|
||||
|
||||
|
||||
def _search_one(source: dict[str, Any], keyword: str, *, mode: str, limit: int) -> list[OnlineSearchItem]:
|
||||
source_name = _string(source.get("bookSourceName") or source.get("sourceName") or source.get("name")) or "未命名源"
|
||||
source_url = _string(source.get("bookSourceUrl") or source.get("sourceUrl") or source.get("url")) or ""
|
||||
search_url = _string(source.get("searchUrl"))
|
||||
rule_search = source.get("ruleSearch") if isinstance(source.get("ruleSearch"), dict) else {}
|
||||
if not search_url:
|
||||
search_url = _string(rule_search.get("url") or rule_search.get("searchUrl")) or ""
|
||||
if not source_url or not search_url:
|
||||
return []
|
||||
|
||||
final_url = _resolve_url(source_url, _build_search_url(search_url, keyword))
|
||||
_assert_public_url(final_url)
|
||||
body = _fetch_text(final_url, source_url, source)
|
||||
list_rule = _first_string(
|
||||
rule_search,
|
||||
"bookList",
|
||||
"comicList",
|
||||
"list",
|
||||
"items",
|
||||
"data",
|
||||
"results",
|
||||
)
|
||||
if list_rule:
|
||||
parsed = _parse_by_rules(body, final_url, source, source_name, source_url, rule_search, mode, limit)
|
||||
if parsed:
|
||||
return parsed
|
||||
if mode == "manga":
|
||||
return _fallback_manga(body, final_url, source_name, source_url, keyword, limit)
|
||||
return _fallback_novel(body, final_url, source_name, source_url, keyword, limit)
|
||||
|
||||
|
||||
def _parse_by_rules(
|
||||
body: str,
|
||||
base_url: str,
|
||||
source: dict[str, Any],
|
||||
source_name: str,
|
||||
source_url: str,
|
||||
rule_search: dict[str, Any],
|
||||
mode: str,
|
||||
limit: int,
|
||||
) -> list[OnlineSearchItem]:
|
||||
fragments = _select_list(body, _first_string(rule_search, "bookList", "comicList", "list", "items", "data", "results"))
|
||||
results: list[OnlineSearchItem] = []
|
||||
seen: set[str] = set()
|
||||
for fragment in fragments:
|
||||
title = _select_one(fragment, _first_string(rule_search, "name", "bookName", "comicName", "title"))
|
||||
detail = _select_one(fragment, _first_string(rule_search, "bookUrl", "comicUrl", "detailUrl", "url", "href"))
|
||||
if not title or not detail:
|
||||
continue
|
||||
detail_url = _resolve_url(base_url, detail)
|
||||
if detail_url in seen:
|
||||
continue
|
||||
seen.add(detail_url)
|
||||
results.append(
|
||||
OnlineSearchItem(
|
||||
source_name=source_name,
|
||||
source_url=source_url,
|
||||
title=title,
|
||||
author=_select_one(fragment, _first_string(rule_search, "author", "bookAuthor", "comicAuthor", "writer")) or None,
|
||||
cover_url=_maybe_url(base_url, _select_one(fragment, _first_string(rule_search, "coverUrl", "cover", "bookCover", "comicCover", "img", "image"))),
|
||||
detail_url=detail_url,
|
||||
kind=mode,
|
||||
)
|
||||
)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
def _fallback_manga(
|
||||
body: str,
|
||||
base_url: str,
|
||||
source_name: str,
|
||||
source_url: str,
|
||||
keyword: str,
|
||||
limit: int,
|
||||
) -> list[OnlineSearchItem]:
|
||||
decoded = _decode_json_like(body)
|
||||
results: list[OnlineSearchItem] = []
|
||||
seen: set[str] = set()
|
||||
if decoded is not None:
|
||||
for node in _walk_dicts(decoded):
|
||||
title = _first_value(node, "title", "name", "comicName", "comic_name", "topic_title", "topicTitle", "bookName")
|
||||
if not _matches(title, keyword):
|
||||
continue
|
||||
detail = _first_value(node, "url", "href", "link", "bookUrl", "book_url", "comicUrl", "comic_url", "detailUrl", "detail_url", "path")
|
||||
if not detail:
|
||||
detail = _first_value(node, "id", "book_id", "comic_id", "topic_id")
|
||||
if not detail:
|
||||
continue
|
||||
detail_url = _resolve_url(base_url, detail)
|
||||
if detail_url in seen:
|
||||
continue
|
||||
seen.add(detail_url)
|
||||
results.append(
|
||||
OnlineSearchItem(
|
||||
source_name=source_name,
|
||||
source_url=source_url,
|
||||
title=title,
|
||||
author=_first_value(node, "author", "writer", "nickname", "user_name", "userName") or None,
|
||||
cover_url=_maybe_url(base_url, _first_value(node, "cover", "coverUrl", "cover_url", "image", "imageUrl", "image_url", "pic", "pic_url", "vertical_image_url")),
|
||||
detail_url=detail_url,
|
||||
kind="manga",
|
||||
)
|
||||
)
|
||||
if len(results) >= limit:
|
||||
return results
|
||||
if results:
|
||||
return results
|
||||
return _fallback_html_cards(body, base_url, source_name, source_url, keyword, "manga", limit)
|
||||
|
||||
|
||||
def _fallback_novel(
|
||||
body: str,
|
||||
base_url: str,
|
||||
source_name: str,
|
||||
source_url: str,
|
||||
keyword: str,
|
||||
limit: int,
|
||||
) -> list[OnlineSearchItem]:
|
||||
decoded = _decode_json_like(body)
|
||||
results: list[OnlineSearchItem] = []
|
||||
if decoded is not None:
|
||||
for node in _walk_dicts(decoded):
|
||||
title = _first_value(node, "title", "name", "bookName", "book_name", "novelName")
|
||||
detail = _first_value(node, "url", "href", "link", "bookUrl", "book_url", "detailUrl", "detail_url")
|
||||
if not _matches(title, keyword) or not detail:
|
||||
continue
|
||||
results.append(
|
||||
OnlineSearchItem(
|
||||
source_name=source_name,
|
||||
source_url=source_url,
|
||||
title=title,
|
||||
author=_first_value(node, "author", "bookAuthor", "writer") or None,
|
||||
cover_url=_maybe_url(base_url, _first_value(node, "cover", "coverUrl", "cover_url", "img", "image")),
|
||||
detail_url=_resolve_url(base_url, detail),
|
||||
kind="novel",
|
||||
)
|
||||
)
|
||||
if len(results) >= limit:
|
||||
return results
|
||||
return results or _fallback_html_cards(body, base_url, source_name, source_url, keyword, "novel", limit)
|
||||
|
||||
|
||||
def _fallback_html_cards(
|
||||
body: str,
|
||||
base_url: str,
|
||||
source_name: str,
|
||||
source_url: str,
|
||||
keyword: str,
|
||||
kind: str,
|
||||
limit: int,
|
||||
) -> list[OnlineSearchItem]:
|
||||
parser = _AnchorParser()
|
||||
parser.feed(body)
|
||||
results: list[OnlineSearchItem] = []
|
||||
seen: set[str] = set()
|
||||
for anchor in parser.anchors:
|
||||
title = _clean(anchor.get("title") or anchor.get("alt") or anchor.get("text") or "")
|
||||
href = _clean(anchor.get("href") or "")
|
||||
if not href or not _matches(title, keyword):
|
||||
continue
|
||||
detail_url = _resolve_url(base_url, href)
|
||||
if detail_url in seen:
|
||||
continue
|
||||
seen.add(detail_url)
|
||||
results.append(
|
||||
OnlineSearchItem(
|
||||
source_name=source_name,
|
||||
source_url=source_url,
|
||||
title=title,
|
||||
author=None,
|
||||
cover_url=_maybe_url(base_url, anchor.get("img") or ""),
|
||||
detail_url=detail_url,
|
||||
kind=kind,
|
||||
)
|
||||
)
|
||||
if len(results) >= limit:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
def _select_list(body: str, rule: str | None) -> list[str]:
|
||||
if not rule:
|
||||
return []
|
||||
for part in _split_rule(rule, "||"):
|
||||
if part.lower().startswith("regex:"):
|
||||
pattern = _strip_replace(part.split(":", 1)[1])
|
||||
matches = re.findall(pattern, body, re.I | re.S)
|
||||
output = [_clean(item[0] if isinstance(item, tuple) else item) for item in matches]
|
||||
if output:
|
||||
return [item for item in output if item]
|
||||
if part.startswith("$.") or part.startswith("$["):
|
||||
decoded = _decode_json_like(body)
|
||||
values = _json_path(decoded, part) if decoded is not None else []
|
||||
output = [json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value) for value in values]
|
||||
if output:
|
||||
return output
|
||||
matches = _html_rule_list(body, part)
|
||||
if matches:
|
||||
return matches
|
||||
return []
|
||||
|
||||
|
||||
def _select_one(fragment: str, rule: str | None) -> str:
|
||||
values = _select_list(fragment, rule)
|
||||
return values[0] if values else ""
|
||||
|
||||
|
||||
def _html_rule_list(body: str, rule: str) -> list[str]:
|
||||
rule = _strip_replace(rule.strip())
|
||||
if not rule:
|
||||
return []
|
||||
tokens = [item.strip() for item in rule.split("@") if item.strip()]
|
||||
selector = tokens[0] if tokens else rule
|
||||
attr = tokens[-1] if tokens and tokens[-1] not in {selector, "text", "html"} else ""
|
||||
if selector.startswith("class."):
|
||||
classes = [item for item in selector[6:].split() if item]
|
||||
class_checks = "".join(rf'(?=[^"\']*{re.escape(item)})' for item in classes[:3])
|
||||
pattern = rf'<(?P<tag>[a-zA-Z0-9]+)\b[^>]*class=["\']{class_checks}[^"\']*["\'][^>]*>.*?</(?P=tag)>'
|
||||
elif selector.startswith("id."):
|
||||
pattern = rf'<(?P<tag>[a-zA-Z0-9]+)\b[^>]*id=["\']{re.escape(selector[3:])}["\'][^>]*>.*?</(?P=tag)>'
|
||||
elif selector.startswith("tag."):
|
||||
tag = re.escape(selector[4:])
|
||||
pattern = rf'<{tag}\b[^>]*>.*?</{tag}>'
|
||||
elif selector.startswith("$.") or selector.startswith("$["):
|
||||
decoded = _decode_json_like(body)
|
||||
return [json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value) for value in _json_path(decoded, selector)]
|
||||
else:
|
||||
pattern = rf'<a\b[^>]*>.*?</a>' if "href" in tokens else ""
|
||||
if not pattern:
|
||||
return []
|
||||
blocks = [m.group(0) for m in re.finditer(pattern, body, re.I | re.S)]
|
||||
if attr and attr not in {"text", "html"}:
|
||||
return [_clean(m.group(1)) for block in blocks for m in [re.search(rf'{re.escape(attr)}=["\']([^"\']+)["\']', block, re.I)] if m]
|
||||
if "text" in tokens:
|
||||
return [_html_text(block) for block in blocks]
|
||||
return blocks
|
||||
|
||||
|
||||
def _json_path(decoded: Any, rule: str) -> list[Any]:
|
||||
if decoded is None:
|
||||
return []
|
||||
path = re.sub(r"^\$\.?", "", rule.strip())
|
||||
if path in {"", "*", "[*]"}:
|
||||
return decoded if isinstance(decoded, list) else [decoded]
|
||||
current = [decoded]
|
||||
for segment in [item for item in path.split(".") if item]:
|
||||
next_values: list[Any] = []
|
||||
star = segment.endswith("[*]")
|
||||
key = segment[:-3] if star else segment
|
||||
for value in current:
|
||||
child = value.get(key) if isinstance(value, dict) and key else value
|
||||
if star and isinstance(child, list):
|
||||
next_values.extend(child)
|
||||
elif child is not None:
|
||||
next_values.append(child)
|
||||
current = next_values
|
||||
return current
|
||||
|
||||
|
||||
class _AnchorParser(HTMLParser):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.anchors: list[dict[str, str]] = []
|
||||
self._current: dict[str, str] | None = None
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
attr = {key: value or "" for key, value in attrs}
|
||||
if tag.lower() == "a" and attr.get("href"):
|
||||
self._current = {"href": attr.get("href", ""), "title": attr.get("title", ""), "text": ""}
|
||||
elif tag.lower() == "img" and self._current is not None:
|
||||
for key in ("data-original", "data-src", "src"):
|
||||
if attr.get(key):
|
||||
self._current["img"] = attr[key]
|
||||
break
|
||||
if attr.get("alt"):
|
||||
self._current["alt"] = attr["alt"]
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._current is not None:
|
||||
self._current["text"] = f'{self._current.get("text", "")} {data}'
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag.lower() == "a" and self._current is not None:
|
||||
self.anchors.append(self._current)
|
||||
self._current = None
|
||||
|
||||
|
||||
def _fetch_text(url: str, referer: str, source: dict[str, Any]) -> str:
|
||||
headers = {
|
||||
"User-Agent": _string(source.get("httpUserAgent")) or "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,application/json;q=0.8,*/*;q=0.7",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9",
|
||||
"Referer": referer,
|
||||
}
|
||||
extra = source.get("headers") or source.get("header")
|
||||
if isinstance(extra, dict):
|
||||
for key, value in extra.items():
|
||||
if str(key).lower() in {"cookie", "authorization"}:
|
||||
continue
|
||||
headers[str(key)] = str(value)
|
||||
with urlopen(Request(url, headers=headers), timeout=12) as response:
|
||||
raw = response.read(2_000_000)
|
||||
content_type = response.headers.get("content-type", "")
|
||||
charset = _string(source.get("charset") or source.get("encoding")) or _charset_from_content_type(content_type) or "utf-8"
|
||||
for candidate in (charset, "utf-8", "gb18030", "gbk", "gb2312"):
|
||||
try:
|
||||
return raw.decode(candidate, errors="replace")
|
||||
except LookupError:
|
||||
continue
|
||||
return raw.decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def _assert_public_url(url: str) -> None:
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
||||
raise OnlineSearchError("Only http/https source URLs are allowed")
|
||||
infos = socket.getaddrinfo(parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80), type=socket.SOCK_STREAM)
|
||||
for item in infos:
|
||||
ip = ipaddress.ip_address(item[4][0])
|
||||
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved:
|
||||
raise OnlineSearchError("Private network source URLs are not allowed")
|
||||
|
||||
|
||||
def _build_search_url(template: str, keyword: str) -> str:
|
||||
encoded = quote(keyword, safe="")
|
||||
value = template.strip()
|
||||
for token in ("{{key}}", "{{keyword}}", "{key}", "{keyword}", "<key>", "<keyword>", "searchKey"):
|
||||
value = value.replace(token, encoded)
|
||||
value = value.replace("%s", encoded, 1)
|
||||
return value.replace("{{page}}", "1").replace("{page}", "1").replace("<page>", "1")
|
||||
|
||||
|
||||
def _resolve_url(base: str, value: str) -> str:
|
||||
return urljoin(base, value.strip())
|
||||
|
||||
|
||||
def _maybe_url(base: str, value: str | None) -> str | None:
|
||||
text = _clean(value or "")
|
||||
if not text or text.lower() == "null":
|
||||
return None
|
||||
return _resolve_url(base, text)
|
||||
|
||||
|
||||
def _decode_json_like(body: str) -> Any:
|
||||
text = body.lstrip("\ufeff \t\r\n")
|
||||
match = re.match(r"^[a-zA-Z_$][\w$]*\s*\((.*)\)\s*;?\s*$", text, re.S)
|
||||
if match:
|
||||
text = match.group(1)
|
||||
try:
|
||||
return json.loads(text)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _walk_dicts(node: Any):
|
||||
if isinstance(node, dict):
|
||||
yield node
|
||||
for value in node.values():
|
||||
yield from _walk_dicts(value)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
yield from _walk_dicts(item)
|
||||
|
||||
|
||||
def _first_value(node: dict[str, Any], *keys: str) -> str:
|
||||
for key in keys:
|
||||
value = node.get(key)
|
||||
if value is not None:
|
||||
text = _clean(str(value))
|
||||
if text and text.lower() != "null":
|
||||
return text
|
||||
return ""
|
||||
|
||||
|
||||
def _first_string(mapping: dict[str, Any], *keys: str) -> str | None:
|
||||
for key in keys:
|
||||
value = _string(mapping.get(key))
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _source_type(source: dict[str, Any]) -> int:
|
||||
value = source.get("bookSourceType", source.get("sourceType", source.get("type", 0)))
|
||||
try:
|
||||
numeric = int(value)
|
||||
except Exception:
|
||||
numeric = 0
|
||||
text = json.dumps(source, ensure_ascii=False).lower()
|
||||
manga_markers = (
|
||||
"comiclist",
|
||||
"comicname",
|
||||
"comicurl",
|
||||
"imageStyle".lower(),
|
||||
"chapterImages".lower(),
|
||||
"manga",
|
||||
"manhua",
|
||||
"漫画",
|
||||
)
|
||||
if numeric == 0 and any(marker in text for marker in manga_markers):
|
||||
return 2
|
||||
return numeric
|
||||
|
||||
|
||||
def _split_rule(rule: str, sep: str) -> list[str]:
|
||||
return [item.strip() for item in rule.split(sep) if item.strip()]
|
||||
|
||||
|
||||
def _strip_replace(rule: str) -> str:
|
||||
return re.split(r"##|###", rule, maxsplit=1)[0].strip()
|
||||
|
||||
|
||||
def _string(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip()
|
||||
return text or None
|
||||
|
||||
|
||||
def _matches(title: str, keyword: str) -> bool:
|
||||
title = _clean(title).lower()
|
||||
keyword = _clean(keyword).lower()
|
||||
return bool(title) and (not keyword or keyword in title)
|
||||
|
||||
|
||||
def _clean(value: str) -> str:
|
||||
return unescape(re.sub(r"\s+", " ", value or "")).strip()
|
||||
|
||||
|
||||
def _html_text(value: str) -> str:
|
||||
return _clean(re.sub(r"<[^>]+>", " ", value))
|
||||
|
||||
|
||||
def _charset_from_content_type(value: str) -> str | None:
|
||||
match = re.search(r"charset=([^;]+)", value or "", re.I)
|
||||
return match.group(1).strip("\"'") if match else None
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,160 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from sakura_cloud.core.config import settings
|
||||
from sakura_cloud.models.follow import UpdateAlert
|
||||
from sakura_cloud.models.mixins import utc_now
|
||||
from sakura_cloud.models.push import PushDeviceToken
|
||||
|
||||
|
||||
PushSender = Callable[[str, str, str, dict[str, str]], bool]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PushSendSummary:
|
||||
attempted: int = 0
|
||||
sent: int = 0
|
||||
disabled: int = 0
|
||||
failed: int = 0
|
||||
|
||||
|
||||
def send_update_alert_push(
|
||||
db: Session,
|
||||
alert: UpdateAlert,
|
||||
*,
|
||||
sender: PushSender | None = None,
|
||||
) -> PushSendSummary:
|
||||
payload = alert.payload_json or {}
|
||||
if alert.platform == "system":
|
||||
title = alert.title
|
||||
body = alert.body or ""
|
||||
else:
|
||||
title = "你关注的作者有新作品"
|
||||
author_name = (
|
||||
payload.get("author_name")
|
||||
or payload.get("creator_name")
|
||||
or payload.get("creator_handle")
|
||||
or payload.get("author")
|
||||
or alert.creator_id
|
||||
)
|
||||
body = f"{author_name} 发布了 {alert.title}"
|
||||
|
||||
data = {
|
||||
"alert_id": alert.id,
|
||||
"source_platform": alert.platform,
|
||||
"source_id": str(payload.get("source_id") or payload.get("id") or alert.creator_id),
|
||||
"resource_type": str(payload.get("resource_type") or alert.content_type or ""),
|
||||
}
|
||||
return send_push_to_user(db, alert.user_id, title, body, data, sender=sender)
|
||||
|
||||
|
||||
def send_push_to_user(
|
||||
db: Session,
|
||||
user_id: str,
|
||||
title: str,
|
||||
body: str,
|
||||
data: dict[str, Any] | None = None,
|
||||
*,
|
||||
sender: PushSender | None = None,
|
||||
) -> PushSendSummary:
|
||||
rows = db.scalars(
|
||||
select(PushDeviceToken).where(
|
||||
PushDeviceToken.user_id == user_id,
|
||||
PushDeviceToken.is_active.is_(True),
|
||||
)
|
||||
).all()
|
||||
summary = PushSendSummary(attempted=len(rows))
|
||||
if not rows:
|
||||
return summary
|
||||
|
||||
safe_data = {key: "" if value is None else str(value) for key, value in (data or {}).items()}
|
||||
disabled_any = False
|
||||
for row in rows:
|
||||
try:
|
||||
ok = sender(row.fcm_token, title, body, safe_data) if sender else _send_fcm(row.fcm_token, title, body, safe_data)
|
||||
except Exception as exc: # pragma: no cover - defensive guard around provider SDK.
|
||||
if _is_invalid_token_error(exc):
|
||||
_disable_token(row)
|
||||
disabled_any = True
|
||||
summary.disabled += 1
|
||||
logger.warning("Disabled invalid FCM token for user={} device={}", user_id, row.device_id)
|
||||
else:
|
||||
summary.failed += 1
|
||||
logger.warning("FCM push failed for user={} device={}: {}", user_id, row.device_id, exc.__class__.__name__)
|
||||
continue
|
||||
|
||||
if ok:
|
||||
summary.sent += 1
|
||||
logger.info("FCM push sent for user={} device={} alert={}", user_id, row.device_id, safe_data.get("alert_id", ""))
|
||||
else:
|
||||
_disable_token(row)
|
||||
disabled_any = True
|
||||
summary.disabled += 1
|
||||
logger.warning("Disabled invalid FCM token for user={} device={}", user_id, row.device_id)
|
||||
|
||||
if disabled_any:
|
||||
db.commit()
|
||||
return summary
|
||||
|
||||
|
||||
def _disable_token(row: PushDeviceToken) -> None:
|
||||
row.is_active = False
|
||||
row.last_seen_at = utc_now()
|
||||
|
||||
|
||||
def _send_fcm(token: str, title: str, body: str, data: dict[str, str]) -> bool:
|
||||
if not settings.fcm_enabled:
|
||||
logger.debug("FCM push skipped because FCM_ENABLED is false")
|
||||
return True
|
||||
app, messaging = _firebase_context()
|
||||
if app is None or messaging is None:
|
||||
logger.debug("FCM push skipped because Firebase credentials are not configured")
|
||||
return True
|
||||
|
||||
message = messaging.Message(
|
||||
token=token,
|
||||
notification=messaging.Notification(title=title, body=body),
|
||||
data=data,
|
||||
android=messaging.AndroidConfig(priority="high"),
|
||||
)
|
||||
messaging.send(message, app=app)
|
||||
return True
|
||||
|
||||
|
||||
def _firebase_context() -> tuple[Any | None, Any | None]:
|
||||
credential_json = settings.firebase_credentials_json.strip()
|
||||
credential_path = settings.firebase_credentials_path.strip()
|
||||
if not credential_json and not credential_path:
|
||||
return None, None
|
||||
|
||||
try:
|
||||
import firebase_admin
|
||||
from firebase_admin import credentials, messaging
|
||||
except ImportError:
|
||||
logger.warning("firebase-admin is not installed; FCM push is disabled")
|
||||
return None, None
|
||||
|
||||
if firebase_admin._apps:
|
||||
return firebase_admin.get_app(), messaging
|
||||
|
||||
if credential_json:
|
||||
cred = credentials.Certificate(json.loads(credential_json))
|
||||
else:
|
||||
cred = credentials.Certificate(credential_path)
|
||||
return firebase_admin.initialize_app(cred), messaging
|
||||
|
||||
|
||||
def _is_invalid_token_error(exc: Exception) -> bool:
|
||||
name = exc.__class__.__name__
|
||||
if name in {"UnregisteredError", "SenderIdMismatchError", "InvalidArgumentError"}:
|
||||
return True
|
||||
text = str(exc).lower()
|
||||
return "registration token" in text or "not a valid fcm" in text or "unregistered" in text
|
||||
@@ -0,0 +1,137 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta, timezone
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from sakura_cloud.models.follow import UpdateAlert
|
||||
from sakura_cloud.models.mixins import utc_now
|
||||
from sakura_cloud.models.user import User
|
||||
from sakura_cloud.services.push import send_update_alert_push
|
||||
|
||||
|
||||
DEFAULT_REMINDER_TIMEZONE = "Asia/Shanghai"
|
||||
DIGEST_PLATFORM = "system"
|
||||
DIGEST_CREATOR_ID = "daily-reminder"
|
||||
|
||||
|
||||
def _resolve_timezone(name: str | None) -> ZoneInfo:
|
||||
try:
|
||||
return ZoneInfo(name or DEFAULT_REMINDER_TIMEZONE)
|
||||
except ZoneInfoNotFoundError:
|
||||
return ZoneInfo(DEFAULT_REMINDER_TIMEZONE)
|
||||
|
||||
|
||||
def _display_platform(platform: str) -> str:
|
||||
if platform.lower() == "pixiv":
|
||||
return "Pixiv"
|
||||
if platform.lower() == "x":
|
||||
return "X"
|
||||
return platform
|
||||
|
||||
|
||||
def run_daily_reminder_digest(db: Session) -> int:
|
||||
now_utc = utc_now()
|
||||
users = db.scalars(select(User).where(User.reminder_enabled.is_(True))).all()
|
||||
reminders_created = 0
|
||||
|
||||
for user in users:
|
||||
zone = _resolve_timezone(user.reminder_timezone)
|
||||
local_now = now_utc.astimezone(zone)
|
||||
scheduled_today = local_now.replace(
|
||||
hour=user.reminder_hour,
|
||||
minute=user.reminder_minute,
|
||||
second=0,
|
||||
microsecond=0,
|
||||
)
|
||||
|
||||
if local_now < scheduled_today:
|
||||
continue
|
||||
|
||||
if user.last_reminder_sent_at is not None:
|
||||
last_sent_local = user.last_reminder_sent_at.astimezone(zone)
|
||||
if last_sent_local.date() == local_now.date():
|
||||
continue
|
||||
|
||||
window_start = (
|
||||
user.last_reminder_sent_at
|
||||
if user.last_reminder_sent_at is not None
|
||||
else (scheduled_today - timedelta(days=1)).astimezone(timezone.utc)
|
||||
)
|
||||
|
||||
new_alert_count = db.scalar(
|
||||
select(func.count(UpdateAlert.id)).where(
|
||||
UpdateAlert.user_id == user.id,
|
||||
UpdateAlert.platform != DIGEST_PLATFORM,
|
||||
UpdateAlert.created_at > window_start,
|
||||
)
|
||||
) or 0
|
||||
|
||||
recent_alerts = db.scalars(
|
||||
select(UpdateAlert)
|
||||
.where(
|
||||
UpdateAlert.user_id == user.id,
|
||||
UpdateAlert.platform != DIGEST_PLATFORM,
|
||||
UpdateAlert.created_at > window_start,
|
||||
)
|
||||
.order_by(UpdateAlert.created_at.desc())
|
||||
.limit(5)
|
||||
).all()
|
||||
|
||||
creator_names: list[str] = []
|
||||
platforms: list[str] = []
|
||||
for alert in recent_alerts:
|
||||
platform = _display_platform(alert.platform)
|
||||
if platform not in platforms:
|
||||
platforms.append(platform)
|
||||
candidate = (
|
||||
alert.payload_json.get("creator_name")
|
||||
or alert.payload_json.get("creator_handle")
|
||||
or alert.payload_json.get("author")
|
||||
or alert.creator_id
|
||||
)
|
||||
if candidate and candidate not in creator_names:
|
||||
creator_names.append(str(candidate))
|
||||
|
||||
date_text = f"{local_now.month}月{local_now.day}日"
|
||||
platforms_text = " / ".join(platforms) if platforms else "Pixiv / X"
|
||||
creators_text = ", ".join(creator_names[:3])
|
||||
title = f"{date_text},未更新" if not new_alert_count else f"{date_text},更新 {new_alert_count} 条记录"
|
||||
if not new_alert_count:
|
||||
body = f"{date_text} 暂未发现关注作者的新内容。云端会在明天 {user.reminder_hour:02d}:{user.reminder_minute:02d} 继续检查。"
|
||||
elif creators_text:
|
||||
body = (
|
||||
f"{date_text},{creators_text} 等作者带来了 {new_alert_count} "
|
||||
f"条新动态,来自 {platforms_text}。"
|
||||
)
|
||||
else:
|
||||
body = f"{date_text},你关注的作者有 {new_alert_count} 条新动态,来自 {platforms_text}。"
|
||||
|
||||
digest_alert = UpdateAlert(
|
||||
user_id=user.id,
|
||||
follow_id=None,
|
||||
platform=DIGEST_PLATFORM,
|
||||
creator_id=DIGEST_CREATOR_ID,
|
||||
title=title,
|
||||
body=body,
|
||||
content_type="digest",
|
||||
payload_json={
|
||||
"kind": "daily_digest",
|
||||
"window_start": window_start.isoformat(),
|
||||
"new_alert_count": int(new_alert_count),
|
||||
"platforms": platforms,
|
||||
"creators": creator_names[:5],
|
||||
"date": local_now.date().isoformat(),
|
||||
"status": "updated" if new_alert_count else "not_updated",
|
||||
},
|
||||
)
|
||||
db.add(digest_alert)
|
||||
user.last_reminder_sent_at = now_utc
|
||||
db.commit()
|
||||
db.refresh(digest_alert)
|
||||
send_update_alert_push(db, digest_alert)
|
||||
reminders_created += 1
|
||||
|
||||
return reminders_created
|
||||
@@ -0,0 +1,294 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import HTTPException, UploadFile, status
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from sakura_cloud.core.config import settings
|
||||
from sakura_cloud.core.safety import is_allowed_upload_mime, is_safe_upload_filename
|
||||
from sakura_cloud.models.file import FileBlob, UploadSession, UserFile
|
||||
from sakura_cloud.models.user import User
|
||||
from sakura_cloud.schemas.upload import UploadCompleteResponse, UploadInitRequest, UploadInitResponse
|
||||
from sakura_cloud.services.thumbnails import generate_webp_thumbnail
|
||||
|
||||
|
||||
def _ensure_storage_quota(user: User, incoming_size: int) -> None:
|
||||
size = max(0, incoming_size)
|
||||
if size and user.storage_used + size > user.storage_limit:
|
||||
raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="Storage quota exceeded")
|
||||
|
||||
|
||||
def _is_video_mime(mime_type: str | None) -> bool:
|
||||
return (mime_type or "").lower().strip().startswith("video/")
|
||||
|
||||
|
||||
def _user_video_storage_used(user_id: str, db: Session) -> int:
|
||||
value = db.scalar(
|
||||
select(func.coalesce(func.sum(FileBlob.file_size), 0))
|
||||
.join(UserFile, UserFile.file_sha256 == FileBlob.sha256)
|
||||
.where(UserFile.user_id == user_id, FileBlob.mime_type.like("video/%"))
|
||||
)
|
||||
return int(value or 0)
|
||||
|
||||
|
||||
def _ensure_video_storage_quota(user: User, incoming_size: int, db: Session) -> None:
|
||||
size = max(0, incoming_size)
|
||||
if size and _user_video_storage_used(user.id, db) + size > settings.video_storage_limit_bytes:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||||
detail=f"Video storage quota exceeded ({settings.video_storage_limit_gb}GB)",
|
||||
)
|
||||
|
||||
|
||||
def _root() -> Path:
|
||||
root = Path(settings.storage_root)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
return root
|
||||
|
||||
|
||||
def temp_upload_path(user_id: str, upload_id: str) -> Path:
|
||||
path = _root() / "tmp" / user_id
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path / f"{upload_id}.part"
|
||||
|
||||
|
||||
def _replacement_temp_upload_path(user_id: str, upload_id: str) -> Path:
|
||||
path = _root() / "tmp" / user_id
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path / f"{upload_id}-{uuid4().hex[:8]}.part"
|
||||
|
||||
|
||||
def blob_storage_path(resource_type: str, user_id: str, sha256: str, file_name: str) -> Path:
|
||||
now = datetime.now(timezone.utc)
|
||||
ext = Path(file_name).suffix.lower() or ".bin"
|
||||
kind = "novels" if resource_type == "novel" else "videos" if resource_type == "video" else "images"
|
||||
path = _root() / "uploads" / kind / user_id / str(now.year) / f"{now.month:02d}"
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path / f"{sha256[:16]}{ext}"
|
||||
|
||||
|
||||
def _open_temp_file_for_append(temp_path: Path):
|
||||
temp_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
return temp_path.open("ab")
|
||||
|
||||
|
||||
def _reset_unwritable_upload_session(session: UploadSession, db: Session) -> Path:
|
||||
old_path = Path(session.temp_path or "")
|
||||
new_path = _replacement_temp_upload_path(session.user_id, session.id)
|
||||
if old_path and old_path != new_path:
|
||||
try:
|
||||
if old_path.is_file():
|
||||
old_path.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
session.temp_path = str(new_path)
|
||||
session.received_size = 0
|
||||
session.status = "pending"
|
||||
db.commit()
|
||||
db.refresh(session)
|
||||
return new_path
|
||||
|
||||
|
||||
def calculate_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as fp:
|
||||
for chunk in iter(lambda: fp.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def cleanup_stale_uploads(db: Session, *, older_than: timedelta | None = None) -> int:
|
||||
cutoff = datetime.now(timezone.utc) - (older_than or timedelta(hours=settings.temp_upload_ttl_hours))
|
||||
sessions = db.scalars(
|
||||
select(UploadSession).where(
|
||||
UploadSession.status != "completed",
|
||||
UploadSession.updated_at < cutoff,
|
||||
)
|
||||
).all()
|
||||
removed = 0
|
||||
for session in sessions:
|
||||
if session.temp_path:
|
||||
Path(session.temp_path).unlink(missing_ok=True)
|
||||
db.delete(session)
|
||||
removed += 1
|
||||
if removed:
|
||||
db.commit()
|
||||
return removed
|
||||
|
||||
|
||||
def init_upload(payload: UploadInitRequest, user: User, db: Session) -> UploadInitResponse:
|
||||
is_video = _is_video_mime(payload.mime_type)
|
||||
resource_type = "video" if is_video else payload.resource_type
|
||||
max_upload_mb = settings.max_video_upload_size_mb if is_video else settings.max_upload_size_mb
|
||||
max_upload_size = max_upload_mb * 1024 * 1024
|
||||
if payload.file_size > max_upload_size:
|
||||
raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File is too large")
|
||||
if not is_safe_upload_filename(payload.file_name):
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid file name")
|
||||
if resource_type not in {"image", "novel", "video"}:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid resource type")
|
||||
if not is_allowed_upload_mime(payload.mime_type):
|
||||
raise HTTPException(status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, detail="Unsupported MIME type")
|
||||
if resource_type == "video" and not is_video:
|
||||
raise HTTPException(status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, detail="Unsupported video MIME type")
|
||||
cleanup_stale_uploads(db)
|
||||
existing_blob = db.get(FileBlob, payload.sha256)
|
||||
existing_user_file = None
|
||||
if existing_blob is not None:
|
||||
existing_user_file = db.scalar(
|
||||
select(UserFile).where(UserFile.user_id == user.id, UserFile.file_sha256 == payload.sha256)
|
||||
)
|
||||
if existing_user_file is None:
|
||||
_ensure_storage_quota(user, existing_blob.file_size)
|
||||
if _is_video_mime(existing_blob.mime_type):
|
||||
_ensure_video_storage_quota(user, existing_blob.file_size, db)
|
||||
user_file = UserFile(user_id=user.id, file_sha256=payload.sha256, original_name=payload.file_name)
|
||||
db.add(user_file)
|
||||
user.storage_used += existing_blob.file_size
|
||||
db.commit()
|
||||
db.refresh(user_file)
|
||||
else:
|
||||
user_file = existing_user_file
|
||||
return UploadInitResponse(
|
||||
upload_id="",
|
||||
exists=True,
|
||||
chunk_size=settings.upload_chunk_size,
|
||||
received_size=payload.file_size,
|
||||
file_id=user_file.id,
|
||||
)
|
||||
|
||||
session = db.scalar(
|
||||
select(UploadSession).where(
|
||||
UploadSession.user_id == user.id,
|
||||
UploadSession.sha256 == payload.sha256,
|
||||
UploadSession.status != "completed",
|
||||
)
|
||||
)
|
||||
if session is None:
|
||||
_ensure_storage_quota(user, payload.file_size)
|
||||
if is_video:
|
||||
_ensure_video_storage_quota(user, payload.file_size, db)
|
||||
session = UploadSession(
|
||||
user_id=user.id,
|
||||
sha256=payload.sha256,
|
||||
file_name=payload.file_name,
|
||||
file_size=payload.file_size,
|
||||
mime_type=payload.mime_type,
|
||||
resource_type=resource_type,
|
||||
chunk_size=settings.upload_chunk_size,
|
||||
temp_path="",
|
||||
)
|
||||
db.add(session)
|
||||
db.flush()
|
||||
session.temp_path = str(temp_upload_path(user.id, session.id))
|
||||
db.commit()
|
||||
db.refresh(session)
|
||||
return UploadInitResponse(
|
||||
upload_id=session.id,
|
||||
exists=False,
|
||||
chunk_size=session.chunk_size,
|
||||
received_size=session.received_size,
|
||||
)
|
||||
|
||||
|
||||
async def append_chunk(upload_id: str, file: UploadFile, user: User, db: Session) -> UploadSession:
|
||||
session = db.get(UploadSession, upload_id)
|
||||
if session is None or session.user_id != user.id:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Upload session not found")
|
||||
if session.status == "completed":
|
||||
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Upload already completed")
|
||||
temp_path = Path(session.temp_path)
|
||||
size = 0
|
||||
try:
|
||||
with _open_temp_file_for_append(temp_path) as fp:
|
||||
while True:
|
||||
chunk = await file.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
size += len(chunk)
|
||||
if session.received_size + size > session.file_size:
|
||||
raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="Upload exceeds declared size")
|
||||
fp.write(chunk)
|
||||
except PermissionError:
|
||||
# Old upload sessions may point to temp files created by another process/user.
|
||||
# Reset the session onto a fresh temp path and let the client restart from 0.
|
||||
temp_parent = temp_path.parent
|
||||
if not temp_parent.exists():
|
||||
temp_parent.mkdir(parents=True, exist_ok=True)
|
||||
if not os.access(temp_parent, os.W_OK):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail=f"Upload temp directory is not writable: {temp_parent}",
|
||||
)
|
||||
new_path = _reset_unwritable_upload_session(session, db)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=f"Upload temp file was not writable. Session reset to {new_path.name}, please retry upload.",
|
||||
)
|
||||
session.received_size += size
|
||||
session.status = "uploading"
|
||||
db.commit()
|
||||
db.refresh(session)
|
||||
return session
|
||||
|
||||
|
||||
def complete_upload(upload_id: str, user: User, db: Session) -> UploadCompleteResponse:
|
||||
session = db.get(UploadSession, upload_id)
|
||||
if session is None or session.user_id != user.id:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Upload session not found")
|
||||
temp_path = Path(session.temp_path)
|
||||
if not temp_path.is_file():
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Upload file is missing")
|
||||
actual_size = temp_path.stat().st_size
|
||||
if actual_size != session.file_size:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Upload size mismatch")
|
||||
actual_sha = calculate_sha256(temp_path)
|
||||
if actual_sha != session.sha256:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="SHA256 mismatch")
|
||||
|
||||
storage_path = blob_storage_path(session.resource_type, user.id, session.sha256, session.file_name)
|
||||
if not storage_path.exists():
|
||||
shutil.move(str(temp_path), storage_path)
|
||||
else:
|
||||
temp_path.unlink(missing_ok=True)
|
||||
|
||||
thumbnail_path = generate_webp_thumbnail(storage_path) if (session.mime_type or "").startswith("image/") else None
|
||||
|
||||
blob = db.get(FileBlob, session.sha256)
|
||||
if blob is None:
|
||||
blob = FileBlob(
|
||||
sha256=session.sha256,
|
||||
storage_path=str(storage_path),
|
||||
thumbnail_path=thumbnail_path,
|
||||
file_size=session.file_size,
|
||||
mime_type=session.mime_type,
|
||||
)
|
||||
db.add(blob)
|
||||
elif thumbnail_path and not blob.thumbnail_path:
|
||||
blob.thumbnail_path = thumbnail_path
|
||||
|
||||
user_file = db.scalar(select(UserFile).where(UserFile.user_id == user.id, UserFile.file_sha256 == session.sha256))
|
||||
if user_file is None:
|
||||
_ensure_storage_quota(user, session.file_size)
|
||||
if _is_video_mime(session.mime_type):
|
||||
_ensure_video_storage_quota(user, session.file_size, db)
|
||||
user_file = UserFile(user_id=user.id, file_sha256=session.sha256, original_name=session.file_name)
|
||||
db.add(user_file)
|
||||
user.storage_used += session.file_size
|
||||
|
||||
session.status = "completed"
|
||||
db.commit()
|
||||
db.refresh(user_file)
|
||||
return UploadCompleteResponse(
|
||||
file_id=user_file.id,
|
||||
sha256=session.sha256,
|
||||
storage_path=str(storage_path),
|
||||
file_size=session.file_size,
|
||||
)
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
from sakura_cloud.core.config import settings
|
||||
|
||||
|
||||
def thumbnail_path_for(storage_path: str | Path) -> Path:
|
||||
source = Path(storage_path)
|
||||
return source.with_name(f"{source.stem}.thumb.webp")
|
||||
|
||||
|
||||
def generate_webp_thumbnail(storage_path: str | Path) -> str | None:
|
||||
source = Path(storage_path)
|
||||
if not source.is_file():
|
||||
return None
|
||||
target = thumbnail_path_for(source)
|
||||
try:
|
||||
with Image.open(source) as image:
|
||||
image = ImageOps.exif_transpose(image)
|
||||
image.thumbnail((settings.thumbnail_max_size, settings.thumbnail_max_size))
|
||||
if image.mode not in {"RGB", "RGBA"}:
|
||||
image = image.convert("RGB")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
image.save(target, "WEBP", quality=settings.thumbnail_quality, method=6)
|
||||
return str(target)
|
||||
except Exception:
|
||||
return None
|
||||
Reference in new issue
Block a user