first commit

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
kunkunxiaomaoandClaude committed 2026-11-16 20:45:22 +08:00
commit 9c5fef1a4d
412 files changed
+94582

No files matched your search

+1
View File
@@ -0,0 +1 @@
"""Application services."""
+45
View File
@@ -0,0 +1,45 @@
from __future__ import annotations
from fastapi import Request
from sqlalchemy.orm import Session
from sakura_cloud.models.admin import AdminAuditLog
from sakura_cloud.models.user import User
def write_admin_audit_log(
db: Session,
admin: User,
*,
action: str,
target_type: str | None = None,
target_id: str | None = None,
request: Request | None = None,
detail: dict | None = None,
) -> AdminAuditLog:
ip_address = request.client.host if request is not None and request.client is not None else None
user_agent = request.headers.get("User-Agent") if request is not None else None
row = AdminAuditLog(
admin_user_id=admin.id,
action=action,
target_type=target_type,
target_id=target_id,
ip_address=ip_address,
user_agent=user_agent,
detail_json=_redact_detail(detail or {}),
)
db.add(row)
db.commit()
db.refresh(row)
return row
def _redact_detail(value: dict) -> dict:
redacted: dict = {}
for key, item in value.items():
lowered = str(key).lower()
if any(marker in lowered for marker in ("password", "token", "secret", "cookie", "key")):
redacted[key] = "***"
else:
redacted[key] = item
return redacted
+21
View File
@@ -0,0 +1,21 @@
from __future__ import annotations
from pathlib import Path
from alembic import command
from alembic.config import Config
from sakura_cloud.core.config import settings
from sakura_cloud.core.logging import logger
def upgrade_database_to_head() -> None:
cloud_dir = Path(__file__).resolve().parents[1]
alembic_ini = cloud_dir / "alembic.ini"
config = Config(str(alembic_ini))
config.set_main_option("script_location", str(cloud_dir / "migrations"))
config.set_main_option("sqlalchemy.url", settings.database_url)
logger.info("Running database migrations to head")
command.upgrade(config, "head")
+163
View File
@@ -0,0 +1,163 @@
from __future__ import annotations
import threading
from pathlib import Path
from typing import Callable
from loguru import logger
from sqlalchemy import select
from sqlalchemy.orm import Session
from sakura_cloud.core.config import settings
from sakura_cloud.db.session import SessionLocal
from sakura_cloud.models.artwork import Artwork
from sakura_cloud.models.ocr import ArtworkOcrText
OcrExtractor = Callable[[Path], tuple[str, float | None, str]]
_semaphore = threading.BoundedSemaphore(settings.ocr_max_concurrency)
def queue_ocr_for_artwork(db: Session, artwork: Artwork) -> ArtworkOcrText | None:
if not settings.ocr_enabled:
return None
row = _get_or_create_row(db, artwork)
if row.status in {"pending", "running", "done"}:
return row
row.status = "pending"
row.error = None
db.commit()
return row
def run_ocr_for_artwork_id(artwork_id: str, user_id: str) -> None:
with SessionLocal() as db:
artwork = db.get(Artwork, artwork_id)
if artwork is None or artwork.user_id != user_id:
return
run_ocr_for_artwork(db, artwork)
def run_ocr_for_artwork(
db: Session,
artwork: Artwork,
*,
extractor: OcrExtractor | None = None,
) -> ArtworkOcrText:
row = _get_or_create_row(db, artwork)
if not settings.ocr_enabled and extractor is None:
row.status = "failed"
row.error = "ocr_disabled"
db.commit()
return row
if not _is_supported_image(artwork):
row.status = "failed"
row.error = "unsupported_file_type"
db.commit()
logger.info("OCR skipped artwork={} mime={}", artwork.id, artwork.mime_type)
return row
if row.status == "running":
return row
if row.attempts >= settings.ocr_max_attempts and extractor is None:
return row
acquired = _semaphore.acquire(blocking=False)
if not acquired:
row.status = "pending"
row.error = "ocr_busy"
db.commit()
return row
try:
row.status = "running"
row.attempts += 1
row.error = None
db.commit()
text, confidence, language = (extractor or _paddle_ocr_extract)(Path(artwork.file_path))
row.ocr_text = text.strip()
row.confidence = confidence
row.language = language or "mixed"
row.engine = "paddleocr"
row.status = "done"
row.error = None
db.commit()
logger.info("OCR done artwork={} text_len={} confidence={}", artwork.id, len(row.ocr_text), confidence)
return row
except Exception as exc:
row.status = "failed"
row.error = exc.__class__.__name__
db.commit()
logger.warning("OCR failed artwork={} attempt={} error={}", artwork.id, row.attempts, exc.__class__.__name__)
return row
finally:
_semaphore.release()
def _get_or_create_row(db: Session, artwork: Artwork) -> ArtworkOcrText:
row = db.scalar(
select(ArtworkOcrText).where(
ArtworkOcrText.user_id == artwork.user_id,
ArtworkOcrText.artwork_id == artwork.id,
)
)
if row is None:
row = ArtworkOcrText(
user_id=artwork.user_id,
artwork_id=artwork.id,
file_id=artwork.file_id,
source_platform=artwork.source_platform,
source_id=artwork.source_id,
status="pending",
)
db.add(row)
db.commit()
db.refresh(row)
else:
row.file_id = artwork.file_id
row.source_platform = artwork.source_platform
row.source_id = artwork.source_id
return row
def _is_supported_image(artwork: Artwork) -> bool:
mime = (artwork.mime_type or "").lower()
if mime in {"image/jpeg", "image/png", "image/webp"}:
return True
suffix = Path(artwork.file_path or "").suffix.lower()
return suffix in {".jpg", ".jpeg", ".png", ".webp"} and mime != "image/gif"
def _paddle_ocr_extract(path: Path) -> tuple[str, float | None, str]:
from PIL import Image
from paddleocr import PaddleOCR
if not path.is_file():
raise FileNotFoundError("image_not_found")
work_path = path
with Image.open(path) as image:
original_size = image.size
image.thumbnail((settings.ocr_max_image_dimension, settings.ocr_max_image_dimension))
if image.size != original_size:
tmp = path.with_suffix(".ocr-resized.jpg")
image.convert("RGB").save(tmp, quality=90)
work_path = tmp
ocr = PaddleOCR(use_angle_cls=True, lang=settings.ocr_language, use_gpu=False)
result = ocr.ocr(str(work_path), cls=True)
texts: list[str] = []
scores: list[float] = []
for page in result or []:
for line in page or []:
if len(line) >= 2 and isinstance(line[1], (list, tuple)):
text = str(line[1][0]).strip()
if text:
texts.append(text)
try:
scores.append(float(line[1][1]))
except (TypeError, ValueError, IndexError):
pass
if work_path != path:
work_path.unlink(missing_ok=True)
confidence = sum(scores) / len(scores) if scores else None
return "\n".join(texts), confidence, settings.ocr_language
+537
View File
@@ -0,0 +1,537 @@
from __future__ import annotations
import ipaddress
import json
import re
import socket
from dataclasses import dataclass
from html import unescape
from html.parser import HTMLParser
from typing import Any
from urllib.parse import quote, urljoin, urlparse
from urllib.request import Request, urlopen
from sakura_cloud.services import online_search_engine
from sakura_cloud.services.online_search_engine import (
OnlineSearchEngine,
SearchEngineError,
assert_public_url,
build_search_url,
parse_comic,
parse_novel,
resolve_resource_url,
search_resource,
source_type,
)
class OnlineSearchError(RuntimeError):
pass
@dataclass(frozen=True)
class OnlineSearchItem:
source_name: str
source_url: str
title: str
author: str | None
cover_url: str | None
detail_url: str
kind: str
def parse_source_payload(payload: Any) -> list[dict[str, Any]]:
if isinstance(payload, str):
payload = json.loads(payload)
if isinstance(payload, list):
rows = payload
elif isinstance(payload, dict):
for key in ("sources", "bookSources", "data", "list", "items", "results"):
value = payload.get(key)
if isinstance(value, list):
rows = value
break
else:
rows = [payload]
else:
rows = []
return [dict(item) for item in rows if isinstance(item, dict)]
def search_sources(
sources: list[dict[str, Any]],
keyword: str,
*,
mode: str,
limit_sources: int = 30,
per_source_limit: int = 20,
) -> list[OnlineSearchItem]:
results: list[OnlineSearchItem] = []
expected_type = 2 if mode == "manga" else 0
engine = OnlineSearchEngine(timeout=10.0)
for source in sources[:limit_sources]:
if _source_type(source) != expected_type:
continue
try:
online_search_engine.assert_public_url = _assert_public_url
engine._request_text = ( # type: ignore[method-assign]
lambda rule, url, data=None, _source=source: _fetch_text(url, rule.base_url, _source)
)
candidates = engine.search_legado_source(source, keyword, mode=mode, limit=per_source_limit)
results.extend(
OnlineSearchItem(
source_name=item.source_name,
source_url=item.source_url,
title=item.title,
author=item.author,
cover_url=item.cover,
detail_url=item.url,
kind=mode,
)
for item in candidates
)
except SearchEngineError:
continue
except Exception:
continue
return results
def _search_one(source: dict[str, Any], keyword: str, *, mode: str, limit: int) -> list[OnlineSearchItem]:
source_name = _string(source.get("bookSourceName") or source.get("sourceName") or source.get("name")) or "未命名源"
source_url = _string(source.get("bookSourceUrl") or source.get("sourceUrl") or source.get("url")) or ""
search_url = _string(source.get("searchUrl"))
rule_search = source.get("ruleSearch") if isinstance(source.get("ruleSearch"), dict) else {}
if not search_url:
search_url = _string(rule_search.get("url") or rule_search.get("searchUrl")) or ""
if not source_url or not search_url:
return []
final_url = _resolve_url(source_url, _build_search_url(search_url, keyword))
_assert_public_url(final_url)
body = _fetch_text(final_url, source_url, source)
list_rule = _first_string(
rule_search,
"bookList",
"comicList",
"list",
"items",
"data",
"results",
)
if list_rule:
parsed = _parse_by_rules(body, final_url, source, source_name, source_url, rule_search, mode, limit)
if parsed:
return parsed
if mode == "manga":
return _fallback_manga(body, final_url, source_name, source_url, keyword, limit)
return _fallback_novel(body, final_url, source_name, source_url, keyword, limit)
def _parse_by_rules(
body: str,
base_url: str,
source: dict[str, Any],
source_name: str,
source_url: str,
rule_search: dict[str, Any],
mode: str,
limit: int,
) -> list[OnlineSearchItem]:
fragments = _select_list(body, _first_string(rule_search, "bookList", "comicList", "list", "items", "data", "results"))
results: list[OnlineSearchItem] = []
seen: set[str] = set()
for fragment in fragments:
title = _select_one(fragment, _first_string(rule_search, "name", "bookName", "comicName", "title"))
detail = _select_one(fragment, _first_string(rule_search, "bookUrl", "comicUrl", "detailUrl", "url", "href"))
if not title or not detail:
continue
detail_url = _resolve_url(base_url, detail)
if detail_url in seen:
continue
seen.add(detail_url)
results.append(
OnlineSearchItem(
source_name=source_name,
source_url=source_url,
title=title,
author=_select_one(fragment, _first_string(rule_search, "author", "bookAuthor", "comicAuthor", "writer")) or None,
cover_url=_maybe_url(base_url, _select_one(fragment, _first_string(rule_search, "coverUrl", "cover", "bookCover", "comicCover", "img", "image"))),
detail_url=detail_url,
kind=mode,
)
)
if len(results) >= limit:
break
return results
def _fallback_manga(
body: str,
base_url: str,
source_name: str,
source_url: str,
keyword: str,
limit: int,
) -> list[OnlineSearchItem]:
decoded = _decode_json_like(body)
results: list[OnlineSearchItem] = []
seen: set[str] = set()
if decoded is not None:
for node in _walk_dicts(decoded):
title = _first_value(node, "title", "name", "comicName", "comic_name", "topic_title", "topicTitle", "bookName")
if not _matches(title, keyword):
continue
detail = _first_value(node, "url", "href", "link", "bookUrl", "book_url", "comicUrl", "comic_url", "detailUrl", "detail_url", "path")
if not detail:
detail = _first_value(node, "id", "book_id", "comic_id", "topic_id")
if not detail:
continue
detail_url = _resolve_url(base_url, detail)
if detail_url in seen:
continue
seen.add(detail_url)
results.append(
OnlineSearchItem(
source_name=source_name,
source_url=source_url,
title=title,
author=_first_value(node, "author", "writer", "nickname", "user_name", "userName") or None,
cover_url=_maybe_url(base_url, _first_value(node, "cover", "coverUrl", "cover_url", "image", "imageUrl", "image_url", "pic", "pic_url", "vertical_image_url")),
detail_url=detail_url,
kind="manga",
)
)
if len(results) >= limit:
return results
if results:
return results
return _fallback_html_cards(body, base_url, source_name, source_url, keyword, "manga", limit)
def _fallback_novel(
body: str,
base_url: str,
source_name: str,
source_url: str,
keyword: str,
limit: int,
) -> list[OnlineSearchItem]:
decoded = _decode_json_like(body)
results: list[OnlineSearchItem] = []
if decoded is not None:
for node in _walk_dicts(decoded):
title = _first_value(node, "title", "name", "bookName", "book_name", "novelName")
detail = _first_value(node, "url", "href", "link", "bookUrl", "book_url", "detailUrl", "detail_url")
if not _matches(title, keyword) or not detail:
continue
results.append(
OnlineSearchItem(
source_name=source_name,
source_url=source_url,
title=title,
author=_first_value(node, "author", "bookAuthor", "writer") or None,
cover_url=_maybe_url(base_url, _first_value(node, "cover", "coverUrl", "cover_url", "img", "image")),
detail_url=_resolve_url(base_url, detail),
kind="novel",
)
)
if len(results) >= limit:
return results
return results or _fallback_html_cards(body, base_url, source_name, source_url, keyword, "novel", limit)
def _fallback_html_cards(
body: str,
base_url: str,
source_name: str,
source_url: str,
keyword: str,
kind: str,
limit: int,
) -> list[OnlineSearchItem]:
parser = _AnchorParser()
parser.feed(body)
results: list[OnlineSearchItem] = []
seen: set[str] = set()
for anchor in parser.anchors:
title = _clean(anchor.get("title") or anchor.get("alt") or anchor.get("text") or "")
href = _clean(anchor.get("href") or "")
if not href or not _matches(title, keyword):
continue
detail_url = _resolve_url(base_url, href)
if detail_url in seen:
continue
seen.add(detail_url)
results.append(
OnlineSearchItem(
source_name=source_name,
source_url=source_url,
title=title,
author=None,
cover_url=_maybe_url(base_url, anchor.get("img") or ""),
detail_url=detail_url,
kind=kind,
)
)
if len(results) >= limit:
break
return results
def _select_list(body: str, rule: str | None) -> list[str]:
if not rule:
return []
for part in _split_rule(rule, "||"):
if part.lower().startswith("regex:"):
pattern = _strip_replace(part.split(":", 1)[1])
matches = re.findall(pattern, body, re.I | re.S)
output = [_clean(item[0] if isinstance(item, tuple) else item) for item in matches]
if output:
return [item for item in output if item]
if part.startswith("$.") or part.startswith("$["):
decoded = _decode_json_like(body)
values = _json_path(decoded, part) if decoded is not None else []
output = [json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value) for value in values]
if output:
return output
matches = _html_rule_list(body, part)
if matches:
return matches
return []
def _select_one(fragment: str, rule: str | None) -> str:
values = _select_list(fragment, rule)
return values[0] if values else ""
def _html_rule_list(body: str, rule: str) -> list[str]:
rule = _strip_replace(rule.strip())
if not rule:
return []
tokens = [item.strip() for item in rule.split("@") if item.strip()]
selector = tokens[0] if tokens else rule
attr = tokens[-1] if tokens and tokens[-1] not in {selector, "text", "html"} else ""
if selector.startswith("class."):
classes = [item for item in selector[6:].split() if item]
class_checks = "".join(rf'(?=[^"\']*{re.escape(item)})' for item in classes[:3])
pattern = rf'<(?P<tag>[a-zA-Z0-9]+)\b[^>]*class=["\']{class_checks}[^"\']*["\'][^>]*>.*?</(?P=tag)>'
elif selector.startswith("id."):
pattern = rf'<(?P<tag>[a-zA-Z0-9]+)\b[^>]*id=["\']{re.escape(selector[3:])}["\'][^>]*>.*?</(?P=tag)>'
elif selector.startswith("tag."):
tag = re.escape(selector[4:])
pattern = rf'<{tag}\b[^>]*>.*?</{tag}>'
elif selector.startswith("$.") or selector.startswith("$["):
decoded = _decode_json_like(body)
return [json.dumps(value, ensure_ascii=False) if isinstance(value, (dict, list)) else str(value) for value in _json_path(decoded, selector)]
else:
pattern = rf'<a\b[^>]*>.*?</a>' if "href" in tokens else ""
if not pattern:
return []
blocks = [m.group(0) for m in re.finditer(pattern, body, re.I | re.S)]
if attr and attr not in {"text", "html"}:
return [_clean(m.group(1)) for block in blocks for m in [re.search(rf'{re.escape(attr)}=["\']([^"\']+)["\']', block, re.I)] if m]
if "text" in tokens:
return [_html_text(block) for block in blocks]
return blocks
def _json_path(decoded: Any, rule: str) -> list[Any]:
if decoded is None:
return []
path = re.sub(r"^\$\.?", "", rule.strip())
if path in {"", "*", "[*]"}:
return decoded if isinstance(decoded, list) else [decoded]
current = [decoded]
for segment in [item for item in path.split(".") if item]:
next_values: list[Any] = []
star = segment.endswith("[*]")
key = segment[:-3] if star else segment
for value in current:
child = value.get(key) if isinstance(value, dict) and key else value
if star and isinstance(child, list):
next_values.extend(child)
elif child is not None:
next_values.append(child)
current = next_values
return current
class _AnchorParser(HTMLParser):
def __init__(self) -> None:
super().__init__()
self.anchors: list[dict[str, str]] = []
self._current: dict[str, str] | None = None
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
attr = {key: value or "" for key, value in attrs}
if tag.lower() == "a" and attr.get("href"):
self._current = {"href": attr.get("href", ""), "title": attr.get("title", ""), "text": ""}
elif tag.lower() == "img" and self._current is not None:
for key in ("data-original", "data-src", "src"):
if attr.get(key):
self._current["img"] = attr[key]
break
if attr.get("alt"):
self._current["alt"] = attr["alt"]
def handle_data(self, data: str) -> None:
if self._current is not None:
self._current["text"] = f'{self._current.get("text", "")} {data}'
def handle_endtag(self, tag: str) -> None:
if tag.lower() == "a" and self._current is not None:
self.anchors.append(self._current)
self._current = None
def _fetch_text(url: str, referer: str, source: dict[str, Any]) -> str:
headers = {
"User-Agent": _string(source.get("httpUserAgent")) or "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,application/json;q=0.8,*/*;q=0.7",
"Accept-Language": "zh-CN,zh;q=0.9",
"Referer": referer,
}
extra = source.get("headers") or source.get("header")
if isinstance(extra, dict):
for key, value in extra.items():
if str(key).lower() in {"cookie", "authorization"}:
continue
headers[str(key)] = str(value)
with urlopen(Request(url, headers=headers), timeout=12) as response:
raw = response.read(2_000_000)
content_type = response.headers.get("content-type", "")
charset = _string(source.get("charset") or source.get("encoding")) or _charset_from_content_type(content_type) or "utf-8"
for candidate in (charset, "utf-8", "gb18030", "gbk", "gb2312"):
try:
return raw.decode(candidate, errors="replace")
except LookupError:
continue
return raw.decode("utf-8", errors="replace")
def _assert_public_url(url: str) -> None:
parsed = urlparse(url)
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
raise OnlineSearchError("Only http/https source URLs are allowed")
infos = socket.getaddrinfo(parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80), type=socket.SOCK_STREAM)
for item in infos:
ip = ipaddress.ip_address(item[4][0])
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_reserved:
raise OnlineSearchError("Private network source URLs are not allowed")
def _build_search_url(template: str, keyword: str) -> str:
encoded = quote(keyword, safe="")
value = template.strip()
for token in ("{{key}}", "{{keyword}}", "{key}", "{keyword}", "<key>", "<keyword>", "searchKey"):
value = value.replace(token, encoded)
value = value.replace("%s", encoded, 1)
return value.replace("{{page}}", "1").replace("{page}", "1").replace("<page>", "1")
def _resolve_url(base: str, value: str) -> str:
return urljoin(base, value.strip())
def _maybe_url(base: str, value: str | None) -> str | None:
text = _clean(value or "")
if not text or text.lower() == "null":
return None
return _resolve_url(base, text)
def _decode_json_like(body: str) -> Any:
text = body.lstrip("\ufeff \t\r\n")
match = re.match(r"^[a-zA-Z_$][\w$]*\s*\((.*)\)\s*;?\s*$", text, re.S)
if match:
text = match.group(1)
try:
return json.loads(text)
except Exception:
return None
def _walk_dicts(node: Any):
if isinstance(node, dict):
yield node
for value in node.values():
yield from _walk_dicts(value)
elif isinstance(node, list):
for item in node:
yield from _walk_dicts(item)
def _first_value(node: dict[str, Any], *keys: str) -> str:
for key in keys:
value = node.get(key)
if value is not None:
text = _clean(str(value))
if text and text.lower() != "null":
return text
return ""
def _first_string(mapping: dict[str, Any], *keys: str) -> str | None:
for key in keys:
value = _string(mapping.get(key))
if value:
return value
return None
def _source_type(source: dict[str, Any]) -> int:
value = source.get("bookSourceType", source.get("sourceType", source.get("type", 0)))
try:
numeric = int(value)
except Exception:
numeric = 0
text = json.dumps(source, ensure_ascii=False).lower()
manga_markers = (
"comiclist",
"comicname",
"comicurl",
"imageStyle".lower(),
"chapterImages".lower(),
"manga",
"manhua",
"漫画",
)
if numeric == 0 and any(marker in text for marker in manga_markers):
return 2
return numeric
def _split_rule(rule: str, sep: str) -> list[str]:
return [item.strip() for item in rule.split(sep) if item.strip()]
def _strip_replace(rule: str) -> str:
return re.split(r"##|###", rule, maxsplit=1)[0].strip()
def _string(value: Any) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None
def _matches(title: str, keyword: str) -> bool:
title = _clean(title).lower()
keyword = _clean(keyword).lower()
return bool(title) and (not keyword or keyword in title)
def _clean(value: str) -> str:
return unescape(re.sub(r"\s+", " ", value or "")).strip()
def _html_text(value: str) -> str:
return _clean(re.sub(r"<[^>]+>", " ", value))
def _charset_from_content_type(value: str) -> str | None:
match = re.search(r"charset=([^;]+)", value or "", re.I)
return match.group(1).strip("\"'") if match else None
File diff suppressed because it is too large. Load diff
+160
View File
@@ -0,0 +1,160 @@
from __future__ import annotations
import json
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from loguru import logger
from sqlalchemy import select
from sqlalchemy.orm import Session
from sakura_cloud.core.config import settings
from sakura_cloud.models.follow import UpdateAlert
from sakura_cloud.models.mixins import utc_now
from sakura_cloud.models.push import PushDeviceToken
PushSender = Callable[[str, str, str, dict[str, str]], bool]
@dataclass
class PushSendSummary:
attempted: int = 0
sent: int = 0
disabled: int = 0
failed: int = 0
def send_update_alert_push(
db: Session,
alert: UpdateAlert,
*,
sender: PushSender | None = None,
) -> PushSendSummary:
payload = alert.payload_json or {}
if alert.platform == "system":
title = alert.title
body = alert.body or ""
else:
title = "你关注的作者有新作品"
author_name = (
payload.get("author_name")
or payload.get("creator_name")
or payload.get("creator_handle")
or payload.get("author")
or alert.creator_id
)
body = f"{author_name} 发布了 {alert.title}"
data = {
"alert_id": alert.id,
"source_platform": alert.platform,
"source_id": str(payload.get("source_id") or payload.get("id") or alert.creator_id),
"resource_type": str(payload.get("resource_type") or alert.content_type or ""),
}
return send_push_to_user(db, alert.user_id, title, body, data, sender=sender)
def send_push_to_user(
db: Session,
user_id: str,
title: str,
body: str,
data: dict[str, Any] | None = None,
*,
sender: PushSender | None = None,
) -> PushSendSummary:
rows = db.scalars(
select(PushDeviceToken).where(
PushDeviceToken.user_id == user_id,
PushDeviceToken.is_active.is_(True),
)
).all()
summary = PushSendSummary(attempted=len(rows))
if not rows:
return summary
safe_data = {key: "" if value is None else str(value) for key, value in (data or {}).items()}
disabled_any = False
for row in rows:
try:
ok = sender(row.fcm_token, title, body, safe_data) if sender else _send_fcm(row.fcm_token, title, body, safe_data)
except Exception as exc: # pragma: no cover - defensive guard around provider SDK.
if _is_invalid_token_error(exc):
_disable_token(row)
disabled_any = True
summary.disabled += 1
logger.warning("Disabled invalid FCM token for user={} device={}", user_id, row.device_id)
else:
summary.failed += 1
logger.warning("FCM push failed for user={} device={}: {}", user_id, row.device_id, exc.__class__.__name__)
continue
if ok:
summary.sent += 1
logger.info("FCM push sent for user={} device={} alert={}", user_id, row.device_id, safe_data.get("alert_id", ""))
else:
_disable_token(row)
disabled_any = True
summary.disabled += 1
logger.warning("Disabled invalid FCM token for user={} device={}", user_id, row.device_id)
if disabled_any:
db.commit()
return summary
def _disable_token(row: PushDeviceToken) -> None:
row.is_active = False
row.last_seen_at = utc_now()
def _send_fcm(token: str, title: str, body: str, data: dict[str, str]) -> bool:
if not settings.fcm_enabled:
logger.debug("FCM push skipped because FCM_ENABLED is false")
return True
app, messaging = _firebase_context()
if app is None or messaging is None:
logger.debug("FCM push skipped because Firebase credentials are not configured")
return True
message = messaging.Message(
token=token,
notification=messaging.Notification(title=title, body=body),
data=data,
android=messaging.AndroidConfig(priority="high"),
)
messaging.send(message, app=app)
return True
def _firebase_context() -> tuple[Any | None, Any | None]:
credential_json = settings.firebase_credentials_json.strip()
credential_path = settings.firebase_credentials_path.strip()
if not credential_json and not credential_path:
return None, None
try:
import firebase_admin
from firebase_admin import credentials, messaging
except ImportError:
logger.warning("firebase-admin is not installed; FCM push is disabled")
return None, None
if firebase_admin._apps:
return firebase_admin.get_app(), messaging
if credential_json:
cred = credentials.Certificate(json.loads(credential_json))
else:
cred = credentials.Certificate(credential_path)
return firebase_admin.initialize_app(cred), messaging
def _is_invalid_token_error(exc: Exception) -> bool:
name = exc.__class__.__name__
if name in {"UnregisteredError", "SenderIdMismatchError", "InvalidArgumentError"}:
return True
text = str(exc).lower()
return "registration token" in text or "not a valid fcm" in text or "unregistered" in text
+137
View File
@@ -0,0 +1,137 @@
from __future__ import annotations
from datetime import timedelta, timezone
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from sakura_cloud.models.follow import UpdateAlert
from sakura_cloud.models.mixins import utc_now
from sakura_cloud.models.user import User
from sakura_cloud.services.push import send_update_alert_push
DEFAULT_REMINDER_TIMEZONE = "Asia/Shanghai"
DIGEST_PLATFORM = "system"
DIGEST_CREATOR_ID = "daily-reminder"
def _resolve_timezone(name: str | None) -> ZoneInfo:
try:
return ZoneInfo(name or DEFAULT_REMINDER_TIMEZONE)
except ZoneInfoNotFoundError:
return ZoneInfo(DEFAULT_REMINDER_TIMEZONE)
def _display_platform(platform: str) -> str:
if platform.lower() == "pixiv":
return "Pixiv"
if platform.lower() == "x":
return "X"
return platform
def run_daily_reminder_digest(db: Session) -> int:
now_utc = utc_now()
users = db.scalars(select(User).where(User.reminder_enabled.is_(True))).all()
reminders_created = 0
for user in users:
zone = _resolve_timezone(user.reminder_timezone)
local_now = now_utc.astimezone(zone)
scheduled_today = local_now.replace(
hour=user.reminder_hour,
minute=user.reminder_minute,
second=0,
microsecond=0,
)
if local_now < scheduled_today:
continue
if user.last_reminder_sent_at is not None:
last_sent_local = user.last_reminder_sent_at.astimezone(zone)
if last_sent_local.date() == local_now.date():
continue
window_start = (
user.last_reminder_sent_at
if user.last_reminder_sent_at is not None
else (scheduled_today - timedelta(days=1)).astimezone(timezone.utc)
)
new_alert_count = db.scalar(
select(func.count(UpdateAlert.id)).where(
UpdateAlert.user_id == user.id,
UpdateAlert.platform != DIGEST_PLATFORM,
UpdateAlert.created_at > window_start,
)
) or 0
recent_alerts = db.scalars(
select(UpdateAlert)
.where(
UpdateAlert.user_id == user.id,
UpdateAlert.platform != DIGEST_PLATFORM,
UpdateAlert.created_at > window_start,
)
.order_by(UpdateAlert.created_at.desc())
.limit(5)
).all()
creator_names: list[str] = []
platforms: list[str] = []
for alert in recent_alerts:
platform = _display_platform(alert.platform)
if platform not in platforms:
platforms.append(platform)
candidate = (
alert.payload_json.get("creator_name")
or alert.payload_json.get("creator_handle")
or alert.payload_json.get("author")
or alert.creator_id
)
if candidate and candidate not in creator_names:
creator_names.append(str(candidate))
date_text = f"{local_now.month}月{local_now.day}日"
platforms_text = " / ".join(platforms) if platforms else "Pixiv / X"
creators_text = ", ".join(creator_names[:3])
title = f"{date_text},未更新" if not new_alert_count else f"{date_text},更新 {new_alert_count} 条记录"
if not new_alert_count:
body = f"{date_text} 暂未发现关注作者的新内容。云端会在明天 {user.reminder_hour:02d}:{user.reminder_minute:02d} 继续检查。"
elif creators_text:
body = (
f"{date_text},{creators_text} 等作者带来了 {new_alert_count} "
f"条新动态,来自 {platforms_text}。"
)
else:
body = f"{date_text},你关注的作者有 {new_alert_count} 条新动态,来自 {platforms_text}。"
digest_alert = UpdateAlert(
user_id=user.id,
follow_id=None,
platform=DIGEST_PLATFORM,
creator_id=DIGEST_CREATOR_ID,
title=title,
body=body,
content_type="digest",
payload_json={
"kind": "daily_digest",
"window_start": window_start.isoformat(),
"new_alert_count": int(new_alert_count),
"platforms": platforms,
"creators": creator_names[:5],
"date": local_now.date().isoformat(),
"status": "updated" if new_alert_count else "not_updated",
},
)
db.add(digest_alert)
user.last_reminder_sent_at = now_utc
db.commit()
db.refresh(digest_alert)
send_update_alert_push(db, digest_alert)
reminders_created += 1
return reminders_created
+294
View File
@@ -0,0 +1,294 @@
from __future__ import annotations
import hashlib
import os
import shutil
from datetime import datetime, timedelta, timezone
from pathlib import Path
from uuid import uuid4
from fastapi import HTTPException, UploadFile, status
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from sakura_cloud.core.config import settings
from sakura_cloud.core.safety import is_allowed_upload_mime, is_safe_upload_filename
from sakura_cloud.models.file import FileBlob, UploadSession, UserFile
from sakura_cloud.models.user import User
from sakura_cloud.schemas.upload import UploadCompleteResponse, UploadInitRequest, UploadInitResponse
from sakura_cloud.services.thumbnails import generate_webp_thumbnail
def _ensure_storage_quota(user: User, incoming_size: int) -> None:
size = max(0, incoming_size)
if size and user.storage_used + size > user.storage_limit:
raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="Storage quota exceeded")
def _is_video_mime(mime_type: str | None) -> bool:
return (mime_type or "").lower().strip().startswith("video/")
def _user_video_storage_used(user_id: str, db: Session) -> int:
value = db.scalar(
select(func.coalesce(func.sum(FileBlob.file_size), 0))
.join(UserFile, UserFile.file_sha256 == FileBlob.sha256)
.where(UserFile.user_id == user_id, FileBlob.mime_type.like("video/%"))
)
return int(value or 0)
def _ensure_video_storage_quota(user: User, incoming_size: int, db: Session) -> None:
size = max(0, incoming_size)
if size and _user_video_storage_used(user.id, db) + size > settings.video_storage_limit_bytes:
raise HTTPException(
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
detail=f"Video storage quota exceeded ({settings.video_storage_limit_gb}GB)",
)
def _root() -> Path:
root = Path(settings.storage_root)
root.mkdir(parents=True, exist_ok=True)
return root
def temp_upload_path(user_id: str, upload_id: str) -> Path:
path = _root() / "tmp" / user_id
path.mkdir(parents=True, exist_ok=True)
return path / f"{upload_id}.part"
def _replacement_temp_upload_path(user_id: str, upload_id: str) -> Path:
path = _root() / "tmp" / user_id
path.mkdir(parents=True, exist_ok=True)
return path / f"{upload_id}-{uuid4().hex[:8]}.part"
def blob_storage_path(resource_type: str, user_id: str, sha256: str, file_name: str) -> Path:
now = datetime.now(timezone.utc)
ext = Path(file_name).suffix.lower() or ".bin"
kind = "novels" if resource_type == "novel" else "videos" if resource_type == "video" else "images"
path = _root() / "uploads" / kind / user_id / str(now.year) / f"{now.month:02d}"
path.mkdir(parents=True, exist_ok=True)
return path / f"{sha256[:16]}{ext}"
def _open_temp_file_for_append(temp_path: Path):
temp_path.parent.mkdir(parents=True, exist_ok=True)
return temp_path.open("ab")
def _reset_unwritable_upload_session(session: UploadSession, db: Session) -> Path:
old_path = Path(session.temp_path or "")
new_path = _replacement_temp_upload_path(session.user_id, session.id)
if old_path and old_path != new_path:
try:
if old_path.is_file():
old_path.unlink()
except OSError:
pass
session.temp_path = str(new_path)
session.received_size = 0
session.status = "pending"
db.commit()
db.refresh(session)
return new_path
def calculate_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as fp:
for chunk in iter(lambda: fp.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def cleanup_stale_uploads(db: Session, *, older_than: timedelta | None = None) -> int:
cutoff = datetime.now(timezone.utc) - (older_than or timedelta(hours=settings.temp_upload_ttl_hours))
sessions = db.scalars(
select(UploadSession).where(
UploadSession.status != "completed",
UploadSession.updated_at < cutoff,
)
).all()
removed = 0
for session in sessions:
if session.temp_path:
Path(session.temp_path).unlink(missing_ok=True)
db.delete(session)
removed += 1
if removed:
db.commit()
return removed
def init_upload(payload: UploadInitRequest, user: User, db: Session) -> UploadInitResponse:
is_video = _is_video_mime(payload.mime_type)
resource_type = "video" if is_video else payload.resource_type
max_upload_mb = settings.max_video_upload_size_mb if is_video else settings.max_upload_size_mb
max_upload_size = max_upload_mb * 1024 * 1024
if payload.file_size > max_upload_size:
raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File is too large")
if not is_safe_upload_filename(payload.file_name):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid file name")
if resource_type not in {"image", "novel", "video"}:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid resource type")
if not is_allowed_upload_mime(payload.mime_type):
raise HTTPException(status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, detail="Unsupported MIME type")
if resource_type == "video" and not is_video:
raise HTTPException(status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, detail="Unsupported video MIME type")
cleanup_stale_uploads(db)
existing_blob = db.get(FileBlob, payload.sha256)
existing_user_file = None
if existing_blob is not None:
existing_user_file = db.scalar(
select(UserFile).where(UserFile.user_id == user.id, UserFile.file_sha256 == payload.sha256)
)
if existing_user_file is None:
_ensure_storage_quota(user, existing_blob.file_size)
if _is_video_mime(existing_blob.mime_type):
_ensure_video_storage_quota(user, existing_blob.file_size, db)
user_file = UserFile(user_id=user.id, file_sha256=payload.sha256, original_name=payload.file_name)
db.add(user_file)
user.storage_used += existing_blob.file_size
db.commit()
db.refresh(user_file)
else:
user_file = existing_user_file
return UploadInitResponse(
upload_id="",
exists=True,
chunk_size=settings.upload_chunk_size,
received_size=payload.file_size,
file_id=user_file.id,
)
session = db.scalar(
select(UploadSession).where(
UploadSession.user_id == user.id,
UploadSession.sha256 == payload.sha256,
UploadSession.status != "completed",
)
)
if session is None:
_ensure_storage_quota(user, payload.file_size)
if is_video:
_ensure_video_storage_quota(user, payload.file_size, db)
session = UploadSession(
user_id=user.id,
sha256=payload.sha256,
file_name=payload.file_name,
file_size=payload.file_size,
mime_type=payload.mime_type,
resource_type=resource_type,
chunk_size=settings.upload_chunk_size,
temp_path="",
)
db.add(session)
db.flush()
session.temp_path = str(temp_upload_path(user.id, session.id))
db.commit()
db.refresh(session)
return UploadInitResponse(
upload_id=session.id,
exists=False,
chunk_size=session.chunk_size,
received_size=session.received_size,
)
async def append_chunk(upload_id: str, file: UploadFile, user: User, db: Session) -> UploadSession:
session = db.get(UploadSession, upload_id)
if session is None or session.user_id != user.id:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Upload session not found")
if session.status == "completed":
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Upload already completed")
temp_path = Path(session.temp_path)
size = 0
try:
with _open_temp_file_for_append(temp_path) as fp:
while True:
chunk = await file.read(1024 * 1024)
if not chunk:
break
size += len(chunk)
if session.received_size + size > session.file_size:
raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="Upload exceeds declared size")
fp.write(chunk)
except PermissionError:
# Old upload sessions may point to temp files created by another process/user.
# Reset the session onto a fresh temp path and let the client restart from 0.
temp_parent = temp_path.parent
if not temp_parent.exists():
temp_parent.mkdir(parents=True, exist_ok=True)
if not os.access(temp_parent, os.W_OK):
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Upload temp directory is not writable: {temp_parent}",
)
new_path = _reset_unwritable_upload_session(session, db)
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"Upload temp file was not writable. Session reset to {new_path.name}, please retry upload.",
)
session.received_size += size
session.status = "uploading"
db.commit()
db.refresh(session)
return session
def complete_upload(upload_id: str, user: User, db: Session) -> UploadCompleteResponse:
session = db.get(UploadSession, upload_id)
if session is None or session.user_id != user.id:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Upload session not found")
temp_path = Path(session.temp_path)
if not temp_path.is_file():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Upload file is missing")
actual_size = temp_path.stat().st_size
if actual_size != session.file_size:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Upload size mismatch")
actual_sha = calculate_sha256(temp_path)
if actual_sha != session.sha256:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="SHA256 mismatch")
storage_path = blob_storage_path(session.resource_type, user.id, session.sha256, session.file_name)
if not storage_path.exists():
shutil.move(str(temp_path), storage_path)
else:
temp_path.unlink(missing_ok=True)
thumbnail_path = generate_webp_thumbnail(storage_path) if (session.mime_type or "").startswith("image/") else None
blob = db.get(FileBlob, session.sha256)
if blob is None:
blob = FileBlob(
sha256=session.sha256,
storage_path=str(storage_path),
thumbnail_path=thumbnail_path,
file_size=session.file_size,
mime_type=session.mime_type,
)
db.add(blob)
elif thumbnail_path and not blob.thumbnail_path:
blob.thumbnail_path = thumbnail_path
user_file = db.scalar(select(UserFile).where(UserFile.user_id == user.id, UserFile.file_sha256 == session.sha256))
if user_file is None:
_ensure_storage_quota(user, session.file_size)
if _is_video_mime(session.mime_type):
_ensure_video_storage_quota(user, session.file_size, db)
user_file = UserFile(user_id=user.id, file_sha256=session.sha256, original_name=session.file_name)
db.add(user_file)
user.storage_used += session.file_size
session.status = "completed"
db.commit()
db.refresh(user_file)
return UploadCompleteResponse(
file_id=user_file.id,
sha256=session.sha256,
storage_path=str(storage_path),
file_size=session.file_size,
)
+30
View File
@@ -0,0 +1,30 @@
from __future__ import annotations
from pathlib import Path
from PIL import Image, ImageOps
from sakura_cloud.core.config import settings
def thumbnail_path_for(storage_path: str | Path) -> Path:
source = Path(storage_path)
return source.with_name(f"{source.stem}.thumb.webp")
def generate_webp_thumbnail(storage_path: str | Path) -> str | None:
source = Path(storage_path)
if not source.is_file():
return None
target = thumbnail_path_for(source)
try:
with Image.open(source) as image:
image = ImageOps.exif_transpose(image)
image.thumbnail((settings.thumbnail_max_size, settings.thumbnail_max_size))
if image.mode not in {"RGB", "RGBA"}:
image = image.convert("RGB")
target.parent.mkdir(parents=True, exist_ok=True)
image.save(target, "WEBP", quality=settings.thumbnail_quality, method=6)
return str(target)
except Exception:
return None