first commit

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
kunkunxiaomaoandClaude committed 2026-11-16 20:45:22 +08:00
commit 9c5fef1a4d
412 files changed
+94582

No files matched your search

+294
View File
@@ -0,0 +1,294 @@
from __future__ import annotations
import hashlib
import os
import shutil
from datetime import datetime, timedelta, timezone
from pathlib import Path
from uuid import uuid4
from fastapi import HTTPException, UploadFile, status
from sqlalchemy import func, select
from sqlalchemy.orm import Session
from sakura_cloud.core.config import settings
from sakura_cloud.core.safety import is_allowed_upload_mime, is_safe_upload_filename
from sakura_cloud.models.file import FileBlob, UploadSession, UserFile
from sakura_cloud.models.user import User
from sakura_cloud.schemas.upload import UploadCompleteResponse, UploadInitRequest, UploadInitResponse
from sakura_cloud.services.thumbnails import generate_webp_thumbnail
def _ensure_storage_quota(user: User, incoming_size: int) -> None:
size = max(0, incoming_size)
if size and user.storage_used + size > user.storage_limit:
raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="Storage quota exceeded")
def _is_video_mime(mime_type: str | None) -> bool:
return (mime_type or "").lower().strip().startswith("video/")
def _user_video_storage_used(user_id: str, db: Session) -> int:
value = db.scalar(
select(func.coalesce(func.sum(FileBlob.file_size), 0))
.join(UserFile, UserFile.file_sha256 == FileBlob.sha256)
.where(UserFile.user_id == user_id, FileBlob.mime_type.like("video/%"))
)
return int(value or 0)
def _ensure_video_storage_quota(user: User, incoming_size: int, db: Session) -> None:
size = max(0, incoming_size)
if size and _user_video_storage_used(user.id, db) + size > settings.video_storage_limit_bytes:
raise HTTPException(
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
detail=f"Video storage quota exceeded ({settings.video_storage_limit_gb}GB)",
)
def _root() -> Path:
root = Path(settings.storage_root)
root.mkdir(parents=True, exist_ok=True)
return root
def temp_upload_path(user_id: str, upload_id: str) -> Path:
path = _root() / "tmp" / user_id
path.mkdir(parents=True, exist_ok=True)
return path / f"{upload_id}.part"
def _replacement_temp_upload_path(user_id: str, upload_id: str) -> Path:
path = _root() / "tmp" / user_id
path.mkdir(parents=True, exist_ok=True)
return path / f"{upload_id}-{uuid4().hex[:8]}.part"
def blob_storage_path(resource_type: str, user_id: str, sha256: str, file_name: str) -> Path:
now = datetime.now(timezone.utc)
ext = Path(file_name).suffix.lower() or ".bin"
kind = "novels" if resource_type == "novel" else "videos" if resource_type == "video" else "images"
path = _root() / "uploads" / kind / user_id / str(now.year) / f"{now.month:02d}"
path.mkdir(parents=True, exist_ok=True)
return path / f"{sha256[:16]}{ext}"
def _open_temp_file_for_append(temp_path: Path):
temp_path.parent.mkdir(parents=True, exist_ok=True)
return temp_path.open("ab")
def _reset_unwritable_upload_session(session: UploadSession, db: Session) -> Path:
old_path = Path(session.temp_path or "")
new_path = _replacement_temp_upload_path(session.user_id, session.id)
if old_path and old_path != new_path:
try:
if old_path.is_file():
old_path.unlink()
except OSError:
pass
session.temp_path = str(new_path)
session.received_size = 0
session.status = "pending"
db.commit()
db.refresh(session)
return new_path
def calculate_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as fp:
for chunk in iter(lambda: fp.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def cleanup_stale_uploads(db: Session, *, older_than: timedelta | None = None) -> int:
cutoff = datetime.now(timezone.utc) - (older_than or timedelta(hours=settings.temp_upload_ttl_hours))
sessions = db.scalars(
select(UploadSession).where(
UploadSession.status != "completed",
UploadSession.updated_at < cutoff,
)
).all()
removed = 0
for session in sessions:
if session.temp_path:
Path(session.temp_path).unlink(missing_ok=True)
db.delete(session)
removed += 1
if removed:
db.commit()
return removed
def init_upload(payload: UploadInitRequest, user: User, db: Session) -> UploadInitResponse:
is_video = _is_video_mime(payload.mime_type)
resource_type = "video" if is_video else payload.resource_type
max_upload_mb = settings.max_video_upload_size_mb if is_video else settings.max_upload_size_mb
max_upload_size = max_upload_mb * 1024 * 1024
if payload.file_size > max_upload_size:
raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File is too large")
if not is_safe_upload_filename(payload.file_name):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid file name")
if resource_type not in {"image", "novel", "video"}:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid resource type")
if not is_allowed_upload_mime(payload.mime_type):
raise HTTPException(status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, detail="Unsupported MIME type")
if resource_type == "video" and not is_video:
raise HTTPException(status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, detail="Unsupported video MIME type")
cleanup_stale_uploads(db)
existing_blob = db.get(FileBlob, payload.sha256)
existing_user_file = None
if existing_blob is not None:
existing_user_file = db.scalar(
select(UserFile).where(UserFile.user_id == user.id, UserFile.file_sha256 == payload.sha256)
)
if existing_user_file is None:
_ensure_storage_quota(user, existing_blob.file_size)
if _is_video_mime(existing_blob.mime_type):
_ensure_video_storage_quota(user, existing_blob.file_size, db)
user_file = UserFile(user_id=user.id, file_sha256=payload.sha256, original_name=payload.file_name)
db.add(user_file)
user.storage_used += existing_blob.file_size
db.commit()
db.refresh(user_file)
else:
user_file = existing_user_file
return UploadInitResponse(
upload_id="",
exists=True,
chunk_size=settings.upload_chunk_size,
received_size=payload.file_size,
file_id=user_file.id,
)
session = db.scalar(
select(UploadSession).where(
UploadSession.user_id == user.id,
UploadSession.sha256 == payload.sha256,
UploadSession.status != "completed",
)
)
if session is None:
_ensure_storage_quota(user, payload.file_size)
if is_video:
_ensure_video_storage_quota(user, payload.file_size, db)
session = UploadSession(
user_id=user.id,
sha256=payload.sha256,
file_name=payload.file_name,
file_size=payload.file_size,
mime_type=payload.mime_type,
resource_type=resource_type,
chunk_size=settings.upload_chunk_size,
temp_path="",
)
db.add(session)
db.flush()
session.temp_path = str(temp_upload_path(user.id, session.id))
db.commit()
db.refresh(session)
return UploadInitResponse(
upload_id=session.id,
exists=False,
chunk_size=session.chunk_size,
received_size=session.received_size,
)
async def append_chunk(upload_id: str, file: UploadFile, user: User, db: Session) -> UploadSession:
session = db.get(UploadSession, upload_id)
if session is None or session.user_id != user.id:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Upload session not found")
if session.status == "completed":
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Upload already completed")
temp_path = Path(session.temp_path)
size = 0
try:
with _open_temp_file_for_append(temp_path) as fp:
while True:
chunk = await file.read(1024 * 1024)
if not chunk:
break
size += len(chunk)
if session.received_size + size > session.file_size:
raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="Upload exceeds declared size")
fp.write(chunk)
except PermissionError:
# Old upload sessions may point to temp files created by another process/user.
# Reset the session onto a fresh temp path and let the client restart from 0.
temp_parent = temp_path.parent
if not temp_parent.exists():
temp_parent.mkdir(parents=True, exist_ok=True)
if not os.access(temp_parent, os.W_OK):
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail=f"Upload temp directory is not writable: {temp_parent}",
)
new_path = _reset_unwritable_upload_session(session, db)
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail=f"Upload temp file was not writable. Session reset to {new_path.name}, please retry upload.",
)
session.received_size += size
session.status = "uploading"
db.commit()
db.refresh(session)
return session
def complete_upload(upload_id: str, user: User, db: Session) -> UploadCompleteResponse:
session = db.get(UploadSession, upload_id)
if session is None or session.user_id != user.id:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Upload session not found")
temp_path = Path(session.temp_path)
if not temp_path.is_file():
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Upload file is missing")
actual_size = temp_path.stat().st_size
if actual_size != session.file_size:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Upload size mismatch")
actual_sha = calculate_sha256(temp_path)
if actual_sha != session.sha256:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="SHA256 mismatch")
storage_path = blob_storage_path(session.resource_type, user.id, session.sha256, session.file_name)
if not storage_path.exists():
shutil.move(str(temp_path), storage_path)
else:
temp_path.unlink(missing_ok=True)
thumbnail_path = generate_webp_thumbnail(storage_path) if (session.mime_type or "").startswith("image/") else None
blob = db.get(FileBlob, session.sha256)
if blob is None:
blob = FileBlob(
sha256=session.sha256,
storage_path=str(storage_path),
thumbnail_path=thumbnail_path,
file_size=session.file_size,
mime_type=session.mime_type,
)
db.add(blob)
elif thumbnail_path and not blob.thumbnail_path:
blob.thumbnail_path = thumbnail_path
user_file = db.scalar(select(UserFile).where(UserFile.user_id == user.id, UserFile.file_sha256 == session.sha256))
if user_file is None:
_ensure_storage_quota(user, session.file_size)
if _is_video_mime(session.mime_type):
_ensure_video_storage_quota(user, session.file_size, db)
user_file = UserFile(user_id=user.id, file_sha256=session.sha256, original_name=session.file_name)
db.add(user_file)
user.storage_used += session.file_size
session.status = "completed"
db.commit()
db.refresh(user_file)
return UploadCompleteResponse(
file_id=user_file.id,
sha256=session.sha256,
storage_path=str(storage_path),
file_size=session.file_size,
)