294 lines
12 KiB
Python
294 lines
12 KiB
Python
from __future__ import annotations
|
|||
|
|
|
||
|
|
import hashlib
|
||
|
|
import os
|
||
|
|
import shutil
|
||
|
|
from datetime import datetime, timedelta, timezone
|
||
|
|
from pathlib import Path
|
||
|
|
from uuid import uuid4
|
||
|
|
|
||
|
|
from fastapi import HTTPException, UploadFile, status
|
||
|
|
from sqlalchemy import func, select
|
||
|
|
from sqlalchemy.orm import Session
|
||
|
|
|
||
|
|
from sakura_cloud.core.config import settings
|
||
|
|
from sakura_cloud.core.safety import is_allowed_upload_mime, is_safe_upload_filename
|
||
|
|
from sakura_cloud.models.file import FileBlob, UploadSession, UserFile
|
||
|
|
from sakura_cloud.models.user import User
|
||
|
|
from sakura_cloud.schemas.upload import UploadCompleteResponse, UploadInitRequest, UploadInitResponse
|
||
|
|
from sakura_cloud.services.thumbnails import generate_webp_thumbnail
|
||
|
|
|
||
|
|
|
||
|
|
def _ensure_storage_quota(user: User, incoming_size: int) -> None:
|
||
|
|
size = max(0, incoming_size)
|
||
|
|
if size and user.storage_used + size > user.storage_limit:
|
||
|
|
raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="Storage quota exceeded")
|
||
|
|
|
||
|
|
|
||
|
|
def _is_video_mime(mime_type: str | None) -> bool:
|
||
|
|
return (mime_type or "").lower().strip().startswith("video/")
|
||
|
|
|
||
|
|
|
||
|
|
def _user_video_storage_used(user_id: str, db: Session) -> int:
|
||
|
|
value = db.scalar(
|
||
|
|
select(func.coalesce(func.sum(FileBlob.file_size), 0))
|
||
|
|
.join(UserFile, UserFile.file_sha256 == FileBlob.sha256)
|
||
|
|
.where(UserFile.user_id == user_id, FileBlob.mime_type.like("video/%"))
|
||
|
|
)
|
||
|
|
return int(value or 0)
|
||
|
|
|
||
|
|
|
||
|
|
def _ensure_video_storage_quota(user: User, incoming_size: int, db: Session) -> None:
|
||
|
|
size = max(0, incoming_size)
|
||
|
|
if size and _user_video_storage_used(user.id, db) + size > settings.video_storage_limit_bytes:
|
||
|
|
raise HTTPException(
|
||
|
|
status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
|
||
|
|
detail=f"Video storage quota exceeded ({settings.video_storage_limit_gb}GB)",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _root() -> Path:
|
||
|
|
root = Path(settings.storage_root)
|
||
|
|
root.mkdir(parents=True, exist_ok=True)
|
||
|
|
return root
|
||
|
|
|
||
|
|
|
||
|
|
def temp_upload_path(user_id: str, upload_id: str) -> Path:
|
||
|
|
path = _root() / "tmp" / user_id
|
||
|
|
path.mkdir(parents=True, exist_ok=True)
|
||
|
|
return path / f"{upload_id}.part"
|
||
|
|
|
||
|
|
|
||
|
|
def _replacement_temp_upload_path(user_id: str, upload_id: str) -> Path:
|
||
|
|
path = _root() / "tmp" / user_id
|
||
|
|
path.mkdir(parents=True, exist_ok=True)
|
||
|
|
return path / f"{upload_id}-{uuid4().hex[:8]}.part"
|
||
|
|
|
||
|
|
|
||
|
|
def blob_storage_path(resource_type: str, user_id: str, sha256: str, file_name: str) -> Path:
|
||
|
|
now = datetime.now(timezone.utc)
|
||
|
|
ext = Path(file_name).suffix.lower() or ".bin"
|
||
|
|
kind = "novels" if resource_type == "novel" else "videos" if resource_type == "video" else "images"
|
||
|
|
path = _root() / "uploads" / kind / user_id / str(now.year) / f"{now.month:02d}"
|
||
|
|
path.mkdir(parents=True, exist_ok=True)
|
||
|
|
return path / f"{sha256[:16]}{ext}"
|
||
|
|
|
||
|
|
|
||
|
|
def _open_temp_file_for_append(temp_path: Path):
|
||
|
|
temp_path.parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
return temp_path.open("ab")
|
||
|
|
|
||
|
|
|
||
|
|
def _reset_unwritable_upload_session(session: UploadSession, db: Session) -> Path:
|
||
|
|
old_path = Path(session.temp_path or "")
|
||
|
|
new_path = _replacement_temp_upload_path(session.user_id, session.id)
|
||
|
|
if old_path and old_path != new_path:
|
||
|
|
try:
|
||
|
|
if old_path.is_file():
|
||
|
|
old_path.unlink()
|
||
|
|
except OSError:
|
||
|
|
pass
|
||
|
|
session.temp_path = str(new_path)
|
||
|
|
session.received_size = 0
|
||
|
|
session.status = "pending"
|
||
|
|
db.commit()
|
||
|
|
db.refresh(session)
|
||
|
|
return new_path
|
||
|
|
|
||
|
|
|
||
|
|
def calculate_sha256(path: Path) -> str:
|
||
|
|
digest = hashlib.sha256()
|
||
|
|
with path.open("rb") as fp:
|
||
|
|
for chunk in iter(lambda: fp.read(1024 * 1024), b""):
|
||
|
|
digest.update(chunk)
|
||
|
|
return digest.hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
def cleanup_stale_uploads(db: Session, *, older_than: timedelta | None = None) -> int:
|
||
|
|
cutoff = datetime.now(timezone.utc) - (older_than or timedelta(hours=settings.temp_upload_ttl_hours))
|
||
|
|
sessions = db.scalars(
|
||
|
|
select(UploadSession).where(
|
||
|
|
UploadSession.status != "completed",
|
||
|
|
UploadSession.updated_at < cutoff,
|
||
|
|
)
|
||
|
|
).all()
|
||
|
|
removed = 0
|
||
|
|
for session in sessions:
|
||
|
|
if session.temp_path:
|
||
|
|
Path(session.temp_path).unlink(missing_ok=True)
|
||
|
|
db.delete(session)
|
||
|
|
removed += 1
|
||
|
|
if removed:
|
||
|
|
db.commit()
|
||
|
|
return removed
|
||
|
|
|
||
|
|
|
||
|
|
def init_upload(payload: UploadInitRequest, user: User, db: Session) -> UploadInitResponse:
|
||
|
|
is_video = _is_video_mime(payload.mime_type)
|
||
|
|
resource_type = "video" if is_video else payload.resource_type
|
||
|
|
max_upload_mb = settings.max_video_upload_size_mb if is_video else settings.max_upload_size_mb
|
||
|
|
max_upload_size = max_upload_mb * 1024 * 1024
|
||
|
|
if payload.file_size > max_upload_size:
|
||
|
|
raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="File is too large")
|
||
|
|
if not is_safe_upload_filename(payload.file_name):
|
||
|
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid file name")
|
||
|
|
if resource_type not in {"image", "novel", "video"}:
|
||
|
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid resource type")
|
||
|
|
if not is_allowed_upload_mime(payload.mime_type):
|
||
|
|
raise HTTPException(status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, detail="Unsupported MIME type")
|
||
|
|
if resource_type == "video" and not is_video:
|
||
|
|
raise HTTPException(status_code=status.HTTP_415_UNSUPPORTED_MEDIA_TYPE, detail="Unsupported video MIME type")
|
||
|
|
cleanup_stale_uploads(db)
|
||
|
|
existing_blob = db.get(FileBlob, payload.sha256)
|
||
|
|
existing_user_file = None
|
||
|
|
if existing_blob is not None:
|
||
|
|
existing_user_file = db.scalar(
|
||
|
|
select(UserFile).where(UserFile.user_id == user.id, UserFile.file_sha256 == payload.sha256)
|
||
|
|
)
|
||
|
|
if existing_user_file is None:
|
||
|
|
_ensure_storage_quota(user, existing_blob.file_size)
|
||
|
|
if _is_video_mime(existing_blob.mime_type):
|
||
|
|
_ensure_video_storage_quota(user, existing_blob.file_size, db)
|
||
|
|
user_file = UserFile(user_id=user.id, file_sha256=payload.sha256, original_name=payload.file_name)
|
||
|
|
db.add(user_file)
|
||
|
|
user.storage_used += existing_blob.file_size
|
||
|
|
db.commit()
|
||
|
|
db.refresh(user_file)
|
||
|
|
else:
|
||
|
|
user_file = existing_user_file
|
||
|
|
return UploadInitResponse(
|
||
|
|
upload_id="",
|
||
|
|
exists=True,
|
||
|
|
chunk_size=settings.upload_chunk_size,
|
||
|
|
received_size=payload.file_size,
|
||
|
|
file_id=user_file.id,
|
||
|
|
)
|
||
|
|
|
||
|
|
session = db.scalar(
|
||
|
|
select(UploadSession).where(
|
||
|
|
UploadSession.user_id == user.id,
|
||
|
|
UploadSession.sha256 == payload.sha256,
|
||
|
|
UploadSession.status != "completed",
|
||
|
|
)
|
||
|
|
)
|
||
|
|
if session is None:
|
||
|
|
_ensure_storage_quota(user, payload.file_size)
|
||
|
|
if is_video:
|
||
|
|
_ensure_video_storage_quota(user, payload.file_size, db)
|
||
|
|
session = UploadSession(
|
||
|
|
user_id=user.id,
|
||
|
|
sha256=payload.sha256,
|
||
|
|
file_name=payload.file_name,
|
||
|
|
file_size=payload.file_size,
|
||
|
|
mime_type=payload.mime_type,
|
||
|
|
resource_type=resource_type,
|
||
|
|
chunk_size=settings.upload_chunk_size,
|
||
|
|
temp_path="",
|
||
|
|
)
|
||
|
|
db.add(session)
|
||
|
|
db.flush()
|
||
|
|
session.temp_path = str(temp_upload_path(user.id, session.id))
|
||
|
|
db.commit()
|
||
|
|
db.refresh(session)
|
||
|
|
return UploadInitResponse(
|
||
|
|
upload_id=session.id,
|
||
|
|
exists=False,
|
||
|
|
chunk_size=session.chunk_size,
|
||
|
|
received_size=session.received_size,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
async def append_chunk(upload_id: str, file: UploadFile, user: User, db: Session) -> UploadSession:
|
||
|
|
session = db.get(UploadSession, upload_id)
|
||
|
|
if session is None or session.user_id != user.id:
|
||
|
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Upload session not found")
|
||
|
|
if session.status == "completed":
|
||
|
|
raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail="Upload already completed")
|
||
|
|
temp_path = Path(session.temp_path)
|
||
|
|
size = 0
|
||
|
|
try:
|
||
|
|
with _open_temp_file_for_append(temp_path) as fp:
|
||
|
|
while True:
|
||
|
|
chunk = await file.read(1024 * 1024)
|
||
|
|
if not chunk:
|
||
|
|
break
|
||
|
|
size += len(chunk)
|
||
|
|
if session.received_size + size > session.file_size:
|
||
|
|
raise HTTPException(status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, detail="Upload exceeds declared size")
|
||
|
|
fp.write(chunk)
|
||
|
|
except PermissionError:
|
||
|
|
# Old upload sessions may point to temp files created by another process/user.
|
||
|
|
# Reset the session onto a fresh temp path and let the client restart from 0.
|
||
|
|
temp_parent = temp_path.parent
|
||
|
|
if not temp_parent.exists():
|
||
|
|
temp_parent.mkdir(parents=True, exist_ok=True)
|
||
|
|
if not os.access(temp_parent, os.W_OK):
|
||
|
|
raise HTTPException(
|
||
|
|
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||
|
|
detail=f"Upload temp directory is not writable: {temp_parent}",
|
||
|
|
)
|
||
|
|
new_path = _reset_unwritable_upload_session(session, db)
|
||
|
|
raise HTTPException(
|
||
|
|
status_code=status.HTTP_409_CONFLICT,
|
||
|
|
detail=f"Upload temp file was not writable. Session reset to {new_path.name}, please retry upload.",
|
||
|
|
)
|
||
|
|
session.received_size += size
|
||
|
|
session.status = "uploading"
|
||
|
|
db.commit()
|
||
|
|
db.refresh(session)
|
||
|
|
return session
|
||
|
|
|
||
|
|
|
||
|
|
def complete_upload(upload_id: str, user: User, db: Session) -> UploadCompleteResponse:
|
||
|
|
session = db.get(UploadSession, upload_id)
|
||
|
|
if session is None or session.user_id != user.id:
|
||
|
|
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Upload session not found")
|
||
|
|
temp_path = Path(session.temp_path)
|
||
|
|
if not temp_path.is_file():
|
||
|
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Upload file is missing")
|
||
|
|
actual_size = temp_path.stat().st_size
|
||
|
|
if actual_size != session.file_size:
|
||
|
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Upload size mismatch")
|
||
|
|
actual_sha = calculate_sha256(temp_path)
|
||
|
|
if actual_sha != session.sha256:
|
||
|
|
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="SHA256 mismatch")
|
||
|
|
|
||
|
|
storage_path = blob_storage_path(session.resource_type, user.id, session.sha256, session.file_name)
|
||
|
|
if not storage_path.exists():
|
||
|
|
shutil.move(str(temp_path), storage_path)
|
||
|
|
else:
|
||
|
|
temp_path.unlink(missing_ok=True)
|
||
|
|
|
||
|
|
thumbnail_path = generate_webp_thumbnail(storage_path) if (session.mime_type or "").startswith("image/") else None
|
||
|
|
|
||
|
|
blob = db.get(FileBlob, session.sha256)
|
||
|
|
if blob is None:
|
||
|
|
blob = FileBlob(
|
||
|
|
sha256=session.sha256,
|
||
|
|
storage_path=str(storage_path),
|
||
|
|
thumbnail_path=thumbnail_path,
|
||
|
|
file_size=session.file_size,
|
||
|
|
mime_type=session.mime_type,
|
||
|
|
)
|
||
|
|
db.add(blob)
|
||
|
|
elif thumbnail_path and not blob.thumbnail_path:
|
||
|
|
blob.thumbnail_path = thumbnail_path
|
||
|
|
|
||
|
|
user_file = db.scalar(select(UserFile).where(UserFile.user_id == user.id, UserFile.file_sha256 == session.sha256))
|
||
|
|
if user_file is None:
|
||
|
|
_ensure_storage_quota(user, session.file_size)
|
||
|
|
if _is_video_mime(session.mime_type):
|
||
|
|
_ensure_video_storage_quota(user, session.file_size, db)
|
||
|
|
user_file = UserFile(user_id=user.id, file_sha256=session.sha256, original_name=session.file_name)
|
||
|
|
db.add(user_file)
|
||
|
|
user.storage_used += session.file_size
|
||
|
|
|
||
|
|
session.status = "completed"
|
||
|
|
db.commit()
|
||
|
|
db.refresh(user_file)
|
||
|
|
return UploadCompleteResponse(
|
||
|
|
file_id=user_file.id,
|
||
|
|
sha256=session.sha256,
|
||
|
|
storage_path=str(storage_path),
|
||
|
|
file_size=session.file_size,
|
||
|
|
)
|