1028 lines
34 KiB
Python
1028 lines
34 KiB
Python
"""
|
|||
|
|
AO3 (Archive of Our Own) novel plugin.
|
||
|
|
|
||
|
|
Parses AO3 work pages, extracts metadata, downloads works via EPUB or HTML,
|
||
|
|
and integrates with the existing library / cloud-sync / update-check pipelines.
|
||
|
|
|
||
|
|
Reuses:
|
||
|
|
- pixiv_app.core.library.PixivLibrary for local storage
|
||
|
|
- pixiv_app.core.cloud_sync.SyncMetadata / SyncManager for cloud sync
|
||
|
|
- pixiv_app.core.cookie_import utilities for cookie management
|
||
|
|
- pixiv_app.core.downloader helpers (sanitize_filename, save_text_file)
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import logging
|
||
|
|
import re
|
||
|
|
import time
|
||
|
|
from dataclasses import dataclass, field
|
||
|
|
from datetime import datetime, timezone
|
||
|
|
from html import unescape as html_unescape
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any
|
||
|
|
from urllib.parse import urlparse
|
||
|
|
|
||
|
|
import requests
|
||
|
|
|
||
|
|
from pixiv_app.core.downloader import (
|
||
|
|
USER_AGENT,
|
||
|
|
DEFAULT_TIMEOUT,
|
||
|
|
DownloadFileRecord,
|
||
|
|
sanitize_filename,
|
||
|
|
save_text_file,
|
||
|
|
)
|
||
|
|
from pixiv_app.core.library import PixivLibrary
|
||
|
|
from pixiv_app.core.plugin.base import (
|
||
|
|
BasePlugin,
|
||
|
|
PluginAuthError,
|
||
|
|
PluginDownloadError,
|
||
|
|
PluginError,
|
||
|
|
PluginParseError,
|
||
|
|
Resource,
|
||
|
|
)
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# Constants
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
AO3_BASE = "https://archiveofourown.org"
|
||
|
|
AO3_DOMAINS = ("archiveofourown.org", "ao3.org")
|
||
|
|
|
||
|
|
# AO3 URL patterns
|
||
|
|
_WORK_URL_RE = re.compile(
|
||
|
|
r"/works/(?P<work_id>\d+)(?:/chapters/(?P<chapter_id>\d+))?",
|
||
|
|
re.IGNORECASE,
|
||
|
|
)
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# AO3-specific request helpers
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
def _ao3_request(
|
||
|
|
url: str,
|
||
|
|
*,
|
||
|
|
cookie: str = "",
|
||
|
|
timeout: int = DEFAULT_TIMEOUT,
|
||
|
|
retries: int = 3,
|
||
|
|
) -> requests.Response:
|
||
|
|
"""GET an AO3 page, handling rate-limit (Retry-After) and transient errors."""
|
||
|
|
headers = {
|
||
|
|
"User-Agent": USER_AGENT,
|
||
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
|
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||
|
|
}
|
||
|
|
if cookie:
|
||
|
|
headers["Cookie"] = cookie
|
||
|
|
|
||
|
|
last_exc: Exception | None = None
|
||
|
|
for attempt in range(1, retries + 1):
|
||
|
|
try:
|
||
|
|
response = requests.get(url, headers=headers, timeout=timeout, allow_redirects=True)
|
||
|
|
except requests.RequestException as exc:
|
||
|
|
last_exc = exc
|
||
|
|
if attempt < retries:
|
||
|
|
time.sleep(min(1.0 * attempt, 3.0))
|
||
|
|
continue
|
||
|
|
|
||
|
|
if response.status_code == 429:
|
||
|
|
retry_after = response.headers.get("Retry-After")
|
||
|
|
try:
|
||
|
|
wait = int(retry_after) if retry_after else 5 * attempt
|
||
|
|
except (TypeError, ValueError):
|
||
|
|
wait = 5 * attempt
|
||
|
|
logger.warning("AO3 rate-limited, waiting %ds", wait)
|
||
|
|
time.sleep(wait)
|
||
|
|
last_exc = PluginError("AO3 rate limited the request; please retry later.")
|
||
|
|
continue
|
||
|
|
|
||
|
|
if response.status_code == 404:
|
||
|
|
raise PluginParseError("AO3 work was not found (404).")
|
||
|
|
|
||
|
|
if response.status_code == 401:
|
||
|
|
raise PluginAuthError("AO3 requires login for this work. Import a valid AO3 Cookie.")
|
||
|
|
|
||
|
|
# AO3 adult content confirmation page
|
||
|
|
if "This work could have adult content" in response.text or "adult check" in response.url.lower():
|
||
|
|
raise PluginAuthError(
|
||
|
|
"This AO3 work requires adult-content confirmation. Import a valid AO3 Cookie."
|
||
|
|
)
|
||
|
|
|
||
|
|
if not response.ok:
|
||
|
|
last_exc = PluginError(f"AO3 request failed HTTP {response.status_code}")
|
||
|
|
if attempt < retries:
|
||
|
|
time.sleep(min(1.0 * attempt, 3.0))
|
||
|
|
continue
|
||
|
|
|
||
|
|
# Check for login wall (page says you need to log in but it's not 401)
|
||
|
|
if "Please log in" in response.text and "adult" not in response.text.lower():
|
||
|
|
if not cookie:
|
||
|
|
raise PluginAuthError("This AO3 work is visible only to logged-in users. Import an AO3 Cookie.")
|
||
|
|
raise PluginAuthError("AO3 Cookie may be expired. Import it again.")
|
||
|
|
|
||
|
|
return response
|
||
|
|
|
||
|
|
if last_exc:
|
||
|
|
raise PluginError(f"AO3 request failed: {last_exc}") from last_exc
|
||
|
|
raise PluginError("AO3 request failed after maximum retries.")
|
||
|
|
|
||
|
|
|
||
|
|
def _ao3_download_file(
|
||
|
|
url: str,
|
||
|
|
save_path: Path,
|
||
|
|
*,
|
||
|
|
cookie: str = "",
|
||
|
|
timeout: int = 60,
|
||
|
|
) -> Path:
|
||
|
|
"""Download a binary file (EPUB) from AO3."""
|
||
|
|
headers = {
|
||
|
|
"User-Agent": USER_AGENT,
|
||
|
|
"Accept": "*/*",
|
||
|
|
}
|
||
|
|
if cookie:
|
||
|
|
headers["Cookie"] = cookie
|
||
|
|
|
||
|
|
if save_path.exists() and save_path.stat().st_size > 1024:
|
||
|
|
return save_path # already downloaded
|
||
|
|
|
||
|
|
temp_path = save_path.with_suffix(save_path.suffix + ".part")
|
||
|
|
response = requests.get(url, headers=headers, timeout=timeout, stream=True)
|
||
|
|
response.raise_for_status()
|
||
|
|
|
||
|
|
with open(temp_path, "wb") as fh:
|
||
|
|
for chunk in response.iter_content(chunk_size=256 * 1024):
|
||
|
|
if chunk:
|
||
|
|
fh.write(chunk)
|
||
|
|
|
||
|
|
if temp_path.stat().st_size <= 1024:
|
||
|
|
temp_path.unlink(missing_ok=True)
|
||
|
|
raise PluginDownloadError("Downloaded file is too small to be a valid EPUB.")
|
||
|
|
|
||
|
|
temp_path.replace(save_path)
|
||
|
|
return save_path
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# AO3 metadata extraction (pure HTML scraping)
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class AO3WorkMeta:
|
||
|
|
"""Parsed AO3 work metadata."""
|
||
|
|
|
||
|
|
work_id: str = ""
|
||
|
|
title: str = ""
|
||
|
|
author: str = ""
|
||
|
|
author_id: str = "" # AO3 username
|
||
|
|
summary: str = ""
|
||
|
|
language: str = ""
|
||
|
|
rating: str = ""
|
||
|
|
warnings: list[str] = field(default_factory=list)
|
||
|
|
fandoms: list[str] = field(default_factory=list)
|
||
|
|
relationships: list[str] = field(default_factory=list)
|
||
|
|
characters: list[str] = field(default_factory=list)
|
||
|
|
tags: list[str] = field(default_factory=list)
|
||
|
|
categories: list[str] = field(default_factory=list)
|
||
|
|
status: str = "" # "Completed" or "In Progress"
|
||
|
|
chapter_count: int = 1
|
||
|
|
word_count: int = 0
|
||
|
|
published_at: str = ""
|
||
|
|
updated_at: str = ""
|
||
|
|
original_url: str = ""
|
||
|
|
cover: str = ""
|
||
|
|
|
||
|
|
def to_dict(self) -> dict[str, Any]:
|
||
|
|
return {
|
||
|
|
"source": "ao3",
|
||
|
|
"work_id": self.work_id,
|
||
|
|
"title": self.title,
|
||
|
|
"author": self.author,
|
||
|
|
"author_id": self.author_id,
|
||
|
|
"summary": self.summary,
|
||
|
|
"language": self.language,
|
||
|
|
"rating": self.rating,
|
||
|
|
"warnings": self.warnings,
|
||
|
|
"fandoms": self.fandoms,
|
||
|
|
"relationships": self.relationships,
|
||
|
|
"characters": self.characters,
|
||
|
|
"tags": self.tags,
|
||
|
|
"categories": self.categories,
|
||
|
|
"status": self.status,
|
||
|
|
"chapter_count": self.chapter_count,
|
||
|
|
"word_count": self.word_count,
|
||
|
|
"published_at": self.published_at,
|
||
|
|
"updated_at": self.updated_at,
|
||
|
|
"cover": self.cover,
|
||
|
|
"original_url": self.original_url,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def _extract_work_id_from_url(url: str) -> str:
|
||
|
|
"""Extract the work_id from an AO3 URL."""
|
||
|
|
match = _WORK_URL_RE.search(url)
|
||
|
|
if match:
|
||
|
|
return match.group("work_id")
|
||
|
|
raise PluginParseError(f"Invalid AO3 work URL: {url}")
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def extract_ao3_ids(url: str) -> tuple[str, str | None]:
|
||
|
|
"""Extract ``(work_id, chapter_id)`` from an AO3 work/chapter URL."""
|
||
|
|
match = _WORK_URL_RE.search(url.strip())
|
||
|
|
if not match:
|
||
|
|
raise PluginParseError(f"Invalid AO3 work URL: {url}")
|
||
|
|
return match.group("work_id"), match.group("chapter_id")
|
||
|
|
|
||
|
|
|
||
|
|
def _is_ao3_url(url: str) -> bool:
|
||
|
|
parsed = urlparse(url.strip())
|
||
|
|
host = parsed.netloc.lower().split("@")[-1].split(":")[0]
|
||
|
|
return host in AO3_DOMAINS and bool(_WORK_URL_RE.search(parsed.path))
|
||
|
|
|
||
|
|
def _meta_tag(html: str, property_name: str) -> str | None:
|
||
|
|
"""Extract a <meta> tag content by property or name."""
|
||
|
|
for attr in ("property", "name"):
|
||
|
|
pattern = re.compile(
|
||
|
|
rf'<meta\s+{attr}=["\']{re.escape(property_name)}["\']\s+content=["\']([^"\']+)["\']',
|
||
|
|
re.IGNORECASE,
|
||
|
|
)
|
||
|
|
match = pattern.search(html)
|
||
|
|
if match:
|
||
|
|
return html_unescape(match.group(1).strip())
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def _text_of_first(html: str, selector_re: str) -> str:
|
||
|
|
"""Return inner text of the first element matching a regex."""
|
||
|
|
pattern = re.compile(selector_re, re.IGNORECASE | re.DOTALL)
|
||
|
|
match = pattern.search(html)
|
||
|
|
if not match:
|
||
|
|
return ""
|
||
|
|
inner = match.group(1)
|
||
|
|
# Strip inner HTML tags
|
||
|
|
inner = re.sub(r"<[^>]+>", " ", inner)
|
||
|
|
inner = html_unescape(inner)
|
||
|
|
return re.sub(r"\s+", " ", inner).strip()
|
||
|
|
|
||
|
|
|
||
|
|
def _dd_texts(html: str, dt_label: str) -> list[str]:
|
||
|
|
"""Extract tag texts from <dd> that follows a <dt> with given label.
|
||
|
|
|
||
|
|
AO3 meta uses two patterns:
|
||
|
|
1. Tag block: <dt class="...">Rating:</dt> <dd class="... tags"><ul><li><a class="tag">...</a></li></ul></dd>
|
||
|
|
2. Stats block: <dt class="...">Language:</dt> <dd class="...">English</dd>
|
||
|
|
"""
|
||
|
|
expected = _normalize_ao3_label(dt_label)
|
||
|
|
match = None
|
||
|
|
for candidate in re.finditer(
|
||
|
|
r"<dt[^>]*>(.*?)</dt>\s*<dd[^>]*>(.*?)</dd>",
|
||
|
|
html,
|
||
|
|
re.IGNORECASE | re.DOTALL,
|
||
|
|
):
|
||
|
|
label_text = _normalize_ao3_label(candidate.group(1))
|
||
|
|
if label_text == expected:
|
||
|
|
match = candidate
|
||
|
|
break
|
||
|
|
if match is None:
|
||
|
|
return []
|
||
|
|
dd_html = match.group(2)
|
||
|
|
# Extract tag text from <a class="tag"> elements
|
||
|
|
tags = re.findall(
|
||
|
|
r'<a[^>]*class="[^"]*tag[^"]*"[^>]*>(.*?)</a>',
|
||
|
|
dd_html,
|
||
|
|
re.IGNORECASE | re.DOTALL,
|
||
|
|
)
|
||
|
|
if tags:
|
||
|
|
return [
|
||
|
|
html_unescape(re.sub(r"<[^>]+>", "", t)).strip()
|
||
|
|
for t in tags if t.strip()
|
||
|
|
]
|
||
|
|
# Fallback: plain text in <dd>
|
||
|
|
inner = re.sub(r"<[^>]+>", "", dd_html)
|
||
|
|
inner = html_unescape(inner).strip()
|
||
|
|
return [inner] if inner else []
|
||
|
|
|
||
|
|
|
||
|
|
def _normalize_ao3_label(value: str) -> str:
|
||
|
|
text = re.sub(r"<[^>]+>", "", value)
|
||
|
|
text = html_unescape(text)
|
||
|
|
text = text.replace("*", "")
|
||
|
|
text = re.sub(r"\s+", " ", text).strip().rstrip(":").lower()
|
||
|
|
return text
|
||
|
|
|
||
|
|
|
||
|
|
def _dd_text_single(html: str, dt_label: str) -> str:
|
||
|
|
"""Extract a single text value from <dd> after <dt>."""
|
||
|
|
texts = _dd_texts(html, dt_label)
|
||
|
|
return texts[0] if texts else ""
|
||
|
|
|
||
|
|
|
||
|
|
def _stats_value(html: str, label: str) -> str:
|
||
|
|
"""Extract a value from the stats block: <dt>Words:</dt><dd>12345</dd>."""
|
||
|
|
return _dd_text_single(html, label)
|
||
|
|
|
||
|
|
|
||
|
|
def parse_work_meta(html: str, url: str = "") -> AO3WorkMeta:
|
||
|
|
"""Parse an AO3 work page HTML into structured metadata."""
|
||
|
|
meta = AO3WorkMeta()
|
||
|
|
|
||
|
|
# Work ID from URL or meta
|
||
|
|
meta.original_url = url
|
||
|
|
try:
|
||
|
|
meta.work_id = _extract_work_id_from_url(url)
|
||
|
|
except PluginParseError:
|
||
|
|
meta.work_id = ""
|
||
|
|
|
||
|
|
# Title
|
||
|
|
meta.title = _text_of_first(html, r'<h2\s+class="[^"]*title[^"]*heading[^"]*"[^>]*>(.*?)</h2>')
|
||
|
|
if not meta.title:
|
||
|
|
meta.title = _meta_tag(html, "og:title") or ""
|
||
|
|
|
||
|
|
# Author
|
||
|
|
meta.author = _text_of_first(html, r'<a\s+[^>]*rel="author"[^>]*>(.*?)</a>')
|
||
|
|
if not meta.author:
|
||
|
|
meta.author = _meta_tag(html, "og:site_name") or ""
|
||
|
|
# Author ID from href
|
||
|
|
author_href_match = re.search(r'<a\s+[^>]*rel="author"[^>]*href="([^"]+)"', html, re.IGNORECASE)
|
||
|
|
if author_href_match:
|
||
|
|
author_path = author_href_match.group(1)
|
||
|
|
author_id_match = re.search(r"/users/([^/]+)", author_path)
|
||
|
|
if author_id_match:
|
||
|
|
meta.author_id = author_id_match.group(1)
|
||
|
|
else:
|
||
|
|
# Pseud path: /users/username/pseuds/name
|
||
|
|
pseud_match = re.search(r"/users/([^/]+)/pseuds", author_path)
|
||
|
|
if pseud_match:
|
||
|
|
meta.author_id = pseud_match.group(1)
|
||
|
|
|
||
|
|
# Summary
|
||
|
|
meta.summary = _text_of_first(html, r'<blockquote\s+class="[^"]*userstuff[^"]*"[^>]*>(.*?)</blockquote>')
|
||
|
|
if not meta.summary:
|
||
|
|
meta.summary = _meta_tag(html, "og:description") or ""
|
||
|
|
|
||
|
|
# Language
|
||
|
|
meta.language = _dd_text_single(html, "Language:")
|
||
|
|
|
||
|
|
# Rating
|
||
|
|
meta.rating = _dd_text_single(html, "Rating:")
|
||
|
|
|
||
|
|
# Warnings
|
||
|
|
meta.warnings = _dd_texts(html, "Archive Warnings*:")
|
||
|
|
|
||
|
|
# Fandoms
|
||
|
|
meta.fandoms = _dd_texts(html, "Fandoms*:")
|
||
|
|
|
||
|
|
# Relationships
|
||
|
|
meta.relationships = _dd_texts(html, "Relationships:")
|
||
|
|
|
||
|
|
# Characters
|
||
|
|
meta.characters = _dd_texts(html, "Characters:")
|
||
|
|
|
||
|
|
# Tags (freeform)
|
||
|
|
meta.tags = _dd_texts(html, "Additional Tags:")
|
||
|
|
|
||
|
|
# Categories
|
||
|
|
meta.categories = _dd_texts(html, "Categories:")
|
||
|
|
|
||
|
|
# Stats
|
||
|
|
raw_chapters = _stats_value(html, "Chapters:")
|
||
|
|
if "/" in raw_chapters:
|
||
|
|
parts = raw_chapters.split("/")
|
||
|
|
meta.chapter_count = int(parts[0].strip() or "1")
|
||
|
|
total = parts[1].strip()
|
||
|
|
if total == "?":
|
||
|
|
meta.status = "In Progress"
|
||
|
|
elif total.isdigit() and int(total) == meta.chapter_count:
|
||
|
|
meta.status = "Completed"
|
||
|
|
else:
|
||
|
|
meta.status = "In Progress"
|
||
|
|
elif raw_chapters.isdigit():
|
||
|
|
meta.chapter_count = int(raw_chapters)
|
||
|
|
else:
|
||
|
|
meta.chapter_count = 1
|
||
|
|
|
||
|
|
raw_words = _stats_value(html, "Words:")
|
||
|
|
meta.word_count = int(raw_words.replace(",", "") or "0")
|
||
|
|
|
||
|
|
# Check completion status explicitly
|
||
|
|
if not meta.status:
|
||
|
|
completed = _dd_texts(html, "Completion Status*:")
|
||
|
|
if completed:
|
||
|
|
meta.status = completed[0]
|
||
|
|
|
||
|
|
if not meta.status:
|
||
|
|
meta.status = "Completed" if meta.chapter_count == 1 else "In Progress"
|
||
|
|
|
||
|
|
# Published date
|
||
|
|
meta.published_at = _stats_value(html, "Published:")
|
||
|
|
# Normalize timezone
|
||
|
|
if meta.published_at:
|
||
|
|
try:
|
||
|
|
# AO3 dates are often like "2024-01-15"
|
||
|
|
dt = datetime.strptime(meta.published_at.strip(), "%Y-%m-%d")
|
||
|
|
meta.published_at = dt.replace(tzinfo=timezone.utc).isoformat()
|
||
|
|
except ValueError:
|
||
|
|
pass
|
||
|
|
|
||
|
|
# Updated date
|
||
|
|
meta.updated_at = _stats_value(html, "Updated:")
|
||
|
|
if meta.updated_at:
|
||
|
|
try:
|
||
|
|
dt = datetime.strptime(meta.updated_at.strip(), "%Y-%m-%d")
|
||
|
|
meta.updated_at = dt.replace(tzinfo=timezone.utc).isoformat()
|
||
|
|
except ValueError:
|
||
|
|
pass
|
||
|
|
|
||
|
|
# Cover: AO3 does not have covers, leave empty.
|
||
|
|
meta.cover = ""
|
||
|
|
|
||
|
|
return meta
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# EPUB text extraction (minimal: extract paragraphs from EPUB)
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
from zipfile import ZipFile
|
||
|
|
|
||
|
|
|
||
|
|
def _extract_epub_text(epub_path: Path) -> str:
|
||
|
|
"""Extract readable text from an EPUB file (XHTML chapters).
|
||
|
|
|
||
|
|
Returns joined text with chapter separators.
|
||
|
|
Paragraphs within a chapter are separated by blank lines.
|
||
|
|
Inline markup (<em>, <strong>, etc.) is preserved as plain text
|
||
|
|
without introducing spurious paragraph breaks.
|
||
|
|
"""
|
||
|
|
from html.parser import HTMLParser
|
||
|
|
|
||
|
|
# Tags whose closing triggers a paragraph break.
|
||
|
|
_BLOCK_TAGS = {"p", "div", "br", "h1", "h2", "h3", "h4", "h5", "h6", "li", "blockquote", "section", "article", "hr"}
|
||
|
|
|
||
|
|
class AO3TextExtractor(HTMLParser):
|
||
|
|
def __init__(self):
|
||
|
|
super().__init__()
|
||
|
|
self.paragraphs: list[str] = []
|
||
|
|
self._buf: list[str] = [] # accumulating text for the current paragraph
|
||
|
|
self._skip = 0 # nesting depth for skip regions (script/style/head)
|
||
|
|
|
||
|
|
def handle_starttag(self, tag, attrs):
|
||
|
|
if tag in ("script", "style", "head"):
|
||
|
|
self._skip += 1
|
||
|
|
|
||
|
|
def handle_endtag(self, tag):
|
||
|
|
if tag in ("script", "style", "head"):
|
||
|
|
self._skip = max(0, self._skip - 1)
|
||
|
|
return
|
||
|
|
if tag in _BLOCK_TAGS:
|
||
|
|
# flush the current in-progress paragraph
|
||
|
|
text = " ".join(self._buf).strip()
|
||
|
|
if text:
|
||
|
|
self.paragraphs.append(text)
|
||
|
|
self._buf.clear()
|
||
|
|
|
||
|
|
def handle_data(self, data):
|
||
|
|
if self._skip:
|
||
|
|
return
|
||
|
|
text = data.strip()
|
||
|
|
if text:
|
||
|
|
self._buf.append(text)
|
||
|
|
|
||
|
|
def finish(self):
|
||
|
|
"""Flush any remaining buffered text."""
|
||
|
|
text = " ".join(self._buf).strip()
|
||
|
|
if text:
|
||
|
|
self.paragraphs.append(text)
|
||
|
|
self._buf.clear()
|
||
|
|
|
||
|
|
chapters: list[str] = []
|
||
|
|
with ZipFile(epub_path, "r") as zf:
|
||
|
|
xhtml_names = sorted(
|
||
|
|
[n for n in zf.namelist() if n.endswith((".xhtml", ".html", ".htm"))],
|
||
|
|
)
|
||
|
|
for name in xhtml_names:
|
||
|
|
try:
|
||
|
|
content = zf.read(name).decode("utf-8", errors="replace")
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
|
||
|
|
# Extract title from <title> tag
|
||
|
|
title_match = re.search(r"<title>(.*?)</title>", content, re.IGNORECASE)
|
||
|
|
chapter_title = html_unescape(title_match.group(1).strip()) if title_match else ""
|
||
|
|
|
||
|
|
extractor = AO3TextExtractor()
|
||
|
|
try:
|
||
|
|
extractor.feed(content)
|
||
|
|
extractor.finish()
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
|
||
|
|
if not extractor.paragraphs:
|
||
|
|
continue
|
||
|
|
|
||
|
|
chapter_text = "\n\n".join(extractor.paragraphs)
|
||
|
|
if chapter_title and chapter_title.lower() != "chapters":
|
||
|
|
chapters.append(f"\n\n=== {chapter_title} ===\n\n")
|
||
|
|
chapters.append(chapter_text)
|
||
|
|
|
||
|
|
return "\n".join(chapters) if chapters else ""
|
||
|
|
|
||
|
|
|
||
|
|
def _extract_html_work_text(html: str) -> str:
|
||
|
|
"""Extract readable AO3 work text from a work HTML page.
|
||
|
|
|
||
|
|
AO3 wraps chapter content in <div class="userstuff module" role="article">
|
||
|
|
(or <div class="userstuff" role="article">). We split on these opening
|
||
|
|
markers, then extract <h3 class="landmark heading"> for the chapter title
|
||
|
|
and all <p> elements for the body.
|
||
|
|
"""
|
||
|
|
# Prefer the actual AO3 chapter article block. Summary/preface blocks also
|
||
|
|
# use ``userstuff``, so the role="article" guard is important.
|
||
|
|
ARTICLE_START_RE = re.compile(
|
||
|
|
r'<div[^>]*class="[^"]*userstuff[^"]*"[^>]*role="article"[^>]*>',
|
||
|
|
re.IGNORECASE,
|
||
|
|
)
|
||
|
|
segments = ARTICLE_START_RE.split(html)[1:] # drop content before first block
|
||
|
|
if not segments:
|
||
|
|
# Fallback 1: try the older AO3 "chapter" div class
|
||
|
|
segments = re.findall(
|
||
|
|
r'<div[^>]*class="[^"]*chapter[^"]*"[^>]*>(.*?)</div>\s*(?=<div[^>]*class="[^"]*chapter|\Z)',
|
||
|
|
html,
|
||
|
|
re.IGNORECASE | re.DOTALL,
|
||
|
|
)
|
||
|
|
if not segments:
|
||
|
|
# Fallback 2: try <div id="chapters"> wrapper
|
||
|
|
segments = re.findall(
|
||
|
|
r'<div[^>]*id="chapters"[^>]*>(.*?)</div>\s*(?=<div[^>]*id="afterword"|\Z)',
|
||
|
|
html,
|
||
|
|
re.IGNORECASE | re.DOTALL,
|
||
|
|
)
|
||
|
|
|
||
|
|
chapters: list[str] = []
|
||
|
|
for segment in segments:
|
||
|
|
# Trim everything after the *next* userstuff article start or common
|
||
|
|
# trailing markers so we stay within this chapter's block.
|
||
|
|
end_match = ARTICLE_START_RE.search(segment)
|
||
|
|
if end_match:
|
||
|
|
segment = segment[: end_match.start()]
|
||
|
|
# Also trim at AO3 main-content/comment/footer markers. Chapter pages
|
||
|
|
# commonly end the body with <!--/main--> before feedback/actions.
|
||
|
|
for trailer_marker in (
|
||
|
|
r"<!--\s*/main\s*-->",
|
||
|
|
r"<!--\s*BEGIN comment section\s*-->",
|
||
|
|
r'<div[^>]*id="feedback"',
|
||
|
|
r'<div[^>]*id="afterword"',
|
||
|
|
r'<div[^>]*class="[^"]*afterword[^"]*"',
|
||
|
|
r'<div[^>]*class="[^"]*notes[^"]*module[^"]*"',
|
||
|
|
):
|
||
|
|
trailer_match = re.search(trailer_marker, segment, re.IGNORECASE)
|
||
|
|
if trailer_match:
|
||
|
|
segment = segment[: trailer_match.start()]
|
||
|
|
|
||
|
|
# Chapter heading: <h3 class="landmark heading" id="work">Chapter Text</h3>
|
||
|
|
title_match = re.search(
|
||
|
|
r'<h3[^>]*class="[^"]*landmark[^"]*heading[^"]*"[^>]*>(.*?)</h3>',
|
||
|
|
segment,
|
||
|
|
re.IGNORECASE | re.DOTALL,
|
||
|
|
)
|
||
|
|
chapter_title = (
|
||
|
|
html_unescape(re.sub(r"<[^>]+>", "", title_match.group(1))).strip()
|
||
|
|
if title_match
|
||
|
|
else ""
|
||
|
|
)
|
||
|
|
if chapter_title.lower() == "chapter text":
|
||
|
|
chapter_title = ""
|
||
|
|
|
||
|
|
# Extract all <p> text
|
||
|
|
raw_paragraphs = re.findall(
|
||
|
|
r"<p[^>]*>(.*?)</p>", segment, re.IGNORECASE | re.DOTALL,
|
||
|
|
)
|
||
|
|
paragraphs: list[str] = []
|
||
|
|
for raw in raw_paragraphs:
|
||
|
|
cleaned = html_unescape(re.sub(r"<[^>]+>", " ", raw))
|
||
|
|
cleaned = re.sub(r"\s+", " ", cleaned).strip()
|
||
|
|
if cleaned:
|
||
|
|
paragraphs.append(cleaned)
|
||
|
|
|
||
|
|
if chapter_title:
|
||
|
|
chapters.append(f"=== {chapter_title} ===")
|
||
|
|
if paragraphs:
|
||
|
|
chapters.append("\n\n".join(paragraphs))
|
||
|
|
|
||
|
|
if chapters:
|
||
|
|
return "\n\n".join(chapters).strip()
|
||
|
|
|
||
|
|
# Last resort: extract from <blockquote class="userstuff">
|
||
|
|
text = _text_of_first(
|
||
|
|
html,
|
||
|
|
r'<blockquote\s+class="[^"]*userstuff[^"]*"[^>]*>(.*?)</blockquote>',
|
||
|
|
)
|
||
|
|
return text.strip()
|
||
|
|
|
||
|
|
|
||
|
|
def _resolve_ao3_epub_url(work_id: str, *, cookie: str = "") -> str | None:
|
||
|
|
"""Resolve the actual AO3 EPUB download link for a work.
|
||
|
|
|
||
|
|
AO3's download page at ``/works/{id}/download`` lists format links like
|
||
|
|
``<a href="/downloads/{id}/Title.epub">EPUB</a>``. We fetch that page
|
||
|
|
and extract the EPUB href. Returns the absolute URL, or *None* when
|
||
|
|
the download page is unreachable or does not offer EPUB.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
download_page_url = f"{AO3_BASE}/works/{work_id}/download"
|
||
|
|
response = _ao3_request(download_page_url, cookie=cookie, retries=2)
|
||
|
|
except PluginError:
|
||
|
|
return None
|
||
|
|
|
||
|
|
# Look for an EPUB link: <a href="/downloads/...">EPUB</a>
|
||
|
|
match = re.search(
|
||
|
|
r'''<a\s+[^>]*href\s*=\s*["'](?P<href>/downloads/[^"']+\.epub)["'][^>]*>\s*EPUB\s*</a>''',
|
||
|
|
response.text,
|
||
|
|
re.IGNORECASE,
|
||
|
|
)
|
||
|
|
if match:
|
||
|
|
return AO3_BASE + match.group("href")
|
||
|
|
|
||
|
|
# Fallback: any <a> whose text is "EPUB"
|
||
|
|
match = re.search(
|
||
|
|
r'''<a\s+[^>]*href\s*=\s*["'](?P<href>[^"']+\.epub)["'][^>]*>\s*EPUB\s*</a>''',
|
||
|
|
response.text,
|
||
|
|
re.IGNORECASE,
|
||
|
|
)
|
||
|
|
if match:
|
||
|
|
href = match.group("href")
|
||
|
|
return href if href.startswith("http") else AO3_BASE + href
|
||
|
|
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def _format_ao3_text_header(resource: Resource, work_id: str) -> str:
|
||
|
|
return "\n".join(
|
||
|
|
[
|
||
|
|
f"Title: {resource.title}",
|
||
|
|
f"Author: {resource.author}",
|
||
|
|
f"AO3 Work ID: {work_id}",
|
||
|
|
f"Source: {resource.url}",
|
||
|
|
f"Summary: {resource.metadata.get('summary', '')}",
|
||
|
|
"",
|
||
|
|
"",
|
||
|
|
]
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# AO3 Plugin
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
|
||
|
|
class AO3Plugin(BasePlugin):
|
||
|
|
domain = "archiveofourown.org"
|
||
|
|
name = "AO3"
|
||
|
|
version = "1.0.0"
|
||
|
|
|
||
|
|
def __init__(self, cookie: str = "") -> None:
|
||
|
|
self.cookie = cookie
|
||
|
|
self._session = requests.Session()
|
||
|
|
|
||
|
|
# ---- auth ----
|
||
|
|
|
||
|
|
def set_auth(self, cookie: str) -> None:
|
||
|
|
self.cookie = cookie or ""
|
||
|
|
|
||
|
|
def get_headers(self) -> dict[str, str]:
|
||
|
|
headers = {
|
||
|
|
"User-Agent": USER_AGENT,
|
||
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
|
|
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||
|
|
}
|
||
|
|
if self.cookie:
|
||
|
|
headers["Cookie"] = self.cookie
|
||
|
|
return headers
|
||
|
|
|
||
|
|
# ---- URL detection ----
|
||
|
|
|
||
|
|
def can_handle(self, url: str) -> bool:
|
||
|
|
return _is_ao3_url(url)
|
||
|
|
|
||
|
|
# ---- parse ----
|
||
|
|
|
||
|
|
def parse(self, url: str) -> list[Resource]:
|
||
|
|
url = url.strip()
|
||
|
|
if not _is_ao3_url(url):
|
||
|
|
raise PluginParseError(f"Invalid AO3 work URL: {url}")
|
||
|
|
work_id, _chapter_id = extract_ao3_ids(url)
|
||
|
|
meta = self._fetch_meta(work_id)
|
||
|
|
|
||
|
|
# Build tags as a flat list for the cloud sync / library
|
||
|
|
all_tags = (
|
||
|
|
meta.fandoms
|
||
|
|
+ meta.warnings
|
||
|
|
+ meta.relationships
|
||
|
|
+ meta.characters
|
||
|
|
+ meta.tags
|
||
|
|
+ meta.categories
|
||
|
|
)
|
||
|
|
|
||
|
|
metadata = meta.to_dict()
|
||
|
|
metadata.update({
|
||
|
|
"kind": "novel",
|
||
|
|
"resource_type": "novel",
|
||
|
|
"content_type": "novel",
|
||
|
|
"source_platform": "ao3",
|
||
|
|
"source_id": work_id,
|
||
|
|
"series_id": work_id, # AO3 work is the series (multi-chapter)
|
||
|
|
"series_title": meta.title,
|
||
|
|
"chapter_title": meta.title,
|
||
|
|
"chapter_order": 1,
|
||
|
|
"tags_flat": all_tags,
|
||
|
|
})
|
||
|
|
|
||
|
|
return [
|
||
|
|
Resource(
|
||
|
|
id=work_id,
|
||
|
|
url=meta.original_url or f"{AO3_BASE}/works/{work_id}",
|
||
|
|
title=meta.title,
|
||
|
|
author=meta.author,
|
||
|
|
author_id=meta.author_id,
|
||
|
|
files=[],
|
||
|
|
metadata=metadata,
|
||
|
|
thumbnail=meta.cover or None,
|
||
|
|
created_at=meta.published_at or None,
|
||
|
|
)
|
||
|
|
]
|
||
|
|
|
||
|
|
# ---- download ----
|
||
|
|
|
||
|
|
def download(self, resource: Resource, save_path: Path) -> list[Path]:
|
||
|
|
work_id = resource.metadata.get("work_id") or resource.id
|
||
|
|
title = sanitize_filename(resource.title, f"ao3_{work_id}")
|
||
|
|
author = sanitize_filename(resource.author, "unknown_author")
|
||
|
|
|
||
|
|
save_path.mkdir(parents=True, exist_ok=True)
|
||
|
|
novel_dir = save_path / "novels" / sanitize_filename(author, f"ao3_{work_id}")
|
||
|
|
novel_dir.mkdir(parents=True, exist_ok=True)
|
||
|
|
|
||
|
|
downloaded_paths: list[Path] = []
|
||
|
|
error_messages: list[str] = []
|
||
|
|
body_text = ""
|
||
|
|
|
||
|
|
text_path = novel_dir / f"{title}_{work_id}.txt"
|
||
|
|
|
||
|
|
# ---- Primary: download full-work HTML page (most reliable) ----
|
||
|
|
try:
|
||
|
|
html_url = (
|
||
|
|
f"{AO3_BASE}/works/{work_id}"
|
||
|
|
f"?view_adult=true&view_full_work=true"
|
||
|
|
)
|
||
|
|
response = _ao3_request(html_url, cookie=self.cookie)
|
||
|
|
body_text = _extract_html_work_text(response.text)
|
||
|
|
if body_text.strip():
|
||
|
|
full_text = _format_ao3_text_header(resource, work_id) + body_text
|
||
|
|
ok, _msg = save_text_file(text_path, full_text)
|
||
|
|
if ok and text_path.exists():
|
||
|
|
downloaded_paths.append(text_path)
|
||
|
|
logger.info(
|
||
|
|
"AO3: HTML full-work text saved for work %s (%d chars)",
|
||
|
|
work_id, len(body_text),
|
||
|
|
)
|
||
|
|
else:
|
||
|
|
error_messages.append("HTML page contained no extractable text.")
|
||
|
|
except Exception as exc:
|
||
|
|
error_messages.append(f"HTML download failed: {exc}")
|
||
|
|
logger.warning(
|
||
|
|
"AO3: HTML full-work download failed for work %s: %s",
|
||
|
|
work_id, exc,
|
||
|
|
)
|
||
|
|
|
||
|
|
# ---- Secondary: try EPUB for archival preservation ----
|
||
|
|
epub_url = _resolve_ao3_epub_url(work_id, cookie=self.cookie)
|
||
|
|
if epub_url:
|
||
|
|
epub_path = novel_dir / f"{title}_{work_id}.epub"
|
||
|
|
try:
|
||
|
|
_ao3_download_file(epub_url, epub_path, cookie=self.cookie)
|
||
|
|
downloaded_paths.append(epub_path)
|
||
|
|
logger.info("AO3: EPUB saved for work %s", work_id)
|
||
|
|
except Exception as exc:
|
||
|
|
logger.warning(
|
||
|
|
"AO3: EPUB download skipped for work %s: %s",
|
||
|
|
work_id, exc,
|
||
|
|
)
|
||
|
|
|
||
|
|
if not downloaded_paths:
|
||
|
|
raise PluginDownloadError(
|
||
|
|
f"AO3 work {work_id} download failed: {'; '.join(error_messages)}"
|
||
|
|
)
|
||
|
|
|
||
|
|
# Update resource metadata
|
||
|
|
resource.metadata["word_count"] = len(body_text)
|
||
|
|
|
||
|
|
return downloaded_paths
|
||
|
|
|
||
|
|
# ---- update checking ----
|
||
|
|
|
||
|
|
def check_update(self, work_id: str) -> dict[str, Any] | None:
|
||
|
|
"""Check if the work has been updated since last import.
|
||
|
|
|
||
|
|
Returns a dict with update info, or None if no update.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
url = f"{AO3_BASE}/works/{work_id}"
|
||
|
|
response = _ao3_request(url, cookie=self.cookie)
|
||
|
|
meta = parse_work_meta(response.text, url)
|
||
|
|
|
||
|
|
return {
|
||
|
|
"work_id": meta.work_id,
|
||
|
|
"title": meta.title,
|
||
|
|
"updated_at": meta.updated_at,
|
||
|
|
"chapter_count": meta.chapter_count,
|
||
|
|
"word_count": meta.word_count,
|
||
|
|
"status": meta.status,
|
||
|
|
"original_url": meta.original_url,
|
||
|
|
"has_update": True, # caller should determine this
|
||
|
|
}
|
||
|
|
except PluginError:
|
||
|
|
return None
|
||
|
|
except Exception:
|
||
|
|
return None
|
||
|
|
|
||
|
|
# ---- internal helpers ----
|
||
|
|
|
||
|
|
def _fetch_meta(self, work_id: str) -> AO3WorkMeta:
|
||
|
|
"""Fetch and parse metadata for a work."""
|
||
|
|
url = f"{AO3_BASE}/works/{work_id}?view_adult=true"
|
||
|
|
response = _ao3_request(url, cookie=self.cookie)
|
||
|
|
return parse_work_meta(response.text, response.url)
|
||
|
|
|
||
|
|
def validate(self) -> bool:
|
||
|
|
return True
|
||
|
|
|
||
|
|
# ---- cookie config (matches Pixiv pattern) ----
|
||
|
|
|
||
|
|
def get_config_schema(self) -> dict[str, Any]:
|
||
|
|
return {
|
||
|
|
"type": "object",
|
||
|
|
"properties": {
|
||
|
|
"cookie": {
|
||
|
|
"type": "string",
|
||
|
|
"title": "AO3 Cookie",
|
||
|
|
"description": "Paste AO3 Cookie text.",
|
||
|
|
},
|
||
|
|
},
|
||
|
|
}
|
||
|
|
|
||
|
|
def load_config(self) -> dict[str, Any]:
|
||
|
|
return {"cookie": self.cookie}
|
||
|
|
|
||
|
|
def save_config(self, data: dict[str, Any]) -> None:
|
||
|
|
self.cookie = str(data.get("cookie") or "")
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# Library integration helper
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
def ao3_to_library_artwork_id(work_id: str) -> int:
|
||
|
|
"""Map an AO3 work_id to the library artwork_id namespace.
|
||
|
|
|
||
|
|
Uses negative integers beyond the Pixiv novel range to avoid collisions.
|
||
|
|
AO3 work IDs can be up to 8+ digits (e.g. 12345678), we offset them:
|
||
|
|
- AO3 prefix: -200000000 (200 million)
|
||
|
|
- artwork_id = -(200000000 + int(work_id))
|
||
|
|
|
||
|
|
This keeps AO3 works clearly distinct from Pixiv novels (-1 to -199999999).
|
||
|
|
"""
|
||
|
|
AO3_OFFSET = 200_000_000
|
||
|
|
return -(AO3_OFFSET + int(work_id))
|
||
|
|
|
||
|
|
|
||
|
|
def upsert_ao3_novel_to_library(
|
||
|
|
library: PixivLibrary,
|
||
|
|
meta: AO3WorkMeta,
|
||
|
|
*,
|
||
|
|
file_records: list[DownloadFileRecord] | None = None,
|
||
|
|
) -> int:
|
||
|
|
"""Insert or update an AO3 work in the local library.
|
||
|
|
|
||
|
|
Returns the library artwork_id.
|
||
|
|
"""
|
||
|
|
from pixiv_app.core.library import library_artwork_id
|
||
|
|
|
||
|
|
work_id = int(meta.work_id)
|
||
|
|
lib_id = library_artwork_id(work_id, artwork_type="novel", source_platform="ao3")
|
||
|
|
|
||
|
|
# Build detail dict compatible with upsert_artwork
|
||
|
|
# Include source / source_platform so upsert_artwork computes the correct
|
||
|
|
# AO3-offset artwork_id via library_artwork_id(id, "novel", "ao3").
|
||
|
|
detail: dict[str, Any] = {
|
||
|
|
"id": work_id,
|
||
|
|
"novelId": work_id,
|
||
|
|
"title": meta.title,
|
||
|
|
"userName": meta.author,
|
||
|
|
"userId": _ao3_author_id_to_int(meta.author_id),
|
||
|
|
"description": meta.summary,
|
||
|
|
"pageCount": meta.chapter_count,
|
||
|
|
"likeCount": 0,
|
||
|
|
"bookmarkCount": 0,
|
||
|
|
"viewCount": 0,
|
||
|
|
"xRestrict": 1 if meta.rating in ("Explicit", "Not Rated") else 0,
|
||
|
|
"aiType": 0,
|
||
|
|
"createDate": meta.published_at,
|
||
|
|
"source_platform": "ao3",
|
||
|
|
"source": "ao3",
|
||
|
|
"tags": [
|
||
|
|
{"tag": tag, "translation": None}
|
||
|
|
for tag in (
|
||
|
|
meta.fandoms
|
||
|
|
+ meta.characters
|
||
|
|
+ meta.tags
|
||
|
|
+ meta.relationships
|
||
|
|
+ meta.warnings
|
||
|
|
+ meta.categories
|
||
|
|
)
|
||
|
|
],
|
||
|
|
}
|
||
|
|
|
||
|
|
# Upsert artist stub
|
||
|
|
artist_id = _ao3_author_id_to_int(meta.author_id)
|
||
|
|
library.ensure_artist_stub(artist_id, name=meta.author)
|
||
|
|
library.upsert_artist_profile(artist_id, name=meta.author)
|
||
|
|
|
||
|
|
# Upsert artwork via the standard method (now AO3-ID-aware)
|
||
|
|
library.ensure_artwork_stub(work_id, artwork_type="novel", title=meta.title, source_platform="ao3")
|
||
|
|
library.upsert_artwork(detail, artwork_type="novel")
|
||
|
|
|
||
|
|
# Record files if provided
|
||
|
|
if file_records:
|
||
|
|
library.record_files(lib_id, file_records)
|
||
|
|
|
||
|
|
return lib_id
|
||
|
|
|
||
|
|
|
||
|
|
def _ao3_author_id_to_int(author_id: str) -> int:
|
||
|
|
"""Convert an AO3 username string to a stable int for the library.
|
||
|
|
|
||
|
|
Uses a simple DJB2 hash constrained to positive 32-bit range.
|
||
|
|
"""
|
||
|
|
if not author_id:
|
||
|
|
return 0
|
||
|
|
h = 5381
|
||
|
|
for ch in author_id:
|
||
|
|
h = ((h << 5) + h) + ord(ch)
|
||
|
|
return h & 0x7FFFFFFF
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# Cloud sync metadata builder
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
def build_ao3_sync_metadata(
|
||
|
|
meta: AO3WorkMeta,
|
||
|
|
file_path: Path,
|
||
|
|
*,
|
||
|
|
chapter_title: str = "",
|
||
|
|
chapter_order: int = 1,
|
||
|
|
) -> dict[str, Any]:
|
||
|
|
"""Build a SyncMetadata-like dict for cloud sync.
|
||
|
|
|
||
|
|
Compatible with pixiv_app.core.cloud_sync.SyncMetadata.
|
||
|
|
"""
|
||
|
|
from pixiv_app.core.cloud_sync import SyncMetadata
|
||
|
|
|
||
|
|
all_tags = (
|
||
|
|
meta.fandoms
|
||
|
|
+ meta.warnings
|
||
|
|
+ meta.relationships
|
||
|
|
+ meta.characters
|
||
|
|
+ meta.tags
|
||
|
|
+ meta.categories
|
||
|
|
)
|
||
|
|
|
||
|
|
return SyncMetadata(
|
||
|
|
resource_type="novel",
|
||
|
|
source_platform="ao3",
|
||
|
|
source_id=meta.work_id,
|
||
|
|
content_type="novel",
|
||
|
|
title=meta.title,
|
||
|
|
author=meta.author,
|
||
|
|
author_id=meta.author_id,
|
||
|
|
tags=all_tags,
|
||
|
|
metadata=meta.to_dict(),
|
||
|
|
page_count=meta.chapter_count,
|
||
|
|
description=meta.summary,
|
||
|
|
word_count=meta.word_count,
|
||
|
|
series_id=meta.work_id, # work = series for AO3
|
||
|
|
series_title=meta.title,
|
||
|
|
chapter_title=chapter_title or meta.title,
|
||
|
|
chapter_order=chapter_order,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# Plugin class export
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
plugin_class = AO3Plugin
|