Files
2026-11-16 20:45:22 +08:00

1029 lines
34 KiB
Python

"""
AO3 (Archive of Our Own) novel plugin.
Parses AO3 work pages, extracts metadata, downloads works via EPUB or HTML,
and integrates with the existing library / cloud-sync / update-check pipelines.
Reuses:
- pixiv_app.core.library.PixivLibrary for local storage
- pixiv_app.core.cloud_sync.SyncMetadata / SyncManager for cloud sync
- pixiv_app.core.cookie_import utilities for cookie management
- pixiv_app.core.downloader helpers (sanitize_filename, save_text_file)
"""
from __future__ import annotations
import logging
import re
import time
from dataclasses import dataclass, field
from datetime import datetime, timezone
from html import unescape as html_unescape
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
import requests
from pixiv_app.core.downloader import (
USER_AGENT,
DEFAULT_TIMEOUT,
DownloadFileRecord,
sanitize_filename,
save_text_file,
)
from pixiv_app.core.library import PixivLibrary
from pixiv_app.core.plugin.base import (
BasePlugin,
PluginAuthError,
PluginDownloadError,
PluginError,
PluginParseError,
Resource,
)
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
AO3_BASE = "https://archiveofourown.org"
AO3_DOMAINS = ("archiveofourown.org", "ao3.org")
# AO3 URL patterns
_WORK_URL_RE = re.compile(
r"/works/(?P<work_id>\d+)(?:/chapters/(?P<chapter_id>\d+))?",
re.IGNORECASE,
)
# ---------------------------------------------------------------------------
# AO3-specific request helpers
# ---------------------------------------------------------------------------
def _ao3_request(
url: str,
*,
cookie: str = "",
timeout: int = DEFAULT_TIMEOUT,
retries: int = 3,
) -> requests.Response:
"""GET an AO3 page, handling rate-limit (Retry-After) and transient errors."""
headers = {
"User-Agent": USER_AGENT,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
if cookie:
headers["Cookie"] = cookie
last_exc: Exception | None = None
for attempt in range(1, retries + 1):
try:
response = requests.get(url, headers=headers, timeout=timeout, allow_redirects=True)
except requests.RequestException as exc:
last_exc = exc
if attempt < retries:
time.sleep(min(1.0 * attempt, 3.0))
continue
if response.status_code == 429:
retry_after = response.headers.get("Retry-After")
try:
wait = int(retry_after) if retry_after else 5 * attempt
except (TypeError, ValueError):
wait = 5 * attempt
logger.warning("AO3 rate-limited, waiting %ds", wait)
time.sleep(wait)
last_exc = PluginError("AO3 rate limited the request; please retry later.")
continue
if response.status_code == 404:
raise PluginParseError("AO3 work was not found (404).")
if response.status_code == 401:
raise PluginAuthError("AO3 requires login for this work. Import a valid AO3 Cookie.")
# AO3 adult content confirmation page
if "This work could have adult content" in response.text or "adult check" in response.url.lower():
raise PluginAuthError(
"This AO3 work requires adult-content confirmation. Import a valid AO3 Cookie."
)
if not response.ok:
last_exc = PluginError(f"AO3 request failed HTTP {response.status_code}")
if attempt < retries:
time.sleep(min(1.0 * attempt, 3.0))
continue
# Check for login wall (page says you need to log in but it's not 401)
if "Please log in" in response.text and "adult" not in response.text.lower():
if not cookie:
raise PluginAuthError("This AO3 work is visible only to logged-in users. Import an AO3 Cookie.")
raise PluginAuthError("AO3 Cookie may be expired. Import it again.")
return response
if last_exc:
raise PluginError(f"AO3 request failed: {last_exc}") from last_exc
raise PluginError("AO3 request failed after maximum retries.")
def _ao3_download_file(
url: str,
save_path: Path,
*,
cookie: str = "",
timeout: int = 60,
) -> Path:
"""Download a binary file (EPUB) from AO3."""
headers = {
"User-Agent": USER_AGENT,
"Accept": "*/*",
}
if cookie:
headers["Cookie"] = cookie
if save_path.exists() and save_path.stat().st_size > 1024:
return save_path # already downloaded
temp_path = save_path.with_suffix(save_path.suffix + ".part")
response = requests.get(url, headers=headers, timeout=timeout, stream=True)
response.raise_for_status()
with open(temp_path, "wb") as fh:
for chunk in response.iter_content(chunk_size=256 * 1024):
if chunk:
fh.write(chunk)
if temp_path.stat().st_size <= 1024:
temp_path.unlink(missing_ok=True)
raise PluginDownloadError("Downloaded file is too small to be a valid EPUB.")
temp_path.replace(save_path)
return save_path
# ---------------------------------------------------------------------------
# AO3 metadata extraction (pure HTML scraping)
# ---------------------------------------------------------------------------
@dataclass
class AO3WorkMeta:
"""Parsed AO3 work metadata."""
work_id: str = ""
title: str = ""
author: str = ""
author_id: str = "" # AO3 username
summary: str = ""
language: str = ""
rating: str = ""
warnings: list[str] = field(default_factory=list)
fandoms: list[str] = field(default_factory=list)
relationships: list[str] = field(default_factory=list)
characters: list[str] = field(default_factory=list)
tags: list[str] = field(default_factory=list)
categories: list[str] = field(default_factory=list)
status: str = "" # "Completed" or "In Progress"
chapter_count: int = 1
word_count: int = 0
published_at: str = ""
updated_at: str = ""
original_url: str = ""
cover: str = ""
def to_dict(self) -> dict[str, Any]:
return {
"source": "ao3",
"work_id": self.work_id,
"title": self.title,
"author": self.author,
"author_id": self.author_id,
"summary": self.summary,
"language": self.language,
"rating": self.rating,
"warnings": self.warnings,
"fandoms": self.fandoms,
"relationships": self.relationships,
"characters": self.characters,
"tags": self.tags,
"categories": self.categories,
"status": self.status,
"chapter_count": self.chapter_count,
"word_count": self.word_count,
"published_at": self.published_at,
"updated_at": self.updated_at,
"cover": self.cover,
"original_url": self.original_url,
}
def _extract_work_id_from_url(url: str) -> str:
"""Extract the work_id from an AO3 URL."""
match = _WORK_URL_RE.search(url)
if match:
return match.group("work_id")
raise PluginParseError(f"Invalid AO3 work URL: {url}")
def extract_ao3_ids(url: str) -> tuple[str, str | None]:
"""Extract ``(work_id, chapter_id)`` from an AO3 work/chapter URL."""
match = _WORK_URL_RE.search(url.strip())
if not match:
raise PluginParseError(f"Invalid AO3 work URL: {url}")
return match.group("work_id"), match.group("chapter_id")
def _is_ao3_url(url: str) -> bool:
parsed = urlparse(url.strip())
host = parsed.netloc.lower().split("@")[-1].split(":")[0]
return host in AO3_DOMAINS and bool(_WORK_URL_RE.search(parsed.path))
def _meta_tag(html: str, property_name: str) -> str | None:
"""Extract a <meta> tag content by property or name."""
for attr in ("property", "name"):
pattern = re.compile(
rf'<meta\s+{attr}=["\']{re.escape(property_name)}["\']\s+content=["\']([^"\']+)["\']',
re.IGNORECASE,
)
match = pattern.search(html)
if match:
return html_unescape(match.group(1).strip())
return None
def _text_of_first(html: str, selector_re: str) -> str:
"""Return inner text of the first element matching a regex."""
pattern = re.compile(selector_re, re.IGNORECASE | re.DOTALL)
match = pattern.search(html)
if not match:
return ""
inner = match.group(1)
# Strip inner HTML tags
inner = re.sub(r"<[^>]+>", " ", inner)
inner = html_unescape(inner)
return re.sub(r"\s+", " ", inner).strip()
def _dd_texts(html: str, dt_label: str) -> list[str]:
"""Extract tag texts from <dd> that follows a <dt> with given label.
AO3 meta uses two patterns:
1. Tag block: <dt class="...">Rating:</dt> <dd class="... tags"><ul><li><a class="tag">...</a></li></ul></dd>
2. Stats block: <dt class="...">Language:</dt> <dd class="...">English</dd>
"""
expected = _normalize_ao3_label(dt_label)
match = None
for candidate in re.finditer(
r"<dt[^>]*>(.*?)</dt>\s*<dd[^>]*>(.*?)</dd>",
html,
re.IGNORECASE | re.DOTALL,
):
label_text = _normalize_ao3_label(candidate.group(1))
if label_text == expected:
match = candidate
break
if match is None:
return []
dd_html = match.group(2)
# Extract tag text from <a class="tag"> elements
tags = re.findall(
r'<a[^>]*class="[^"]*tag[^"]*"[^>]*>(.*?)</a>',
dd_html,
re.IGNORECASE | re.DOTALL,
)
if tags:
return [
html_unescape(re.sub(r"<[^>]+>", "", t)).strip()
for t in tags if t.strip()
]
# Fallback: plain text in <dd>
inner = re.sub(r"<[^>]+>", "", dd_html)
inner = html_unescape(inner).strip()
return [inner] if inner else []
def _normalize_ao3_label(value: str) -> str:
text = re.sub(r"<[^>]+>", "", value)
text = html_unescape(text)
text = text.replace("*", "")
text = re.sub(r"\s+", " ", text).strip().rstrip(":").lower()
return text
def _dd_text_single(html: str, dt_label: str) -> str:
"""Extract a single text value from <dd> after <dt>."""
texts = _dd_texts(html, dt_label)
return texts[0] if texts else ""
def _stats_value(html: str, label: str) -> str:
"""Extract a value from the stats block: <dt>Words:</dt><dd>12345</dd>."""
return _dd_text_single(html, label)
def parse_work_meta(html: str, url: str = "") -> AO3WorkMeta:
"""Parse an AO3 work page HTML into structured metadata."""
meta = AO3WorkMeta()
# Work ID from URL or meta
meta.original_url = url
try:
meta.work_id = _extract_work_id_from_url(url)
except PluginParseError:
meta.work_id = ""
# Title
meta.title = _text_of_first(html, r'<h2\s+class="[^"]*title[^"]*heading[^"]*"[^>]*>(.*?)</h2>')
if not meta.title:
meta.title = _meta_tag(html, "og:title") or ""
# Author
meta.author = _text_of_first(html, r'<a\s+[^>]*rel="author"[^>]*>(.*?)</a>')
if not meta.author:
meta.author = _meta_tag(html, "og:site_name") or ""
# Author ID from href
author_href_match = re.search(r'<a\s+[^>]*rel="author"[^>]*href="([^"]+)"', html, re.IGNORECASE)
if author_href_match:
author_path = author_href_match.group(1)
author_id_match = re.search(r"/users/([^/]+)", author_path)
if author_id_match:
meta.author_id = author_id_match.group(1)
else:
# Pseud path: /users/username/pseuds/name
pseud_match = re.search(r"/users/([^/]+)/pseuds", author_path)
if pseud_match:
meta.author_id = pseud_match.group(1)
# Summary
meta.summary = _text_of_first(html, r'<blockquote\s+class="[^"]*userstuff[^"]*"[^>]*>(.*?)</blockquote>')
if not meta.summary:
meta.summary = _meta_tag(html, "og:description") or ""
# Language
meta.language = _dd_text_single(html, "Language:")
# Rating
meta.rating = _dd_text_single(html, "Rating:")
# Warnings
meta.warnings = _dd_texts(html, "Archive Warnings*:")
# Fandoms
meta.fandoms = _dd_texts(html, "Fandoms*:")
# Relationships
meta.relationships = _dd_texts(html, "Relationships:")
# Characters
meta.characters = _dd_texts(html, "Characters:")
# Tags (freeform)
meta.tags = _dd_texts(html, "Additional Tags:")
# Categories
meta.categories = _dd_texts(html, "Categories:")
# Stats
raw_chapters = _stats_value(html, "Chapters:")
if "/" in raw_chapters:
parts = raw_chapters.split("/")
meta.chapter_count = int(parts[0].strip() or "1")
total = parts[1].strip()
if total == "?":
meta.status = "In Progress"
elif total.isdigit() and int(total) == meta.chapter_count:
meta.status = "Completed"
else:
meta.status = "In Progress"
elif raw_chapters.isdigit():
meta.chapter_count = int(raw_chapters)
else:
meta.chapter_count = 1
raw_words = _stats_value(html, "Words:")
meta.word_count = int(raw_words.replace(",", "") or "0")
# Check completion status explicitly
if not meta.status:
completed = _dd_texts(html, "Completion Status*:")
if completed:
meta.status = completed[0]
if not meta.status:
meta.status = "Completed" if meta.chapter_count == 1 else "In Progress"
# Published date
meta.published_at = _stats_value(html, "Published:")
# Normalize timezone
if meta.published_at:
try:
# AO3 dates are often like "2024-01-15"
dt = datetime.strptime(meta.published_at.strip(), "%Y-%m-%d")
meta.published_at = dt.replace(tzinfo=timezone.utc).isoformat()
except ValueError:
pass
# Updated date
meta.updated_at = _stats_value(html, "Updated:")
if meta.updated_at:
try:
dt = datetime.strptime(meta.updated_at.strip(), "%Y-%m-%d")
meta.updated_at = dt.replace(tzinfo=timezone.utc).isoformat()
except ValueError:
pass
# Cover: AO3 does not have covers, leave empty.
meta.cover = ""
return meta
# ---------------------------------------------------------------------------
# EPUB text extraction (minimal: extract paragraphs from EPUB)
# ---------------------------------------------------------------------------
from zipfile import ZipFile
def _extract_epub_text(epub_path: Path) -> str:
"""Extract readable text from an EPUB file (XHTML chapters).
Returns joined text with chapter separators.
Paragraphs within a chapter are separated by blank lines.
Inline markup (<em>, <strong>, etc.) is preserved as plain text
without introducing spurious paragraph breaks.
"""
from html.parser import HTMLParser
# Tags whose closing triggers a paragraph break.
_BLOCK_TAGS = {"p", "div", "br", "h1", "h2", "h3", "h4", "h5", "h6", "li", "blockquote", "section", "article", "hr"}
class AO3TextExtractor(HTMLParser):
def __init__(self):
super().__init__()
self.paragraphs: list[str] = []
self._buf: list[str] = [] # accumulating text for the current paragraph
self._skip = 0 # nesting depth for skip regions (script/style/head)
def handle_starttag(self, tag, attrs):
if tag in ("script", "style", "head"):
self._skip += 1
def handle_endtag(self, tag):
if tag in ("script", "style", "head"):
self._skip = max(0, self._skip - 1)
return
if tag in _BLOCK_TAGS:
# flush the current in-progress paragraph
text = " ".join(self._buf).strip()
if text:
self.paragraphs.append(text)
self._buf.clear()
def handle_data(self, data):
if self._skip:
return
text = data.strip()
if text:
self._buf.append(text)
def finish(self):
"""Flush any remaining buffered text."""
text = " ".join(self._buf).strip()
if text:
self.paragraphs.append(text)
self._buf.clear()
chapters: list[str] = []
with ZipFile(epub_path, "r") as zf:
xhtml_names = sorted(
[n for n in zf.namelist() if n.endswith((".xhtml", ".html", ".htm"))],
)
for name in xhtml_names:
try:
content = zf.read(name).decode("utf-8", errors="replace")
except Exception:
continue
# Extract title from <title> tag
title_match = re.search(r"<title>(.*?)</title>", content, re.IGNORECASE)
chapter_title = html_unescape(title_match.group(1).strip()) if title_match else ""
extractor = AO3TextExtractor()
try:
extractor.feed(content)
extractor.finish()
except Exception:
continue
if not extractor.paragraphs:
continue
chapter_text = "\n\n".join(extractor.paragraphs)
if chapter_title and chapter_title.lower() != "chapters":
chapters.append(f"\n\n=== {chapter_title} ===\n\n")
chapters.append(chapter_text)
return "\n".join(chapters) if chapters else ""
def _extract_html_work_text(html: str) -> str:
"""Extract readable AO3 work text from a work HTML page.
AO3 wraps chapter content in <div class="userstuff module" role="article">
(or <div class="userstuff" role="article">). We split on these opening
markers, then extract <h3 class="landmark heading"> for the chapter title
and all <p> elements for the body.
"""
# Prefer the actual AO3 chapter article block. Summary/preface blocks also
# use ``userstuff``, so the role="article" guard is important.
ARTICLE_START_RE = re.compile(
r'<div[^>]*class="[^"]*userstuff[^"]*"[^>]*role="article"[^>]*>',
re.IGNORECASE,
)
segments = ARTICLE_START_RE.split(html)[1:] # drop content before first block
if not segments:
# Fallback 1: try the older AO3 "chapter" div class
segments = re.findall(
r'<div[^>]*class="[^"]*chapter[^"]*"[^>]*>(.*?)</div>\s*(?=<div[^>]*class="[^"]*chapter|\Z)',
html,
re.IGNORECASE | re.DOTALL,
)
if not segments:
# Fallback 2: try <div id="chapters"> wrapper
segments = re.findall(
r'<div[^>]*id="chapters"[^>]*>(.*?)</div>\s*(?=<div[^>]*id="afterword"|\Z)',
html,
re.IGNORECASE | re.DOTALL,
)
chapters: list[str] = []
for segment in segments:
# Trim everything after the *next* userstuff article start or common
# trailing markers so we stay within this chapter's block.
end_match = ARTICLE_START_RE.search(segment)
if end_match:
segment = segment[: end_match.start()]
# Also trim at AO3 main-content/comment/footer markers. Chapter pages
# commonly end the body with <!--/main--> before feedback/actions.
for trailer_marker in (
r"<!--\s*/main\s*-->",
r"<!--\s*BEGIN comment section\s*-->",
r'<div[^>]*id="feedback"',
r'<div[^>]*id="afterword"',
r'<div[^>]*class="[^"]*afterword[^"]*"',
r'<div[^>]*class="[^"]*notes[^"]*module[^"]*"',
):
trailer_match = re.search(trailer_marker, segment, re.IGNORECASE)
if trailer_match:
segment = segment[: trailer_match.start()]
# Chapter heading: <h3 class="landmark heading" id="work">Chapter Text</h3>
title_match = re.search(
r'<h3[^>]*class="[^"]*landmark[^"]*heading[^"]*"[^>]*>(.*?)</h3>',
segment,
re.IGNORECASE | re.DOTALL,
)
chapter_title = (
html_unescape(re.sub(r"<[^>]+>", "", title_match.group(1))).strip()
if title_match
else ""
)
if chapter_title.lower() == "chapter text":
chapter_title = ""
# Extract all <p> text
raw_paragraphs = re.findall(
r"<p[^>]*>(.*?)</p>", segment, re.IGNORECASE | re.DOTALL,
)
paragraphs: list[str] = []
for raw in raw_paragraphs:
cleaned = html_unescape(re.sub(r"<[^>]+>", " ", raw))
cleaned = re.sub(r"\s+", " ", cleaned).strip()
if cleaned:
paragraphs.append(cleaned)
if chapter_title:
chapters.append(f"=== {chapter_title} ===")
if paragraphs:
chapters.append("\n\n".join(paragraphs))
if chapters:
return "\n\n".join(chapters).strip()
# Last resort: extract from <blockquote class="userstuff">
text = _text_of_first(
html,
r'<blockquote\s+class="[^"]*userstuff[^"]*"[^>]*>(.*?)</blockquote>',
)
return text.strip()
def _resolve_ao3_epub_url(work_id: str, *, cookie: str = "") -> str | None:
"""Resolve the actual AO3 EPUB download link for a work.
AO3's download page at ``/works/{id}/download`` lists format links like
``<a href="/downloads/{id}/Title.epub">EPUB</a>``. We fetch that page
and extract the EPUB href. Returns the absolute URL, or *None* when
the download page is unreachable or does not offer EPUB.
"""
try:
download_page_url = f"{AO3_BASE}/works/{work_id}/download"
response = _ao3_request(download_page_url, cookie=cookie, retries=2)
except PluginError:
return None
# Look for an EPUB link: <a href="/downloads/...">EPUB</a>
match = re.search(
r'''<a\s+[^>]*href\s*=\s*["'](?P<href>/downloads/[^"']+\.epub)["'][^>]*>\s*EPUB\s*</a>''',
response.text,
re.IGNORECASE,
)
if match:
return AO3_BASE + match.group("href")
# Fallback: any <a> whose text is "EPUB"
match = re.search(
r'''<a\s+[^>]*href\s*=\s*["'](?P<href>[^"']+\.epub)["'][^>]*>\s*EPUB\s*</a>''',
response.text,
re.IGNORECASE,
)
if match:
href = match.group("href")
return href if href.startswith("http") else AO3_BASE + href
return None
def _format_ao3_text_header(resource: Resource, work_id: str) -> str:
return "\n".join(
[
f"Title: {resource.title}",
f"Author: {resource.author}",
f"AO3 Work ID: {work_id}",
f"Source: {resource.url}",
f"Summary: {resource.metadata.get('summary', '')}",
"",
"",
]
)
# ---------------------------------------------------------------------------
# AO3 Plugin
# ---------------------------------------------------------------------------
class AO3Plugin(BasePlugin):
domain = "archiveofourown.org"
name = "AO3"
version = "1.0.0"
def __init__(self, cookie: str = "") -> None:
self.cookie = cookie
self._session = requests.Session()
# ---- auth ----
def set_auth(self, cookie: str) -> None:
self.cookie = cookie or ""
def get_headers(self) -> dict[str, str]:
headers = {
"User-Agent": USER_AGENT,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
if self.cookie:
headers["Cookie"] = self.cookie
return headers
# ---- URL detection ----
def can_handle(self, url: str) -> bool:
return _is_ao3_url(url)
# ---- parse ----
def parse(self, url: str) -> list[Resource]:
url = url.strip()
if not _is_ao3_url(url):
raise PluginParseError(f"Invalid AO3 work URL: {url}")
work_id, _chapter_id = extract_ao3_ids(url)
meta = self._fetch_meta(work_id)
# Build tags as a flat list for the cloud sync / library
all_tags = (
meta.fandoms
+ meta.warnings
+ meta.relationships
+ meta.characters
+ meta.tags
+ meta.categories
)
metadata = meta.to_dict()
metadata.update({
"kind": "novel",
"resource_type": "novel",
"content_type": "novel",
"source_platform": "ao3",
"source_id": work_id,
"series_id": work_id, # AO3 work is the series (multi-chapter)
"series_title": meta.title,
"chapter_title": meta.title,
"chapter_order": 1,
"tags_flat": all_tags,
})
return [
Resource(
id=work_id,
url=meta.original_url or f"{AO3_BASE}/works/{work_id}",
title=meta.title,
author=meta.author,
author_id=meta.author_id,
files=[],
metadata=metadata,
thumbnail=meta.cover or None,
created_at=meta.published_at or None,
)
]
# ---- download ----
def download(self, resource: Resource, save_path: Path) -> list[Path]:
work_id = resource.metadata.get("work_id") or resource.id
title = sanitize_filename(resource.title, f"ao3_{work_id}")
author = sanitize_filename(resource.author, "unknown_author")
save_path.mkdir(parents=True, exist_ok=True)
novel_dir = save_path / "novels" / sanitize_filename(author, f"ao3_{work_id}")
novel_dir.mkdir(parents=True, exist_ok=True)
downloaded_paths: list[Path] = []
error_messages: list[str] = []
body_text = ""
text_path = novel_dir / f"{title}_{work_id}.txt"
# ---- Primary: download full-work HTML page (most reliable) ----
try:
html_url = (
f"{AO3_BASE}/works/{work_id}"
f"?view_adult=true&view_full_work=true"
)
response = _ao3_request(html_url, cookie=self.cookie)
body_text = _extract_html_work_text(response.text)
if body_text.strip():
full_text = _format_ao3_text_header(resource, work_id) + body_text
ok, _msg = save_text_file(text_path, full_text)
if ok and text_path.exists():
downloaded_paths.append(text_path)
logger.info(
"AO3: HTML full-work text saved for work %s (%d chars)",
work_id, len(body_text),
)
else:
error_messages.append("HTML page contained no extractable text.")
except Exception as exc:
error_messages.append(f"HTML download failed: {exc}")
logger.warning(
"AO3: HTML full-work download failed for work %s: %s",
work_id, exc,
)
# ---- Secondary: try EPUB for archival preservation ----
epub_url = _resolve_ao3_epub_url(work_id, cookie=self.cookie)
if epub_url:
epub_path = novel_dir / f"{title}_{work_id}.epub"
try:
_ao3_download_file(epub_url, epub_path, cookie=self.cookie)
downloaded_paths.append(epub_path)
logger.info("AO3: EPUB saved for work %s", work_id)
except Exception as exc:
logger.warning(
"AO3: EPUB download skipped for work %s: %s",
work_id, exc,
)
if not downloaded_paths:
raise PluginDownloadError(
f"AO3 work {work_id} download failed: {'; '.join(error_messages)}"
)
# Update resource metadata
resource.metadata["word_count"] = len(body_text)
return downloaded_paths
# ---- update checking ----
def check_update(self, work_id: str) -> dict[str, Any] | None:
"""Check if the work has been updated since last import.
Returns a dict with update info, or None if no update.
"""
try:
url = f"{AO3_BASE}/works/{work_id}"
response = _ao3_request(url, cookie=self.cookie)
meta = parse_work_meta(response.text, url)
return {
"work_id": meta.work_id,
"title": meta.title,
"updated_at": meta.updated_at,
"chapter_count": meta.chapter_count,
"word_count": meta.word_count,
"status": meta.status,
"original_url": meta.original_url,
"has_update": True, # caller should determine this
}
except PluginError:
return None
except Exception:
return None
# ---- internal helpers ----
def _fetch_meta(self, work_id: str) -> AO3WorkMeta:
"""Fetch and parse metadata for a work."""
url = f"{AO3_BASE}/works/{work_id}?view_adult=true"
response = _ao3_request(url, cookie=self.cookie)
return parse_work_meta(response.text, response.url)
def validate(self) -> bool:
return True
# ---- cookie config (matches Pixiv pattern) ----
def get_config_schema(self) -> dict[str, Any]:
return {
"type": "object",
"properties": {
"cookie": {
"type": "string",
"title": "AO3 Cookie",
"description": "Paste AO3 Cookie text.",
},
},
}
def load_config(self) -> dict[str, Any]:
return {"cookie": self.cookie}
def save_config(self, data: dict[str, Any]) -> None:
self.cookie = str(data.get("cookie") or "")
# ---------------------------------------------------------------------------
# Library integration helper
# ---------------------------------------------------------------------------
def ao3_to_library_artwork_id(work_id: str) -> int:
"""Map an AO3 work_id to the library artwork_id namespace.
Uses negative integers beyond the Pixiv novel range to avoid collisions.
AO3 work IDs can be up to 8+ digits (e.g. 12345678), we offset them:
- AO3 prefix: -200000000 (200 million)
- artwork_id = -(200000000 + int(work_id))
This keeps AO3 works clearly distinct from Pixiv novels (-1 to -199999999).
"""
AO3_OFFSET = 200_000_000
return -(AO3_OFFSET + int(work_id))
def upsert_ao3_novel_to_library(
library: PixivLibrary,
meta: AO3WorkMeta,
*,
file_records: list[DownloadFileRecord] | None = None,
) -> int:
"""Insert or update an AO3 work in the local library.
Returns the library artwork_id.
"""
from pixiv_app.core.library import library_artwork_id
work_id = int(meta.work_id)
lib_id = library_artwork_id(work_id, artwork_type="novel", source_platform="ao3")
# Build detail dict compatible with upsert_artwork
# Include source / source_platform so upsert_artwork computes the correct
# AO3-offset artwork_id via library_artwork_id(id, "novel", "ao3").
detail: dict[str, Any] = {
"id": work_id,
"novelId": work_id,
"title": meta.title,
"userName": meta.author,
"userId": _ao3_author_id_to_int(meta.author_id),
"description": meta.summary,
"pageCount": meta.chapter_count,
"likeCount": 0,
"bookmarkCount": 0,
"viewCount": 0,
"xRestrict": 1 if meta.rating in ("Explicit", "Not Rated") else 0,
"aiType": 0,
"createDate": meta.published_at,
"source_platform": "ao3",
"source": "ao3",
"tags": [
{"tag": tag, "translation": None}
for tag in (
meta.fandoms
+ meta.characters
+ meta.tags
+ meta.relationships
+ meta.warnings
+ meta.categories
)
],
}
# Upsert artist stub
artist_id = _ao3_author_id_to_int(meta.author_id)
library.ensure_artist_stub(artist_id, name=meta.author)
library.upsert_artist_profile(artist_id, name=meta.author)
# Upsert artwork via the standard method (now AO3-ID-aware)
library.ensure_artwork_stub(work_id, artwork_type="novel", title=meta.title, source_platform="ao3")
library.upsert_artwork(detail, artwork_type="novel")
# Record files if provided
if file_records:
library.record_files(lib_id, file_records)
return lib_id
def _ao3_author_id_to_int(author_id: str) -> int:
"""Convert an AO3 username string to a stable int for the library.
Uses a simple DJB2 hash constrained to positive 32-bit range.
"""
if not author_id:
return 0
h = 5381
for ch in author_id:
h = ((h << 5) + h) + ord(ch)
return h & 0x7FFFFFFF
# ---------------------------------------------------------------------------
# Cloud sync metadata builder
# ---------------------------------------------------------------------------
def build_ao3_sync_metadata(
meta: AO3WorkMeta,
file_path: Path,
*,
chapter_title: str = "",
chapter_order: int = 1,
) -> dict[str, Any]:
"""Build a SyncMetadata-like dict for cloud sync.
Compatible with pixiv_app.core.cloud_sync.SyncMetadata.
"""
from pixiv_app.core.cloud_sync import SyncMetadata
all_tags = (
meta.fandoms
+ meta.warnings
+ meta.relationships
+ meta.characters
+ meta.tags
+ meta.categories
)
return SyncMetadata(
resource_type="novel",
source_platform="ao3",
source_id=meta.work_id,
content_type="novel",
title=meta.title,
author=meta.author,
author_id=meta.author_id,
tags=all_tags,
metadata=meta.to_dict(),
page_count=meta.chapter_count,
description=meta.summary,
word_count=meta.word_count,
series_id=meta.work_id, # work = series for AO3
series_title=meta.title,
chapter_title=chapter_title or meta.title,
chapter_order=chapter_order,
)
# ---------------------------------------------------------------------------
# Plugin class export
# ---------------------------------------------------------------------------
plugin_class = AO3Plugin