Files

1028 lines
34 KiB
Python
Raw Permalink Normal View History

2026-11-16 20:45:22 +08:00
"""
AO3 (Archive of Our Own) novel plugin.
Parses AO3 work pages, extracts metadata, downloads works via EPUB or HTML,
and integrates with the existing library / cloud-sync / update-check pipelines.
Reuses:
- pixiv_app.core.library.PixivLibrary for local storage
- pixiv_app.core.cloud_sync.SyncMetadata / SyncManager for cloud sync
- pixiv_app.core.cookie_import utilities for cookie management
- pixiv_app.core.downloader helpers (sanitize_filename, save_text_file)
"""
from __future__ import annotations
import logging
import re
import time
from dataclasses import dataclass, field
from datetime import datetime, timezone
from html import unescape as html_unescape
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
import requests
from pixiv_app.core.downloader import (
USER_AGENT,
DEFAULT_TIMEOUT,
DownloadFileRecord,
sanitize_filename,
save_text_file,
)
from pixiv_app.core.library import PixivLibrary
from pixiv_app.core.plugin.base import (
BasePlugin,
PluginAuthError,
PluginDownloadError,
PluginError,
PluginParseError,
Resource,
)
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
AO3_BASE = "https://archiveofourown.org"
AO3_DOMAINS = ("archiveofourown.org", "ao3.org")
# AO3 URL patterns
_WORK_URL_RE = re.compile(
r"/works/(?P<work_id>\d+)(?:/chapters/(?P<chapter_id>\d+))?",
re.IGNORECASE,
)
# ---------------------------------------------------------------------------
# AO3-specific request helpers
# ---------------------------------------------------------------------------
def _ao3_request(
url: str,
*,
cookie: str = "",
timeout: int = DEFAULT_TIMEOUT,
retries: int = 3,
) -> requests.Response:
"""GET an AO3 page, handling rate-limit (Retry-After) and transient errors."""
headers = {
"User-Agent": USER_AGENT,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
if cookie:
headers["Cookie"] = cookie
last_exc: Exception | None = None
for attempt in range(1, retries + 1):
try:
response = requests.get(url, headers=headers, timeout=timeout, allow_redirects=True)
except requests.RequestException as exc:
last_exc = exc
if attempt < retries:
time.sleep(min(1.0 * attempt, 3.0))
continue
if response.status_code == 429:
retry_after = response.headers.get("Retry-After")
try:
wait = int(retry_after) if retry_after else 5 * attempt
except (TypeError, ValueError):
wait = 5 * attempt
logger.warning("AO3 rate-limited, waiting %ds", wait)
time.sleep(wait)
last_exc = PluginError("AO3 rate limited the request; please retry later.")
continue
if response.status_code == 404:
raise PluginParseError("AO3 work was not found (404).")
if response.status_code == 401:
raise PluginAuthError("AO3 requires login for this work. Import a valid AO3 Cookie.")
# AO3 adult content confirmation page
if "This work could have adult content" in response.text or "adult check" in response.url.lower():
raise PluginAuthError(
"This AO3 work requires adult-content confirmation. Import a valid AO3 Cookie."
)
if not response.ok:
last_exc = PluginError(f"AO3 request failed HTTP {response.status_code}")
if attempt < retries:
time.sleep(min(1.0 * attempt, 3.0))
continue
# Check for login wall (page says you need to log in but it's not 401)
if "Please log in" in response.text and "adult" not in response.text.lower():
if not cookie:
raise PluginAuthError("This AO3 work is visible only to logged-in users. Import an AO3 Cookie.")
raise PluginAuthError("AO3 Cookie may be expired. Import it again.")
return response
if last_exc:
raise PluginError(f"AO3 request failed: {last_exc}") from last_exc
raise PluginError("AO3 request failed after maximum retries.")
def _ao3_download_file(
url: str,
save_path: Path,
*,
cookie: str = "",
timeout: int = 60,
) -> Path:
"""Download a binary file (EPUB) from AO3."""
headers = {
"User-Agent": USER_AGENT,
"Accept": "*/*",
}
if cookie:
headers["Cookie"] = cookie
if save_path.exists() and save_path.stat().st_size > 1024:
return save_path # already downloaded
temp_path = save_path.with_suffix(save_path.suffix + ".part")
response = requests.get(url, headers=headers, timeout=timeout, stream=True)
response.raise_for_status()
with open(temp_path, "wb") as fh:
for chunk in response.iter_content(chunk_size=256 * 1024):
if chunk:
fh.write(chunk)
if temp_path.stat().st_size <= 1024:
temp_path.unlink(missing_ok=True)
raise PluginDownloadError("Downloaded file is too small to be a valid EPUB.")
temp_path.replace(save_path)
return save_path
# ---------------------------------------------------------------------------
# AO3 metadata extraction (pure HTML scraping)
# ---------------------------------------------------------------------------
@dataclass
class AO3WorkMeta:
"""Parsed AO3 work metadata."""
work_id: str = ""
title: str = ""
author: str = ""
author_id: str = "" # AO3 username
summary: str = ""
language: str = ""
rating: str = ""
warnings: list[str] = field(default_factory=list)
fandoms: list[str] = field(default_factory=list)
relationships: list[str] = field(default_factory=list)
characters: list[str] = field(default_factory=list)
tags: list[str] = field(default_factory=list)
categories: list[str] = field(default_factory=list)
status: str = "" # "Completed" or "In Progress"
chapter_count: int = 1
word_count: int = 0
published_at: str = ""
updated_at: str = ""
original_url: str = ""
cover: str = ""
def to_dict(self) -> dict[str, Any]:
return {
"source": "ao3",
"work_id": self.work_id,
"title": self.title,
"author": self.author,
"author_id": self.author_id,
"summary": self.summary,
"language": self.language,
"rating": self.rating,
"warnings": self.warnings,
"fandoms": self.fandoms,
"relationships": self.relationships,
"characters": self.characters,
"tags": self.tags,
"categories": self.categories,
"status": self.status,
"chapter_count": self.chapter_count,
"word_count": self.word_count,
"published_at": self.published_at,
"updated_at": self.updated_at,
"cover": self.cover,
"original_url": self.original_url,
}
def _extract_work_id_from_url(url: str) -> str:
"""Extract the work_id from an AO3 URL."""
match = _WORK_URL_RE.search(url)
if match:
return match.group("work_id")
raise PluginParseError(f"Invalid AO3 work URL: {url}")
def extract_ao3_ids(url: str) -> tuple[str, str | None]:
"""Extract ``(work_id, chapter_id)`` from an AO3 work/chapter URL."""
match = _WORK_URL_RE.search(url.strip())
if not match:
raise PluginParseError(f"Invalid AO3 work URL: {url}")
return match.group("work_id"), match.group("chapter_id")
def _is_ao3_url(url: str) -> bool:
parsed = urlparse(url.strip())
host = parsed.netloc.lower().split("@")[-1].split(":")[0]
return host in AO3_DOMAINS and bool(_WORK_URL_RE.search(parsed.path))
def _meta_tag(html: str, property_name: str) -> str | None:
"""Extract a <meta> tag content by property or name."""
for attr in ("property", "name"):
pattern = re.compile(
rf'<meta\s+{attr}=["\']{re.escape(property_name)}["\']\s+content=["\']([^"\']+)["\']',
re.IGNORECASE,
)
match = pattern.search(html)
if match:
return html_unescape(match.group(1).strip())
return None
def _text_of_first(html: str, selector_re: str) -> str:
"""Return inner text of the first element matching a regex."""
pattern = re.compile(selector_re, re.IGNORECASE | re.DOTALL)
match = pattern.search(html)
if not match:
return ""
inner = match.group(1)
# Strip inner HTML tags
inner = re.sub(r"<[^>]+>", " ", inner)
inner = html_unescape(inner)
return re.sub(r"\s+", " ", inner).strip()
def _dd_texts(html: str, dt_label: str) -> list[str]:
"""Extract tag texts from <dd> that follows a <dt> with given label.
AO3 meta uses two patterns:
1. Tag block: <dt class="...">Rating:</dt> <dd class="... tags"><ul><li><a class="tag">...</a></li></ul></dd>
2. Stats block: <dt class="...">Language:</dt> <dd class="...">English</dd>
"""
expected = _normalize_ao3_label(dt_label)
match = None
for candidate in re.finditer(
r"<dt[^>]*>(.*?)</dt>\s*<dd[^>]*>(.*?)</dd>",
html,
re.IGNORECASE | re.DOTALL,
):
label_text = _normalize_ao3_label(candidate.group(1))
if label_text == expected:
match = candidate
break
if match is None:
return []
dd_html = match.group(2)
# Extract tag text from <a class="tag"> elements
tags = re.findall(
r'<a[^>]*class="[^"]*tag[^"]*"[^>]*>(.*?)</a>',
dd_html,
re.IGNORECASE | re.DOTALL,
)
if tags:
return [
html_unescape(re.sub(r"<[^>]+>", "", t)).strip()
for t in tags if t.strip()
]
# Fallback: plain text in <dd>
inner = re.sub(r"<[^>]+>", "", dd_html)
inner = html_unescape(inner).strip()
return [inner] if inner else []
def _normalize_ao3_label(value: str) -> str:
text = re.sub(r"<[^>]+>", "", value)
text = html_unescape(text)
text = text.replace("*", "")
text = re.sub(r"\s+", " ", text).strip().rstrip(":").lower()
return text
def _dd_text_single(html: str, dt_label: str) -> str:
"""Extract a single text value from <dd> after <dt>."""
texts = _dd_texts(html, dt_label)
return texts[0] if texts else ""
def _stats_value(html: str, label: str) -> str:
"""Extract a value from the stats block: <dt>Words:</dt><dd>12345</dd>."""
return _dd_text_single(html, label)
def parse_work_meta(html: str, url: str = "") -> AO3WorkMeta:
"""Parse an AO3 work page HTML into structured metadata."""
meta = AO3WorkMeta()
# Work ID from URL or meta
meta.original_url = url
try:
meta.work_id = _extract_work_id_from_url(url)
except PluginParseError:
meta.work_id = ""
# Title
meta.title = _text_of_first(html, r'<h2\s+class="[^"]*title[^"]*heading[^"]*"[^>]*>(.*?)</h2>')
if not meta.title:
meta.title = _meta_tag(html, "og:title") or ""
# Author
meta.author = _text_of_first(html, r'<a\s+[^>]*rel="author"[^>]*>(.*?)</a>')
if not meta.author:
meta.author = _meta_tag(html, "og:site_name") or ""
# Author ID from href
author_href_match = re.search(r'<a\s+[^>]*rel="author"[^>]*href="([^"]+)"', html, re.IGNORECASE)
if author_href_match:
author_path = author_href_match.group(1)
author_id_match = re.search(r"/users/([^/]+)", author_path)
if author_id_match:
meta.author_id = author_id_match.group(1)
else:
# Pseud path: /users/username/pseuds/name
pseud_match = re.search(r"/users/([^/]+)/pseuds", author_path)
if pseud_match:
meta.author_id = pseud_match.group(1)
# Summary
meta.summary = _text_of_first(html, r'<blockquote\s+class="[^"]*userstuff[^"]*"[^>]*>(.*?)</blockquote>')
if not meta.summary:
meta.summary = _meta_tag(html, "og:description") or ""
# Language
meta.language = _dd_text_single(html, "Language:")
# Rating
meta.rating = _dd_text_single(html, "Rating:")
# Warnings
meta.warnings = _dd_texts(html, "Archive Warnings*:")
# Fandoms
meta.fandoms = _dd_texts(html, "Fandoms*:")
# Relationships
meta.relationships = _dd_texts(html, "Relationships:")
# Characters
meta.characters = _dd_texts(html, "Characters:")
# Tags (freeform)
meta.tags = _dd_texts(html, "Additional Tags:")
# Categories
meta.categories = _dd_texts(html, "Categories:")
# Stats
raw_chapters = _stats_value(html, "Chapters:")
if "/" in raw_chapters:
parts = raw_chapters.split("/")
meta.chapter_count = int(parts[0].strip() or "1")
total = parts[1].strip()
if total == "?":
meta.status = "In Progress"
elif total.isdigit() and int(total) == meta.chapter_count:
meta.status = "Completed"
else:
meta.status = "In Progress"
elif raw_chapters.isdigit():
meta.chapter_count = int(raw_chapters)
else:
meta.chapter_count = 1
raw_words = _stats_value(html, "Words:")
meta.word_count = int(raw_words.replace(",", "") or "0")
# Check completion status explicitly
if not meta.status:
completed = _dd_texts(html, "Completion Status*:")
if completed:
meta.status = completed[0]
if not meta.status:
meta.status = "Completed" if meta.chapter_count == 1 else "In Progress"
# Published date
meta.published_at = _stats_value(html, "Published:")
# Normalize timezone
if meta.published_at:
try:
# AO3 dates are often like "2024-01-15"
dt = datetime.strptime(meta.published_at.strip(), "%Y-%m-%d")
meta.published_at = dt.replace(tzinfo=timezone.utc).isoformat()
except ValueError:
pass
# Updated date
meta.updated_at = _stats_value(html, "Updated:")
if meta.updated_at:
try:
dt = datetime.strptime(meta.updated_at.strip(), "%Y-%m-%d")
meta.updated_at = dt.replace(tzinfo=timezone.utc).isoformat()
except ValueError:
pass
# Cover: AO3 does not have covers, leave empty.
meta.cover = ""
return meta
# ---------------------------------------------------------------------------
# EPUB text extraction (minimal: extract paragraphs from EPUB)
# ---------------------------------------------------------------------------
from zipfile import ZipFile
def _extract_epub_text(epub_path: Path) -> str:
"""Extract readable text from an EPUB file (XHTML chapters).
Returns joined text with chapter separators.
Paragraphs within a chapter are separated by blank lines.
Inline markup (<em>, <strong>, etc.) is preserved as plain text
without introducing spurious paragraph breaks.
"""
from html.parser import HTMLParser
# Tags whose closing triggers a paragraph break.
_BLOCK_TAGS = {"p", "div", "br", "h1", "h2", "h3", "h4", "h5", "h6", "li", "blockquote", "section", "article", "hr"}
class AO3TextExtractor(HTMLParser):
def __init__(self):
super().__init__()
self.paragraphs: list[str] = []
self._buf: list[str] = [] # accumulating text for the current paragraph
self._skip = 0 # nesting depth for skip regions (script/style/head)
def handle_starttag(self, tag, attrs):
if tag in ("script", "style", "head"):
self._skip += 1
def handle_endtag(self, tag):
if tag in ("script", "style", "head"):
self._skip = max(0, self._skip - 1)
return
if tag in _BLOCK_TAGS:
# flush the current in-progress paragraph
text = " ".join(self._buf).strip()
if text:
self.paragraphs.append(text)
self._buf.clear()
def handle_data(self, data):
if self._skip:
return
text = data.strip()
if text:
self._buf.append(text)
def finish(self):
"""Flush any remaining buffered text."""
text = " ".join(self._buf).strip()
if text:
self.paragraphs.append(text)
self._buf.clear()
chapters: list[str] = []
with ZipFile(epub_path, "r") as zf:
xhtml_names = sorted(
[n for n in zf.namelist() if n.endswith((".xhtml", ".html", ".htm"))],
)
for name in xhtml_names:
try:
content = zf.read(name).decode("utf-8", errors="replace")
except Exception:
continue
# Extract title from <title> tag
title_match = re.search(r"<title>(.*?)</title>", content, re.IGNORECASE)
chapter_title = html_unescape(title_match.group(1).strip()) if title_match else ""
extractor = AO3TextExtractor()
try:
extractor.feed(content)
extractor.finish()
except Exception:
continue
if not extractor.paragraphs:
continue
chapter_text = "\n\n".join(extractor.paragraphs)
if chapter_title and chapter_title.lower() != "chapters":
chapters.append(f"\n\n=== {chapter_title} ===\n\n")
chapters.append(chapter_text)
return "\n".join(chapters) if chapters else ""
def _extract_html_work_text(html: str) -> str:
"""Extract readable AO3 work text from a work HTML page.
AO3 wraps chapter content in <div class="userstuff module" role="article">
(or <div class="userstuff" role="article">). We split on these opening
markers, then extract <h3 class="landmark heading"> for the chapter title
and all <p> elements for the body.
"""
# Prefer the actual AO3 chapter article block. Summary/preface blocks also
# use ``userstuff``, so the role="article" guard is important.
ARTICLE_START_RE = re.compile(
r'<div[^>]*class="[^"]*userstuff[^"]*"[^>]*role="article"[^>]*>',
re.IGNORECASE,
)
segments = ARTICLE_START_RE.split(html)[1:] # drop content before first block
if not segments:
# Fallback 1: try the older AO3 "chapter" div class
segments = re.findall(
r'<div[^>]*class="[^"]*chapter[^"]*"[^>]*>(.*?)</div>\s*(?=<div[^>]*class="[^"]*chapter|\Z)',
html,
re.IGNORECASE | re.DOTALL,
)
if not segments:
# Fallback 2: try <div id="chapters"> wrapper
segments = re.findall(
r'<div[^>]*id="chapters"[^>]*>(.*?)</div>\s*(?=<div[^>]*id="afterword"|\Z)',
html,
re.IGNORECASE | re.DOTALL,
)
chapters: list[str] = []
for segment in segments:
# Trim everything after the *next* userstuff article start or common
# trailing markers so we stay within this chapter's block.
end_match = ARTICLE_START_RE.search(segment)
if end_match:
segment = segment[: end_match.start()]
# Also trim at AO3 main-content/comment/footer markers. Chapter pages
# commonly end the body with <!--/main--> before feedback/actions.
for trailer_marker in (
r"<!--\s*/main\s*-->",
r"<!--\s*BEGIN comment section\s*-->",
r'<div[^>]*id="feedback"',
r'<div[^>]*id="afterword"',
r'<div[^>]*class="[^"]*afterword[^"]*"',
r'<div[^>]*class="[^"]*notes[^"]*module[^"]*"',
):
trailer_match = re.search(trailer_marker, segment, re.IGNORECASE)
if trailer_match:
segment = segment[: trailer_match.start()]
# Chapter heading: <h3 class="landmark heading" id="work">Chapter Text</h3>
title_match = re.search(
r'<h3[^>]*class="[^"]*landmark[^"]*heading[^"]*"[^>]*>(.*?)</h3>',
segment,
re.IGNORECASE | re.DOTALL,
)
chapter_title = (
html_unescape(re.sub(r"<[^>]+>", "", title_match.group(1))).strip()
if title_match
else ""
)
if chapter_title.lower() == "chapter text":
chapter_title = ""
# Extract all <p> text
raw_paragraphs = re.findall(
r"<p[^>]*>(.*?)</p>", segment, re.IGNORECASE | re.DOTALL,
)
paragraphs: list[str] = []
for raw in raw_paragraphs:
cleaned = html_unescape(re.sub(r"<[^>]+>", " ", raw))
cleaned = re.sub(r"\s+", " ", cleaned).strip()
if cleaned:
paragraphs.append(cleaned)
if chapter_title:
chapters.append(f"=== {chapter_title} ===")
if paragraphs:
chapters.append("\n\n".join(paragraphs))
if chapters:
return "\n\n".join(chapters).strip()
# Last resort: extract from <blockquote class="userstuff">
text = _text_of_first(
html,
r'<blockquote\s+class="[^"]*userstuff[^"]*"[^>]*>(.*?)</blockquote>',
)
return text.strip()
def _resolve_ao3_epub_url(work_id: str, *, cookie: str = "") -> str | None:
"""Resolve the actual AO3 EPUB download link for a work.
AO3's download page at ``/works/{id}/download`` lists format links like
``<a href="/downloads/{id}/Title.epub">EPUB</a>``. We fetch that page
and extract the EPUB href. Returns the absolute URL, or *None* when
the download page is unreachable or does not offer EPUB.
"""
try:
download_page_url = f"{AO3_BASE}/works/{work_id}/download"
response = _ao3_request(download_page_url, cookie=cookie, retries=2)
except PluginError:
return None
# Look for an EPUB link: <a href="/downloads/...">EPUB</a>
match = re.search(
r'''<a\s+[^>]*href\s*=\s*["'](?P<href>/downloads/[^"']+\.epub)["'][^>]*>\s*EPUB\s*</a>''',
response.text,
re.IGNORECASE,
)
if match:
return AO3_BASE + match.group("href")
# Fallback: any <a> whose text is "EPUB"
match = re.search(
r'''<a\s+[^>]*href\s*=\s*["'](?P<href>[^"']+\.epub)["'][^>]*>\s*EPUB\s*</a>''',
response.text,
re.IGNORECASE,
)
if match:
href = match.group("href")
return href if href.startswith("http") else AO3_BASE + href
return None
def _format_ao3_text_header(resource: Resource, work_id: str) -> str:
return "\n".join(
[
f"Title: {resource.title}",
f"Author: {resource.author}",
f"AO3 Work ID: {work_id}",
f"Source: {resource.url}",
f"Summary: {resource.metadata.get('summary', '')}",
"",
"",
]
)
# ---------------------------------------------------------------------------
# AO3 Plugin
# ---------------------------------------------------------------------------
class AO3Plugin(BasePlugin):
domain = "archiveofourown.org"
name = "AO3"
version = "1.0.0"
def __init__(self, cookie: str = "") -> None:
self.cookie = cookie
self._session = requests.Session()
# ---- auth ----
def set_auth(self, cookie: str) -> None:
self.cookie = cookie or ""
def get_headers(self) -> dict[str, str]:
headers = {
"User-Agent": USER_AGENT,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}
if self.cookie:
headers["Cookie"] = self.cookie
return headers
# ---- URL detection ----
def can_handle(self, url: str) -> bool:
return _is_ao3_url(url)
# ---- parse ----
def parse(self, url: str) -> list[Resource]:
url = url.strip()
if not _is_ao3_url(url):
raise PluginParseError(f"Invalid AO3 work URL: {url}")
work_id, _chapter_id = extract_ao3_ids(url)
meta = self._fetch_meta(work_id)
# Build tags as a flat list for the cloud sync / library
all_tags = (
meta.fandoms
+ meta.warnings
+ meta.relationships
+ meta.characters
+ meta.tags
+ meta.categories
)
metadata = meta.to_dict()
metadata.update({
"kind": "novel",
"resource_type": "novel",
"content_type": "novel",
"source_platform": "ao3",
"source_id": work_id,
"series_id": work_id, # AO3 work is the series (multi-chapter)
"series_title": meta.title,
"chapter_title": meta.title,
"chapter_order": 1,
"tags_flat": all_tags,
})
return [
Resource(
id=work_id,
url=meta.original_url or f"{AO3_BASE}/works/{work_id}",
title=meta.title,
author=meta.author,
author_id=meta.author_id,
files=[],
metadata=metadata,
thumbnail=meta.cover or None,
created_at=meta.published_at or None,
)
]
# ---- download ----
def download(self, resource: Resource, save_path: Path) -> list[Path]:
work_id = resource.metadata.get("work_id") or resource.id
title = sanitize_filename(resource.title, f"ao3_{work_id}")
author = sanitize_filename(resource.author, "unknown_author")
save_path.mkdir(parents=True, exist_ok=True)
novel_dir = save_path / "novels" / sanitize_filename(author, f"ao3_{work_id}")
novel_dir.mkdir(parents=True, exist_ok=True)
downloaded_paths: list[Path] = []
error_messages: list[str] = []
body_text = ""
text_path = novel_dir / f"{title}_{work_id}.txt"
# ---- Primary: download full-work HTML page (most reliable) ----
try:
html_url = (
f"{AO3_BASE}/works/{work_id}"
f"?view_adult=true&view_full_work=true"
)
response = _ao3_request(html_url, cookie=self.cookie)
body_text = _extract_html_work_text(response.text)
if body_text.strip():
full_text = _format_ao3_text_header(resource, work_id) + body_text
ok, _msg = save_text_file(text_path, full_text)
if ok and text_path.exists():
downloaded_paths.append(text_path)
logger.info(
"AO3: HTML full-work text saved for work %s (%d chars)",
work_id, len(body_text),
)
else:
error_messages.append("HTML page contained no extractable text.")
except Exception as exc:
error_messages.append(f"HTML download failed: {exc}")
logger.warning(
"AO3: HTML full-work download failed for work %s: %s",
work_id, exc,
)
# ---- Secondary: try EPUB for archival preservation ----
epub_url = _resolve_ao3_epub_url(work_id, cookie=self.cookie)
if epub_url:
epub_path = novel_dir / f"{title}_{work_id}.epub"
try:
_ao3_download_file(epub_url, epub_path, cookie=self.cookie)
downloaded_paths.append(epub_path)
logger.info("AO3: EPUB saved for work %s", work_id)
except Exception as exc:
logger.warning(
"AO3: EPUB download skipped for work %s: %s",
work_id, exc,
)
if not downloaded_paths:
raise PluginDownloadError(
f"AO3 work {work_id} download failed: {'; '.join(error_messages)}"
)
# Update resource metadata
resource.metadata["word_count"] = len(body_text)
return downloaded_paths
# ---- update checking ----
def check_update(self, work_id: str) -> dict[str, Any] | None:
"""Check if the work has been updated since last import.
Returns a dict with update info, or None if no update.
"""
try:
url = f"{AO3_BASE}/works/{work_id}"
response = _ao3_request(url, cookie=self.cookie)
meta = parse_work_meta(response.text, url)
return {
"work_id": meta.work_id,
"title": meta.title,
"updated_at": meta.updated_at,
"chapter_count": meta.chapter_count,
"word_count": meta.word_count,
"status": meta.status,
"original_url": meta.original_url,
"has_update": True, # caller should determine this
}
except PluginError:
return None
except Exception:
return None
# ---- internal helpers ----
def _fetch_meta(self, work_id: str) -> AO3WorkMeta:
"""Fetch and parse metadata for a work."""
url = f"{AO3_BASE}/works/{work_id}?view_adult=true"
response = _ao3_request(url, cookie=self.cookie)
return parse_work_meta(response.text, response.url)
def validate(self) -> bool:
return True
# ---- cookie config (matches Pixiv pattern) ----
def get_config_schema(self) -> dict[str, Any]:
return {
"type": "object",
"properties": {
"cookie": {
"type": "string",
"title": "AO3 Cookie",
"description": "Paste AO3 Cookie text.",
},
},
}
def load_config(self) -> dict[str, Any]:
return {"cookie": self.cookie}
def save_config(self, data: dict[str, Any]) -> None:
self.cookie = str(data.get("cookie") or "")
# ---------------------------------------------------------------------------
# Library integration helper
# ---------------------------------------------------------------------------
def ao3_to_library_artwork_id(work_id: str) -> int:
"""Map an AO3 work_id to the library artwork_id namespace.
Uses negative integers beyond the Pixiv novel range to avoid collisions.
AO3 work IDs can be up to 8+ digits (e.g. 12345678), we offset them:
- AO3 prefix: -200000000 (200 million)
- artwork_id = -(200000000 + int(work_id))
This keeps AO3 works clearly distinct from Pixiv novels (-1 to -199999999).
"""
AO3_OFFSET = 200_000_000
return -(AO3_OFFSET + int(work_id))
def upsert_ao3_novel_to_library(
library: PixivLibrary,
meta: AO3WorkMeta,
*,
file_records: list[DownloadFileRecord] | None = None,
) -> int:
"""Insert or update an AO3 work in the local library.
Returns the library artwork_id.
"""
from pixiv_app.core.library import library_artwork_id
work_id = int(meta.work_id)
lib_id = library_artwork_id(work_id, artwork_type="novel", source_platform="ao3")
# Build detail dict compatible with upsert_artwork
# Include source / source_platform so upsert_artwork computes the correct
# AO3-offset artwork_id via library_artwork_id(id, "novel", "ao3").
detail: dict[str, Any] = {
"id": work_id,
"novelId": work_id,
"title": meta.title,
"userName": meta.author,
"userId": _ao3_author_id_to_int(meta.author_id),
"description": meta.summary,
"pageCount": meta.chapter_count,
"likeCount": 0,
"bookmarkCount": 0,
"viewCount": 0,
"xRestrict": 1 if meta.rating in ("Explicit", "Not Rated") else 0,
"aiType": 0,
"createDate": meta.published_at,
"source_platform": "ao3",
"source": "ao3",
"tags": [
{"tag": tag, "translation": None}
for tag in (
meta.fandoms
+ meta.characters
+ meta.tags
+ meta.relationships
+ meta.warnings
+ meta.categories
)
],
}
# Upsert artist stub
artist_id = _ao3_author_id_to_int(meta.author_id)
library.ensure_artist_stub(artist_id, name=meta.author)
library.upsert_artist_profile(artist_id, name=meta.author)
# Upsert artwork via the standard method (now AO3-ID-aware)
library.ensure_artwork_stub(work_id, artwork_type="novel", title=meta.title, source_platform="ao3")
library.upsert_artwork(detail, artwork_type="novel")
# Record files if provided
if file_records:
library.record_files(lib_id, file_records)
return lib_id
def _ao3_author_id_to_int(author_id: str) -> int:
"""Convert an AO3 username string to a stable int for the library.
Uses a simple DJB2 hash constrained to positive 32-bit range.
"""
if not author_id:
return 0
h = 5381
for ch in author_id:
h = ((h << 5) + h) + ord(ch)
return h & 0x7FFFFFFF
# ---------------------------------------------------------------------------
# Cloud sync metadata builder
# ---------------------------------------------------------------------------
def build_ao3_sync_metadata(
meta: AO3WorkMeta,
file_path: Path,
*,
chapter_title: str = "",
chapter_order: int = 1,
) -> dict[str, Any]:
"""Build a SyncMetadata-like dict for cloud sync.
Compatible with pixiv_app.core.cloud_sync.SyncMetadata.
"""
from pixiv_app.core.cloud_sync import SyncMetadata
all_tags = (
meta.fandoms
+ meta.warnings
+ meta.relationships
+ meta.characters
+ meta.tags
+ meta.categories
)
return SyncMetadata(
resource_type="novel",
source_platform="ao3",
source_id=meta.work_id,
content_type="novel",
title=meta.title,
author=meta.author,
author_id=meta.author_id,
tags=all_tags,
metadata=meta.to_dict(),
page_count=meta.chapter_count,
description=meta.summary,
word_count=meta.word_count,
series_id=meta.work_id, # work = series for AO3
series_title=meta.title,
chapter_title=chapter_title or meta.title,
chapter_order=chapter_order,
)
# ---------------------------------------------------------------------------
# Plugin class export
# ---------------------------------------------------------------------------
plugin_class = AO3Plugin