29f71dd330
- kubera-sync.py: /r/kubera → Google Docs → CBZ manga translation pipeline. Scans Reddit RSS, extracts translated images from linked Google Docs, packages as CBZ. old.reddit.com endpoint (www.reddit.com returns 429 from datacenter IPs). Idempotent, 6h systemd timer. - aurelio-print.py: CBZ/PDF → tankobon booklet imposition → IPP submit to CUPS print server. Saddle-stitch ordering, signature folding, pure-Python PDF writer (no external deps). Targets CT244 print server. Co-authored-by: Álvaro de Campos <campos@portugalfuturista.org>
302 lines
11 KiB
Python
302 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
kubera-sync — Download /r/kubera fan translations from Google Docs → CBZ.
|
|
|
|
Pipeline:
|
|
1. Scan /r/kubera RSS for [RAW] posts with Google Docs links
|
|
2. Download each Google Doc's embedded images (base64 PNGs in export HTML)
|
|
3. Package as CBZ into the Kavita manga library
|
|
|
|
Usage:
|
|
kubera-sync scan # Scan Reddit, show available chapters
|
|
kubera-sync download --all # Download all found translations
|
|
kubera-sync download --ch 434 # Download specific chapter
|
|
kubera-sync sync # Scan + download missing chapters
|
|
|
|
Config: environment variables or defaults
|
|
MANGA_OUTPUT_DIR — where CBZ files go (default: /mnt/media/manga/Kubera (Translated))
|
|
REDDIT_SUB — subreddit to scan (default: kubera)
|
|
"""
|
|
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
import zipfile
|
|
import xml.etree.ElementTree as ET
|
|
from pathlib import Path
|
|
|
|
# ── Config ────────────────────────────────────────────────────────────────
|
|
|
|
REDDIT_SUB = os.environ.get("REDDIT_SUB", "kubera")
|
|
OUTPUT_DIR = Path(os.environ.get(
|
|
"MANGA_OUTPUT_DIR",
|
|
"/mnt/media/manga/Kubera (Translated)"
|
|
))
|
|
CACHE_FILE = Path(os.environ.get(
|
|
"KUBERA_CACHE",
|
|
str(Path.home() / ".cache" / "kubera-sync.json")
|
|
))
|
|
|
|
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0"
|
|
|
|
|
|
# ── Reddit scanner ────────────────────────────────────────────────────────
|
|
|
|
def scan_reddit(sub=REDDIT_SUB, limit=100):
|
|
"""Scan subreddit RSS for [RAW] posts with Google Docs links.
|
|
|
|
Returns list of dicts: {chapter, season, title, gdoc_id, reddit_url}
|
|
"""
|
|
# old.reddit.com RSS is not datacenter-IP-blocked; www.reddit.com returns
|
|
# 429 from server IPs. Fall back across both.
|
|
url = f"https://old.reddit.com/r/{sub}/new/.rss?limit={limit}"
|
|
req = urllib.request.Request(url, headers={
|
|
"User-Agent": USER_AGENT,
|
|
"Accept": "text/xml, application/rss+xml",
|
|
})
|
|
|
|
# Retry with backoff (Reddit rate-limits)
|
|
xml_data = b""
|
|
for attempt in range(3):
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
xml_data = resp.read()
|
|
break
|
|
except urllib.error.HTTPError as e:
|
|
if e.code == 429 and attempt < 2:
|
|
time.sleep(5 * (attempt + 1))
|
|
continue
|
|
raise
|
|
|
|
root = ET.fromstring(xml_data)
|
|
ns = {"atom": "http://www.w3.org/2005/Atom"}
|
|
|
|
chapters = []
|
|
|
|
for entry in root.findall("atom:entry", ns):
|
|
title_el = entry.find("atom:title", ns)
|
|
title = title_el.text if title_el is not None else ""
|
|
|
|
content_el = entry.find("atom:content", ns)
|
|
content_html = content_el.text if content_el is not None else ""
|
|
|
|
link_el = entry.find("atom:link", ns)
|
|
reddit_url = link_el.get("href") if link_el is not None else ""
|
|
|
|
# Only [RAW] posts with chapter numbers
|
|
if not title or not title.lower().startswith("[raw]"):
|
|
continue
|
|
|
|
# Extract Google Doc ID
|
|
gdoc_match = re.search(
|
|
r"docs\.google\.com/document/d/([a-zA-Z0-9_-]+)",
|
|
content_html or ""
|
|
)
|
|
if not gdoc_match:
|
|
continue
|
|
|
|
gdoc_id = gdoc_match.group(1)
|
|
|
|
# Parse chapter number from title
|
|
# Format: "[RAW] Kubera S03 - 434: Finale (36)"
|
|
ch_match = re.search(r"S(\d+)\s*[-:]\s*(\d+)", title)
|
|
if ch_match:
|
|
season = int(ch_match.group(1))
|
|
chapter = int(ch_match.group(2))
|
|
else:
|
|
ch_match = re.search(r"(\d+)", title)
|
|
if ch_match:
|
|
chapter = int(ch_match.group(1))
|
|
season = 3
|
|
else:
|
|
continue
|
|
|
|
chapters.append({
|
|
"season": season,
|
|
"chapter": chapter,
|
|
"title": title,
|
|
"gdoc_id": gdoc_id,
|
|
"reddit_url": reddit_url,
|
|
})
|
|
|
|
# Sort by chapter number
|
|
chapters.sort(key=lambda c: c["chapter"])
|
|
return chapters
|
|
|
|
|
|
# ── Google Docs downloader ────────────────────────────────────────────────
|
|
|
|
def download_gdoc_images(gdoc_id):
|
|
"""Download images from a Google Doc via export HTML.
|
|
|
|
Returns list of (filename, bytes) tuples.
|
|
"""
|
|
url = f"https://docs.google.com/document/d/{gdoc_id}/export?format=html"
|
|
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
|
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
html = resp.read().decode("utf-8", errors="replace")
|
|
|
|
# Extract base64-encoded images
|
|
images = []
|
|
img_pattern = re.compile(r'src="(data:image/(png|jpeg|jpg);base64,([a-zA-Z0-9+/=]+))"')
|
|
|
|
for i, match in enumerate(img_pattern.finditer(html), 1):
|
|
mime = match.group(2)
|
|
b64_data = match.group(3)
|
|
ext = "png" if mime == "png" else "jpg"
|
|
img_bytes = base64.b64decode(b64_data)
|
|
filename = f"{i:03d}.{ext}"
|
|
images.append((filename, img_bytes))
|
|
|
|
return images
|
|
|
|
|
|
# ── CBZ packager ──────────────────────────────────────────────────────────
|
|
|
|
def package_cbz(images, output_path):
|
|
"""Package images into a CBZ file (which is just a ZIP)."""
|
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
|
for filename, img_bytes in sorted(images):
|
|
zf.writestr(filename, img_bytes)
|
|
return output_path
|
|
|
|
|
|
def chapter_filename(chapter_info):
|
|
"""Generate CBZ filename from chapter info."""
|
|
s = chapter_info["season"]
|
|
ch = chapter_info["chapter"]
|
|
return f"Kubera_S{s:02d}_{ch:03d}_translated.cbz"
|
|
|
|
|
|
# ── Cache ─────────────────────────────────────────────────────────────────
|
|
|
|
def load_cache():
|
|
if CACHE_FILE.exists():
|
|
with open(CACHE_FILE) as f:
|
|
return json.load(f)
|
|
return {"downloaded": {}}
|
|
|
|
|
|
def save_cache(cache):
|
|
CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(CACHE_FILE, "w") as f:
|
|
json.dump(cache, f, indent=2)
|
|
|
|
|
|
# ── Commands ──────────────────────────────────────────────────────────────
|
|
|
|
def cmd_scan(args):
|
|
"""Scan Reddit for available translations."""
|
|
print(f"Scanning /r/{REDDIT_SUB}...")
|
|
chapters = scan_reddit(limit=args.limit)
|
|
|
|
if not chapters:
|
|
print("No translation chapters found.")
|
|
return
|
|
|
|
# Check which are already downloaded
|
|
cache = load_cache()
|
|
|
|
print(f"\n{'Ch':>5} {'Status':8s} {'GDoc ID':30s} Title")
|
|
print("-" * 90)
|
|
for ch in chapters:
|
|
filename = chapter_filename(ch)
|
|
status = "✅ done" if str(ch["chapter"]) in cache["downloaded"] else "⬇ ready"
|
|
if (OUTPUT_DIR / filename).exists():
|
|
status = "✅ file"
|
|
print(f"S{ch['season']:02d}ch{ch['chapter']:03d} {status:8s} {ch['gdoc_id'][:30]:30s} {ch['title'][:40]}")
|
|
|
|
|
|
def cmd_download(args):
|
|
"""Download translations from Google Docs."""
|
|
chapters = scan_reddit(limit=args.limit)
|
|
cache = load_cache()
|
|
|
|
if args.chapter:
|
|
chapters = [c for c in chapters if c["chapter"] == args.chapter]
|
|
if not chapters:
|
|
print(f"Chapter {args.chapter} not found in /r/{REDDIT_SUB}")
|
|
return
|
|
|
|
print(f"Downloading {len(chapters)} chapters to {OUTPUT_DIR}...")
|
|
|
|
for ch in chapters:
|
|
filename = chapter_filename(ch)
|
|
output_path = OUTPUT_DIR / filename
|
|
|
|
if output_path.exists() and not args.force:
|
|
print(f" ⏭ S{ch['season']:02d}ch{ch['chapter']:03d} (already exists)")
|
|
continue
|
|
|
|
print(f" ⬇ S{ch['season']:02d}ch{ch['chapter']:03d} ({ch['gdoc_id'][:16]}...)")
|
|
|
|
try:
|
|
images = download_gdoc_images(ch["gdoc_id"])
|
|
if not images:
|
|
print(f" ⚠ No images found in doc")
|
|
continue
|
|
|
|
package_cbz(images, output_path)
|
|
size_mb = output_path.stat().st_size / (1024 * 1024)
|
|
print(f" ✅ {len(images)} pages → {filename} ({size_mb:.1f}MB)")
|
|
|
|
cache["downloaded"][str(ch["chapter"])] = {
|
|
"filename": filename,
|
|
"gdoc_id": ch["gdoc_id"],
|
|
"reddit_url": ch["reddit_url"],
|
|
"downloaded_at": time.time(),
|
|
}
|
|
save_cache(cache)
|
|
|
|
except Exception as e:
|
|
print(f" ❌ Error: {e}")
|
|
|
|
# Be polite to Google
|
|
time.sleep(2)
|
|
|
|
|
|
def cmd_sync(args):
|
|
"""Scan + download all missing chapters."""
|
|
cmd_scan(args)
|
|
print()
|
|
args.force = False
|
|
cmd_download(args)
|
|
|
|
|
|
# ── Main ──────────────────────────────────────────────────────────────────
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
prog="kubera-sync",
|
|
description="Download /r/kubera fan translations from Google Docs → CBZ"
|
|
)
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
|
|
p_scan = sub.add_parser("scan", help="Scan Reddit for available translations")
|
|
p_scan.add_argument("--limit", type=int, default=100)
|
|
p_scan.set_defaults(func=cmd_scan)
|
|
|
|
p_dl = sub.add_parser("download", help="Download translations")
|
|
p_dl.add_argument("--chapter", "-c", type=int, help="Specific chapter number")
|
|
p_dl.add_argument("--all", action="store_true", help="Download all found")
|
|
p_dl.add_argument("--force", action="store_true", help="Re-download even if exists")
|
|
p_dl.add_argument("--limit", type=int, default=100)
|
|
p_dl.set_defaults(func=cmd_download)
|
|
|
|
p_sync = sub.add_parser("sync", help="Scan + download missing chapters")
|
|
p_sync.add_argument("--limit", type=int, default=100)
|
|
p_sync.set_defaults(func=cmd_sync)
|
|
|
|
args = parser.parse_args()
|
|
args.func(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|