diff --git a/tools/extract_pikeos_docs.py b/tools/extract_pikeos_docs.py new file mode 100644 index 000000000..18550701b --- /dev/null +++ b/tools/extract_pikeos_docs.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Extract PikeOS PDF manuals to markdown files. + +Usage: + python3 tools/extract_pikeos_docs.py [ ...] + +Each PDF is converted to a markdown file under docs-extracted/ with the same +relative path and YAML frontmatter containing title, source, and page count. +""" + +import argparse +import os +import re +import subprocess +import sys +from datetime import datetime +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parent.parent +DOCS_DIR = REPO_ROOT / "docs" +OUTPUT_DIR = REPO_ROOT / "docs-extracted" + + +def sanitize_title(filename: str) -> str: + """Convert a filename like 'kernel-reference-manual.pdf' to a title.""" + base = Path(filename).stem + base = base.replace("-", " ").replace("_", " ") + return base.title() + + +def count_pages(pdf_path: Path) -> int: + """Return the number of pages in a PDF using pdfinfo if available.""" + try: + result = subprocess.run( + ["pdfinfo", str(pdf_path)], + capture_output=True, + text=True, + check=True, + ) + for line in result.stdout.splitlines(): + if line.startswith("Pages:"): + return int(line.split(":", 1)[1].strip()) + except Exception: + pass + return 0 + + +def extract_pdf(pdf_path: Path) -> Path: + """Extract a single PDF to a markdown file.""" + rel_path = pdf_path.resolve().relative_to(DOCS_DIR.resolve()) + out_path = OUTPUT_DIR / rel_path.with_suffix(".md") + out_path.parent.mkdir(parents=True, exist_ok=True) + + title = sanitize_title(pdf_path.name) + pages = count_pages(pdf_path) + category = rel_path.parent.as_posix() if rel_path.parent != Path(".") else "general" + + # Extract plain text with layout preservation. + text = subprocess.run( + ["pdftotext", "-layout", "-nopgbrk", str(pdf_path), "-"], + capture_output=True, + text=True, + check=True, + ).stdout + + # Normalize whitespace slightly without destroying structure. + text = re.sub(r"\n{4,}", "\n\n\n", text) + + frontmatter = f"""--- +title: "{title}" +source: "docs/{rel_path.as_posix()}" +category: "{category}" +pages: {pages} +extracted: "{datetime.now().isoformat()}" +--- + +# {title} + +> Extracted from `docs/{rel_path.as_posix()}` ({pages} pages). +> Figures, diagrams, and tables may not render accurately in plain text. + +{text} +""" + + out_path.write_text(frontmatter, encoding="utf-8") + return out_path + + +def main() -> int: + parser = argparse.ArgumentParser(description="Extract PikeOS PDFs to markdown.") + parser.add_argument("pdfs", nargs="+", type=Path, help="PDF files to extract.") + args = parser.parse_args() + + created = [] + errors = [] + for pdf in args.pdfs: + if not pdf.exists(): + errors.append(f"Not found: {pdf}") + continue + try: + out = extract_pdf(pdf) + created.append(out) + print(f"EXTRACTED: {out.relative_to(REPO_ROOT)}") + except Exception as exc: + errors.append(f"{pdf}: {exc}") + print(f"ERROR: {pdf}: {exc}", file=sys.stderr) + + if errors: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main())