Fábio Coutada
aa516bded6
- Add tools/extract_pikeos_docs.py to convert PDFs to markdown - Generates YAML frontmatter with title, source, category, and page count - Uses pdftotext for reliable plain-text extraction
115 lines
3.1 KiB
Python
115 lines
3.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract PikeOS PDF manuals to markdown files.
|
|
|
|
Usage:
|
|
python3 tools/extract_pikeos_docs.py <pdf> [<pdf> ...]
|
|
|
|
Each PDF is converted to a markdown file under docs-extracted/ with the same
|
|
relative path and YAML frontmatter containing title, source, and page count.
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
DOCS_DIR = REPO_ROOT / "docs"
|
|
OUTPUT_DIR = REPO_ROOT / "docs-extracted"
|
|
|
|
|
|
def sanitize_title(filename: str) -> str:
|
|
"""Convert a filename like 'kernel-reference-manual.pdf' to a title."""
|
|
base = Path(filename).stem
|
|
base = base.replace("-", " ").replace("_", " ")
|
|
return base.title()
|
|
|
|
|
|
def count_pages(pdf_path: Path) -> int:
|
|
"""Return the number of pages in a PDF using pdfinfo if available."""
|
|
try:
|
|
result = subprocess.run(
|
|
["pdfinfo", str(pdf_path)],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
for line in result.stdout.splitlines():
|
|
if line.startswith("Pages:"):
|
|
return int(line.split(":", 1)[1].strip())
|
|
except Exception:
|
|
pass
|
|
return 0
|
|
|
|
|
|
def extract_pdf(pdf_path: Path) -> Path:
|
|
"""Extract a single PDF to a markdown file."""
|
|
rel_path = pdf_path.resolve().relative_to(DOCS_DIR.resolve())
|
|
out_path = OUTPUT_DIR / rel_path.with_suffix(".md")
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
title = sanitize_title(pdf_path.name)
|
|
pages = count_pages(pdf_path)
|
|
category = rel_path.parent.as_posix() if rel_path.parent != Path(".") else "general"
|
|
|
|
# Extract plain text with layout preservation.
|
|
text = subprocess.run(
|
|
["pdftotext", "-layout", "-nopgbrk", str(pdf_path), "-"],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
).stdout
|
|
|
|
# Normalize whitespace slightly without destroying structure.
|
|
text = re.sub(r"\n{4,}", "\n\n\n", text)
|
|
|
|
frontmatter = f"""---
|
|
title: "{title}"
|
|
source: "docs/{rel_path.as_posix()}"
|
|
category: "{category}"
|
|
pages: {pages}
|
|
extracted: "{datetime.now().isoformat()}"
|
|
---
|
|
|
|
# {title}
|
|
|
|
> Extracted from `docs/{rel_path.as_posix()}` ({pages} pages).
|
|
> Figures, diagrams, and tables may not render accurately in plain text.
|
|
|
|
{text}
|
|
"""
|
|
|
|
out_path.write_text(frontmatter, encoding="utf-8")
|
|
return out_path
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Extract PikeOS PDFs to markdown.")
|
|
parser.add_argument("pdfs", nargs="+", type=Path, help="PDF files to extract.")
|
|
args = parser.parse_args()
|
|
|
|
created = []
|
|
errors = []
|
|
for pdf in args.pdfs:
|
|
if not pdf.exists():
|
|
errors.append(f"Not found: {pdf}")
|
|
continue
|
|
try:
|
|
out = extract_pdf(pdf)
|
|
created.append(out)
|
|
print(f"EXTRACTED: {out.relative_to(REPO_ROOT)}")
|
|
except Exception as exc:
|
|
errors.append(f"{pdf}: {exc}")
|
|
print(f"ERROR: {pdf}: {exc}", file=sys.stderr)
|
|
|
|
if errors:
|
|
return 1
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|