feat(docs): add PikeOS PDF extraction helper script
- Add tools/extract_pikeos_docs.py to convert PDFs to markdown - Generates YAML frontmatter with title, source, category, and page count - Uses pdftotext for reliable plain-text extraction
This commit is contained in:
parent
bafa872415
commit
aa516bded6
1 changed files with 115 additions and 0 deletions
115
tools/extract_pikeos_docs.py
Normal file
115
tools/extract_pikeos_docs.py
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Extract PikeOS PDF manuals to markdown files.
|
||||
|
||||
Usage:
|
||||
python3 tools/extract_pikeos_docs.py <pdf> [<pdf> ...]
|
||||
|
||||
Each PDF is converted to a markdown file under docs-extracted/ with the same
|
||||
relative path and YAML frontmatter containing title, source, and page count.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
DOCS_DIR = REPO_ROOT / "docs"
|
||||
OUTPUT_DIR = REPO_ROOT / "docs-extracted"
|
||||
|
||||
|
||||
def sanitize_title(filename: str) -> str:
|
||||
"""Convert a filename like 'kernel-reference-manual.pdf' to a title."""
|
||||
base = Path(filename).stem
|
||||
base = base.replace("-", " ").replace("_", " ")
|
||||
return base.title()
|
||||
|
||||
|
||||
def count_pages(pdf_path: Path) -> int:
|
||||
"""Return the number of pages in a PDF using pdfinfo if available."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pdfinfo", str(pdf_path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
for line in result.stdout.splitlines():
|
||||
if line.startswith("Pages:"):
|
||||
return int(line.split(":", 1)[1].strip())
|
||||
except Exception:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
def extract_pdf(pdf_path: Path) -> Path:
|
||||
"""Extract a single PDF to a markdown file."""
|
||||
rel_path = pdf_path.resolve().relative_to(DOCS_DIR.resolve())
|
||||
out_path = OUTPUT_DIR / rel_path.with_suffix(".md")
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
title = sanitize_title(pdf_path.name)
|
||||
pages = count_pages(pdf_path)
|
||||
category = rel_path.parent.as_posix() if rel_path.parent != Path(".") else "general"
|
||||
|
||||
# Extract plain text with layout preservation.
|
||||
text = subprocess.run(
|
||||
["pdftotext", "-layout", "-nopgbrk", str(pdf_path), "-"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
).stdout
|
||||
|
||||
# Normalize whitespace slightly without destroying structure.
|
||||
text = re.sub(r"\n{4,}", "\n\n\n", text)
|
||||
|
||||
frontmatter = f"""---
|
||||
title: "{title}"
|
||||
source: "docs/{rel_path.as_posix()}"
|
||||
category: "{category}"
|
||||
pages: {pages}
|
||||
extracted: "{datetime.now().isoformat()}"
|
||||
---
|
||||
|
||||
# {title}
|
||||
|
||||
> Extracted from `docs/{rel_path.as_posix()}` ({pages} pages).
|
||||
> Figures, diagrams, and tables may not render accurately in plain text.
|
||||
|
||||
{text}
|
||||
"""
|
||||
|
||||
out_path.write_text(frontmatter, encoding="utf-8")
|
||||
return out_path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Extract PikeOS PDFs to markdown.")
|
||||
parser.add_argument("pdfs", nargs="+", type=Path, help="PDF files to extract.")
|
||||
args = parser.parse_args()
|
||||
|
||||
created = []
|
||||
errors = []
|
||||
for pdf in args.pdfs:
|
||||
if not pdf.exists():
|
||||
errors.append(f"Not found: {pdf}")
|
||||
continue
|
||||
try:
|
||||
out = extract_pdf(pdf)
|
||||
created.append(out)
|
||||
print(f"EXTRACTED: {out.relative_to(REPO_ROOT)}")
|
||||
except Exception as exc:
|
||||
errors.append(f"{pdf}: {exc}")
|
||||
print(f"ERROR: {pdf}: {exc}", file=sys.stderr)
|
||||
|
||||
if errors:
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Reference in a new issue