feat(docs): extend extraction script to support arbitrary base directories

- Add --base-dir argument to extract_pikeos_docs.py
- Preserve source path relative to the chosen base directory
- Fix Path('.') resolution so --base-dir . works correctly
This commit is contained in:
Fábio Coutada 2026-07-06 23:16:27 +01:00
parent 480fbc9805
commit 0d101203e4

View file

@ -2,14 +2,14 @@
"""Extract PikeOS PDF manuals to markdown files. """Extract PikeOS PDF manuals to markdown files.
Usage: Usage:
python3 tools/extract_pikeos_docs.py <pdf> [<pdf> ...] python3 tools/extract_pikeos_docs.py [--base-dir <dir>] <pdf> [<pdf> ...]
Each PDF is converted to a markdown file under docs-extracted/ with the same Each PDF is converted to a markdown file under docs-extracted/ preserving the
relative path and YAML frontmatter containing title, source, and page count. relative path from <dir> (default: docs/). YAML frontmatter contains title,
source, category, and page count.
""" """
import argparse import argparse
import os
import re import re
import subprocess import subprocess
import sys import sys
@ -18,7 +18,6 @@ from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parent.parent REPO_ROOT = Path(__file__).resolve().parent.parent
DOCS_DIR = REPO_ROOT / "docs"
OUTPUT_DIR = REPO_ROOT / "docs-extracted" OUTPUT_DIR = REPO_ROOT / "docs-extracted"
@ -46,9 +45,9 @@ def count_pages(pdf_path: Path) -> int:
return 0 return 0
def extract_pdf(pdf_path: Path) -> Path: def extract_pdf(pdf_path: Path, base_dir: Path) -> Path:
"""Extract a single PDF to a markdown file.""" """Extract a single PDF to a markdown file."""
rel_path = pdf_path.resolve().relative_to(DOCS_DIR.resolve()) rel_path = pdf_path.resolve().relative_to(base_dir.resolve())
out_path = OUTPUT_DIR / rel_path.with_suffix(".md") out_path = OUTPUT_DIR / rel_path.with_suffix(".md")
out_path.parent.mkdir(parents=True, exist_ok=True) out_path.parent.mkdir(parents=True, exist_ok=True)
@ -56,6 +55,9 @@ def extract_pdf(pdf_path: Path) -> Path:
pages = count_pages(pdf_path) pages = count_pages(pdf_path)
category = rel_path.parent.as_posix() if rel_path.parent != Path(".") else "general" category = rel_path.parent.as_posix() if rel_path.parent != Path(".") else "general"
source_prefix = base_dir.resolve().relative_to(REPO_ROOT).as_posix()
source_display = f"{source_prefix}/{rel_path.as_posix()}"
# Extract plain text with layout preservation. # Extract plain text with layout preservation.
text = subprocess.run( text = subprocess.run(
["pdftotext", "-layout", "-nopgbrk", str(pdf_path), "-"], ["pdftotext", "-layout", "-nopgbrk", str(pdf_path), "-"],
@ -69,7 +71,7 @@ def extract_pdf(pdf_path: Path) -> Path:
frontmatter = f"""--- frontmatter = f"""---
title: "{title}" title: "{title}"
source: "docs/{rel_path.as_posix()}" source: "{source_display}"
category: "{category}" category: "{category}"
pages: {pages} pages: {pages}
extracted: "{datetime.now().isoformat()}" extracted: "{datetime.now().isoformat()}"
@ -77,7 +79,7 @@ extracted: "{datetime.now().isoformat()}"
# {title} # {title}
> Extracted from `docs/{rel_path.as_posix()}` ({pages} pages). > Extracted from `{source_display}` ({pages} pages).
> Figures, diagrams, and tables may not render accurately in plain text. > Figures, diagrams, and tables may not render accurately in plain text.
{text} {text}
@ -89,6 +91,12 @@ extracted: "{datetime.now().isoformat()}"
def main() -> int: def main() -> int:
parser = argparse.ArgumentParser(description="Extract PikeOS PDFs to markdown.") parser = argparse.ArgumentParser(description="Extract PikeOS PDFs to markdown.")
parser.add_argument(
"--base-dir",
type=Path,
default=REPO_ROOT / "docs",
help="Base directory for resolving PDF relative paths (default: docs/)",
)
parser.add_argument("pdfs", nargs="+", type=Path, help="PDF files to extract.") parser.add_argument("pdfs", nargs="+", type=Path, help="PDF files to extract.")
args = parser.parse_args() args = parser.parse_args()
@ -99,7 +107,7 @@ def main() -> int:
errors.append(f"Not found: {pdf}") errors.append(f"Not found: {pdf}")
continue continue
try: try:
out = extract_pdf(pdf) out = extract_pdf(pdf, args.base_dir)
created.append(out) created.append(out)
print(f"EXTRACTED: {out.relative_to(REPO_ROOT)}") print(f"EXTRACTED: {out.relative_to(REPO_ROOT)}")
except Exception as exc: except Exception as exc: