feat(docs): extend extraction script to support arbitrary base directories
- Add --base-dir argument to extract_pikeos_docs.py
- Preserve source path relative to the chosen base directory
- Fix Path('.') resolution so --base-dir . works correctly
This commit is contained in:
parent
480fbc9805
commit
0d101203e4
1 changed files with 18 additions and 10 deletions
|
|
@ -2,14 +2,14 @@
|
||||||
"""Extract PikeOS PDF manuals to markdown files.
|
"""Extract PikeOS PDF manuals to markdown files.
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
python3 tools/extract_pikeos_docs.py <pdf> [<pdf> ...]
|
python3 tools/extract_pikeos_docs.py [--base-dir <dir>] <pdf> [<pdf> ...]
|
||||||
|
|
||||||
Each PDF is converted to a markdown file under docs-extracted/ with the same
|
Each PDF is converted to a markdown file under docs-extracted/ preserving the
|
||||||
relative path and YAML frontmatter containing title, source, and page count.
|
relative path from <dir> (default: docs/). YAML frontmatter contains title,
|
||||||
|
source, category, and page count.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
|
@ -18,7 +18,6 @@ from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||||
DOCS_DIR = REPO_ROOT / "docs"
|
|
||||||
OUTPUT_DIR = REPO_ROOT / "docs-extracted"
|
OUTPUT_DIR = REPO_ROOT / "docs-extracted"
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -46,9 +45,9 @@ def count_pages(pdf_path: Path) -> int:
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def extract_pdf(pdf_path: Path) -> Path:
|
def extract_pdf(pdf_path: Path, base_dir: Path) -> Path:
|
||||||
"""Extract a single PDF to a markdown file."""
|
"""Extract a single PDF to a markdown file."""
|
||||||
rel_path = pdf_path.resolve().relative_to(DOCS_DIR.resolve())
|
rel_path = pdf_path.resolve().relative_to(base_dir.resolve())
|
||||||
out_path = OUTPUT_DIR / rel_path.with_suffix(".md")
|
out_path = OUTPUT_DIR / rel_path.with_suffix(".md")
|
||||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
|
@ -56,6 +55,9 @@ def extract_pdf(pdf_path: Path) -> Path:
|
||||||
pages = count_pages(pdf_path)
|
pages = count_pages(pdf_path)
|
||||||
category = rel_path.parent.as_posix() if rel_path.parent != Path(".") else "general"
|
category = rel_path.parent.as_posix() if rel_path.parent != Path(".") else "general"
|
||||||
|
|
||||||
|
source_prefix = base_dir.resolve().relative_to(REPO_ROOT).as_posix()
|
||||||
|
source_display = f"{source_prefix}/{rel_path.as_posix()}"
|
||||||
|
|
||||||
# Extract plain text with layout preservation.
|
# Extract plain text with layout preservation.
|
||||||
text = subprocess.run(
|
text = subprocess.run(
|
||||||
["pdftotext", "-layout", "-nopgbrk", str(pdf_path), "-"],
|
["pdftotext", "-layout", "-nopgbrk", str(pdf_path), "-"],
|
||||||
|
|
@ -69,7 +71,7 @@ def extract_pdf(pdf_path: Path) -> Path:
|
||||||
|
|
||||||
frontmatter = f"""---
|
frontmatter = f"""---
|
||||||
title: "{title}"
|
title: "{title}"
|
||||||
source: "docs/{rel_path.as_posix()}"
|
source: "{source_display}"
|
||||||
category: "{category}"
|
category: "{category}"
|
||||||
pages: {pages}
|
pages: {pages}
|
||||||
extracted: "{datetime.now().isoformat()}"
|
extracted: "{datetime.now().isoformat()}"
|
||||||
|
|
@ -77,7 +79,7 @@ extracted: "{datetime.now().isoformat()}"
|
||||||
|
|
||||||
# {title}
|
# {title}
|
||||||
|
|
||||||
> Extracted from `docs/{rel_path.as_posix()}` ({pages} pages).
|
> Extracted from `{source_display}` ({pages} pages).
|
||||||
> Figures, diagrams, and tables may not render accurately in plain text.
|
> Figures, diagrams, and tables may not render accurately in plain text.
|
||||||
|
|
||||||
{text}
|
{text}
|
||||||
|
|
@ -89,6 +91,12 @@ extracted: "{datetime.now().isoformat()}"
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser(description="Extract PikeOS PDFs to markdown.")
|
parser = argparse.ArgumentParser(description="Extract PikeOS PDFs to markdown.")
|
||||||
|
parser.add_argument(
|
||||||
|
"--base-dir",
|
||||||
|
type=Path,
|
||||||
|
default=REPO_ROOT / "docs",
|
||||||
|
help="Base directory for resolving PDF relative paths (default: docs/)",
|
||||||
|
)
|
||||||
parser.add_argument("pdfs", nargs="+", type=Path, help="PDF files to extract.")
|
parser.add_argument("pdfs", nargs="+", type=Path, help="PDF files to extract.")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
|
@ -99,7 +107,7 @@ def main() -> int:
|
||||||
errors.append(f"Not found: {pdf}")
|
errors.append(f"Not found: {pdf}")
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
out = extract_pdf(pdf)
|
out = extract_pdf(pdf, args.base_dir)
|
||||||
created.append(out)
|
created.append(out)
|
||||||
print(f"EXTRACTED: {out.relative_to(REPO_ROOT)}")
|
print(f"EXTRACTED: {out.relative_to(REPO_ROOT)}")
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue