replica-omnisciente/infrastructure/fleet/pfctl.py

#!/usr/bin/env python3
"""
pfctl — Portugal Futurista unified fleet control CLI.

Single command interface for ALL fleet nodes:
  - Proxmox LXC containers (via pct on the hypervisor host)
  - Proxmox QEMU VMs (via qm)
  - Hetzner cloud VMs (via hcloud API)
  - Bare-metal hosts (via SSH)

All nodes treated equally. pfctl routes commands to the right backend.

Usage:
  pfctl ls                          # list all nodes
  pfctl ls --running                # only running nodes
  pfctl ls --host gigabyte          # filter by physical host
  pfctl ssh 223                     # SSH into CT 223
  pfctl ssh pf-aurelio              # SSH into named node
  pfctl exec 223 "uptime"           # run command on a node
  pfctl exec all "uname -r"         # run on every running node
  pfctl start 208                   # start a stopped CT/VM
  pfctl stop 223                    # stop a running CT/VM
  pfctl restart 217                 # restart
  pfctl status 223                  # detailed status of one node
  pfctl dashboard                   # live fleet overview (auto-refresh)
  pfctl cost                        # cloud cost summary
  pfctl update-inventory            # rescan Proxmox API + refresh fleet.yaml

The fleet inventory lives at infrastructure/fleet/fleet.yaml.
"""

import argparse
import json
import os
import subprocess
import sys
import time
from pathlib import Path

import yaml

# ── Constants ─────────────────────────────────────────────────────────────

FLEET_YAML = Path(__file__).resolve().parent.parent / "fleet" / "fleet.yaml"
PROXMOX_SSH_TARGET = "root@192.168.0.38"  # ASUS — cluster API node
PROXMOX_SSH_OPTS = [
    "-o", "StrictHostKeyChecking=no",
    "-o", "UserKnownHostsFile=/dev/null",
    "-o", "LogLevel=ERROR",
    "-o", "ConnectTimeout=10",
]
# Cloudflared tunnel proxy command for reaching ASUS from the laptop
PROXMOX_PROXY_CMD = 'cloudflared access ssh --hostname ssh.portugalfuturista.org'


# ── Inventory ─────────────────────────────────────────────────────────────

def load_fleet() -> dict:
    if not FLEET_YAML.exists():
        die(f"Fleet inventory not found: {FLEET_YAML}")
    with open(FLEET_YAML) as f:
        data = yaml.safe_load(f)
    # The YAML has a top-level `fleet:` key — unwrap it
    return data.get("fleet", data)


def all_nodes(fleet: dict) -> list[dict]:
    """Flatten all managed nodes (hosts + cloud + containers) into one list."""
    nodes = []
    for name, info in fleet.get("hosts", {}).items():
        nodes.append({"_key": name, "_type": "host", "name": name, **info})
    for name, info in fleet.get("cloud", {}).items():
        nodes.append({"_key": name, "_type": "cloud", "name": name, **info})
    for vmid, info in fleet.get("nodes", {}).items():
        nodes.append({"_key": vmid, "_type": info.get("provider", "?"),
                      "vmid": vmid, **info})
    return nodes


def find_node(fleet: dict, identifier: str) -> dict | None:
    """Find a node by VMID, name, or key."""
    for n in all_nodes(fleet):
        if identifier in (str(n.get("vmid", "")), n.get("name", ""),
                          n.get("_key", "")):
            return n
    return None


# ── Proxmox helpers ───────────────────────────────────────────────────────

def proxmox_ssh(cmd: str, timeout: int = 30) -> tuple[str, int]:
    """Run a command on the Proxmox cluster primary (ASUS) via SSH tunnel."""
    full_cmd = [
        "ssh", *PROXMOX_SSH_OPTS,
        "-o", f"ProxyCommand={PROXMOX_PROXY_CMD}",
        PROXMOX_SSH_TARGET, cmd
    ]
    r = subprocess.run(full_cmd, capture_output=True, text=True, timeout=timeout)
    return r.stdout + r.stderr, r.returncode


def proxmox_ct_cmd(vmid: str, action: str, node_host: str = None) -> str:
    """Execute a pct command. If node_host given, SSH to that host first."""
    if node_host and node_host != "asus":
        # Remote node — SSH chain through ASUS
        inner = f"ssh -o StrictHostKeyChecking=no root@{node_host} pct {action} {vmid}"
        return proxmox_ssh(inner)[0]
    return proxmox_ssh(f"pct {action} {vmid}")[0]


def proxmox_vm_cmd(vmid: str, action: str, node_host: str = None) -> str:
    if node_host and node_host != "asus":
        inner = f"ssh -o StrictHostKeyChecking=no root@{node_host} qm {action} {vmid}"
        return proxmox_ssh(inner)[0]
    return proxmox_ssh(f"qm {action} {vmid}")[0]


# ── Cloud helpers ─────────────────────────────────────────────────────────

def hcloud_cmd(args: list[str]) -> tuple[str, int]:
    r = subprocess.run(["hcloud"] + args, capture_output=True, text=True)
    return r.stdout + r.stderr, r.returncode


# ── Commands ──────────────────────────────────────────────────────────────

def cmd_ls(args):
    """List all fleet nodes."""
    fleet = load_fleet()
    nodes = all_nodes(fleet)

    if args.running:
        nodes = [n for n in nodes if n.get("status") == "running"]
    if args.host:
        nodes = [n for n in nodes if n.get("host") == args.host]
    if args.type:
        nodes = [n for n in nodes if n["_type"].startswith(args.type)]

    # Table
    print(f"{'ID':>4}  {'TYPE':12s} {'HOST':12s} {'STATUS':8s} {'NAME':30s} {'IP':16s}")
    print("-" * 90)
    for n in sorted(nodes, key=lambda x: str(x.get("host", x.get("_type", ""))) + str(x.get("vmid", x.get("_key", 0)))):
        vid = n.get("vmid", n.get("_key", ""))
        ntype = n["_type"]
        host = n.get("host", n.get("_type", ""))
        status = n.get("status", "?")
        name = n.get("name", "")
        ip = n.get("ip") or n.get("wireguard_ip") or ""
        if n.get("status") == "planned":
            status = "PLANNED"
        print(f"{str(vid):>4}  {ntype:12s} {host:12s} {status:8s} {name:30s} {ip:16s}")

    print(f"\n{len(nodes)} nodes")


def cmd_ssh(args):
    """SSH into a node."""
    fleet = load_fleet()
    node = find_node(fleet, args.node)
    if not node:
        die(f"Node not found: {args.node}")

    ip = node.get("ip") or node.get("wireguard_ip")
    if not ip:
        die(f"No IP for node {args.node}")

    if node["_type"] == "host":
        # Proxmox host — go through tunnel
        os.execvp("ssh", ["ssh", *PROXMOX_SSH_OPTS,
                          f"-o ProxyCommand={PROXMOX_PROXY_CMD}",
                          f"root@{ip}"])
    elif node["_type"] == "cloud" and node.get("provider") == "hetzner":
        os.execvp("ssh", ["ssh", "-o", "StrictHostKeyChecking=no", f"root@{ip}"])
    elif node["_type"] in ("proxmox-lxc", "proxmox-vm"):
        # CT/VM on LAN — SSH directly if on LAN, or through tunnel
        if is_on_lan():
            os.execvp("ssh", ["ssh", "-o", "StrictHostKeyChecking=no", f"root@{ip}"])
        else:
            os.execvp("ssh", ["ssh", *PROXMOX_SSH_OPTS,
                              f"-o ProxyCommand={PROXMOX_PROXY_CMD}",
                              PROXMOX_SSH_TARGET,
                              f"ssh -o StrictHostKeyChecking=no root@{ip}"])
    else:
        die(f"Don't know how to SSH into {node['_type']}")


def cmd_exec(args):
    """Execute a command on one or all nodes."""
    fleet = load_fleet()

    if args.node == "all":
        nodes = [n for n in all_nodes(fleet)
                 if n.get("status") == "running" and n.get("ip")]
        for n in nodes:
            result = _exec_on_node(n, args.command)
            label = n.get("name", n.get("vmid", n["_key"]))
            print(f"── {label} ({n.get('ip', '?')}) ──")
            print(result.rstrip())
            print()
    else:
        node = find_node(fleet, args.node)
        if not node:
            die(f"Node not found: {args.node}")
        print(_exec_on_node(node, args.command).rstrip())


def _exec_on_node(node: dict, command: str) -> str:
    """Run a shell command on a node, return stdout."""
    ip = node.get("ip") or node.get("wireguard_ip")
    if not ip:
        return "(no IP)"

    if node["_type"] in ("proxmox-lxc", "proxmox-vm", "host"):
        # Use pct/qm exec from the physical Proxmox host (no SSH needed into the CT).
        host = node.get("host", "asus")
        vmid = node.get("vmid")
        host_ip = {"asus": "192.168.0.38", "gigabyte": "192.168.0.104",
                   "dell": "192.168.0.41", "lattepanda": "192.168.0.200"}.get(host, "192.168.0.38")

        if node["_type"] == "proxmox-lxc":
            pct_cmd = f"pct exec {vmid} -- bash -c '{command}'"
        elif node["_type"] == "proxmox-vm":
            pct_cmd = f"qm guest exec {vmid} -- bash -c '{command}'"
        else:
            # Physical host
            pct_cmd = command

        if host == "asus":
            full_cmd = pct_cmd
        else:
            # Chain: laptop → ASUS tunnel → physical host → pct exec
            full_cmd = f"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=8 root@{host_ip} '{pct_cmd}'"

        try:
            r = subprocess.run(
                ["ssh", *PROXMOX_SSH_OPTS,
                 f"-o ProxyCommand={PROXMOX_PROXY_CMD}",
                 PROXMOX_SSH_TARGET, full_cmd],
                capture_output=True, text=True, timeout=25)
            return r.stdout + r.stderr
        except subprocess.TimeoutExpired:
            return "(timeout — node may be unreachable)"
    elif node["_type"] == "cloud":
        try:
            r = subprocess.run(["ssh", "-o", "StrictHostKeyChecking=no",
                                "-o", "ConnectTimeout=8",
                                f"root@{ip}", command],
                               capture_output=True, text=True, timeout=20)
            return r.stdout + r.stderr
        except subprocess.TimeoutExpired:
            return "(timeout)"
    return "(unsupported)"


def cmd_start(args):
    """Start a stopped node."""
    fleet = load_fleet()
    node = find_node(fleet, args.node)
    if not node:
        die(f"Node not found: {args.node}")

    if node["_type"] == "proxmox-lxc":
        out = proxmox_ct_cmd(node["vmid"], "start", node.get("host"))
        print(f"Started CT {node['vmid']} ({node.get('name', '')})")
    elif node["_type"] == "proxmox-vm":
        out = proxmox_vm_cmd(node["vmid"], "start", node.get("host"))
        print(f"Started VM {node['vmid']} ({node.get('name', '')})")
    elif node["_type"] == "cloud" and node.get("provider") == "hetzner":
        out, _ = hcloud_cmd(["server", "power-on", node["name"]])
        print(f"Started Hetzner VM {node['name']}")
    else:
        die(f"Cannot start {node['_type']}")


def cmd_stop(args):
    """Stop a running node."""
    fleet = load_fleet()
    node = find_node(fleet, args.node)
    if not node:
        die(f"Node not found: {args.node}")

    if node["_type"] == "proxmox-lxc":
        proxmox_ct_cmd(node["vmid"], "stop", node.get("host"))
        print(f"Stopped CT {node['vmid']} ({node.get('name', '')})")
    elif node["_type"] == "proxmox-vm":
        proxmox_vm_cmd(node["vmid"], "stop", node.get("host"))
        print(f"Stopped VM {node['vmid']} ({node.get('name', '')})")
    elif node["_type"] == "cloud" and node.get("provider") == "hetzner":
        hcloud_cmd(["server", "power-off", node["name"]])
        print(f"Stopped Hetzner VM {node['name']}")
    else:
        die(f"Cannot stop {node['_type']}")


def cmd_restart(args):
    """Restart a node."""
    fleet = load_fleet()
    node = find_node(fleet, args.node)
    if not node:
        die(f"Node not found: {args.node}")

    if node["_type"] == "proxmox-lxc":
        proxmox_ct_cmd(node["vmid"], "reboot", node.get("host"))
        print(f"Restarted CT {node['vmid']}")
    elif node["_type"] == "cloud" and node.get("provider") == "hetzner":
        hcloud_cmd(["server", "reboot", node["name"]])
        print(f"Restarted Hetzner VM {node['name']}")


def cmd_status(args):
    """Show detailed status of one node."""
    fleet = load_fleet()
    node = find_node(fleet, args.node)
    if not node:
        die(f"Node not found: {args.node}")

    print(f"  ID:       {node.get('vmid', node.get('_key', '?'))}")
    print(f"  Name:     {node.get('name', '?')}")
    print(f"  Type:     {node['_type']}")
    print(f"  Provider: {node.get('provider', '?')}")
    print(f"  Host:     {node.get('host', '?')}")
    print(f"  IP:       {node.get('ip', '?')}")
    if node.get("wireguard_ip"):
        print(f"  WG IP:    {node['wireguard_ip']}")
    print(f"  Status:   {node.get('status', '?')}")
    if node.get("specs"):
        s = node["specs"]
        print(f"  CPU:      {s.get('cores', '?')} cores")
        print(f"  RAM:      {s.get('ram_gb', '?')} GB")
        print(f"  Disk:     {s.get('disk_gb', '?')} GB")
    if node.get("cost_eur_month"):
        print(f"  Cost:     €{node['cost_eur_month']}/mo")
    if node.get("services"):
        print(f"  Services: {', '.join(node['services'])}")
    if node.get("notes"):
        print(f"  Notes:    {node['notes']}")

    # Live status if running
    if node.get("status") == "running" and node.get("ip"):
        print()
        uptime = _exec_on_node(node, "uptime 2>/dev/null || echo '(unreachable)'")
        print(f"  Live:     {uptime.strip()}")


def cmd_cost(args):
    """Show cloud cost summary."""
    fleet = load_fleet()
    total = 0
    print(f"{'Node':20s} {'Provider':10s} {'Type':8s} {'€/mo':>8s}")
    print("-" * 50)
    for n in all_nodes(fleet):
        cost = n.get("cost_eur_month")
        if cost:
            print(f"{n.get('name', n['_key']):20s} {n.get('provider','?'):10s} "
                  f"{n.get('type','?'):8s} €{cost:>7.2f}")
            total += cost
    print("-" * 50)
    print(f"{'TOTAL':20s} {'':10s} {'':8s} €{total:>7.2f}/mo")


def cmd_update_inventory(args):
    """Rescan Proxmox API and update fleet.yaml with live state."""
    print("Scanning Proxmox cluster...")
    out, rc = proxmox_ssh(
        "pvesh get /cluster/resources --type vm --output-format json 2>/dev/null"
    )
    if rc != 0:
        die(f"Failed to query Proxmox API: {out}")

    vms = json.loads(out)
    fleet = load_fleet()

    updated = 0
    for vm in vms:
        vmid = str(vm["vmid"])
        status = vm.get("status", "unknown")
        name = vm.get("name", "?")
        node = vm.get("node", "?")
        vtype = vm.get("type", "lxc")

        # Update existing entries
        if vmid in fleet.get("nodes", {}):
            entry = fleet["nodes"][vmid]
            entry["status"] = status
            entry["host"] = node
            if "provider" not in entry:
                entry["provider"] = f"proxmox-{vtype}"
            updated += 1
        else:
            # New CT/VM not in inventory — add it
            if "nodes" not in fleet:
                fleet["nodes"] = {}
            fleet["nodes"][vmid] = {
                "name": name,
                "host": node,
                "provider": f"proxmox-{vtype}",
                "specs": {
                    "cores": vm.get("cpus", 0),
                    "ram_gb": round(vm.get("maxmem", 0) / 1e9, 1),
                    "disk_gb": round(vm.get("maxdisk", 0) / 1e9, 0),
                },
                "status": status,
            }
            print(f"  + NEW: {vmid} {name} ({node}, {status})")
            updated += 1

    with open(FLEET_YAML, "w") as f:
        yaml.dump(fleet, f, default_flow_style=False, sort_keys=False, allow_unicode=True)

    print(f"\nUpdated {updated} entries in {FLEET_YAML}")


def cmd_dashboard(args):
    """Live fleet dashboard — auto-refreshing overview."""
    interval = args.refresh or 5
    try:
        while True:
            os.system("clear")
            fleet = load_fleet()
            nodes = all_nodes(fleet)
            running = [n for n in nodes if n.get("status") == "running"]
            stopped = [n for n in nodes if n.get("status") == "stopped"]
            planned = [n for n in nodes if n.get("status") == "planned"]
            cloud_cost = sum(n.get("cost_eur_month", 0) for n in nodes)

            print("╔════════════════════════════════════════════════════════════╗")
            print("║         PORTUGAL FUTURISTA — FLEET CONTROL                 ║")
            print("╠════════════════════════════════════════════════════════════╣")
            print(f"║  Nodes: {len(nodes):3d}  │  Running: {len(running):3d}  │  "
                  f"Stopped: {len(stopped):2d}  │  Planned: {len(planned):1d}  │  "
                  f"Cloud: €{cloud_cost:.2f}/mo  ║")
            print("╠══════╦════════════╦════════╦═══════════════╦═════════════╣")
            print("║  ID  │   Host     │ Status │    Name       │     IP      ║")
            print("╠══════╬════════════╬════════╬═══════════════╬═════════════╣")

            for n in sorted(nodes, key=lambda x: (x.get("host", ""),
                                                  x.get("vmid", 0))):
                vid = str(n.get("vmid", n.get("_key", "")))[:4]
                host = n.get("host", n.get("_type", ""))[:10]
                status = n.get("status", "?")[:8]
                name = n.get("name", "")[:13]
                ip = (n.get("ip") or n.get("wireguard_ip") or "")[:11]
                icon = "🟢" if status == "running" else "🔴" if status == "stopped" else "⚪"
                print(f"║ {icon}{vid:<3}│ {host:<10s} │ {status:<6s} │ {name:<13s} │ {ip:<11s} ║")

            print("╚══════╩════════════╩════════╩═══════════════╩═════════════╝")
            print(f"  Refresh: {interval}s | Ctrl+C to exit | {time.strftime('%H:%M:%S')}")

            if not args.no_refresh:
                time.sleep(interval)
            else:
                break
    except KeyboardInterrupt:
        print("\nExited.")


# ── Utilities ─────────────────────────────────────────────────────────────

def is_on_lan() -> bool:
    """Check if we're on the LAN (can reach 192.168.0.x directly)."""
    r = subprocess.run(["ip", "route", "get", "192.168.0.38"],
                       capture_output=True, text=True, timeout=2)
    return "192.168.0.38" in r.stdout and "via" not in r.stdout.split("\n")[0]


def die(msg: str):
    print(f"ERROR: {msg}", file=sys.stderr)
    sys.exit(1)


# ── Main ──────────────────────────────────────────────────────────────────

def main():
    parser = argparse.ArgumentParser(
        prog="pfctl",
        description="Portugal Futurista unified fleet control",
    )
    sub = parser.add_subparsers(dest="command", required=True)

    # ls
    p_ls = sub.add_parser("ls", help="List all fleet nodes")
    p_ls.add_argument("--running", action="store_true")
    p_ls.add_argument("--host", help="Filter by physical host")
    p_ls.add_argument("--type", help="Filter by provider type")
    p_ls.set_defaults(func=cmd_ls)

    # ssh
    p_ssh = sub.add_parser("ssh", help="SSH into a node")
    p_ssh.add_argument("node", help="VMID or name")
    p_ssh.set_defaults(func=cmd_ssh)

    # exec
    p_exec = sub.add_parser("exec", help="Execute command on node(s)")
    p_exec.add_argument("node", help="VMID, name, or 'all'")
    p_exec.add_argument("command", help="Shell command to run")
    p_exec.set_defaults(func=cmd_exec)

    # start / stop / restart
    for cmd_name, func in [("start", cmd_start), ("stop", cmd_stop),
                            ("restart", cmd_restart)]:
        p = sub.add_parser(cmd_name)
        p.add_argument("node", help="VMID or name")
        p.set_defaults(func=func)

    # status
    p_status = sub.add_parser("status", help="Detailed status of one node")
    p_status.add_argument("node", help="VMID or name")
    p_status.set_defaults(func=cmd_status)

    # cost
    p_cost = sub.add_parser("cost", help="Cloud cost summary")
    p_cost.set_defaults(func=cmd_cost)

    # update-inventory
    p_upd = sub.add_parser("update-inventory", help="Rescan Proxmox and update fleet.yaml")
    p_upd.set_defaults(func=cmd_update_inventory)

    # dashboard
    p_dash = sub.add_parser("dashboard", help="Live fleet overview")
    p_dash.add_argument("--refresh", type=int, default=5, help="Refresh interval (s)")
    p_dash.add_argument("--no-refresh", action="store_true", help="Print once and exit")
    p_dash.set_defaults(func=cmd_dashboard)

    args = parser.parse_args()
    args.func(args)


if __name__ == "__main__":
    main()