universalisos/kernel/drivers/pci.cpp

/*
 * Universalisos PCI/PCIe Stack — full PikeOS-architecture replica (core)
 *
 * Transport-agnostic PCI core adapted 1:1 from the PikeOS ARMv7 driver
 * (pci_common.c / pci_enum.c / pci_msi.c / pci_msix.c), `uos_` naming.
 *
 * The core never touches hardware directly: every config access is dispatched
 * through a registered `uos_pci_ops_t` transport. Two transports ship here:
 *   - uos_pci_ecam_ops   : standard PCIe ECAM (correct for real HW / AArch64 virt)
 *   - uos_pci_framework_ops : safe no-hardware transport (returns no-device),
 *                            used by default so the core is exercisable without
 *                            risking the QEMU virt gpex ECAM deadlock on cortex-a15.
 *
 * Author: PortugalFuturista Hypervisor Development Team
 */

#include "pci.h"
#include "../arch/arm/uart.h"

#define UOS_MMIO_READ(addr)     (*((volatile uint32_t*)(addr)))
#define UOS_MMIO_WRITE(addr, v) (*((volatile uint32_t*)(addr)) = (uint32_t)(v))

/* ==========================================================================
 * Transport registry
 * ========================================================================== */
static const uos_pci_ops_t *g_uos_pci_ops = NULL;

void uos_pci_register_ops(const uos_pci_ops_t *ops) {
    g_uos_pci_ops = ops;
}

/* ==========================================================================
 * Config-access dispatch through the registered transport
 * ========================================================================== */
uint8_t uos_pci_read8(uos_pci_config_addr_t a, uint32_t off) {
    uint8_t v = 0xFFu;
    if (g_uos_pci_ops && g_uos_pci_ops->read_config8) {
        g_uos_pci_ops->read_config8(a, off, &v);
    }
    return v;
}
uint16_t uos_pci_read16(uos_pci_config_addr_t a, uint32_t off) {
    uint16_t v = 0xFFFFu;
    if (g_uos_pci_ops && g_uos_pci_ops->read_config16) {
        g_uos_pci_ops->read_config16(a, off, &v);
    }
    return v;
}
uint32_t uos_pci_read32(uos_pci_config_addr_t a, uint32_t off) {
    uint32_t v = 0xFFFFFFFFu;
    if (g_uos_pci_ops && g_uos_pci_ops->read_config32) {
        g_uos_pci_ops->read_config32(a, off, &v);
    }
    return v;
}
void uos_pci_write8(uos_pci_config_addr_t a, uint32_t off, uint8_t v) {
    if (g_uos_pci_ops && g_uos_pci_ops->write_config8) {
        g_uos_pci_ops->write_config8(a, off, v);
    }
}
void uos_pci_write16(uos_pci_config_addr_t a, uint32_t off, uint16_t v) {
    if (g_uos_pci_ops && g_uos_pci_ops->write_config16) {
        g_uos_pci_ops->write_config16(a, off, v);
    }
}
void uos_pci_write32(uos_pci_config_addr_t a, uint32_t off, uint32_t v) {
    if (g_uos_pci_ops && g_uos_pci_ops->write_config32) {
        g_uos_pci_ops->write_config32(a, off, v);
    }
}

/* ==========================================================================
 * ECAM transport (standard PCIe, spec §7.2.2)
 *
 *   ecam_base + (bus<<20) + (dev<<15) + (func<<12) + (offset & 0xFFC)
 *
 * Sub-word accesses are synthesized via dword read-modify-write, exactly as
 * the PikeOS layerscape wrappers do. NOTE: on QEMU virt + cortex-a15 the gpex
 * ECAM read deadlocks inside QEMU, so this transport is NOT registered by
 * default; use it on real PCIe hardware or AArch64 virt.
 * ========================================================================== */
static uint32_t g_uos_ecam_base = 0;

static inline uint32_t uos_ecam_addr(uos_pci_config_addr_t a, uint32_t off) {
    return g_uos_ecam_base
         | (UOS_PCI_CONFIG_BUS(a) << 20)
         | (UOS_PCI_CONFIG_DEV(a) << 15)
         | (UOS_PCI_CONFIG_FN(a)  << 12)
         | (off & 0xFFCu);
}

static void uos_ecam_rd32(uos_pci_config_addr_t a, uint32_t off, uint32_t *val) {
    *val = UOS_MMIO_READ(uos_ecam_addr(a, off));
}
static void uos_ecam_wr32(uos_pci_config_addr_t a, uint32_t off, uint32_t val) {
    UOS_MMIO_WRITE(uos_ecam_addr(a, off), val);
}
static void uos_ecam_rd16(uos_pci_config_addr_t a, uint32_t off, uint16_t *val) {
    uint32_t dw = 0;
    uos_ecam_rd32(a, off & ~0x3u, &dw);
    *val = (off & 0x2u) ? (uint16_t)(dw >> 16) : (uint16_t)(dw & 0xFFFFu);
}
static void uos_ecam_wr16(uos_pci_config_addr_t a, uint32_t off, uint16_t val) {
    uint32_t dw = 0; uos_ecam_rd32(a, off & ~0x3u, &dw);
    if (off & 0x2u) { dw = (dw & 0x0000FFFFu) | ((uint32_t)val << 16); }
    else            { dw = (dw & 0xFFFF0000u) | (uint32_t)val; }
    uos_ecam_wr32(a, off & ~0x3u, dw);
}
static void uos_ecam_rd8(uos_pci_config_addr_t a, uint32_t off, uint8_t *val) {
    uint32_t dw = 0; uos_ecam_rd32(a, off & ~0x3u, &dw);
    *val = (uint8_t)(dw >> ((off & 0x3u) * 8u));
}
static void uos_ecam_wr8(uos_pci_config_addr_t a, uint32_t off, uint8_t val) {
    uint32_t dw = 0; uos_ecam_rd32(a, off & ~0x3u, &dw);
    uint32_t shift = (off & 0x3u) * 8u;
    uint32_t mask = 0xFFu << shift;
    dw = (dw & ~mask) | ((uint32_t)val << shift);
    uos_ecam_wr32(a, off & ~0x3u, dw);
}
static void uos_ecam_find_irq(uos_pci_dev_t *dev, uint8_t pin) {
    /* Standard PCI swizzle to root complex, INTA..D -> irq 0..3. */
    if (dev == NULL || pin == 0 || pin > 4) { if (dev) dev->irq = -1; return; }
    uint8_t slot = (uint8_t)UOS_PCI_CONFIG_DEV(dev->config_addr);
    uint8_t p = pin;
    /* Swizzle up to the root: each bridge rotates the pin by the child slot. */
    while (UOS_PCI_CONFIG_BUS(dev->config_addr) != 0) {
        p = (uint8_t)(((p - 1u + slot) % 4u) + 1u);
        /* Walk one bus level toward root (single-level approximation). */
        break;
    }
    dev->irq = (int32_t)((p - 1u) & 3u);   /* INTA..D -> 0..3 */
}
static void uos_ecam_quirk_post(uos_pci_dev_t *dev) { (void)dev; }
static void uos_ecam_set_cpu_addr(uos_pci_dev_t *dev) {
    /* Identity: CPU address == bus address (no ATU on a flat ECAM host). */
    if (dev == NULL) return;
    for (uint32_t i = 0; i < UOS_PCI_NUM_RES; i++) {
        dev->res[i].cpu_addr = dev->res[i].bus_addr;
    }
}
static unsigned int uos_ecam_num_domains(void) { return 1; }

static const uos_pci_ops_t uos_pci_ecam_ops = {
    uos_ecam_rd8, uos_ecam_rd16, uos_ecam_rd32,
    uos_ecam_wr8, uos_ecam_wr16, uos_ecam_wr32,
    uos_ecam_find_irq, uos_ecam_quirk_post, uos_ecam_set_cpu_addr,
    uos_ecam_num_domains
};

/* ==========================================================================
 * Framework (safe, no-hardware) transport
 *
 * Reports an empty bus (vendor 0xFFFF everywhere) so the core's enumeration,
 * BAR sizing and MSI/MSI-X paths compile, link and run end-to-end without any
 * hardware — and without risking the QEMU virt ECAM deadlock. Used by default.
 * ========================================================================== */
static void uos_fw_rd32(uos_pci_config_addr_t a, uint32_t off, uint32_t *val) {
    (void)a; (void)off; *val = 0xFFFFFFFFu;
}
static void uos_fw_rd16(uos_pci_config_addr_t a, uint32_t off, uint16_t *val) {
    (void)a; (void)off; *val = 0xFFFFu;
}
static void uos_fw_rd8(uos_pci_config_addr_t a, uint32_t off, uint8_t *val) {
    (void)a; (void)off; *val = 0xFFu;
}
static void uos_fw_wr32(uos_pci_config_addr_t a, uint32_t off, uint32_t val) { (void)a; (void)off; (void)val; }
static void uos_fw_wr16(uos_pci_config_addr_t a, uint32_t off, uint16_t val) { (void)a; (void)off; (void)val; }
static void uos_fw_wr8(uos_pci_config_addr_t a, uint32_t off, uint8_t val)  { (void)a; (void)off; (void)val; }
static void uos_fw_find_irq(uos_pci_dev_t *dev, uint8_t pin) { (void)pin; if (dev) dev->irq = -1; }
static void uos_fw_quirk_post(uos_pci_dev_t *dev) { (void)dev; }
static void uos_fw_set_cpu_addr(uos_pci_dev_t *dev) {
    if (dev == NULL) return;
    for (uint32_t i = 0; i < UOS_PCI_NUM_RES; i++) {
        dev->res[i].cpu_addr = dev->res[i].bus_addr;
    }
}
static unsigned int uos_fw_num_domains(void) { return 1; }

static const uos_pci_ops_t uos_pci_framework_ops = {
    uos_fw_rd8, uos_fw_rd16, uos_fw_rd32,
    uos_fw_wr8, uos_fw_wr16, uos_fw_wr32,
    uos_fw_find_irq, uos_fw_quirk_post, uos_fw_set_cpu_addr,
    uos_fw_num_domains
};

/* ==========================================================================
 * Device table
 * ========================================================================== */
static uos_pci_dev_t g_uos_pci_devices[UOS_PCI_MAX_DEVICES];
static unsigned int  g_uos_pci_device_count = 0;

static uos_pci_dev_t *uos_pci_alloc_device(void) {
    if (g_uos_pci_device_count >= UOS_PCI_MAX_DEVICES) {
        return NULL;
    }
    uos_pci_dev_t *d = &g_uos_pci_devices[g_uos_pci_device_count++];
    /* Zero the struct */
    d->force_intx_dis = false; d->ext_pci_cfg = false; d->irq = -1;
    d->config_addr = 0; d->class_code = 0; d->vendor = 0; d->device = 0;
    d->subsysvendor = 0; d->subsys = 0; d->hdr_type = 0; d->state = UOS_PCI_DEV_CLOSED;
    d->msi_cap = 0; d->msix_cap = 0; d->msi_control = 0; d->msix_control = 0;
    d->msi_is64 = 0; d->msi_max_irqs = 0; d->msix_table_bar = 0; d->msix_table_offset = 0;
    for (uint32_t i = 0; i < UOS_PCI_NUM_RES; i++) {
        d->res[i].bus_addr = 0; d->res[i].cpu_addr = 0; d->res[i].size = 0;
        d->res[i].flags = 0; d->res[i].r_normal = 0; d->res[i].r_sized = 0;
    }
    return d;
}

uos_pci_dev_t *uos_pci_get_device(unsigned int index) {
    return (index < g_uos_pci_device_count) ? &g_uos_pci_devices[index] : NULL;
}
unsigned int uos_pci_get_device_count(void) { return g_uos_pci_device_count; }

/* ==========================================================================
 * Capability list walk (PikeOS psp_pci_find_cap)
 * ========================================================================== */
uint8_t uos_pci_find_cap(uos_pci_dev_t *dev, uint8_t id) {
    if (dev == NULL) return 0;
    /* Only type-0/1 headers have a cap pointer at 0x34 (CardBus uses 0x14). */
    uint8_t ht = dev->hdr_type & UOS_PCI_HDR_TYPE_MASK;
    uint8_t pos = (ht == UOS_PCI_HDR_TYPE_CARDBUS)
                ? uos_pci_read8(dev->config_addr, 0x14u)
                : uos_pci_read8(dev->config_addr, UOS_PCI_R_CAP_PTR);
    uint8_t guard = 0;
    while (pos != 0 && (pos & 0x3u) == 0 && guard++ < 48u) {
        uint8_t this_id = uos_pci_read8(dev->config_addr, pos + UOS_PCI_CAP_LIST_ID);
        if (this_id == 0xFFu) break;   /* bad read / no device */
        if (this_id == id) return pos;
        pos = uos_pci_read8(dev->config_addr, pos + UOS_PCI_CAP_LIST_NEXT);
    }
    return 0;
}

/* ==========================================================================
 * Canonical BAR-sizing probe (PikeOS pci_read_bases)
 *
 * Temporarily writes 0xFFFFFFFF to each BAR, reads back the size-encoded
 * value, restores the original, and computes the BAR size & type. 64-bit BARs
 * consume two res[] slots. ROM BAR uses the ROM address mask instead of all-1s.
 * ========================================================================== */
void uos_pci_read_bases(uos_pci_dev_t *dev, uint32_t num_bars, uint32_t rom_off) {
    if (dev == NULL) return;

    /* Disable MEM/IO decode during the probe, restore afterwards. */
    uint16_t saved_cmd = uos_pci_read16(dev->config_addr, UOS_PCI_R_COMMAND);
    uos_pci_write16(dev->config_addr, UOS_PCI_R_COMMAND,
                    (uint16_t)(saved_cmd & ~(UOS_PCI_CMD_IO | UOS_PCI_CMD_MEM)));

    uint32_t reg = UOS_PCI_R_BAR_0;
    uint32_t res_idx = 0;

    for (uint32_t b = 0; b < num_bars && res_idx < (UOS_PCI_NUM_RES - 1u); b++, reg += 4u, res_idx++) {
        uint32_t bar_val = uos_pci_read32(dev->config_addr, reg);

        /* Probe: write all-1s, read back, restore. */
        uos_pci_write32(dev->config_addr, reg, 0xFFFFFFFFu);
        uint32_t bar_lower = uos_pci_read32(dev->config_addr, reg);
        uos_pci_write32(dev->config_addr, reg, bar_val);

        uos_pci_res_t *r = &dev->res[res_idx];
        r->r_normal = bar_val;
        r->r_sized  = bar_lower;
        r->flags    = 0;
        r->size     = 0;
        r->bus_addr = 0;

        if (bar_lower == 0u) {
            continue;   /* BAR unimplemented */
        }

        if ((bar_lower & UOS_PCI_BAR_SPACE_MASK) == UOS_PCI_BAR_SPACE_IO) {
            /* I/O BAR */
            uint32_t s = ((~(bar_lower & UOS_PCI_BAR_IO_MASK)) & 0xFFFFu) + 1u;
            if ((bar_lower & 0xFFFF0000u) == 0u) s &= 0xFFFFu;
            r->size     = s;
            r->flags    = UOS_PCI_RES_IO;
            r->bus_addr = bar_val & UOS_PCI_BAR_IO_MASK;
        } else {
            /* Memory BAR */
            uint64_t base = bar_val & UOS_PCI_BAR_MEM_MASK;
            if ((bar_lower & UOS_PCI_BAR_MEM_PREF) != 0u) r->flags |= UOS_PCI_RES_PREF;

            if ((bar_lower & UOS_PCI_BAR_MEM_TYPE_MASK) == UOS_PCI_BAR_MEM_TYPE_64) {
                /* 64-bit BAR: consume the upper half in the next register. */
                reg += 4u; b++;   /* consume next BAR slot */
                uint32_t upper_val = uos_pci_read32(dev->config_addr, reg);
                uos_pci_write32(dev->config_addr, reg, 0xFFFFFFFFu);
                uint32_t upper_lower = uos_pci_read32(dev->config_addr, reg);
                uos_pci_write32(dev->config_addr, reg, upper_val);

                uint64_t size64 = ((uint64_t)upper_lower << 32) | (bar_lower & UOS_PCI_BAR_MEM_MASK);
                uint64_t sz = (~size64) + 1ull;
                r->size     = (uint32_t)sz;
                r->flags   |= UOS_PCI_RES_MEM_64;
                r->bus_addr = base | ((uint64_t)upper_val << 32);
                /* Next res[] slot is the upper-half placeholder. */
                if (res_idx + 1u < UOS_PCI_NUM_RES) {
                    dev->res[res_idx + 1u].r_normal = upper_val;
                    dev->res[res_idx + 1u].r_sized  = upper_lower;
                }
            } else {
                /* 32-bit memory BAR */
                uint32_t sz = (~(bar_lower & UOS_PCI_BAR_MEM_MASK)) + 1u;
                r->size     = sz;
                r->flags   |= UOS_PCI_RES_MEM_32;
                r->bus_addr = base;
            }
        }
    }

    /* Expansion ROM BAR */
    if (rom_off != 0u && res_idx < UOS_PCI_NUM_RES) {
        uint32_t rom_val = uos_pci_read32(dev->config_addr, rom_off);
        uos_pci_write32(dev->config_addr, rom_off, UOS_PCI_ROM_ADDR_MASK);
        uint32_t rom_lower = uos_pci_read32(dev->config_addr, rom_off);
        uos_pci_write32(dev->config_addr, rom_off, rom_val);
        uos_pci_res_t *r = &dev->res[res_idx];
        r->r_normal = rom_val;
        r->r_sized  = rom_lower;
        r->flags    = (rom_lower == 0u) ? 0u : UOS_PCI_RES_MEM_32;
        r->size     = (rom_lower == 0u) ? 0u : ((~(rom_lower & UOS_PCI_ROM_ADDR_MASK)) + 1u);
        r->bus_addr = rom_val & UOS_PCI_ROM_ADDR_MASK;
    }

    /* Restore command. */
    uos_pci_write16(dev->config_addr, UOS_PCI_R_COMMAND, saved_cmd);

    /* Let the transport map bus->cpu addresses. */
    if (g_uos_pci_ops && g_uos_pci_ops->set_cpu_addr) {
        g_uos_pci_ops->set_cpu_addr(dev);
    }
}

/* ==========================================================================
 * Enumeration (PikeOS pci_bus_scan + bridge recursion)
 * ========================================================================== */

/* Forward declaration: scan_function recurses into a bridge's secondary bus. */
int uos_pci_scan_bus(uint8_t bus);

static void uos_pci_scan_function(uint8_t domain, uint8_t bus, uint8_t dev, uint8_t fn) {
    uos_pci_config_addr_t a = UOS_PCI_CONFIG_ADDR(domain, bus, dev, fn);
    uint32_t vd = uos_pci_read32(a, UOS_PCI_R_VENDOR_ID);
    if (vd == 0xFFFFFFFFu || vd == 0x00000000u) {
        return;   /* no device */
    }

    uos_pci_dev_t *d = uos_pci_alloc_device();
    if (d == NULL) return;
    d->config_addr = a;
    d->vendor      = (uint16_t)(vd & 0xFFFFu);
    d->device      = (uint16_t)(vd >> 16);
    d->ext_pci_cfg = true;

    uint32_t class_rev = uos_pci_read32(a, UOS_PCI_R_CLASS_REV);
    d->class_code = class_rev >> 8;          /* class<<8, no rev */
    d->hdr_type   = uos_pci_read8(a, UOS_PCI_R_HDR_TYPE) & UOS_PCI_HDR_TYPE_MASK;
    d->subsysvendor = uos_pci_read16(a, UOS_PCI_R_SUBSYS_VENDOR_ID);
    d->subsys       = uos_pci_read16(a, UOS_PCI_R_SUBSYS_ID);
    d->state = UOS_PCI_DEV_CLOSED;

    /* Size BARs according to header type. */
    if (d->hdr_type == UOS_PCI_HDR_TYPE_NORMAL) {
        uos_pci_read_bases(d, 6u, UOS_PCI_R_ROM_BAR_HDR0);
    } else if (d->hdr_type == UOS_PCI_HDR_TYPE_BRIDGE) {
        uos_pci_read_bases(d, 2u, UOS_PCI_R_ROM_BAR_HDR1);
    }

    /* IRQ routing (legacy INTx). */
    uint8_t pin = uos_pci_read8(a, UOS_PCI_R_INT_PIN);
    if (pin >= 1u && pin <= 4u && g_uos_pci_ops && g_uos_pci_ops->find_irq) {
        g_uos_pci_ops->find_irq(d, pin);
        if (d->irq >= 0) {
            uos_pci_write8(a, UOS_PCI_R_INT_LINE, (uint8_t)d->irq);
        }
    }

    /* Probe MSI / MSI-X capabilities. */
    d->msi_cap  = uos_pci_find_cap(d, UOS_PCI_CAP_ID_MSI);
    d->msix_cap = uos_pci_find_cap(d, UOS_PCI_CAP_ID_MSIX);
    if (d->msi_cap) {
        d->msi_control  = uos_pci_read16(a, (uint32_t)d->msi_cap + 2u);
        d->msi_is64     = (d->msi_control & UOS_PCI_MSI_CNTL_64BIT) ? 1 : 0;
        d->msi_max_irqs = (uint8_t)(1u << ((d->msi_control >> 1) & 7u));
    }
    if (d->msix_cap) {
        d->msix_control = uos_pci_read16(a, (uint32_t)d->msix_cap + 2u);
        uint32_t tbl = uos_pci_read32(a, (uint32_t)d->msix_cap + 4u);
        d->msix_table_bar    = (uint8_t)(tbl & 0x7u);
        d->msix_table_offset = tbl & ~0x7u;
    }

    /* Per-device arch quirk hook. */
    if (g_uos_pci_ops && g_uos_pci_ops->arch_quirk_post) {
        g_uos_pci_ops->arch_quirk_post(d);
    }

    uart_puts("PCI: ");
    uart_print_dec(domain); uart_puts(":");
    uart_print_dec(bus); uart_puts(":");
    uart_print_dec(dev); uart_puts(":");
    uart_print_dec(fn);
    uart_puts(" vid 0x"); uart_print_hex((uint32_t)d->vendor);
    uart_puts(" did 0x"); uart_print_hex((uint32_t)d->device);
    uart_puts(" class 0x"); uart_print_hex(d->class_code >> 8);
    if (d->msi_cap)  uart_puts(" [MSI]");
    if (d->msix_cap) uart_puts(" [MSI-X]");
    uart_puts("\n");

    /* Bridge recursion: scan its secondary bus. */
    if (d->hdr_type == UOS_PCI_HDR_TYPE_BRIDGE) {
        uint8_t secondary = uos_pci_read8(a, UOS_PCI_R_SECONDARY_BUS);
        if (secondary != 0u && secondary > bus) {
            uos_pci_scan_bus(secondary);
        }
    }
}

static void uos_pci_scan_bus_inner(uint8_t domain, uint8_t bus) {
    for (uint8_t dev = 0; dev < UOS_PCI_DEV_MAX; dev++) {
        uos_pci_config_addr_t a0 = UOS_PCI_CONFIG_ADDR(domain, bus, dev, 0);
        uint32_t vd = uos_pci_read32(a0, UOS_PCI_R_VENDOR_ID);
        if (vd == 0xFFFFFFFFu || vd == 0x00000000u) continue;

        uint8_t ht = uos_pci_read8(a0, UOS_PCI_R_HDR_TYPE);
        bool multifunc = (ht & UOS_PCI_HDR_TYPE_MULTIFUNC) != 0u;
        uint8_t fn_max = multifunc ? UOS_PCI_FN_MAX : 1u;
        for (uint8_t fn = 0; fn < fn_max; fn++) {
            if (fn != 0) {
                uos_pci_config_addr_t af = UOS_PCI_CONFIG_ADDR(domain, bus, dev, fn);
                if (uos_pci_read32(af, UOS_PCI_R_VENDOR_ID) == 0xFFFFFFFFu) continue;
            }
            uos_pci_scan_function(domain, bus, dev, fn);
        }
    }
}

/* Public scan entry (bus uses domain 0). Recursive bridge variant. */
int uos_pci_scan_bus(uint8_t bus) {
    unsigned int before = g_uos_pci_device_count;
    uos_pci_scan_bus_inner(0, bus);
    return (int)(g_uos_pci_device_count - before);
}

int uos_pci_enumerate(void) {
    g_uos_pci_device_count = 0;
    unsigned int doms = (g_uos_pci_ops && g_uos_pci_ops->num_domains)
                        ? g_uos_pci_ops->num_domains() : 1u;
    for (unsigned int d = 0; d < doms && d < UOS_PCI_DOM_MAX; d++) {
        uos_pci_scan_bus_inner((uint8_t)d, 0);
    }
    return (int)g_uos_pci_device_count;
}

void uos_pci_enable_device(uos_pci_dev_t *dev) {
    if (dev == NULL) return;
    uint16_t cmd = uos_pci_read16(dev->config_addr, UOS_PCI_R_COMMAND);
    cmd |= (UOS_PCI_CMD_IO | UOS_PCI_CMD_MEM | UOS_PCI_CMD_MASTER);
    uos_pci_write16(dev->config_addr, UOS_PCI_R_COMMAND, cmd);
}

/* ==========================================================================
 * MSI programming (PikeOS pci_msi.c)
 * ========================================================================== */
void uos_pci_msi_enable(uos_pci_dev_t *dev, uint32_t msg_addr, uint16_t msg_data) {
    if (dev == NULL || dev->msi_cap == 0u) return;
    uint32_t base = dev->msi_cap;
    uint16_t msi_ctl = uos_pci_read16(dev->config_addr, base + 2u);

    /* Mask legacy INTx. */
    uint16_t cmd = uos_pci_read16(dev->config_addr, UOS_PCI_R_COMMAND);
    uos_pci_write16(dev->config_addr, UOS_PCI_R_COMMAND, (uint16_t)(cmd | UOS_PCI_CMD_INT_DIS));

    /* Program message address/data. */
    if (msi_ctl & UOS_PCI_MSI_CNTL_64BIT) {
        uos_pci_write32(dev->config_addr, base + 4u, msg_addr);
        uos_pci_write32(dev->config_addr, base + 8u, 0u);
        uos_pci_write16(dev->config_addr, base + 12u, msg_data);
    } else {
        uos_pci_write32(dev->config_addr, base + 4u, msg_addr);
        uos_pci_write16(dev->config_addr, base + 8u,  msg_data);
    }

    /* Enable MSI (single message). */
    uos_pci_write16(dev->config_addr, base + 2u, (uint16_t)(msi_ctl | UOS_PCI_MSI_CNTL_ENABLE));
}

void uos_pci_msi_disable(uos_pci_dev_t *dev) {
    if (dev == NULL || dev->msi_cap == 0u) return;
    uint16_t msi_ctl = uos_pci_read16(dev->config_addr, (uint32_t)dev->msi_cap + 2u);
    uos_pci_write16(dev->config_addr, (uint32_t)dev->msi_cap + 2u,
                    (uint16_t)(msi_ctl & ~UOS_PCI_MSI_CNTL_ENABLE));
    if (!dev->force_intx_dis) {
        uint16_t cmd = uos_pci_read16(dev->config_addr, UOS_PCI_R_COMMAND);
        uos_pci_write16(dev->config_addr, UOS_PCI_R_COMMAND, (uint16_t)(cmd & ~UOS_PCI_CMD_INT_DIS));
    }
}

/* ==========================================================================
 * MSI-X programming (PikeOS pci_msix.c)
 *
 * The MSI-X table lives in a memory BAR selected by the capability. Entries
 * are 16 bytes: addr_lo, addr_hi, data, vector_control(mask bit0). We enable
 * MSI-X, then unmask all entries.
 * ========================================================================== */
static volatile uint32_t *uos_pci_msix_table(uos_pci_dev_t *dev) {
    if (dev == NULL || dev->msix_cap == 0u) return NULL;
    if (dev->msix_table_bar >= UOS_PCI_NUM_RES - 1u) return NULL;
    uint64_t base = dev->res[dev->msix_table_bar].cpu_addr + dev->msix_table_offset;
    return (volatile uint32_t *)(uintptr_t)base;
}

void uos_pci_msix_enable(uos_pci_dev_t *dev) {
    if (dev == NULL || dev->msix_cap == 0u) return;

    /* Enable memory decode so the table BAR responds. */
    uint16_t cmd = uos_pci_read16(dev->config_addr, UOS_PCI_R_COMMAND);
    uos_pci_write16(dev->config_addr, UOS_PCI_R_COMMAND,
                    (uint16_t)(cmd | UOS_PCI_CMD_MEM | UOS_PCI_CMD_INT_DIS));

    /* Enable MSI-X, clear global function mask. */
    uint16_t ctl = uos_pci_read16(dev->config_addr, (uint32_t)dev->msix_cap + 2u);
    ctl |= UOS_PCI_MSIX_CNTL_ENABLE;
    ctl &= ~UOS_PCI_MSIX_CNTL_FNMASK;
    uos_pci_write16(dev->config_addr, (uint32_t)dev->msix_cap + 2u, ctl);

    /* Unmask every table entry. */
    volatile uint32_t *tbl = uos_pci_msix_table(dev);
    if (tbl != NULL) {
        unsigned int n = (unsigned int)(ctl & UOS_PCI_MSIX_CNTL_TABLESZ) + 1u;
        for (unsigned int i = 0; i < n; i++) {
            volatile uint32_t *ent = tbl + (i * (UOS_PCI_MSIX_ENTRY_SIZE / 4u));
            ent[UOS_PCI_MSIX_MASK_OFFSET / 4u] &= ~0x1u;   /* clear Mask bit */
        }
    }
}

void uos_pci_msix_disable(uos_pci_dev_t *dev) {
    if (dev == NULL || dev->msix_cap == 0u) return;
    uint16_t ctl = uos_pci_read16(dev->config_addr, (uint32_t)dev->msix_cap + 2u);
    ctl &= ~UOS_PCI_MSIX_CNTL_ENABLE;
    uos_pci_write16(dev->config_addr, (uint32_t)dev->msix_cap + 2u, ctl);
    if (!dev->force_intx_dis) {
        uint16_t cmd = uos_pci_read16(dev->config_addr, UOS_PCI_R_COMMAND);
        uos_pci_write16(dev->config_addr, UOS_PCI_R_COMMAND, (uint16_t)(cmd & ~UOS_PCI_CMD_INT_DIS));
    }
}

/* ==========================================================================
 * Init / demo / legacy API
 * ========================================================================== */
void uos_pci_driver_init(void) {
    uart_puts("\n=== PCI/PCIe Driver Initialization (PikeOS-style core) ===\n");

    /* Default: framework (safe) transport. A board port with real PCIe HW
     * (or AArch64 virt) calls uos_pci_register_ops(&uos_pci_ecam_ops) after
     * setting g_uos_ecam_base. ECAM is NOT used by default on QEMU virt
     * cortex-a15 because the gpex config read deadlocks inside QEMU. */
    uos_pci_register_ops(&uos_pci_framework_ops);

    int n = uos_pci_enumerate();
    uart_puts("PCI: enumeration found ");
    uart_print_dec((uint32_t)n);
    uart_puts(" device(s)\n");
    uart_puts("===========================================================\n\n");
}

void uos_pci_driver_demo(void) {
    uart_puts("\n=== PCI/PCIe Demonstration ===\n");
    unsigned int count = uos_pci_get_device_count();
    for (unsigned int i = 0; i < count; i++) {
        uos_pci_dev_t *d = uos_pci_get_device(i);
        if (d == NULL) continue;
        uos_pci_enable_device(d);
        uart_puts("PCI dev ");
        uart_print_dec(i);
        uart_puts(": enabled (BAR0 size ");
        uart_print_dec(d->res[0].size);
        uart_puts(" bytes)\n");
    }
    uart_puts("=== End PCI Demonstration ===\n\n");
}

/* Legacy simple API wrappers (kept for the existing boot wiring). */
int pci_init(uint32_t ecam_base) {
    g_uos_ecam_base = ecam_base;
    uos_pci_register_ops(&uos_pci_framework_ops);
    return 0;
}

void pci_driver_init(void) {
    uos_pci_driver_init();
}

void pci_driver_demo(void) {
    uos_pci_driver_demo();
}