diff --git a/Kconfig b/Kconfig index ab9e308c5..88f83e1e2 100644 --- a/Kconfig +++ b/Kconfig @@ -241,6 +241,15 @@ config ARM_HYPERVISOR_SUPPORT help Support for TK1 SoC-specific SystemMMU + config ARM_SMMU_VM_DEFAULT_MAPPING + bool "Enable SystemMMU default mapping for guest VM" + depends on PLAT_TK1 && ARM_SMMU && ARM_HYPERVISOR_SUPPORT + help + Enable the default translation for guest VM: all + physical memory reserved for a VM is allowed to + be accessed with read/write/nonsecure permissions. + + source "$KERNEL_PATH/src/arch/arm/Kconfig" source "$KERNEL_PATH/src/plat/pc99/Kconfig" diff --git a/include/arch/arm/arch/32/mode/machine_pl2.h b/include/arch/arm/arch/32/mode/machine_pl2.h index 33edb75d0..29d8430ec 100644 --- a/include/arch/arm/arch/32/mode/machine_pl2.h +++ b/include/arch/arm/arch/32/mode/machine_pl2.h @@ -188,7 +188,10 @@ static inline void setCurrentPDPL2(paddr_t pa) {} static inline void invalidateHypTLB(void) {} static inline void writeContextIDPL2(word_t pd) {} static inline void writeContextIDAndPD(word_t id, word_t pd) {} -static inline paddr_t addressTranslateS1CPR(vptr_t vaddr) { return vaddr; } +static inline paddr_t addressTranslateS1CPR(vptr_t vaddr) +{ + return vaddr; +} #endif /* !ARM_HYP */ #endif /* __ARCH_MACHINE_PL2_32_H */ diff --git a/include/arch/arm/arch/32/mode/object/structures.bf b/include/arch/arm/arch/32/mode/object/structures.bf index 35e4ee304..771f166db 100644 --- a/include/arch/arm/arch/32/mode/object/structures.bf +++ b/include/arch/arm/arch/32/mode/object/structures.bf @@ -8,6 +8,7 @@ -- @TAG(GD_GPL) -- +#include -- Default base size: uint32_t base 32 @@ -25,7 +26,12 @@ block small_frame_cap { field capFVMRights 2 field_high capFMappedAddress 20 +#ifdef CONFIG_ARM_SMMU + field capFIsIOSpace 1 + field capFMappedASIDHigh 7 +#else field capFMappedASIDHigh 8 +#endif field_high capFBasePtr 20 field capType 4 } @@ -127,14 +133,15 @@ block io_page_directory_cap (capType, capIOPDIsMapped, capIOPDASID, capIOPDBaseP field capType 8 } -block io_page_table_cap (capType, capIOPTIsMapped, capIOPTASID, capIOPTBasePtr) { - field_high capIOPTBasePtr 20 - padding 12 +block io_page_table_cap (capType, capIOPTIsMapped, capIOPTASID, capIOPTBasePtr, capIOPTMappedAddress) { + field_high capIOPTBasePtr 20 + padding 12 - padding 16 - field capIOPTASID 7 - field capIOPTIsMapped 1 - field capType 8 + field_high capIOPTMappedAddress 10 + padding 6 + field capIOPTASID 7 + field capIOPTIsMapped 1 + field capType 8 } -- NB: odd numbers are arch caps (see isArchCap()) diff --git a/include/arch/arm/arch/32/mode/object/structures.h b/include/arch/arm/arch/32/mode/object/structures.h index 92fbe63d2..21e7942a6 100644 --- a/include/arch/arm/arch/32/mode/object/structures.h +++ b/include/arch/arm/arch/32/mode/object/structures.h @@ -112,7 +112,11 @@ struct user_data { typedef struct user_data user_data_t; enum asidSizeConstants { +#ifdef CONFIG_ARM_SMMU + asidHighBits = 7, +#else asidHighBits = 8, +#endif asidLowBits = 10 }; @@ -325,6 +329,8 @@ cap_get_archCapSizeBits(cap_t cap) case cap_vcpu_cap: return VCPU_SIZE_BITS; #endif + case cap_io_page_table_cap: + return seL4_IOPageTableBits; default: /* Unreachable, but GCC can't figure that out */ @@ -362,6 +368,9 @@ cap_get_archCapPtr(cap_t cap) return VCPU_PTR(cap_vcpu_cap_get_capVCPUPtr(cap)); #endif + case cap_io_page_table_cap: + return (void *)(cap_io_page_table_cap_get_capIOPTBasePtr(cap)); + default: /* Unreachable, but GCC can't figure that out */ return NULL; diff --git a/include/arch/arm/arch/kernel/vspace.h b/include/arch/arm/arch/kernel/vspace.h index 3dbf8ab86..2043b736b 100644 --- a/include/arch/arm/arch/kernel/vspace.h +++ b/include/arch/arm/arch/kernel/vspace.h @@ -95,4 +95,6 @@ exception_t decodeARMMMUInvocation(word_t invLabel, word_t length, cptr_t cptr, word_t *buffer); +bool_t CONST isIOSpaceFrame(cap_t cap); + #endif diff --git a/include/arch/arm/arch/machine.h b/include/arch/arm/arch/machine.h index 473020ca9..4631bc91d 100644 --- a/include/arch/arm/arch/machine.h +++ b/include/arch/arm/arch/machine.h @@ -45,6 +45,13 @@ static inline void clearMemory(word_t* ptr, word_t bits) cleanCacheRange_PoU((word_t)ptr, (word_t)ptr + BIT(bits) - 1, addrFromPPtr(ptr)); } + +static inline void clearMemoryRAM(word_t* ptr, word_t bits) +{ + memzero(ptr, BIT(bits)); + cleanCacheRange_RAM((word_t)ptr, (word_t)ptr + BIT(bits) - 1, + addrFromPPtr(ptr)); +} #endif /* __ASSEMBLER__ */ #endif /* __ARCH_MACHINE_H */ diff --git a/include/arch/arm/arch/object/iospace.h b/include/arch/arm/arch/object/iospace.h new file mode 100644 index 000000000..f51870103 --- /dev/null +++ b/include/arch/arm/arch/object/iospace.h @@ -0,0 +1,29 @@ +/* + * Copyright 2016, General Dynamics C4 Systems + * + * This software may be distributed and modified according to the terms of + * the GNU General Public License version 2. Note that NO WARRANTY is provided. + * See "LICENSE_GPLv2.txt" for details. + * + * @TAG(GD_GPL) + */ + +#ifndef __ARCH_OBJECT_IOSPACE_H +#define __ARCH_OBJECT_IOSPACE_H + +#include +#include +#include + + +seL4_SlotRegion create_iospace_caps(cap_t root_cnode_cap); +exception_t decodeARMIOPTInvocation(word_t invLabel, uint32_t length, cte_t* slot, cap_t cap, extra_caps_t excaps, word_t* buffer); +exception_t decodeARMIOMapInvocation(word_t invLabel, uint32_t length, cte_t* slot, cap_t cap, extra_caps_t excaps, word_t* buffer); +exception_t decodeARMIOUnMapInvocation(word_t invLabel, uint32_t length, cte_t* slot, cap_t cap, extra_caps_t excaps); +exception_t decodeARMIOSpaceInvocation(word_t invLabel, cap_t cap); +void unmapIOPage(cap_t cap); +void deleteIOPageTable(cap_t cap); +void clearIOPageDirectory(cap_t cap); + +#endif + diff --git a/include/arch/arm/arch/object/vcpu.h b/include/arch/arm/arch/object/vcpu.h index 906bf5dee..80e01a961 100644 --- a/include/arch/arm/arch/object/vcpu.h +++ b/include/arch/arm/arch/object/vcpu.h @@ -72,7 +72,7 @@ exception_t invokeVCPUSetTCB(vcpu_t *vcpu, tcb_t *tcb); #else /* end of ARM_HYP */ /* used in boot.c with a guard, use a marco to avoid exposing vcpu_t */ -#define vcpu_restore(x) +#define vcpu_restore(x) #define vcpu_switch(x) static inline void VGICMaintenance(void) {} diff --git a/include/plat/tk1/plat/machine.h b/include/plat/tk1/plat/machine.h index d46e31212..e94483b87 100644 --- a/include/plat/tk1/plat/machine.h +++ b/include/plat/tk1/plat/machine.h @@ -216,6 +216,8 @@ enum IRQConstants { #define INTERRUPT_NS_PGPT INTERRUPT_PPI_14 #define INTERRUPT_VGPT INTERRUPT_PPI_11 #define INTERRUPT_HGPT INTERRUPT_PPI_10 +#define INTERRUPT_SMMU INTERRUPT_MC + /* the kernel runs in secure supervisor mode by default */ diff --git a/include/plat/tk1/plat/machine/devices.h b/include/plat/tk1/plat/machine/devices.h index c24d5569e..c6df01268 100644 --- a/include/plat/tk1/plat/machine/devices.h +++ b/include/plat/tk1/plat/machine/devices.h @@ -21,6 +21,8 @@ #define GIC_CONTROLLER_PPTR 0xfff04000 /* HYP mode kernel devices */ #define GIC_VCPUCTRL_PPTR 0xfff06000 +/* SMMU registers */ +#define SMMU_PPTR 0Xfff07000 #define GIC_PL390_CONTROLLER_PPTR GIC_CONTROLLER_PPTR #define GIC_PL390_DISTRIBUTOR_PPTR GIC_DISTRIBUTOR_PPTR @@ -41,6 +43,14 @@ #define VGICI_REQ_PADDR (ARM_PERIPHBASE + 0x4000) /* hyp view for requesting CPU */ #define VGICI_ALL_PADDR (ARM_PERIPHBASE + 0x5000) /* hyp view for all CPUs */ #define VGICI_VM_PADDR (ARM_PERIPHBASE + 0x6000) /* hyp view for VM view */ +#define PCIE_0_CFG_PADDR (0x01000000) /* PCIE lane 0 config register */ +#define PCIE_1_CFG_PADDR (0x01001000) /* PCIE lane 1 config register */ +#define PCIE_PCA0_1_PADDR (0x01002000) /* PCA 0 and 1 */ +#define PCIE_PADS_AFI_PADDR (0x01003000) /* Pads and AFI */ +#define PCIE_A1_PADDR (0x01000000) /* 16 MB */ +#define PCIE_A2_PADDR (0x02000000) /* 224 MB */ +#define PCIE_A3_PADDR (0x10000000) /* 763 MB */ +#define R8169_NIC_PADDR (0x13000000) /* r8169 NIC 1 MB */ #define GRAPH_HOST_PADDR (0x54000000) /* 16 MB */ #define GPU_PADDR (0x57000000) /* 144 MB */ #define UP_TAG_PADDR (0x60000000) /* 4 KB */ @@ -81,7 +91,8 @@ #define HDA_PADDR (0x70030000) /* 64 KB */ #define MIOBFM_PADDR (0x70200000) /* 64 KB */ #define AUDIO_PADDR (0x70300000) /* 64 KB */ -#define XUSB_HOST_PADDR (0x70090000) /* 40 KB */ +#define XUSB_HOST_PADDR (0x70090000) /* 36 KB */ +#define XUSB_PADCTL_PADDR (0x7009f000) /* 4 KB */ #define XUSB_DEV_PADDR (0x700d0000) /* 40 KB */ #define DDS_PADDR (0x700a0000) /* 8KB 4608 bytes */ #define SDMMC_1_PADDR (0x700b0000) /* 4KB 512 bytes */ @@ -104,4 +115,9 @@ #define USB3_PADDR (0x7d008000) /* 8 KB region, 6 KB */ #define UARTA_PADDR (0x70006000) +/* physical memory regions allocated for virtual machines */ +#define VM_HOST_PA_START 0xb0000000 +#define VM_GUEST_PA_START 0x80000000 +#define VM_HOST_PA_SIZE 0x40000000 + #endif diff --git a/include/plat/tk1/plat/machine/hardware.bf b/include/plat/tk1/plat/machine/hardware.bf index b20c6fa6b..3b00d75b9 100644 --- a/include/plat/tk1/plat/machine/hardware.bf +++ b/include/plat/tk1/plat/machine/hardware.bf @@ -27,19 +27,23 @@ tagged_union iopde page_size { block iopde_pt { field read 1 field write 1 - field secure 1 + field nonsecure 1 field page_size 1 padding 6 field_high address 22 } +-- adding a 2-bit padding before address is because +-- we do not support addressing more than 4 GiB memory + block iopde_4m { field read 1 field write 1 - field secure 1 + field nonsecure 1 field page_size 1 padding 6 - field_high address 12 + padding 2 + field_high address 10 padding 10 } @@ -48,7 +52,8 @@ block iopde_4m { block iopte { field read 1 field write 1 - field secure 1 + field nonsecure 1 padding 7 - field_high address 22 + padding 2 + field_high address 20 } diff --git a/include/plat/tk1/plat/machine/smmu.h b/include/plat/tk1/plat/machine/smmu.h new file mode 100644 index 000000000..cad5ef98a --- /dev/null +++ b/include/plat/tk1/plat/machine/smmu.h @@ -0,0 +1,167 @@ +/* + * Copyright 2016, General Dynamics C4 Systems + * + * This software may be distributed and modified according to the terms of + * the GNU General Public License version 2. Note that NO WARRANTY is provided. + * See "LICENSE_GPLv2.txt" for details. + * + * @TAG(GD_GPL) + */ + +#ifndef __PLAT_SMMU_H +#define __PLAT_SMMU_H + +#include + +#define IOASID_SIZE_BITS 7 + + + +/* The SystemMMU control registers are part of memory controller */ + +typedef struct { + uint32_t intstatus; /* 0x00 */ + uint32_t intmask; /* 0x04 */ + uint32_t err_status; /* 0x08 */ + uint32_t err_adr; /* 0x0c */ + uint32_t smmu_config; /* 0x10 */ + uint32_t smmu_tlb_config; /* 0x14 */ + uint32_t smmu_ptc_config; /* 0x18 */ + uint32_t smmu_ptb_asid; /* 0x1c */ + uint32_t smmu_ptb_data; /* 0x20 */ + uint32_t reserved0; /* 0x24 */ + uint32_t reserved1; /* 0x28 */ + uint32_t reserved2; /* 0x2c */ + uint32_t smmu_tlb_flush; /* 0x30 */ + uint32_t smmu_ptc_flush; /* 0x34 */ + uint32_t reserved3[124]; + uint32_t smmu_translation_enable_0; /* 0x228 */ + uint32_t smmu_translation_enable_1; /* 0x22c */ + uint32_t smmu_translation_enable_2; /* 0x230 */ + uint32_t smmu_translation_enable_3; /* 0x234 */ + uint32_t smmu_afi_asid; /* 0x238 */ + uint32_t smmu_avpc_asid; /* 0x23c */ + uint32_t smmu_dc_asid; /* 0x240 */ + uint32_t smmu_dcb_asid; /* 0x244 */ + uint32_t reserved4; /* 0x248 */ + uint32_t reserved5; /* 0x24c */ + uint32_t smmu_hc_asid; /* 0x250 */ + uint32_t smmu_hda_asid; /* 0x254 */ + uint32_t smmu_isp2_asid; /* 0x258 */ + uint32_t reserved6; /* 0x25c */ + uint32_t reserved7; /* 0x260 */ + uint32_t smmu_msenc_asid; /* 0x264 */ + uint32_t smmu_nv_asid; /* 0x268 */ + uint32_t smmu_nv2_asid; /* 0x26c */ + uint32_t smmu_ppcs_asid; /* 0x270 */ + uint32_t smmu_sata_asid; /* 0x274 */ + uint32_t smmu_vde_asid; /* 0x27c */ + uint32_t smmu_vi_asid; /* 0x280 */ + uint32_t smmu_vic_asid; /* 0x284 */ + uint32_t smmu_xusb_host_asid; /* 0x288 */ + uint32_t smmu_xusb_dev_asid; /* 0x28c */ + uint32_t reserved8; /* 0x290 */ + uint32_t smmu_tsec_asid; /* 0x294 */ + uint32_t smmu_ppcs1_asid; /* 0x298 */ + uint32_t reserved9[217]; + uint32_t smmu_tlb_set_sel_mask; /* 0x600 */ + uint32_t reserved10[237]; + uint32_t smmu_ptc_flush_1; /* 0x9b8 */ + uint32_t reserved11[51]; + uint32_t smmu_dc1_asid; /* 0xa88 */ + uint32_t reserved12; /* 0xa8c */ + uint32_t reserved13; /* 0xa90 */ + uint32_t smmu_sdmmc1a_asid; /* 0xa94 */ + uint32_t smmu_sdmmc2a_asid; /* 0xa98 */ + uint32_t smmu_sdmmc3a_asid; /* 0xa9c */ + uint32_t smmu_sdmmc4a_asid; /* 0xaa0 */ + uint32_t smmu_isp2b_asid; /* 0xaa4 */ + uint32_t smmu_gpu_asid; /* 0xaa8 */ + uint32_t smmu_gpub_asid; /* 0xaac */ + uint32_t smmu_ppcs2_asid; /* 0xab0 */ +} tk1_mc_regs_t; + +/* we start to allocate IO ASIDs from 1, and each module's ASID + * is fixed (i.e. users are not allowed to dynamically allocate + * ASIDs and assign them to devices). + */ +#define SMMU_FIRST_ASID 1 +#define SMMU_AFI_ASID 1 +#define SMMU_AVPC_ASID 2 +#define SMMU_DC_ASID 3 +#define SMMU_DCB_ASID 4 +#define SMMU_HC_ASID 5 +#define SMMU_HDA_ASID 6 +#define SMMU_ISP2_ASID 7 +#define SMMU_MSENC_ASID 8 +#define SMMU_NV_ASID 9 +#define SMMU_NV2_ASID 10 +#define SMMU_PPCS_ASID 11 +#define SMMU_SATA_ASID 12 +#define SMMU_VDE_ASID 13 +#define SMMU_VI_ASID 14 +#define SMMU_VIC_ASID 15 +#define SMMU_XUSB_HOST_ASID 16 +#define SMMU_XUSB_DEV_ASID 17 +#define SMMU_TSEC_ASID 18 +#define SMMU_PPCS1_ASID 19 +#define SMMU_DC1_ASID 20 +#define SMMU_SDMMC1A_ASID 21 +#define SMMU_SDMMC2A_ASID 22 +#define SMMU_SDMMC3A_ASID 23 +#define SMMU_SDMMC4A_ASID 24 +#define SMMU_ISP2B_ASID 25 +#define SMMU_GPU_ASID 26 +#define SMMU_GPUB_ASID 27 +#define SMMU_PPCS2_ASID 28 +#define SMMU_LAST_ASID 28 + +#define ARM_PLAT_NUM_SMMU 28 + +#define SMMU_PD_BITS 12 +#define SMMU_PT_BITS 12 + +#define SMMU_IOPD_INDEX_MASK 0xffc00000 +#define SMMU_IOPD_INDEX_SHIFT 20 +#define SMMU_IOPT_INDEX_MASK 0x3ff000 +#define SMMU_IOPT_INDEX_SHIFT 12 + +inline static uint32_t +plat_smmu_iopd_index(word_t io_address) +{ + uint32_t ret = (io_address & SMMU_IOPD_INDEX_MASK) >> SMMU_IOPD_INDEX_SHIFT; + return ret; +} + +inline static uint32_t +plat_smmu_iopt_index(word_t io_address) +{ + uint32_t ret = (io_address & SMMU_IOPT_INDEX_MASK) >> SMMU_IOPT_INDEX_SHIFT; + return ret; +} + +inline static uint32_t +plat_smmu_get_asid_by_module_id(uint32_t mid) +{ + if (mid < SMMU_FIRST_ASID || mid > SMMU_LAST_ASID) { + return asidInvalid; + } + + /* we have one-to-one mapping from module id to ASID */ + return mid; + +} + +/** MODIFIES: [*] */ +int plat_smmu_init(void); + +/** MODIFIES: [*] */ +void plat_smmu_tlb_flush_all(void); + +/** MODIFIES: [*] */ +void plat_smmu_ptc_flush_all(void); + +iopde_t *plat_smmu_lookup_iopd_by_asid(uint32_t asid); +void plat_smmu_handle_interrupt(void); + +#endif diff --git a/libsel4/arch_include/arm/interfaces/sel4arch.xml b/libsel4/arch_include/arm/interfaces/sel4arch.xml index 6c3ed354a..a1575aac6 100644 --- a/libsel4/arch_include/arm/interfaces/sel4arch.xml +++ b/libsel4/arch_include/arm/interfaces/sel4arch.xml @@ -18,6 +18,14 @@ + + + + + + + + @@ -31,6 +39,11 @@ + + + + + diff --git a/libsel4/arch_include/arm/sel4/arch/objecttype.h b/libsel4/arch_include/arm/sel4/arch/objecttype.h index be0393b84..aa8451b1a 100644 --- a/libsel4/arch_include/arm/sel4/arch/objecttype.h +++ b/libsel4/arch_include/arm/sel4/arch/objecttype.h @@ -20,11 +20,18 @@ typedef enum _object { seL4_ARM_PageDirectoryObject, #ifdef ARM_HYP seL4_ARM_VCPUObject, +#endif +#ifdef CONFIG_ARM_SMMU + seL4_ARM_IOPageTableObject, #endif seL4_ObjectTypeCount } seL4_ArchObjectType; typedef seL4_Word object_t; +#ifndef CONFIG_ARM_SMMU +#define seL4_ARM_IOPageTableObject 0xffff +#endif + #endif /* __ARCH_OBJECTTYPE_H */ diff --git a/libsel4/arch_include/arm/sel4/arch/types.h b/libsel4/arch_include/arm/sel4/arch/types.h index 2b7019124..8e3316913 100644 --- a/libsel4/arch_include/arm/sel4/arch/types.h +++ b/libsel4/arch_include/arm/sel4/arch/types.h @@ -20,5 +20,8 @@ typedef seL4_CPtr seL4_ARM_PageDirectory; typedef seL4_CPtr seL4_ARM_ASIDControl; typedef seL4_CPtr seL4_ARM_ASIDPool; typedef seL4_CPtr seL4_ARM_VCPU; +typedef seL4_CPtr seL4_ARM_IOSpace; +typedef seL4_CPtr seL4_ARM_IOPageTable; + #endif /* __ARCH_SEL4TYPES_H__ */ diff --git a/libsel4/include/sel4/bootinfo_types.h b/libsel4/include/sel4/bootinfo_types.h index 5b302022f..06ae109c0 100644 --- a/libsel4/include/sel4/bootinfo_types.h +++ b/libsel4/include/sel4/bootinfo_types.h @@ -57,6 +57,7 @@ typedef struct { seL4_SlotRegion userImageFrames; /* userland-image frame caps */ seL4_SlotRegion userImagePaging; /* userland-image paging structure caps */ seL4_SlotRegion untyped; /* untyped-object caps (untyped caps) */ + seL4_SlotRegion ioSpaceCaps; /* IOSpace caps for ARM SMMU */ seL4_PAddr untypedPaddrList [CONFIG_MAX_NUM_BOOTINFO_UNTYPED_CAPS]; /* physical address of each untyped cap */ seL4_Uint8 untypedSizeBitsList[CONFIG_MAX_NUM_BOOTINFO_UNTYPED_CAPS]; /* size (2^n) bytes of each untyped cap */ seL4_Uint8 initThreadCNodeSizeBits; /* initial thread's root CNode size (2^n slots) */ diff --git a/libsel4/sel4_arch_include/aarch32/sel4/sel4_arch/constants.h b/libsel4/sel4_arch_include/aarch32/sel4/sel4_arch/constants.h index 708abd448..3fd946324 100644 --- a/libsel4/sel4_arch_include/aarch32/sel4/sel4_arch/constants.h +++ b/libsel4/sel4_arch_include/aarch32/sel4/sel4_arch/constants.h @@ -29,6 +29,7 @@ enum { #define seL4_PageDirBits 14 #define seL4_ASIDPoolBits 12 #define seL4_ARM_VCPUBits 12 +#define seL4_IOPageTableBits 12 /* word size */ #define seL4_WordBits (sizeof(seL4_Word) * 8) diff --git a/libsel4/tools/syscall_stub_gen.py b/libsel4/tools/syscall_stub_gen.py index 5327a750f..a7af603f2 100644 --- a/libsel4/tools/syscall_stub_gen.py +++ b/libsel4/tools/syscall_stub_gen.py @@ -242,6 +242,8 @@ def InitTypes(): CapType("seL4_ARM_ASIDControl"), CapType("seL4_ARM_ASIDPool"), CapType("seL4_ARM_VCPU"), + CapType("seL4_ARM_IOSpace"), + CapType("seL4_ARM_IOPageTable"), StructType("seL4_UserContext", WORD_SIZE_BITS * 17), ], diff --git a/src/arch/arm/32/kernel/vspace.c b/src/arch/arm/32/kernel/vspace.c index ebdc40cc9..6b82b8e14 100644 --- a/src/arch/arm/32/kernel/vspace.c +++ b/src/arch/arm/32/kernel/vspace.c @@ -26,6 +26,7 @@ #include #include #include +#include /* ARM uses multiple identical mappings in a page table / page directory to construct * large mappings. In both cases it happens to be 16 entries, which can be calculated by @@ -513,6 +514,9 @@ create_it_frame_cap(pptr_t pptr, vptr_t vptr, asid_t asid, bool_t use_large) ASID_LOW(asid), /* capFMappedASIDLow */ wordFromVMRights(VMReadWrite), /* capFVMRights */ vptr, /* capFMappedAddress */ +#ifdef CONFIG_ARM_SMMU + 0, /* IOSpace */ +#endif ASID_HIGH(asid), /* capFMappedASIDHigh */ pptr /* capFBasePtr */ ); @@ -1128,6 +1132,16 @@ isValidVTableRoot(cap_t cap) cap_page_directory_cap_get_capPDIsMapped(cap); } +bool_t CONST +isIOSpaceFrame(cap_t cap) +{ +#ifdef CONFIG_ARM_SMMU + return cap_get_capType(cap) == cap_small_frame_cap && cap_small_frame_cap_get_capFIsIOSpace(cap); +#else + return false; +#endif +} + void setVMRoot(tcb_t *tcb) { @@ -2544,6 +2558,13 @@ decodeARMFrameInvocation(word_t invLabel, word_t length, vm_page_size_t frameSize; vm_attributes_t attr; + if (isIOSpaceFrame(cap)) { + userError("ARMFrameRemap: Attempting to remap frame mapped into an IOSpace"); + current_syscall_error.type = seL4_IllegalOperation; + + return EXCEPTION_SYSCALL_ERROR; + } + if (unlikely(length < 2 || excaps.excaprefs[0] == NULL)) { current_syscall_error.type = seL4_TruncatedMessage; @@ -2644,8 +2665,16 @@ decodeARMFrameInvocation(word_t invLabel, word_t length, } case ARMPageUnmap: { - setThreadState(ksCurThread, ThreadState_Restart); - return performPageInvocationUnmap(cap, cte); + if (isIOSpaceFrame(cap)) { + return decodeARMIOUnMapInvocation(invLabel, length, cte, cap, excaps); + } else { + setThreadState(ksCurThread, ThreadState_Restart); + return performPageInvocationUnmap(cap, cte); + } + } + + case ARMPageMapIO: { + return decodeARMIOMapInvocation(invLabel, length, cte, cap, excaps, buffer); } case ARMPageClean_Data: diff --git a/src/arch/arm/32/object/objecttype.c b/src/arch/arm/32/object/objecttype.c index 0cac096b4..8a4e3f421 100644 --- a/src/arch/arm/32/object/objecttype.c +++ b/src/arch/arm/32/object/objecttype.c @@ -74,6 +74,22 @@ Arch_deriveCap(cte_t *slot, cap_t cap) return ret; #endif + case cap_io_space_cap: + ret.cap = cap; + ret.status = EXCEPTION_NONE; + return ret; + + case cap_io_page_table_cap: + if (cap_io_page_table_cap_get_capIOPTIsMapped(cap)) { + ret.cap = cap; + ret.status = EXCEPTION_NONE; + } else { + userError("Deriving a IOPT cap without an assigned IOASID"); + current_syscall_error.type = seL4_IllegalOperation; + ret.cap = cap_null_cap_new(); + ret.status = EXCEPTION_SYSCALL_ERROR; + } + return ret; default: /* This assert has no equivalent in haskell, * as the options are restricted by type */ @@ -140,6 +156,10 @@ Arch_finaliseCap(cap_t cap, bool_t final) case cap_small_frame_cap: if (cap_small_frame_cap_get_capFMappedASID(cap)) { + if (isIOSpaceFrame(cap)) { + unmapIOPage(cap); + break; + } unmapPage(ARMSmallPage, cap_small_frame_cap_get_capFMappedASID(cap), cap_small_frame_cap_get_capFMappedAddress(cap), @@ -163,6 +183,21 @@ Arch_finaliseCap(cap_t cap, bool_t final) } break; #endif + + case cap_io_space_cap: + if (final) { + clearIOPageDirectory(cap); + } + break; + + case cap_io_page_table_cap: + if (final && cap_io_page_table_cap_get_capIOPTIsMapped(cap)) { + deleteIOPageTable(cap); + } + break; + + default: + break; } return cap_null_cap_new(); @@ -182,6 +217,8 @@ resetMemMapping(cap_t cap) case cap_page_directory_cap: /* We don't need to worry about clearing ASID and Address here, only whether it is mapped */ return cap_page_directory_cap_set_capPDIsMapped(cap, 0); + case cap_io_page_table_cap: + return cap_io_page_table_cap_set_capIOPTIsMapped(cap, 0); } return cap; @@ -270,6 +307,15 @@ Arch_recycleCap(bool_t is_final, cap_t cap) return cap; #endif + case cap_io_space_cap: + Arch_finaliseCap(cap, true); + return cap; + + case cap_io_page_table_cap: + clearMemoryRAM(cap_get_capPtr(cap), cap_get_capSizeBits(cap)); + Arch_finaliseCap(cap, is_final); + return resetMemMapping(cap); + default: fail("Arch_recycleCap: invalid cap type"); } @@ -344,6 +390,19 @@ Arch_sameRegionAs(cap_t cap_a, cap_t cap_b) break; #endif + case cap_io_space_cap: + if (cap_get_capType(cap_b) == cap_io_space_cap) { + return cap_io_space_cap_get_capModuleID(cap_a) == + cap_io_space_cap_get_capModuleID(cap_b); + } + break; + + case cap_io_page_table_cap: + if (cap_get_capType(cap_b) == cap_io_page_table_cap) { + return cap_io_page_table_cap_get_capIOPTBasePtr(cap_a) == + cap_io_page_table_cap_get_capIOPTBasePtr(cap_b); + } + break; } return false; @@ -390,6 +449,8 @@ Arch_getObjectSize(word_t t) return PTE_SIZE_BITS + PT_BITS; case seL4_ARM_PageDirectoryObject: return PDE_SIZE_BITS + PD_BITS; + case seL4_ARM_IOPageTableObject: + return seL4_IOPageTableBits; #ifdef ARM_HYP case seL4_ARM_VCPUObject: return VCPU_SIZE_BITS; @@ -417,7 +478,11 @@ Arch_createObject(object_t t, void *regionBase, word_t userSize) return cap_small_frame_cap_new( ASID_LOW(asidInvalid), VMReadWrite, - 0, ASID_HIGH(asidInvalid), + 0, +#ifdef CONFIG_ARM_SMMU + 0, +#endif + ASID_HIGH(asidInvalid), (word_t)regionBase); case seL4_ARM_LargePageObject: @@ -497,6 +562,12 @@ Arch_createObject(object_t t, void *regionBase, word_t userSize) return cap_vcpu_cap_new(VCPU_REF(regionBase)); #endif + case seL4_ARM_IOPageTableObject: + memzero(regionBase, 1 << seL4_IOPageTableBits); + cleanCacheRange_RAM((word_t)regionBase, + (word_t)regionBase + (1 << seL4_IOPageTableBits) - 1, + addrFromPPtr(regionBase)); + return cap_io_page_table_cap_new(0, asidInvalid, (word_t)regionBase, 0); default: /* * This is a conflation of the haskell error: "Arch.createNewCaps @@ -512,13 +583,18 @@ Arch_decodeInvocation(word_t invLabel, word_t length, cptr_t cptr, cte_t *slot, cap_t cap, extra_caps_t excaps, word_t *buffer) { + switch (cap_get_capType(cap)) { + case cap_io_space_cap: + return decodeARMIOSpaceInvocation(invLabel, cap); + case cap_io_page_table_cap: + return decodeARMIOPTInvocation(invLabel, length, slot, cap, excaps, buffer); #ifdef ARM_HYP - if (cap_get_capType(cap) == cap_vcpu_cap) { + case cap_vcpu_cap: return decodeARMVCPUInvocation(invLabel, length, cptr, slot, cap, excaps, buffer); - +#endif /* end of ARM_HYP */ + default: + return decodeARMMMUInvocation(invLabel, length, cptr, slot, cap, excaps, buffer); } -#endif /* ARM_HYP */ - return decodeARMMMUInvocation(invLabel, length, cptr, slot, cap, excaps, buffer); } void diff --git a/src/arch/arm/kernel/boot.c b/src/arch/arm/kernel/boot.c index b7296891e..7363ca65c 100644 --- a/src/arch/arm/kernel/boot.c +++ b/src/arch/arm/kernel/boot.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -134,6 +135,9 @@ init_irqs(cap_t root_cnode_cap) if (config_set(ARM_HYP)) { setIRQState(IRQReserved, INTERRUPT_VGIC_MAINTENANCE); } + if (config_set(CONFIG_ARM_SMMU)) { + setIRQState(IRQReserved, INTERRUPT_SMMU); + } /* provide the IRQ control cap */ write_slot(SLOT_PTR(pptr_of_cap(root_cnode_cap), seL4_CapIRQControl), cap_irq_control_cap_new()); @@ -234,6 +238,14 @@ try_init_kernel( return false; } + if (config_set(CONFIG_ARM_SMMU)) { + ndks_boot.bi_frame->ioSpaceCaps = create_iospace_caps(root_cnode_cap); + if (ndks_boot.bi_frame->ioSpaceCaps.start == 0 && + ndks_boot.bi_frame->ioSpaceCaps.end == 0) { + return false; + } + } + /* Construct an initial address space with enough virtual addresses * to cover the user image + ipc buffer and bootinfo frames */ it_pd_cap = create_it_address_space(root_cnode_cap, it_v_reg); diff --git a/src/arch/arm/object/Makefile b/src/arch/arm/object/Makefile index 3c259eb96..13aa54560 100644 --- a/src/arch/arm/object/Makefile +++ b/src/arch/arm/object/Makefile @@ -10,7 +10,7 @@ DIRECTORIES += src/arch/arm/object -ARCH_C_SOURCES += object/interrupt.c object/tcb.c +ARCH_C_SOURCES += object/interrupt.c object/tcb.c object/iospace.c ifneq (ARM_HYP,) ARCH_C_SOURCES += object/vcpu.c diff --git a/src/arch/arm/object/iospace.c b/src/arch/arm/object/iospace.c new file mode 100644 index 000000000..d931bb3e5 --- /dev/null +++ b/src/arch/arm/object/iospace.c @@ -0,0 +1,539 @@ +/* + * Copyright 2016, General Dynamics C4 Systems + * + * This software may be distributed and modified according to the terms of + * the GNU General Public License version 2. Note that NO WARRANTY is provided. + * See "LICENSE_GPLv2.txt" for details. + * + * @TAG(GD_GPL) + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +typedef struct lookupIOPDSlot_ret { + exception_t status; + iopde_t *iopdSlot; +} lookupIOPDSlot_ret_t; + +typedef struct lookupIOPTSlot_ret { + exception_t status; + iopte_t *ioptSlot; +} lookupIOPTSlot_ret_t; + + +#define IOPDE_VALID_MASK 0xe0000000 +#define IOPTE_EMPTY_MASK 0xe0000000 + +static bool_t +isIOPDEValid(iopde_t *iopde) +{ + assert(iopde != 0); + if ((iopde->words[0] & IOPDE_VALID_MASK) == 0) { + return false; + } + return true; +} + +static bool_t +isIOPTEEmpty(iopte_t *iopte) +{ + assert(iopte != 0); + if ((iopte->words[0] & IOPTE_EMPTY_MASK) == 0) { + return true; + } + return false; +} + + +static lookupIOPDSlot_ret_t +lookupIOPDSlot(iopde_t *iopd, word_t io_address) +{ + lookupIOPDSlot_ret_t ret; + uint32_t index = plat_smmu_iopd_index(io_address); + ret.status = EXCEPTION_NONE; + ret.iopdSlot = iopd + index; + return ret; +} + +static lookupIOPTSlot_ret_t +lookupIOPTSlot(iopde_t *iopd, word_t io_address) +{ + lookupIOPTSlot_ret_t pt_ret; + uint32_t index; + iopte_t *pt; + + lookupIOPDSlot_ret_t pd_ret = lookupIOPDSlot(iopd, io_address); + if (pd_ret.status != EXCEPTION_NONE) { + pt_ret.status = EXCEPTION_LOOKUP_FAULT; + pt_ret.ioptSlot = 0; + return pt_ret; + } + + if (!isIOPDEValid(pd_ret.iopdSlot) || + iopde_ptr_get_page_size(pd_ret.iopdSlot) != iopde_iopde_pt) { + pt_ret.status = EXCEPTION_LOOKUP_FAULT; + pt_ret.ioptSlot = 0; + return pt_ret; + } + + index = plat_smmu_iopt_index(io_address); + pt = (iopte_t *)paddr_to_pptr(iopde_iopde_pt_ptr_get_address(pd_ret.iopdSlot)); + + if (pt == 0) { + pt_ret.status = EXCEPTION_LOOKUP_FAULT; + pt_ret.ioptSlot = 0; + return pt_ret; + } + + pt_ret.status = EXCEPTION_NONE; + pt_ret.ioptSlot = pt + index; + return pt_ret; +} + +BOOT_CODE seL4_SlotRegion +create_iospace_caps(cap_t root_cnode_cap) +{ + seL4_SlotPos start = ndks_boot.slot_pos_cur; + seL4_SlotPos end = 0; + cap_t io_space_cap; + int i = 0; + int num_smmu = plat_smmu_init(); + + if (num_smmu == 0) { + printf("SMMU init failuer\n"); + return (seL4_SlotRegion) S_REG_EMPTY; + } + + /* the 0 is reserved as an invalidASID, + * assuming each module is assigned an unique ASID + * and the ASIDs are contiguous + * */ + for (i = 1; i <= num_smmu; i++) { + io_space_cap = cap_io_space_cap_new(i, i); + if (!provide_cap(root_cnode_cap, io_space_cap)) { + return (seL4_SlotRegion) S_REG_EMPTY; + } + } + end = ndks_boot.slot_pos_cur; + printf("Region [%x to %x) for SMMU caps\n", (unsigned int)start, (unsigned int)end); + return (seL4_SlotRegion) { + start, end + }; +} + +static exception_t +performARMIOPTInvocationMap(cap_t cap, cte_t *slot, iopde_t *iopdSlot, + iopde_t iopde, uint32_t asid, paddr_t io_address) +{ + + + *iopdSlot = iopde; + cleanCacheRange_RAM((word_t)iopdSlot, + ((word_t)iopdSlot) + sizeof(iopde_t), + addrFromPPtr(iopdSlot)); + + plat_smmu_tlb_flush_all(); + plat_smmu_ptc_flush_all(); + + cap = cap_io_page_table_cap_set_capIOPTIsMapped(cap, 1); + cap = cap_io_page_table_cap_set_capIOPTASID(cap, asid); + cap = cap_io_page_table_cap_set_capIOPTMappedAddress(cap, io_address); + + slot->cap = cap; + return EXCEPTION_NONE; +} + + +exception_t +decodeARMIOPTInvocation( + word_t invLabel, + uint32_t length, + cte_t* slot, + cap_t cap, + extra_caps_t excaps, + word_t* buffer +) +{ + cap_t io_space; + word_t io_address; + word_t paddr; + uint16_t module_id; + uint32_t asid; + iopde_t *pd; + iopde_t iopde; + lookupIOPDSlot_ret_t lu_ret; + + if (invLabel == ARMIOPageTableUnmap) { + deleteIOPageTable(slot->cap); + slot->cap = cap_io_page_table_cap_set_capIOPTIsMapped(slot->cap, 0); + + setThreadState(ksCurThread, ThreadState_Restart); + return EXCEPTION_NONE; + } + + if (excaps.excaprefs[0] == NULL || length < 1) { + userError("IOPTInvocation: Truncated message."); + current_syscall_error.type = seL4_TruncatedMessage; + return EXCEPTION_SYSCALL_ERROR; + } + + if (invLabel != ARMIOPageTableMap ) { + userError("IOPTInvocation: Invalid operation."); + current_syscall_error.type = seL4_IllegalOperation; + return EXCEPTION_SYSCALL_ERROR; + } + + io_space = excaps.excaprefs[0]->cap; + io_address = getSyscallArg(0, buffer) & ~MASK(PAGE_BITS); + + if (cap_io_page_table_cap_get_capIOPTIsMapped(cap)) { + userError("IOPTMap: Cap already mapped."); + current_syscall_error.type = seL4_InvalidCapability; + current_syscall_error.invalidCapNumber = 0; + return EXCEPTION_SYSCALL_ERROR; + } + + if (cap_get_capType(io_space) != cap_io_space_cap) { + userError("IOPTMap: Invalid IOSpace cap."); + current_syscall_error.type = seL4_InvalidCapability; + current_syscall_error.invalidCapNumber = 1; + return EXCEPTION_SYSCALL_ERROR; + } + + module_id = cap_io_space_cap_get_capModuleID(io_space); + asid = plat_smmu_get_asid_by_module_id(module_id); + if (asid == asidInvalid) { + userError("IOPTMap: Invalid IOASID."); + current_syscall_error.type = seL4_InvalidCapability; + current_syscall_error.invalidCapNumber = 1; + + return EXCEPTION_SYSCALL_ERROR; + } + + paddr = pptr_to_paddr((void *)cap_io_page_table_cap_get_capIOPTBasePtr(cap)); + pd = (iopde_t *)plat_smmu_lookup_iopd_by_asid(asid); + + if (pd == 0) { + userError("IOPTMap: IOPD not found."); + current_syscall_error.type = seL4_InvalidCapability; + current_syscall_error.invalidCapNumber = 1; + + return EXCEPTION_SYSCALL_ERROR; + } + + lu_ret = lookupIOPDSlot(pd, io_address); + + if (isIOPDEValid(lu_ret.iopdSlot)) { + userError("IOPTMap: Delete first."); + current_syscall_error.type = seL4_DeleteFirst; + return EXCEPTION_SYSCALL_ERROR; + } + + iopde = iopde_iopde_pt_new( + 1, /* read */ + 1, /* write */ + 1, /* nonsecure */ + paddr + ); + + setThreadState(ksCurThread, ThreadState_Restart); + return performARMIOPTInvocationMap(cap, slot, lu_ret.iopdSlot, iopde, asid, io_address); +} + +static exception_t +performARMIOMapInvocation(cap_t cap, cte_t *slot, iopte_t *ioptSlot, + iopte_t iopte, uint32_t asid, paddr_t io_address) +{ + *ioptSlot = iopte; + cleanCacheRange_RAM((word_t)ioptSlot, + ((word_t)ioptSlot) + sizeof(iopte_t), + addrFromPPtr(ioptSlot)); + + plat_smmu_tlb_flush_all(); + plat_smmu_ptc_flush_all(); + +#ifdef CONFIG_ARM_SMMU + cap = cap_small_frame_cap_set_capFIsIOSpace(cap, 1); +#endif + cap = cap_small_frame_cap_set_capFMappedASID(cap, asid); + cap = cap_small_frame_cap_set_capFMappedAddress(cap, io_address); + slot->cap = cap; + + return EXCEPTION_NONE; +} + +exception_t +decodeARMIOMapInvocation( + word_t invLabel, + uint32_t length, + cte_t* slot, + cap_t cap, + extra_caps_t excaps, + word_t* buffer +) +{ + cap_t io_space; + paddr_t io_address; + paddr_t paddr; + uint32_t module_id; + uint32_t asid; + iopde_t *pd; + iopte_t iopte; + vm_rights_t frame_cap_rights; + cap_rights_t dma_cap_rights_mask; + lookupIOPTSlot_ret_t lu_ret; + + if (excaps.excaprefs[0] == NULL || length < 2) { + userError("IOMap: Truncated message."); + current_syscall_error.type = seL4_TruncatedMessage; + return EXCEPTION_SYSCALL_ERROR; + } + + if (generic_frame_cap_get_capFSize(cap) != ARMSmallPage) { + userError("IOMap: Invalid cap type."); + current_syscall_error.type = seL4_InvalidCapability; + current_syscall_error.invalidCapNumber = 0; + return EXCEPTION_SYSCALL_ERROR; + } + + if (cap_small_frame_cap_get_capFMappedASID(cap) != asidInvalid) { + userError("IOMap: Frame all ready mapped."); + current_syscall_error.type = seL4_InvalidCapability; + current_syscall_error.invalidCapNumber = 0; + return EXCEPTION_SYSCALL_ERROR; + } + + io_space = excaps.excaprefs[0]->cap; + io_address = getSyscallArg(1, buffer) & ~MASK(PAGE_BITS); + paddr = pptr_to_paddr((void*)cap_small_frame_cap_get_capFBasePtr(cap)); + + if (cap_get_capType(io_space) != cap_io_space_cap) { + userError("IOMap: Invalid IOSpace cap."); + current_syscall_error.type = seL4_InvalidCapability; + current_syscall_error.invalidCapNumber = 1; + return EXCEPTION_SYSCALL_ERROR; + } + + module_id = cap_io_space_cap_get_capModuleID(io_space); + asid = plat_smmu_get_asid_by_module_id(module_id); + + if (asid == asidInvalid) { + userError("IOMap: Invalid IOSpace ASID."); + current_syscall_error.type = seL4_InvalidCapability; + current_syscall_error.invalidCapNumber = 1; + return EXCEPTION_SYSCALL_ERROR; + } + + pd = (iopde_t *)plat_smmu_lookup_iopd_by_asid(asid); + if (pd == 0) { + userError("IOMap: Invalid IOSpace cap."); + current_syscall_error.type = seL4_InvalidCapability; + current_syscall_error.invalidCapNumber = 1; + return EXCEPTION_SYSCALL_ERROR; + } + + lu_ret = lookupIOPTSlot(pd, io_address); + if (lu_ret.status != EXCEPTION_NONE) { + userError("IOMap: Lookup failed."); + current_syscall_error.type = seL4_FailedLookup; + current_syscall_error.failedLookupWasSource = false; + return EXCEPTION_SYSCALL_ERROR; + } + + if (!isIOPTEEmpty(lu_ret.ioptSlot)) { + userError("IOMap: Delete first."); + current_syscall_error.type = seL4_DeleteFirst; + return EXCEPTION_SYSCALL_ERROR; + } + frame_cap_rights = cap_small_frame_cap_get_capFVMRights(cap); + dma_cap_rights_mask = rightsFromWord(getSyscallArg(0, buffer)); + + if ((frame_cap_rights == VMReadOnly) && cap_rights_get_capAllowRead(dma_cap_rights_mask)) { + /* read only */ + iopte = iopte_new( + 1, /* read */ + 0, /* write */ + 1, /* nonsecure */ + paddr + ); + } else if (frame_cap_rights == VMReadWrite) { + if (cap_rights_get_capAllowRead(dma_cap_rights_mask) && + !cap_rights_get_capAllowWrite(dma_cap_rights_mask)) { + /* read only */ + iopte = iopte_new( + 1, /* read */ + 0, /* write */ + 1, /* nonsecure */ + paddr + ); + } else if (!cap_rights_get_capAllowRead(dma_cap_rights_mask) && + cap_rights_get_capAllowWrite(dma_cap_rights_mask)) { + /* write only */ + iopte = iopte_new( + 0, /* read */ + 1, /* write */ + 1, /* nonsecure */ + paddr + ); + } else if (cap_rights_get_capAllowRead(dma_cap_rights_mask) && + cap_rights_get_capAllowWrite(dma_cap_rights_mask)) { + /* read write */ + iopte = iopte_new( + 1, /* read */ + 1, /* write */ + 1, /* nonsecure */ + paddr + ); + } else { + userError("IOMap: Invalid argument."); + current_syscall_error.type = seL4_InvalidArgument; + current_syscall_error.invalidArgumentNumber = 0; + return EXCEPTION_SYSCALL_ERROR; + } + + } else { + /* VMKernelOnly */ + userError("IOMap: Invalid argument."); + current_syscall_error.type = seL4_InvalidArgument; + current_syscall_error.invalidArgumentNumber = 0; + return EXCEPTION_SYSCALL_ERROR; + } + + setThreadState(ksCurThread, ThreadState_Restart); + return performARMIOMapInvocation(cap, slot, lu_ret.ioptSlot, iopte, asid, io_address); +} + + +void +deleteIOPageTable(cap_t io_pt_cap) +{ + + uint32_t asid; + iopde_t *pd; + lookupIOPDSlot_ret_t lu_ret; + word_t io_address; + if (cap_io_page_table_cap_get_capIOPTIsMapped(io_pt_cap)) { + io_pt_cap = cap_io_page_table_cap_set_capIOPTIsMapped(io_pt_cap, 0); + asid = cap_io_page_table_cap_get_capIOPTASID(io_pt_cap); + pd = (iopde_t *)plat_smmu_lookup_iopd_by_asid(asid); + io_address = cap_io_page_table_cap_get_capIOPTMappedAddress(io_pt_cap); + + if (pd == 0) { + return; + } + + lu_ret = lookupIOPDSlot(pd, io_address); + if (lu_ret.status != EXCEPTION_NONE) { + return; + } + + if (isIOPDEValid(lu_ret.iopdSlot) && + iopde_ptr_get_page_size(lu_ret.iopdSlot) == iopde_iopde_pt && + iopde_iopde_pt_ptr_get_address(lu_ret.iopdSlot) != (pptr_to_paddr((void *)cap_io_page_table_cap_get_capIOPTBasePtr(io_pt_cap)))) { + return; + } + + iopde_iopde_pt_ptr_new(lu_ret.iopdSlot, 0, 0, 0, 0); + cleanCacheRange_RAM((word_t)lu_ret.iopdSlot, + ((word_t)lu_ret.iopdSlot) + sizeof(iopde_t), + addrFromPPtr(lu_ret.iopdSlot)); + + + /* nice to have: flush by address and asid */ + plat_smmu_tlb_flush_all(); + plat_smmu_ptc_flush_all(); + } +} + +void +unmapIOPage(cap_t cap) +{ + lookupIOPTSlot_ret_t lu_ret; + iopde_t *pd; + word_t io_address; + uint32_t asid; + + io_address = cap_small_frame_cap_get_capFMappedAddress(cap); + asid = cap_small_frame_cap_get_capFMappedASID(cap); + pd = (iopde_t *)plat_smmu_lookup_iopd_by_asid(asid); + + if (pd == 0) { + return; + } + + lu_ret = lookupIOPTSlot(pd, io_address); + + if (lu_ret.status != EXCEPTION_NONE) { + return; + } + if (iopte_ptr_get_address(lu_ret.ioptSlot) != pptr_to_paddr((void *)cap_small_frame_cap_get_capFBasePtr(cap))) { + return; + } + + iopte_ptr_new(lu_ret.ioptSlot, 0, 0, 0, 0); + cleanCacheRange_RAM((word_t)lu_ret.ioptSlot, + ((word_t)lu_ret.ioptSlot) + sizeof(iopte_t), + addrFromPPtr(lu_ret.ioptSlot)); + + plat_smmu_tlb_flush_all(); + plat_smmu_ptc_flush_all(); + return; +} + +void clearIOPageDirectory(cap_t cap) +{ + iopde_t *pd; + uint32_t asid = cap_io_space_cap_get_capModuleID(cap); + word_t size = BIT((SMMU_PD_BITS)); + pd = (iopde_t *)plat_smmu_lookup_iopd_by_asid(asid); + + if (pd == 0) { + return; + } + memset((void *)pd, 0, size); + cleanCacheRange_RAM((word_t)pd, (word_t)pd + size, addrFromPPtr(pd)); + + plat_smmu_tlb_flush_all(); + plat_smmu_ptc_flush_all(); + return; +} + +exception_t +decodeARMIOUnMapInvocation( + word_t invLabel, + uint32_t length, + cte_t* slot, + cap_t cap, + extra_caps_t excaps +) +{ + unmapIOPage(slot->cap); + slot->cap = cap_small_frame_cap_set_capFMappedAddress(slot->cap, 0); +#ifdef CONFIG_ARM_SMMU + slot->cap = cap_small_frame_cap_set_capFIsIOSpace(slot->cap, 0); +#endif + slot->cap = cap_small_frame_cap_set_capFMappedASID(slot->cap, asidInvalid); + + setThreadState(ksCurThread, ThreadState_Restart); + return EXCEPTION_NONE; +} + +exception_t +decodeARMIOSpaceInvocation(word_t invLabel, cap_t cap) +{ + userError("IOSpace capability has no invocations"); + current_syscall_error.type = seL4_IllegalOperation; + return EXCEPTION_SYSCALL_ERROR; +} + diff --git a/src/plat/tk1/machine/Makefile b/src/plat/tk1/machine/Makefile index 1af784eaf..346ecd7ad 100644 --- a/src/plat/tk1/machine/Makefile +++ b/src/plat/tk1/machine/Makefile @@ -11,7 +11,8 @@ DIRECTORIES += src/plat/$(PLAT)/machine PLAT_C_SOURCES += machine/hardware.c \ - machine/l2cache.c + machine/l2cache.c \ + machine/smmu.c ifdef DEBUG PLAT_C_SOURCES += machine/io.c diff --git a/src/plat/tk1/machine/hardware.c b/src/plat/tk1/machine/hardware.c index ef2063dfb..9fa00ea0c 100644 --- a/src/plat/tk1/machine/hardware.c +++ b/src/plat/tk1/machine/hardware.c @@ -21,9 +21,9 @@ /* Available physical memory regions on platform (RAM minus kernel image). */ /* NOTE: Regions are not allowed to be adjacent! */ +/* 1 MiB starting from 0xa7f00000 is reserved by the elfloader for monitor mode hooks */ const p_region_t BOOT_RODATA avail_p_regs[] = { -// { .start = 0x80000000, .end = 0xf0000000 } - { .start = 0x80000000, .end = 0xb0000000 } + { .start = 0x80000000, .end = 0xa7f00000 } }; BOOT_CODE int get_num_avail_p_regs(void) @@ -40,13 +40,17 @@ BOOT_CODE p_region_t get_avail_p_reg(word_t i) #define PAGE_SIZE (1 << PAGE_BITS) #define SECTION_SIZE (1 << SECTION_BITS) -#define VM_PA_START 0xb0000000 -#define VM_PA_SIZE 0x10000000 const p_region_t BOOT_RODATA dev_p_regs[] = { - { VM_PA_START, VM_PA_START + VM_PA_SIZE }, + { VM_HOST_PA_START, VM_HOST_PA_START + VM_HOST_PA_SIZE }, { VGICI_VM_PADDR, VGICI_VM_PADDR + PAGE_SIZE }, + { PCIE_0_CFG_PADDR, PCIE_0_CFG_PADDR + PAGE_SIZE }, + { PCIE_1_CFG_PADDR, PCIE_1_CFG_PADDR + PAGE_SIZE }, + { PCIE_PCA0_1_PADDR, PCIE_PCA0_1_PADDR + PAGE_SIZE }, + { PCIE_PADS_AFI_PADDR, PCIE_PADS_AFI_PADDR + PAGE_SIZE }, + { PCIE_A2_PADDR, PCIE_A2_PADDR + SECTION_SIZE }, + { R8169_NIC_PADDR, R8169_NIC_PADDR + SECTION_SIZE}, { GRAPH_HOST_PADDR, GRAPH_HOST_PADDR + (SECTION_SIZE * 16) }, /* 16 MB */ { GPU_PADDR, GPU_PADDR + (SECTION_SIZE * 144) }, /* 144 MB */ { UP_TAG_PADDR, UP_TAG_PADDR + PAGE_SIZE }, /* 4 KB */ @@ -81,13 +85,16 @@ const p_region_t BOOT_RODATA dev_p_regs[] = { { TSENSOR_PADDR, TSENSOR_PADDR + PAGE_SIZE }, /* 4 KB */ { CEC_PADDR, CEC_PADDR + PAGE_SIZE }, /* 4 KB */ { ATOMICS_PADDR, ATOMICS_PADDR + (PAGE_SIZE * 2) }, /* 8 KB */ +#ifndef CONFIG_ARM_SMMU { MC_PADDR, MC_PADDR + PAGE_SIZE }, /* 4 KB */ +#endif { EMC_PADDR, EMC_PADDR + PAGE_SIZE }, /* 4 KB */ { SATA_PADDR, SATA_PADDR + (PAGE_SIZE * 16) }, /* 64 KB */ { HDA_PADDR, HDA_PADDR + (PAGE_SIZE * 16) }, /* 64 KB */ { MIOBFM_PADDR, MIOBFM_PADDR + (PAGE_SIZE * 16) }, /* 64 KB */ { AUDIO_PADDR, AUDIO_PADDR + (PAGE_SIZE * 16) }, /* 64 KB */ - { XUSB_HOST_PADDR, XUSB_HOST_PADDR + (PAGE_SIZE * 10) }, /* 40 KB */ + { XUSB_HOST_PADDR, XUSB_HOST_PADDR + (PAGE_SIZE * 9) }, /* 36 KB */ + { XUSB_PADCTL_PADDR, XUSB_PADCTL_PADDR + PAGE_SIZE }, /* 4 KB */ { XUSB_DEV_PADDR, XUSB_DEV_PADDR + (PAGE_SIZE * 10) }, /* 40 KB */ { DDS_PADDR, DDS_PADDR + (PAGE_SIZE * 2) }, /* 8KB 4608 bytes */ { SDMMC_1_PADDR, SDMMC_1_PADDR + PAGE_SIZE }, /* 4KB 512 bytes */ @@ -137,6 +144,11 @@ handleReservedIRQ(irq_t irq) return; } + if (config_set(CONFIG_ARM_SMMU) && (irq == INTERRUPT_SMMU)) { + plat_smmu_handle_interrupt(); + return; + } + printf("Received reserved IRQ: %d\n", (int)irq); } @@ -167,14 +179,27 @@ map_kernel_devices(void) if (config_set(ARM_HYP)) { map_kernel_frame( - GIC_VCPUCTRL_PADDR, - GIC_VCPUCTRL_PPTR, - VMKernelOnly, - vm_attributes_new( - false, - false, - false - ) + GIC_VCPUCTRL_PADDR, + GIC_VCPUCTRL_PPTR, + VMKernelOnly, + vm_attributes_new( + false, + false, + false + ) + ); + } + + if (config_set(CONFIG_ARM_SMMU)) { + map_kernel_frame( + MC_PADDR, + SMMU_PPTR, + VMKernelOnly, + vm_attributes_new( + false, + false, + false + ) ); } diff --git a/src/plat/tk1/machine/smmu.c b/src/plat/tk1/machine/smmu.c new file mode 100644 index 000000000..8c8580de9 --- /dev/null +++ b/src/plat/tk1/machine/smmu.c @@ -0,0 +1,324 @@ +#include +#include +#include +#include +#include +#include +#include + + + +#define SMMU_CONFIG_OFFSET 0x10 + +#define PTB_DATA_BASE_SHIFT 12 +#define PTB_DATA_READ BIT(31) +#define PTB_DATA_WRITE BIT(30) +#define PTB_DATA_NONSECURE BIT(29) +#define PTB_DATA_BASE_PD_MASK 0x3fffff + +#define MODULE_ASID_ENABLE BIT(31) + +#define PTC_FLUSH_ALL 0 +#define PTC_FLUSH_ADR 1 + +#define TLB_ASID_MATCH BIT(31) +#define TLB_FLUSH_ALL (0) +#define TLB_FLUSH_SECTION (2) +#define TLB_FLUSH_GROUP (3) + +#define MC_DECERR_MTS_BIT 16u +#define MC_SECERR_SEC_BIT 13u +#define MC_DECERR_VPR_BIT 12u +#define MC_APB_ASID_UPDATE_BIT 11u +#define MC_SMMU_PAGE_BIT 10u +#define MC_ARBITRATION_EMEM_BIT 9u +#define MC_SECURITY_BIT 8u +#define MC_DECERR_EMEM_BIT 6u + + +#define MC_ERR_ID_MASK 0x7f +#define MC_ERR_ADR_MASK 0x7000 +#define MC_ERR_RW_MASK 0x10000 +#define MC_ERR_SEC_MASK 0x20000 +#define MC_ERR_SWAP_MASK 0x40000 +#define MC_ERR_ADR_HI_MASK 0x300000 +#define MC_ERR_INVALID_SMMU_PAGE_NONSECURE_MASK 0x2000000 +#define MC_ERR_INVALID_SMMU_PAGE_WRITE_MASK 0x4000000 +#define MC_ERR_INVALID_SMMU_PAGE_READ_MASK 0x8000000 +#define MC_ERR_TYPE_MASK 0x70000000 +#define MC_ERR_TYPE_SHIFT 28 + +#define MC_ERR_TYPE_RSVD 0 +#define MC_ERR_TYPE_DECERR_EMEM 2 +#define MC_ERR_TYPE_SECURITY 3 +#define MC_ERR_TYPE_SECURITY_CARVEOUT 4 +#define MC_ERR_TYPE_INVALID_SMMU_PAGE 6 + +#define IOPDE_4M_INDEX_SHIFT 22 + +static volatile tk1_mc_regs_t *smmu_regs = (volatile tk1_mc_regs_t *)(SMMU_PPTR); + +static void +do_smmu_enable(void) +{ + volatile uint32_t *config = (volatile uint32_t *)(MC_PADDR + SMMU_CONFIG_OFFSET); + *config = 1; +} + +static void +do_smmu_disable(void) +{ + volatile uint32_t *config = (volatile uint32_t *)(MC_PADDR + SMMU_CONFIG_OFFSET); + *config = 0; +} + +static inline void +smmu_disable(void) +{ + if (config_set(ARM_HYP)) { + /* in hyp mode, we need call the hook in monitor mode */ + /* we need physical address here */ + paddr_t addr = addrFromPPtr(&do_smmu_disable); + asm (".arch_extension sec\n"); + asm volatile ("mov r0, %0\n\t" + "dsb\nisb\n" + "smc #0\n" + ::"r"(addr)); + } else { + /* in secure mode, can enable it directly */ + smmu_regs->smmu_config = 0; + } + + return; +} + +static inline void +smmu_enable(void) +{ + if (config_set(ARM_HYP)) { + paddr_t addr = addrFromPPtr(&do_smmu_enable); + asm (".arch_extension sec\n"); + asm volatile ("mov r0, %0\n\t" + "dsb\nisb\n" + "smc #0\n" + ::"r"(addr)); + } else { + smmu_regs->smmu_config = 1; + } + + return; +} + + +static uint32_t +make_ptb_data(uint32_t pd_base, bool_t read, bool_t write, bool_t nonsecure) +{ + uint32_t ret = 0; + ret = (pd_base >> PTB_DATA_BASE_SHIFT); + + if (read) { + ret |= PTB_DATA_READ; + } + if (write) { + ret |= PTB_DATA_WRITE; + } + if (nonsecure) { + ret |= PTB_DATA_NONSECURE; + } + + return ret; +} + +static uint32_t +ptb_data_get_pd_base(uint32_t data) +{ + uint32_t ret = data; + ret &= PTB_DATA_BASE_PD_MASK; + ret <<= PTB_DATA_BASE_SHIFT; + return ret; +} + +void +plat_smmu_ptc_flush_all(void) +{ + uint32_t cmd = PTC_FLUSH_ALL; + smmu_regs->smmu_ptc_flush = cmd; +} + +void +plat_smmu_tlb_flush_all(void) +{ + uint32_t cmd = TLB_FLUSH_ALL; + smmu_regs->smmu_tlb_flush = cmd; +} + + + +/* Using 4 MiB mapping for the Linxu guest VM. + * This is a temporary solution for enabling guest VM + * devices that need DMA while still providing some + * protections. Once the device untyped feature is done, + * this code should be replaced with proper user-mode + * VM initialisation code. + */ + +static void +plat_smmu_vm_mapping(word_t iopd, word_t gpa, word_t pa, word_t size) +{ + iopde_t *iopde = (iopde_t *)iopd; + while (size > 0) { + word_t index = gpa >> IOPDE_4M_INDEX_SHIFT; + iopde_iopde_4m_ptr_new( + iopde + index, + 1, + 1, + 1, + pa + ); + gpa += BIT(IOPDE_4M_INDEX_SHIFT); + pa += BIT(IOPDE_4M_INDEX_SHIFT); + size -= BIT(IOPDE_4M_INDEX_SHIFT); + } +} + +BOOT_CODE int +plat_smmu_init(void) +{ + uint32_t asid = 1; + int i = 0; + + smmu_disable(); + + for (i = 0; i < ARM_PLAT_NUM_SMMU; i++) { + iopde_t *pd = (iopde_t *)alloc_region(SMMU_PD_BITS); + + if (pd == 0) { + printf("Failed to allocate SMMU IOPageDirectory for ASID %d\n", asid); + return 0; + } + + memset(pd, 0, BIT(SMMU_PD_BITS)); + if (config_set(CONFIG_ARM_SMMU_VM_DEFAULT_MAPPING)) { + plat_smmu_vm_mapping((word_t)pd, VM_GUEST_PA_START, VM_HOST_PA_START, VM_HOST_PA_SIZE); + } + cleanCacheRange_RAM((word_t)pd, ((word_t)pd + BIT(SMMU_PD_BITS)), + addrFromPPtr(pd)); + + smmu_regs->smmu_ptb_asid = asid; + + /* make it read/write/nonsecure but all translation entries are invalid */ + smmu_regs->smmu_ptb_data = make_ptb_data(pptr_to_paddr(pd), true, true, true); + asid++; + } + printf("Total %d IOASID set up\n", (asid - 1)); + + /* now assign IOASID to each module */ + smmu_regs->smmu_afi_asid = SMMU_AFI_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_avpc_asid = SMMU_AVPC_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_dc_asid = SMMU_DC_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_dcb_asid = SMMU_DCB_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_hc_asid = SMMU_HC_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_hda_asid = SMMU_HDA_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_isp2_asid = SMMU_ISP2_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_msenc_asid = SMMU_MSENC_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_nv_asid = SMMU_NV_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_nv2_asid = SMMU_NV2_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_ppcs_asid = SMMU_PPCS_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_sata_asid = SMMU_SATA_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_vde_asid = SMMU_VDE_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_vi_asid = SMMU_VI_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_vic_asid = SMMU_VIC_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_xusb_host_asid = SMMU_XUSB_HOST_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_xusb_dev_asid = SMMU_XUSB_DEV_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_tsec_asid = SMMU_TSEC_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_ppcs1_asid = SMMU_PPCS1_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_sdmmc1a_asid = SMMU_SDMMC1A_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_sdmmc2a_asid = SMMU_SDMMC2A_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_sdmmc3a_asid = SMMU_SDMMC3A_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_sdmmc4a_asid = SMMU_SDMMC4A_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_isp2b_asid = SMMU_ISP2B_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_gpu_asid = SMMU_GPU_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_gpub_asid = SMMU_GPUB_ASID | MODULE_ASID_ENABLE; + smmu_regs->smmu_ppcs2_asid = SMMU_PPCS2_ASID | MODULE_ASID_ENABLE; + + /* flush page table cache */ + plat_smmu_ptc_flush_all(); + /* flush TLB */ + plat_smmu_tlb_flush_all(); + smmu_enable(); + + /* also need to unmask interrupts */ + smmu_regs->intmask = BIT(MC_APB_ASID_UPDATE_BIT) | BIT(MC_SMMU_PAGE_BIT) | + BIT(MC_DECERR_MTS_BIT) | BIT(MC_SECERR_SEC_BIT) | + BIT(MC_DECERR_VPR_BIT) | BIT(MC_ARBITRATION_EMEM_BIT) | + BIT(MC_SECURITY_BIT) | BIT(MC_DECERR_EMEM_BIT); + return ARM_PLAT_NUM_SMMU; +} + + +iopde_t * +plat_smmu_lookup_iopd_by_asid(uint32_t asid) +{ + iopde_t *pd = 0; + uint32_t data = 0; + if (asid < SMMU_FIRST_ASID || asid > SMMU_LAST_ASID) { + return 0; + } + + smmu_regs->smmu_ptb_asid = asid; + data = smmu_regs->smmu_ptb_data; + pd = (iopde_t *)(paddr_to_pptr(ptb_data_get_pd_base(data))); + return pd; +} + +void +plat_smmu_handle_interrupt(void) +{ + uint32_t status = smmu_regs->intstatus; + uint32_t clear_status = 0; + + if (status & BIT(MC_DECERR_MTS_BIT)) { + clear_status |= BIT(MC_DECERR_MTS_BIT); + } + if (status & BIT(MC_SECERR_SEC_BIT)) { + clear_status |= BIT(MC_SECERR_SEC_BIT); + } + if (status & BIT(MC_DECERR_VPR_BIT)) { + clear_status |= BIT(MC_DECERR_VPR_BIT); + } + if (status & BIT(MC_ARBITRATION_EMEM_BIT)) { + clear_status |= BIT(MC_ARBITRATION_EMEM_BIT); + } + if (status & BIT(MC_SECURITY_BIT)) { + clear_status |= BIT(MC_SECURITY_BIT); + } + if (status & BIT(MC_DECERR_EMEM_BIT)) { + clear_status |= BIT(MC_DECERR_EMEM_BIT); + } + if (status & BIT(MC_APB_ASID_UPDATE_BIT)) { + clear_status |= BIT(MC_APB_ASID_UPDATE_BIT); + } + + /* we only care about SMMU translation failures */ + if (status & BIT(MC_SMMU_PAGE_BIT)) { + if (config_set(DEBUG)) { + uint32_t err_status = smmu_regs->err_status; + uint32_t UNUSED err_adr = smmu_regs->err_adr; + uint32_t UNUSED id = err_status & MC_ERR_ID_MASK; + uint32_t UNUSED rw = (err_status & MC_ERR_RW_MASK); + uint32_t UNUSED read = (err_status & MC_ERR_INVALID_SMMU_PAGE_READ_MASK); + uint32_t UNUSED write = (err_status & MC_ERR_INVALID_SMMU_PAGE_WRITE_MASK); + uint32_t UNUSED nonsecure = (err_status & MC_ERR_INVALID_SMMU_PAGE_NONSECURE_MASK); + uint32_t UNUSED type = (err_status & MC_ERR_TYPE_MASK) >> MC_ERR_TYPE_SHIFT; + + printf("SMMU Address translation error:\n"); + printf("ID: %d address: 0x%x type: %d direction: 0x%x\n", id, err_adr, type, rw); + printf("IOPT permission: read 0x%x write 0x%x nonsecure 0x%x\n", read, write, nonsecure); + } + clear_status |= BIT(MC_SMMU_PAGE_BIT); + } + + /* write 1 to clear the interrupt */ + smmu_regs->intstatus = clear_status; +} +