From 2c49729da586eddeef968fc65abd48b6f458dcda Mon Sep 17 00:00:00 2001 From: Adrian Danis Date: Wed, 9 Nov 2016 15:22:09 +1100 Subject: [PATCH] x86: Refactor tlb_bitmap to be mode generic Refactors the TLB bitmap code to be generic across ia32 and x86-64. --- .../arch/x86/arch/32/mode/fastpath/fastpath.h | 2 +- .../arch/x86/arch/32/mode/kernel/tlb_bitmap.h | 77 ------------------ .../arch/x86/arch/32/mode/object/structures.h | 2 + .../arch/x86/arch/64/mode/kernel/tlb_bitmap.h | 18 ----- .../arch/x86/arch/64/mode/object/structures.h | 1 + include/arch/x86/arch/kernel/tlb_bitmap.h | 78 +++++++++++++++++++ .../pc99/plat/32/plat_mode/machine/hardware.h | 34 +++++--- src/arch/x86/32/kernel/vspace.c | 4 +- src/arch/x86/kernel/vspace.c | 2 +- 9 files changed, 108 insertions(+), 110 deletions(-) delete mode 100644 include/arch/x86/arch/32/mode/kernel/tlb_bitmap.h delete mode 100644 include/arch/x86/arch/64/mode/kernel/tlb_bitmap.h create mode 100644 include/arch/x86/arch/kernel/tlb_bitmap.h diff --git a/include/arch/x86/arch/32/mode/fastpath/fastpath.h b/include/arch/x86/arch/32/mode/fastpath/fastpath.h index 40bd3e430..7df09919e 100644 --- a/include/arch/x86/arch/32/mode/fastpath/fastpath.h +++ b/include/arch/x86/arch/32/mode/fastpath/fastpath.h @@ -18,7 +18,7 @@ #include #include #include -#include +#include static inline void switchToThread_fp(tcb_t *thread, vspace_root_t *pd, pde_t stored_hw_asid) diff --git a/include/arch/x86/arch/32/mode/kernel/tlb_bitmap.h b/include/arch/x86/arch/32/mode/kernel/tlb_bitmap.h deleted file mode 100644 index 225c8090e..000000000 --- a/include/arch/x86/arch/32/mode/kernel/tlb_bitmap.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2016, Data61 - * Commonwealth Scientific and Industrial Research Organisation (CSIRO) - * ABN 41 687 119 230. - * - * This software may be distributed and modified according to the terms of - * the GNU General Public License version 2. Note that NO WARRANTY is provided. - * See "LICENSE_GPLv2.txt" for details. - * - * @TAG(D61_GPL) - */ - -#ifndef __ARCH_MODE_KERNEL_TLB_BITMAP_H_ -#define __ARCH_MODE_KERNEL_TLB_BITMAP_H_ - -#include -#include -#include - -#if CONFIG_MAX_NUM_NODES > 1 -/* Bit zero for entries in PD are present bits and should always be 0 */ -#define TLBBITMAP_ENTRIES_PER_PD 31 - -/* Number of entries in PD reserved for TLB bitmap */ -#define TLBBITMAP_PD_ENTRIES (((CONFIG_MAX_NUM_NODES - 1) / TLBBITMAP_ENTRIES_PER_PD) + 1) -#define TLBBITMAP_PD_RESERVED (TLBBITMAP_PD_ENTRIES * BIT(LARGE_PAGE_BITS)) -#define TLBBITMAP_PD_INDEX (TLBBITMAP_PPTR >> LARGE_PAGE_BITS) - -/* Total number of usable bits in TLBbitmap */ -#define TLBBITMAP_PD_BITS (TLBBITMAP_ENTRIES_PER_PD * TLBBITMAP_PD_ENTRIES) - -#define TLBBITMAP_PD_MAKE_INDEX(_cpu) (TLBBITMAP_PD_INDEX + ((_cpu) / (wordBits - 1))) -#define TLBBITMAP_PD_MAKE_BIT(_cpu) BIT(((_cpu) % (wordBits - 1)) + 1) - -static inline void -tlb_bitmap_init(pde_t *pd) -{ - for (int i = 0; i < TLBBITMAP_PD_ENTRIES; i++) { - pd[TLBBITMAP_PD_INDEX + i] = makeUserPDEPageTableInvalid(); - } -} - -static inline void -tlb_bitmap_set(pde_t *pd, word_t cpu) -{ - assert(cpu < TLBBITMAP_PD_BITS && cpu <= wordBits); - pd[TLBBITMAP_PD_MAKE_INDEX(cpu)].words[0] |= TLBBITMAP_PD_MAKE_BIT(cpu); -} - -static inline void -tlb_bitmap_unset(pde_t *pd, word_t cpu) -{ - assert(cpu < TLBBITMAP_PD_BITS && cpu <= wordBits); - pd[TLBBITMAP_PD_MAKE_INDEX(cpu)].words[0] &= ~TLBBITMAP_PD_MAKE_BIT(cpu); -} - -static inline word_t -tlb_bitmap_get(pde_t *pd) -{ - word_t bitmap = 0; - - for (int i = 0; i < TLBBITMAP_PD_ENTRIES; i++) { - word_t entry = pd[TLBBITMAP_PD_INDEX + i].words[0]; - // skip present bit - entry >>= 1; - - int shift = i * TLBBITMAP_ENTRIES_PER_PD; - bitmap |= entry << shift; - } - return bitmap; -} - -#else -#define TLBBITMAP_PD_RESERVED 0 -#endif /* CONFIG_MAX_NUM_NODES */ - -#endif /* __ARCH_MODE_KERNEL_TLB_BITMAP_H_ */ diff --git a/include/arch/x86/arch/32/mode/object/structures.h b/include/arch/x86/arch/32/mode/object/structures.h index ae19a174d..989236bdc 100644 --- a/include/arch/x86/arch/32/mode/object/structures.h +++ b/include/arch/x86/arch/32/mode/object/structures.h @@ -43,6 +43,8 @@ typedef pdpte_t vspace_root_t; typedef pde_t vspace_root_t; #endif +#define GET_VSPACE_ROOT_INDEX(x) ((x) >> (seL4_PageBits + PT_INDEX_BITS)) + #define PDPTE_PTR(r) ((pdpte_t *)(r)) #define PDPTE_PTR_PTR(r) ((pdpte_t**)(r)) #define PDPTE_REF(p) ((word_t)(p)) diff --git a/include/arch/x86/arch/64/mode/kernel/tlb_bitmap.h b/include/arch/x86/arch/64/mode/kernel/tlb_bitmap.h deleted file mode 100644 index 09e09148d..000000000 --- a/include/arch/x86/arch/64/mode/kernel/tlb_bitmap.h +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright 2016, Data61 - * Commonwealth Scientific and Industrial Research Organisation (CSIRO) - * ABN 41 687 119 230. - * - * This software may be distributed and modified according to the terms of - * the GNU General Public License version 2. Note that NO WARRANTY is provided. - * See "LICENSE_GPLv2.txt" for details. - * - * @TAG(D61_GPL) - */ - -#ifndef __ARCH_MODE_KERNEL_TLB_BITMAP_H_ -#define __ARCH_MODE_KERNEL_TLB_BITMAP_H_ - -/* This is placeholder file for 64 bit TLB bitmap implementation */ - -#endif /* __ARCH_MODE_KERNEL_TLB_BITMAP_H_ */ diff --git a/include/arch/x86/arch/64/mode/object/structures.h b/include/arch/x86/arch/64/mode/object/structures.h index 65e606d1d..f6a79a28d 100644 --- a/include/arch/x86/arch/64/mode/object/structures.h +++ b/include/arch/x86/arch/64/mode/object/structures.h @@ -61,6 +61,7 @@ compile_assert(uint64_t_size_64, typedef pml4e_t vspace_root_t; #define GET_PML4_INDEX(x) ( ((x) >> (PML4_INDEX_OFFSET)) & MASK(PML4_INDEX_BITS)) +#define GET_VSPACE_ROOT_INDEX GET_PML4_INDEX #define GET_PDPT_INDEX(x) ( ((x) >> (PDPT_INDEX_OFFSET)) & MASK(PDPT_INDEX_BITS)) #define GET_PD_INDEX(x) ( ((x) >> (PD_INDEX_OFFSET)) & MASK(PD_INDEX_BITS)) #define GET_PT_INDEX(x) ( ((x) >> (PT_INDEX_OFFSET)) & MASK(PT_INDEX_BITS)) diff --git a/include/arch/x86/arch/kernel/tlb_bitmap.h b/include/arch/x86/arch/kernel/tlb_bitmap.h new file mode 100644 index 000000000..b0a072ee5 --- /dev/null +++ b/include/arch/x86/arch/kernel/tlb_bitmap.h @@ -0,0 +1,78 @@ +/* + * Copyright 2016, Data61 + * Commonwealth Scientific and Industrial Research Organisation (CSIRO) + * ABN 41 687 119 230. + * + * This software may be distributed and modified according to the terms of + * the GNU General Public License version 2. Note that NO WARRANTY is provided. + * See "LICENSE_GPLv2.txt" for details. + * + * @TAG(D61_GPL) + */ + +#ifndef __ARCH_KERNEL_TLB_BITMAP_H_ +#define __ARCH_KERNEL_TLB_BITMAP_H_ + +#include +#include +#include +#include + +#if CONFIG_MAX_NUM_NODES > 1 +/* Bit zero for entries in PD are present bits and should always be 0 */ +#define TLBBITMAP_ENTRIES_PER_ROOT (wordBits - 1) + +/* Number of entries in PD reserved for TLB bitmap */ +#define TLBBITMAP_ROOT_ENTRIES (((CONFIG_MAX_NUM_NODES - 1) / TLBBITMAP_ENTRIES_PER_ROOT) + 1) +#define TLBBITMAP_RESERVED_VSPACE (TLBBITMAP_ROOT_ENTRIES * BIT(TLBBITMAP_ROOT_ENTRY_SIZE)) +#define TLBBITMAP_ROOT_INDEX GET_VSPACE_ROOT_INDEX(TLBBITMAP_PPTR) + +/* Total number of usable bits in TLBbitmap */ +#define TLBBITMAP_ROOT_BITS (TLBBITMAP_ENTRIES_PER_ROOT * TLBBITMAP_ROOT_ENTRIES) + +#define TLBBITMAP_ROOT_MAKE_INDEX(_cpu) (TLBBITMAP_ROOT_INDEX + ((_cpu) / TLBBITMAP_ENTRIES_PER_ROOT)) +#define TLBBITMAP_ROOT_MAKE_BIT(_cpu) BIT(((_cpu) % TLBBITMAP_ENTRIES_PER_ROOT) + 1) + +static inline void +tlb_bitmap_init(vspace_root_t *root) +{ + for (int i = 0; i < TLBBITMAP_ROOT_ENTRIES; i++) { + root[TLBBITMAP_ROOT_INDEX + i] = x86_make_empty_root_mapping(); + } +} + +static inline void +tlb_bitmap_set(vspace_root_t *root, word_t cpu) +{ + assert(cpu < TLBBITMAP_ROOT_BITS && cpu <= wordBits); + root[TLBBITMAP_ROOT_MAKE_INDEX(cpu)].words[0] |= TLBBITMAP_ROOT_MAKE_BIT(cpu); +} + +static inline void +tlb_bitmap_unset(vspace_root_t *root, word_t cpu) +{ + assert(cpu < TLBBITMAP_ROOT_BITS && cpu <= wordBits); + root[TLBBITMAP_ROOT_MAKE_INDEX(cpu)].words[0] &= ~TLBBITMAP_ROOT_MAKE_BIT(cpu); +} + +static inline word_t +tlb_bitmap_get(vspace_root_t *root) +{ + word_t bitmap = 0; + + for (int i = 0; i < TLBBITMAP_ROOT_ENTRIES; i++) { + word_t entry = root[TLBBITMAP_ROOT_INDEX + i].words[0]; + // skip present bit + entry >>= 1; + + int shift = i * TLBBITMAP_ENTRIES_PER_ROOT; + bitmap |= entry << shift; + } + return bitmap; +} + +#else +#define TLBBITMAP_ROOT_ENTRIES 0 +#endif /* CONFIG_MAX_NUM_NODES */ + +#endif /* __ARCH_KERNEL_TLB_BITMAP_H_ */ diff --git a/include/plat/pc99/plat/32/plat_mode/machine/hardware.h b/include/plat/pc99/plat/32/plat_mode/machine/hardware.h index 47e658550..ae3cd406f 100644 --- a/include/plat/pc99/plat/32/plat_mode/machine/hardware.h +++ b/include/plat/pc99/plat/32/plat_mode/machine/hardware.h @@ -11,28 +11,40 @@ #ifndef __PLAT_MODE_MACHINE_HARDWARE_H #define __PLAT_MODE_MACHINE_HARDWARE_H +#include + /* WARNING: some of these constants are also defined in linker.lds */ #define PADDR_BASE 0x00000000 #define PADDR_LOAD 0x00100000 #define PPTR_BASE 0xe0000000 + #ifdef CONFIG_PAE_PAGING #define PPTR_USER_TOP (PPTR_BASE & (~MASK(seL4_HugePageBits))) #else #define PPTR_USER_TOP (PPTR_BASE & (~MASK(seL4_LargePageBits))) #endif + +/* Calculate virtual address space reserved for TLB Bitmap. ROOT_ENTRIES + * will be zero in the case where the bitmap is unused */ +#define TLBBITMAP_PD_RESERVED (TLBBITMAP_ROOT_ENTRIES * BIT(seL4_LargePageBits)) + +/* Calculate virtual address space reserved for dynamic log buffer mapping */ #ifdef CONFIG_BENCHMARK_USE_KERNEL_LOG_BUFFER -#define PPTR_TOP (-BIT(LARGE_PAGE_BITS + 1) - TLBBITMAP_PD_RESERVED) -#define KS_LOG_PPTR PPTR_TOP -#if CONFIG_MAX_NUM_NODES > 1 -/* The start address which is unusable and reserved for TLBBitmap */ -#define TLBBITMAP_PPTR (PPTR_TOP + BIT(LARGE_PAGE_BITS)) -#endif /* CONFIG_MAX_NUM_NODES */ +#define LOGBUFFER_PD_RESERVED BIT(seL4_LargePageBits) #else -#define PPTR_TOP (-BIT(LARGE_PAGE_BITS) - TLBBITMAP_PD_RESERVED) -#if CONFIG_MAX_NUM_NODES > 1 -#define TLBBITMAP_PPTR PPTR_TOP -#endif /* CONFIG_MAX_NUM_NODES */ -#endif /* CONFIG_BENCHMARK_USE_KERNEL_LOG_BUFFER */ +#define LOGBUFFER_PD_RESERVED 0 +#endif + +/* There is a page table (where the kernel devices will go) in the last + * slot in memory */ +#define PPTR_KERNEL_PT_BASE (-BIT(LARGE_PAGE_BITS)) +/* TLB bitmap is next after the PT */ +#define TLBBITMAP_PPTR (PPTR_KERNEL_PT_BASE - TLBBITMAP_PD_RESERVED) +/* After TLB bitmap is the log buffer */ +#define KERNEL_LOG_PPTR (TLBBITMAP_PPTR - LOGBUFFER_PD_RESERVED) +/* This then marks the end of where physical memory gets mapped */ +#define PPTR_TOP KERNEL_LOG_PPTR + #define PPTR_KDEV 0xffff0000 #define BASE_OFFSET (PPTR_BASE - PADDR_BASE) #define kernelBase PPTR_USER_TOP diff --git a/src/arch/x86/32/kernel/vspace.c b/src/arch/x86/32/kernel/vspace.c index 9c84c215e..c12c96b56 100644 --- a/src/arch/x86/32/kernel/vspace.c +++ b/src/arch/x86/32/kernel/vspace.c @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include /* 'gdt_idt_ptr' is declared globally because of a C-subset restriction. @@ -287,7 +287,7 @@ map_kernel_window( tlb_bitmap_init(ia32KSGlobalPD); phys += TLBBITMAP_PD_RESERVED; - idx += TLBBITMAP_PD_ENTRIES; + idx += TLBBITMAP_ROOT_ENTRIES; #endif /* CONFIG_MAX_NUM_NODES */ /* map page table of last 4M of virtual address space to page directory */ diff --git a/src/arch/x86/kernel/vspace.c b/src/arch/x86/kernel/vspace.c index 6cc1cea34..05c8338db 100644 --- a/src/arch/x86/kernel/vspace.c +++ b/src/arch/x86/kernel/vspace.c @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include