From f050e6a9c58be7b118dcfb88c25db29cb7db392d Mon Sep 17 00:00:00 2001 From: amrzar Date: Fri, 21 Oct 2016 15:44:23 +1100 Subject: [PATCH] implement layout of ipi interrupt handling --- include/arch/arm/arch/kernel/ipi.h | 15 ++++ include/arch/x86/arch/kernel/apic.h | 1 + include/arch/x86/arch/kernel/ipi.h | 56 +++++++++++++ include/arch/x86/arch/kernel/lock.h | 29 +++++-- include/object/structures.h | 5 +- include/plat/pc99/plat/machine.h | 28 +++++-- src/arch/x86/kernel/Makefile | 3 +- src/arch/x86/kernel/apic.c | 29 +++++++ src/arch/x86/kernel/boot.c | 4 + src/arch/x86/kernel/ipi.c | 125 ++++++++++++++++++++++++++++ src/object/interrupt.c | 7 ++ 11 files changed, 282 insertions(+), 20 deletions(-) create mode 100644 include/arch/arm/arch/kernel/ipi.h create mode 100644 include/arch/x86/arch/kernel/ipi.h create mode 100644 src/arch/x86/kernel/ipi.c diff --git a/include/arch/arm/arch/kernel/ipi.h b/include/arch/arm/arch/kernel/ipi.h new file mode 100644 index 000000000..585861c19 --- /dev/null +++ b/include/arch/arm/arch/kernel/ipi.h @@ -0,0 +1,15 @@ +/* + * Copyright 2016, Data61 + * Commonwealth Scientific and Industrial Research Organisation (CSIRO) + * ABN 41 687 119 230. + * + * This software may be distributed and modified according to the terms of + * the GNU General Public License version 2. Note that NO WARRANTY is provided. + * See "LICENSE_GPLv2.txt" for details. + * + * @TAG(D61_GPL) + */ + +#pragma once + +/* placeholder for future arm multicore implementation */ diff --git a/include/arch/x86/arch/kernel/apic.h b/include/arch/x86/arch/kernel/apic.h index fd3ebf166..096d166b8 100644 --- a/include/arch/x86/arch/kernel/apic.h +++ b/include/arch/x86/arch/kernel/apic.h @@ -21,6 +21,7 @@ cpu_id_t apic_get_id(void); bool_t apic_is_interrupt_pending(void); void apic_ack_active_interrupt(void); +void apic_send_ipi(irq_t vector, cpu_id_t cpu_id); void apic_send_init_ipi(cpu_id_t cpu_id); void apic_send_startup_ipi(cpu_id_t cpu_id, paddr_t startup_addr); diff --git a/include/arch/x86/arch/kernel/ipi.h b/include/arch/x86/arch/kernel/ipi.h new file mode 100644 index 000000000..80d441fd3 --- /dev/null +++ b/include/arch/x86/arch/kernel/ipi.h @@ -0,0 +1,56 @@ +/* + * Copyright 2016, Data61 + * Commonwealth Scientific and Industrial Research Organisation (CSIRO) + * ABN 41 687 119 230. + * + * This software may be distributed and modified according to the terms of + * the GNU General Public License version 2. Note that NO WARRANTY is provided. + * See "LICENSE_GPLv2.txt" for details. + * + * @TAG(D61_GPL) + */ + +#pragma once + +#include +#include +#include + +#if CONFIG_MAX_NUM_NODES > 1 + +void Arch_handleIPI(irq_t irq); + +typedef enum { + IpiRemoteCall_Null +} IpiRemoteCall_t; + +/* + * Run a synchronous function on all cores specified by mask. Return when target cores + * have all executed the function. Caller must hold the lock. + * + * @param func the function to run + * @param data passed to the function + * @param mask cores to run function on + */ +void doRemoteMaskOp(IpiRemoteCall_t func, word_t data, word_t mask); + +/* This is asynchronous call and could be called outside the lock. + * Returns immediately. + * + * @param cpu core to request rescheduling + */ +void doReschedule(word_t cpu); + +/* Run a synchronous function on a core specified by cpu. + * + * @param func the function to run + * @param data passed to the function + * @param cpu core to run function on + */ +static void inline FORCE_INLINE +doRemoteOp(IpiRemoteCall_t func, word_t data, word_t cpu) +{ + doRemoteMaskOp(func, data, BIT(cpu)); +} + +#endif diff --git a/include/arch/x86/arch/kernel/lock.h b/include/arch/x86/arch/kernel/lock.h index 1e78bdd03..8fb66f33d 100644 --- a/include/arch/x86/arch/kernel/lock.h +++ b/include/arch/x86/arch/kernel/lock.h @@ -18,6 +18,7 @@ #include #include #include +#include #if CONFIG_MAX_NUM_NODES > 1 @@ -38,8 +39,12 @@ typedef struct clh_qnode { typedef struct clh_qnode_p { volatile clh_qnode_t *node; volatile clh_qnode_t *next; + /* This is the software IPI flag */ + volatile word_t ipi; - PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_t *) + sizeof(clh_qnode_t *)); + PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_t *) + + sizeof(clh_qnode_t *) + + sizeof(word_t)); } clh_qnode_p_t; typedef struct clh_lock { @@ -53,6 +58,12 @@ typedef struct clh_lock { extern clh_lock_t big_kernel_lock; BOOT_CODE void clh_lock_init(void); +static inline bool_t FORCE_INLINE +clh_is_ipi_pending(word_t cpu) +{ + return big_kernel_lock.node_owners[cpu].ipi == 1; +} + static inline void FORCE_INLINE clh_lock_acquire(word_t cpu) { @@ -65,6 +76,11 @@ clh_lock_acquire(word_t cpu) big_kernel_lock.node_owners[cpu].next = prev; while (big_kernel_lock.node_owners[cpu].next->value != CLHState_Granted) { + if (clh_is_ipi_pending(cpu)) { + /* we only handle irq_remote_call_ipi here as other type of IPIs + * are async and could be delayed */ + Arch_handleIPI(irq_remote_call_ipi); + } asm volatile("pause"); } @@ -84,10 +100,10 @@ clh_lock_release(word_t cpu) big_kernel_lock.node_owners[cpu].next; } -static inline clh_qnode_state_t FORCE_INLINE -clh_lock_test(void) +static inline bool_t FORCE_INLINE +clk_is_self_in_queue(void) { - return big_kernel_lock.head->value; + return big_kernel_lock.node_owners[getCurrentCPUIndex()].node->value == CLHState_Pending; } #define NODE_LOCK do { \ @@ -98,15 +114,10 @@ clh_lock_test(void) clh_lock_release(getCurrentCPUIndex()); \ } while(0) -#define LOCK_TEST do { \ - clh_lock_test() \ - } while(0) - #else #define NODE_LOCK do {} while (0) #define NODE_UNLOCK do {} while (0) -#define LOCK_TEST do {} while (0) #endif diff --git a/include/object/structures.h b/include/object/structures.h index 9fd206fe3..5b95c3dfc 100644 --- a/include/object/structures.h +++ b/include/object/structures.h @@ -22,7 +22,10 @@ enum irq_state { IRQInactive = 0, IRQSignal = 1, IRQTimer = 2, - IRQReserved = 3, +#if CONFIG_MAX_NUM_NODES > 1 + IRQIPI = 3, +#endif + IRQReserved }; typedef word_t irq_state_t; diff --git a/include/plat/pc99/plat/machine.h b/include/plat/pc99/plat/machine.h index 065af4fed..b50cd0444 100644 --- a/include/plat/pc99/plat/machine.h +++ b/include/plat/pc99/plat/machine.h @@ -31,22 +31,32 @@ typedef enum _interrupt_t { int_irq_user_max = 157, int_iommu = 158, int_timer = 159, +#if CONFIG_MAX_NUM_NODES > 1 + int_remote_call_ipi = 160, + int_reschedule_ipi = 161, + int_irq_max = 161, /* int_reschedule_ipi is the max irq */ +#else int_irq_max = 159, /* int_timer is the max irq */ - int_trap_min = 160, +#endif + int_trap_min, int_trap_max = 254, int_spurious = 255, int_max = 255 } interrupt_t; typedef enum _irq_t { - irqInvalid = -1, - irq_isa_min = int_irq_isa_min - IRQ_INT_OFFSET, - irq_isa_max = int_irq_isa_max - IRQ_INT_OFFSET, - irq_user_min = int_irq_user_min - IRQ_INT_OFFSET, - irq_user_max = int_irq_user_max - IRQ_INT_OFFSET, - irq_iommu = int_iommu - IRQ_INT_OFFSET, - irq_timer = int_timer - IRQ_INT_OFFSET, - maxIRQ = int_irq_max - IRQ_INT_OFFSET + irqInvalid = -1, + irq_isa_min = int_irq_isa_min - IRQ_INT_OFFSET, + irq_isa_max = int_irq_isa_max - IRQ_INT_OFFSET, + irq_user_min = int_irq_user_min - IRQ_INT_OFFSET, + irq_user_max = int_irq_user_max - IRQ_INT_OFFSET, + irq_iommu = int_iommu - IRQ_INT_OFFSET, + irq_timer = int_timer - IRQ_INT_OFFSET, +#if CONFIG_MAX_NUM_NODES > 1 + irq_remote_call_ipi = int_remote_call_ipi - IRQ_INT_OFFSET, + irq_reschedule_ipi = int_reschedule_ipi - IRQ_INT_OFFSET, +#endif + maxIRQ = int_irq_max - IRQ_INT_OFFSET } irq_t; #define BIOS_PADDR_START 0x0e0000 diff --git a/src/arch/x86/kernel/Makefile b/src/arch/x86/kernel/Makefile index d4cb0fed4..1a8c3455d 100644 --- a/src/arch/x86/kernel/Makefile +++ b/src/arch/x86/kernel/Makefile @@ -16,4 +16,5 @@ ARCH_C_SOURCES += kernel/vspace.c \ kernel/smp_sys.c \ kernel/boot.c \ kernel/cmdline.c \ - kernel/lock.c + kernel/lock.c \ + kernel/ipi.c diff --git a/src/arch/x86/kernel/apic.c b/src/arch/x86/kernel/apic.c index e0e72fd8d..4ceddaeb5 100644 --- a/src/arch/x86/kernel/apic.c +++ b/src/arch/x86/kernel/apic.c @@ -277,3 +277,32 @@ apic_send_startup_ipi(cpu_id_t cpu_id, paddr_t startup_addr) ).words[0] ); } + +void +apic_send_ipi(irq_t vector, cpu_id_t cpu_id) +{ + apic_icr1_t icr1; + /* wait till we can send an IPI */ + do { + icr1.words[0] = apic_read_reg(APIC_ICR1); + } while (icr1.words[0] & BIT(12)); + + apic_write_reg( + APIC_ICR2, + apic_icr2_new( + cpu_id /* dest */ + ).words[0] + ); + apic_write_reg( + APIC_ICR1, + apic_icr1_new( + 0, /* dest_shorthand */ + 0, /* trigger_mode */ + 0, /* level */ + 0, /* delivery_status */ + 0, /* dest_mode */ + 0, /* delivery_mode */ + vector /* vector */ + ).words[0] + ); +} diff --git a/src/arch/x86/kernel/boot.c b/src/arch/x86/kernel/boot.c index 7355edeae..30ed912d2 100644 --- a/src/arch/x86/kernel/boot.c +++ b/src/arch/x86/kernel/boot.c @@ -36,6 +36,10 @@ init_irqs(cap_t root_cnode_cap) for (i = 0; i <= maxIRQ; i++) { if (i == irq_timer) { setIRQState(IRQTimer, i); +#if CONFIG_MAX_NUM_NODES > 1 + } else if (i == irq_remote_call_ipi || i == irq_reschedule_ipi) { + setIRQState(IRQIPI, i); +#endif } else if (i == irq_iommu) { setIRQState(IRQReserved, i); } else if (i == 2 && config_set(CONFIG_IRQ_PIC)) { diff --git a/src/arch/x86/kernel/ipi.c b/src/arch/x86/kernel/ipi.c new file mode 100644 index 000000000..539fc417b --- /dev/null +++ b/src/arch/x86/kernel/ipi.c @@ -0,0 +1,125 @@ +/* + * Copyright 2016, Data61 + * Commonwealth Scientific and Industrial Research Organisation (CSIRO) + * ABN 41 687 119 230. + * + * This software may be distributed and modified according to the terms of + * the GNU General Public License version 2. Note that NO WARRANTY is provided. + * See "LICENSE_GPLv2.txt" for details. + * + * @TAG(D61_GPL) + */ + +#include +#include +#include +#include + +#if CONFIG_MAX_NUM_NODES > 1 + +static volatile struct { + word_t count; + word_t globalsense; + + PAD_TO_NEXT_CACHE_LN(sizeof(word_t) + sizeof(word_t)); +} ipiSyncBarrier = {0}; /* IPI barrier for remote call synchronization */ + +static volatile word_t totalCoreBarrier; /* number of cores involved in IPI 'in progress' */ +static IpiRemoteCall_t remoteCall; /* the remote call being requested */ +static word_t remoteCallData; /* data to be passed to the remote call function */ + +static inline void ipi_wait(word_t cores) +{ + word_t localsense = ipiSyncBarrier.globalsense; + + if (__sync_fetch_and_add(&ipiSyncBarrier.count, 1) == cores) { + ipiSyncBarrier.count = 0; + ipiSyncBarrier.globalsense = + ~ipiSyncBarrier.globalsense; + } + + while (localsense == ipiSyncBarrier.globalsense) { + asm volatile("pause"); + } +} + +static void handleRemoteCall(void) +{ + /* we gets spurious irq_remote_call_ipi calls, e.g. when handling IPI + * in lock while hardware IPI is pending. Guard against spurious IPIs! */ + if (clh_is_ipi_pending(getCurrentCPUIndex())) { + switch (remoteCall) { + case IpiRemoteCall_Null: + + break; + default: + fail("Invalid remote call"); + } + big_kernel_lock.node_owners[getCurrentCPUIndex()].ipi = 0; + ipi_wait(totalCoreBarrier); + } +} + +static void handleReschedule(void) +{ + rescheduleRequired(); +} + +void Arch_handleIPI(irq_t irq) +{ + if (irq == irq_remote_call_ipi) { + handleRemoteCall(); + } else if (irq == irq_reschedule_ipi) { + handleReschedule(); + } else { + fail("Invalid IPI"); + } +} + +/* make sure all cpu IDs for number of core fit in bitwise word */ +compile_assert(invalid_number_of_supported_nodes, CONFIG_MAX_NUM_NODES <= wordBits); + +void doRemoteMaskOp(IpiRemoteCall_t func, word_t data, word_t mask) +{ + word_t nr_target_cores = 0; + uint16_t target_cores[CONFIG_MAX_NUM_NODES]; + + /* make sure the current core is not set in the mask */ + mask &= ~BIT(getCurrentCPUIndex()); + + /* this may happen, e.g. the caller tries to map a pagetable in + * newly created PD which has not been run yet. Guard against them! */ + if (mask != 0) { + + /* setup the data and choose the requested cpu to send IPI*/ + remoteCall = func; + remoteCallData = data; + while (mask) { + int index = wordBits - 1 - clzl(mask); + target_cores[nr_target_cores] = index; + nr_target_cores++; + mask &= ~BIT(index); + } + + /* sending IPIs... */ + totalCoreBarrier = nr_target_cores; + asm volatile("" ::: "memory"); + + for (int i = 0; i < nr_target_cores; i++) { + big_kernel_lock.node_owners[target_cores[i]].ipi = 1; + apic_send_ipi(int_remote_call_ipi, cpuIndexToID(target_cores[i])); + } + + ipi_wait(totalCoreBarrier); + } +} + +void doReschedule(word_t cpu) +{ + if (cpu != getCurrentCPUIndex() && + cpu < CONFIG_MAX_NUM_NODES) { + apic_send_ipi(int_remote_call_ipi, cpuIndexToID(cpu)); + } +} + +#endif diff --git a/src/object/interrupt.c b/src/object/interrupt.c index 0f9d055db..6460f0066 100644 --- a/src/object/interrupt.c +++ b/src/object/interrupt.c @@ -22,6 +22,7 @@ #include #include #include +#include exception_t decodeIRQControlInvocation(word_t invLabel, word_t length, @@ -214,6 +215,12 @@ handleInterrupt(irq_t irq) resetTimer(); break; +#if CONFIG_MAX_NUM_NODES > 1 + case IRQIPI: + Arch_handleIPI(irq); + break; +#endif + case IRQReserved: #ifdef CONFIG_IRQ_REPORTING printf("Received reserved IRQ: %d", (int)irq);