implement layout of ipi interrupt handling

include/arch/arm/arch/kernel/ipi.h

@@ -0,0 +1,15 @@
+/*
+ * Copyright 2016, Data61
+ * Commonwealth Scientific and Industrial Research Organisation (CSIRO)
+ * ABN 41 687 119 230.
+ *
+ * This software may be distributed and modified according to the terms of
+ * the GNU General Public License version 2. Note that NO WARRANTY is provided.
+ * See "LICENSE_GPLv2.txt" for details.
+ *
+ * @TAG(D61_GPL)
+ */
+
+#pragma once
+
+/* placeholder for future arm multicore implementation */

include/arch/x86/arch/kernel/apic.h

@@ -21,6 +21,7 @@ cpu_id_t apic_get_id(void);
 bool_t apic_is_interrupt_pending(void);
 void apic_ack_active_interrupt(void);
 
+void apic_send_ipi(irq_t vector, cpu_id_t cpu_id);
 void apic_send_init_ipi(cpu_id_t cpu_id);
 void apic_send_startup_ipi(cpu_id_t cpu_id, paddr_t startup_addr);
 

include/arch/x86/arch/kernel/ipi.h

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2016, Data61
+ * Commonwealth Scientific and Industrial Research Organisation (CSIRO)
+ * ABN 41 687 119 230.
+ *
+ * This software may be distributed and modified according to the terms of
+ * the GNU General Public License version 2. Note that NO WARRANTY is provided.
+ * See "LICENSE_GPLv2.txt" for details.
+ *
+ * @TAG(D61_GPL)
+ */
+
+#pragma once
+
+#include <config.h>
+#include <types.h>
+#include <plat/machine.h>
+
+#if CONFIG_MAX_NUM_NODES > 1
+
+void Arch_handleIPI(irq_t irq);
+
+typedef enum {
+    IpiRemoteCall_Null
+} IpiRemoteCall_t;
+
+/*
+ * Run a synchronous function on all cores specified by mask. Return when target cores
+ * have all executed the function. Caller must hold the lock.
+ *
+ * @param func the function to run
+ * @param data passed to the function
+ * @param mask cores to run function on
+ */
+void doRemoteMaskOp(IpiRemoteCall_t func, word_t data, word_t mask);
+
+/* This is asynchronous call and could be called outside the lock.
+ * Returns immediately.
+ *
+ * @param cpu core to request rescheduling
+ */
+void doReschedule(word_t cpu);
+
+/* Run a synchronous function on a core specified by cpu.
+ *
+ * @param func the function to run
+ * @param data passed to the function
+ * @param cpu core to run function on
+ */
+static void inline FORCE_INLINE
+doRemoteOp(IpiRemoteCall_t func, word_t data, word_t cpu)
+{
+    doRemoteMaskOp(func, data, BIT(cpu));
+}
+
+#endif

include/arch/x86/arch/kernel/lock.h

@@ -18,6 +18,7 @@
 #include <util.h>
 #include <mode/machine.h>
 #include <arch/model/statedata.h>
+#include <arch/kernel/ipi.h>
 
 #if CONFIG_MAX_NUM_NODES > 1
 
@@ -38,8 +39,12 @@ typedef struct clh_qnode {
 typedef struct clh_qnode_p {
     volatile clh_qnode_t *node;
     volatile clh_qnode_t *next;
+    /* This is the software IPI flag */
+    volatile word_t ipi;
 
-    PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_t *) + sizeof(clh_qnode_t *));
+    PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_t *) +
+                         sizeof(clh_qnode_t *) +
+                         sizeof(word_t));
 } clh_qnode_p_t;
 
 typedef struct clh_lock {
@@ -53,6 +58,12 @@ typedef struct clh_lock {
 extern clh_lock_t big_kernel_lock;
 BOOT_CODE void clh_lock_init(void);
 
+static inline bool_t FORCE_INLINE
+clh_is_ipi_pending(word_t cpu)
+{
+    return big_kernel_lock.node_owners[cpu].ipi == 1;
+}
+
 static inline void FORCE_INLINE
 clh_lock_acquire(word_t cpu)
 {
@@ -65,6 +76,11 @@ clh_lock_acquire(word_t cpu)
     big_kernel_lock.node_owners[cpu].next = prev;
 
     while (big_kernel_lock.node_owners[cpu].next->value != CLHState_Granted) {
+        if (clh_is_ipi_pending(cpu)) {
+            /* we only handle irq_remote_call_ipi here as other type of IPIs
+             * are async and could be delayed */
+            Arch_handleIPI(irq_remote_call_ipi);
+        }
         asm volatile("pause");
     }
 
@@ -84,10 +100,10 @@ clh_lock_release(word_t cpu)
         big_kernel_lock.node_owners[cpu].next;
 }
 
-static inline clh_qnode_state_t FORCE_INLINE
-clh_lock_test(void)
+static inline bool_t FORCE_INLINE
+clk_is_self_in_queue(void)
 {
-    return big_kernel_lock.head->value;
+    return big_kernel_lock.node_owners[getCurrentCPUIndex()].node->value == CLHState_Pending;
 }
 
 #define NODE_LOCK do {                          \
@@ -98,15 +114,10 @@ clh_lock_test(void)
     clh_lock_release(getCurrentCPUIndex());     \
     } while(0)
 
-#define LOCK_TEST do {                          \
-    clh_lock_test()                             \
-    } while(0)
-
 #else
 
 #define NODE_LOCK do {} while (0)
 #define NODE_UNLOCK do {} while (0)
-#define LOCK_TEST do {} while (0)
 
 #endif
 

include/object/structures.h

@@ -22,7 +22,10 @@ enum irq_state {
     IRQInactive  = 0,
     IRQSignal    = 1,
     IRQTimer     = 2,
-    IRQReserved  = 3,
+#if CONFIG_MAX_NUM_NODES > 1
+    IRQIPI       = 3,
+#endif
+    IRQReserved
 };
 typedef word_t irq_state_t;
 

include/plat/pc99/plat/machine.h

@@ -31,22 +31,32 @@ typedef enum _interrupt_t {
     int_irq_user_max            = 157,
     int_iommu                   = 158,
     int_timer                   = 159,
+#if CONFIG_MAX_NUM_NODES > 1
+    int_remote_call_ipi         = 160,
+    int_reschedule_ipi          = 161,
+    int_irq_max                 = 161, /* int_reschedule_ipi is the max irq */
+#else
     int_irq_max                 = 159, /* int_timer is the max irq */
-    int_trap_min                = 160,
+#endif
+    int_trap_min,
     int_trap_max                = 254,
     int_spurious                = 255,
     int_max                     = 255
 } interrupt_t;
 
 typedef enum _irq_t {
-    irqInvalid   = -1,
-    irq_isa_min  = int_irq_isa_min  - IRQ_INT_OFFSET,
-    irq_isa_max  = int_irq_isa_max  - IRQ_INT_OFFSET,
-    irq_user_min = int_irq_user_min - IRQ_INT_OFFSET,
-    irq_user_max = int_irq_user_max - IRQ_INT_OFFSET,
-    irq_iommu    = int_iommu        - IRQ_INT_OFFSET,
-    irq_timer    = int_timer        - IRQ_INT_OFFSET,
-    maxIRQ       = int_irq_max      - IRQ_INT_OFFSET
+    irqInvalid                  = -1,
+    irq_isa_min                 = int_irq_isa_min     - IRQ_INT_OFFSET,
+    irq_isa_max                 = int_irq_isa_max     - IRQ_INT_OFFSET,
+    irq_user_min                = int_irq_user_min    - IRQ_INT_OFFSET,
+    irq_user_max                = int_irq_user_max    - IRQ_INT_OFFSET,
+    irq_iommu                   = int_iommu           - IRQ_INT_OFFSET,
+    irq_timer                   = int_timer           - IRQ_INT_OFFSET,
+#if CONFIG_MAX_NUM_NODES > 1
+    irq_remote_call_ipi         = int_remote_call_ipi - IRQ_INT_OFFSET,
+    irq_reschedule_ipi          = int_reschedule_ipi  - IRQ_INT_OFFSET,
+#endif
+    maxIRQ                      = int_irq_max         - IRQ_INT_OFFSET
 } irq_t;
 
 #define BIOS_PADDR_START 0x0e0000

src/arch/x86/kernel/Makefile

@@ -16,4 +16,5 @@ ARCH_C_SOURCES += kernel/vspace.c \
                   kernel/smp_sys.c \
                   kernel/boot.c \
                   kernel/cmdline.c \
-                  kernel/lock.c
+                  kernel/lock.c \
+                  kernel/ipi.c

src/arch/x86/kernel/apic.c

@@ -277,3 +277,32 @@ apic_send_startup_ipi(cpu_id_t cpu_id, paddr_t startup_addr)
         ).words[0]
     );
 }
+
+void
+apic_send_ipi(irq_t vector, cpu_id_t cpu_id)
+{
+    apic_icr1_t icr1;
+    /* wait till we can send an IPI */
+    do {
+        icr1.words[0] = apic_read_reg(APIC_ICR1);
+    } while (icr1.words[0] & BIT(12));
+
+    apic_write_reg(
+        APIC_ICR2,
+        apic_icr2_new(
+            cpu_id /* dest */
+        ).words[0]
+    );
+    apic_write_reg(
+        APIC_ICR1,
+        apic_icr1_new(
+            0,     /* dest_shorthand  */
+            0,     /* trigger_mode    */
+            0,     /* level           */
+            0,     /* delivery_status */
+            0,     /* dest_mode       */
+            0,     /* delivery_mode   */
+            vector /* vector          */
+        ).words[0]
+    );
+}

src/arch/x86/kernel/boot.c

@@ -36,6 +36,10 @@ init_irqs(cap_t root_cnode_cap)
     for (i = 0; i <= maxIRQ; i++) {
         if (i == irq_timer) {
             setIRQState(IRQTimer, i);
+#if CONFIG_MAX_NUM_NODES > 1
+        } else if (i == irq_remote_call_ipi || i == irq_reschedule_ipi) {
+            setIRQState(IRQIPI, i);
+#endif
         } else if (i == irq_iommu) {
             setIRQState(IRQReserved, i);
         } else if (i == 2 && config_set(CONFIG_IRQ_PIC)) {

src/arch/x86/kernel/ipi.c

@@ -0,0 +1,125 @@
+/*
+ * Copyright 2016, Data61
+ * Commonwealth Scientific and Industrial Research Organisation (CSIRO)
+ * ABN 41 687 119 230.
+ *
+ * This software may be distributed and modified according to the terms of
+ * the GNU General Public License version 2. Note that NO WARRANTY is provided.
+ * See "LICENSE_GPLv2.txt" for details.
+ *
+ * @TAG(D61_GPL)
+ */
+
+#include <config.h>
+#include <arch/kernel/ipi.h>
+#include <arch/kernel/lock.h>
+#include <model/smp.h>
+
+#if CONFIG_MAX_NUM_NODES > 1
+
+static volatile struct {
+    word_t count;
+    word_t globalsense;
+
+    PAD_TO_NEXT_CACHE_LN(sizeof(word_t) + sizeof(word_t));
+} ipiSyncBarrier = {0};                  /* IPI barrier for remote call synchronization */
+
+static volatile word_t totalCoreBarrier; /* number of cores involved in IPI 'in progress' */
+static IpiRemoteCall_t remoteCall;       /* the remote call being requested */
+static word_t remoteCallData;            /* data to be passed to the remote call function */
+
+static inline void ipi_wait(word_t cores)
+{
+    word_t localsense = ipiSyncBarrier.globalsense;
+
+    if (__sync_fetch_and_add(&ipiSyncBarrier.count, 1) == cores) {
+        ipiSyncBarrier.count = 0;
+        ipiSyncBarrier.globalsense =
+            ~ipiSyncBarrier.globalsense;
+    }
+
+    while (localsense == ipiSyncBarrier.globalsense) {
+        asm volatile("pause");
+    }
+}
+
+static void handleRemoteCall(void)
+{
+    /* we gets spurious irq_remote_call_ipi calls, e.g. when handling IPI
+     * in lock while hardware IPI is pending. Guard against spurious IPIs! */
+    if (clh_is_ipi_pending(getCurrentCPUIndex())) {
+        switch (remoteCall) {
+        case IpiRemoteCall_Null:
+
+            break;
+        default:
+            fail("Invalid remote call");
+        }
+        big_kernel_lock.node_owners[getCurrentCPUIndex()].ipi = 0;
+        ipi_wait(totalCoreBarrier);
+    }
+}
+
+static void handleReschedule(void)
+{
+    rescheduleRequired();
+}
+
+void Arch_handleIPI(irq_t irq)
+{
+    if (irq == irq_remote_call_ipi) {
+        handleRemoteCall();
+    } else if (irq == irq_reschedule_ipi) {
+        handleReschedule();
+    } else {
+        fail("Invalid IPI");
+    }
+}
+
+/* make sure all cpu IDs for number of core fit in bitwise word */
+compile_assert(invalid_number_of_supported_nodes, CONFIG_MAX_NUM_NODES <= wordBits);
+
+void doRemoteMaskOp(IpiRemoteCall_t func, word_t data, word_t mask)
+{
+    word_t nr_target_cores = 0;
+    uint16_t target_cores[CONFIG_MAX_NUM_NODES];
+
+    /* make sure the current core is not set in the mask */
+    mask &= ~BIT(getCurrentCPUIndex());
+
+    /* this may happen, e.g. the caller tries to map a pagetable in
+     * newly created PD which has not been run yet. Guard against them! */
+    if (mask != 0) {
+
+        /* setup the data and choose the requested cpu to send IPI*/
+        remoteCall = func;
+        remoteCallData = data;
+        while (mask) {
+            int index = wordBits - 1 - clzl(mask);
+            target_cores[nr_target_cores] = index;
+            nr_target_cores++;
+            mask &= ~BIT(index);
+        }
+
+        /* sending IPIs... */
+        totalCoreBarrier = nr_target_cores;
+        asm volatile("" ::: "memory");
+
+        for (int i = 0; i < nr_target_cores; i++) {
+            big_kernel_lock.node_owners[target_cores[i]].ipi = 1;
+            apic_send_ipi(int_remote_call_ipi, cpuIndexToID(target_cores[i]));
+        }
+
+        ipi_wait(totalCoreBarrier);
+    }
+}
+
+void doReschedule(word_t cpu)
+{
+    if (cpu != getCurrentCPUIndex() &&
+            cpu < CONFIG_MAX_NUM_NODES) {
+        apic_send_ipi(int_remote_call_ipi, cpuIndexToID(cpu));
+    }
+}
+
+#endif

src/object/interrupt.c

@@ -22,6 +22,7 @@
 #include <kernel/thread.h>
 #include <model/statedata.h>
 #include <machine/timer.h>
+#include <arch/kernel/ipi.h>
 
 exception_t
 decodeIRQControlInvocation(word_t invLabel, word_t length,
@@ -214,6 +215,12 @@ handleInterrupt(irq_t irq)
         resetTimer();
         break;
 
+#if CONFIG_MAX_NUM_NODES > 1
+    case IRQIPI:
+        Arch_handleIPI(irq);
+        break;
+#endif
+
     case IRQReserved:
 #ifdef CONFIG_IRQ_REPORTING
         printf("Received reserved IRQ: %d", (int)irq);