portugalfuturista/seL4
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

SMP: Cleanup kernel lock

Browse source 
Remove cpu argument from clh_lock_acquire/release, it is never
valid to call them with something else than getCurrentCPUIndex().

Use clh_is_self_in_queue() instead of poking into lock internals
in ipiStallCoreCallback().

Add some comments from the paper to make it easier to match the
implementation with the description in the paper it's based on.
Don't use confusingly different naming, but follow naming from
the paper: Rename 'node' to 'myreq' and 'next' to 'watch', etc.

Signed-off-by: Indan Zupancic <Indan.Zupancic@mep-info.com>
This commit is contained in:
Indan Zupancic 2022-06-14 18:53:45 +02:00 committed by Kent McLeod
parent 18944f5072
commit 4e74d029cc
6 changed files with 52 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

69
include/smp/lock.h
View file

@ -22,31 +22,33 @@
typedef enum {
CLHState_Granted = 0,
CLHState_Pending
} clh_qnode_state_t;
} clh_req_state_t;
typedef struct clh_qnode {
clh_qnode_state_t value;
/* Lock request */
typedef struct clh_req {
clh_req_state_t state;
PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_state_t));
} clh_qnode_t;
PAD_TO_NEXT_CACHE_LN(sizeof(clh_req_state_t));
} clh_req_t;
typedef struct clh_qnode_p {
clh_qnode_t *node;
clh_qnode_t *next;
/* This is the software IPI flag */
/* Our node (called "Process" in the paper) */
typedef struct clh_node {
clh_req_t *watch; // Used by predecessor to grant the lock to us.
clh_req_t *myreq; // Used to grant the lock to our successor.
/* This is the software blocking IPI flag */
word_t ipi;
PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_t *) +
sizeof(clh_qnode_t *) +
PAD_TO_NEXT_CACHE_LN(sizeof(clh_req_t *) +
sizeof(clh_req_t *) +
sizeof(word_t));
} clh_qnode_p_t;
} clh_node_t;
typedef struct clh_lock {
clh_qnode_t nodes[CONFIG_MAX_NUM_NODES + 1];
clh_qnode_p_t node_owners[CONFIG_MAX_NUM_NODES];
clh_req_t request[CONFIG_MAX_NUM_NODES + 1];
clh_node_t node[CONFIG_MAX_NUM_NODES];
clh_qnode_t *head;
PAD_TO_NEXT_CACHE_LN(sizeof(clh_qnode_t *));
clh_req_t *tail;
PAD_TO_NEXT_CACHE_LN(sizeof(clh_req_t *));
} clh_lock_t;
extern clh_lock_t big_kernel_lock;
@ -54,19 +56,21 @@ BOOT_CODE void clh_lock_init(void);
static inline bool_t FORCE_INLINE clh_is_ipi_pending(word_t cpu)
{
return big_kernel_lock.node_owners[cpu].ipi == 1;
return big_kernel_lock.node[cpu].ipi == 1;
}
static inline void FORCE_INLINE clh_lock_acquire(word_t cpu, bool_t irqPath)
static inline void FORCE_INLINE clh_lock_acquire(bool_t irqPath)
{
clh_qnode_p_t volatile *node_owner = &big_kernel_lock.node_owners[cpu];
word_t cpu = getCurrentCPUIndex();
clh_node_t *node = &big_kernel_lock.node[cpu];
node_owner->node->value = CLHState_Pending;
node_owner->next = __atomic_exchange_n(&big_kernel_lock.head, node_owner->node, __ATOMIC_ACQ_REL);
/* Tell successor to wait */
node->myreq->state = CLHState_Pending;
/* Enqueue our request */
node->watch = __atomic_exchange_n(&big_kernel_lock.tail, node->myreq, __ATOMIC_ACQ_REL);
/* We do not have an __atomic_thread_fence here as this is already handled by the
* atomic_exchange just above */
while (node_owner->next->value != CLHState_Granted) {
/* Wait until predecessor finishes */
while (node->watch->state != CLHState_Granted) {
/* As we are in a loop we need to ensure that any loads of future iterations of the
* loop are performed after this one */
__atomic_thread_fence(__ATOMIC_ACQUIRE);
@ -85,27 +89,30 @@ static inline void FORCE_INLINE clh_lock_acquire(word_t cpu, bool_t irqPath)
__atomic_thread_fence(__ATOMIC_ACQUIRE);
}
static inline void FORCE_INLINE clh_lock_release(word_t cpu)
static inline void FORCE_INLINE clh_lock_release(void)
{
clh_node_t *node = &big_kernel_lock.node[getCurrentCPUIndex()];
/* make sure no resource access passes from this point */
__atomic_thread_fence(__ATOMIC_RELEASE);
big_kernel_lock.node_owners[cpu].node->value = CLHState_Granted;
big_kernel_lock.node_owners[cpu].node =
big_kernel_lock.node_owners[cpu].next;
/* Pass lock to successor */
node->myreq->state = CLHState_Granted;
/* Take ownership of watched request, to use next time we take the lock */
node->myreq = node->watch;
}
static inline bool_t FORCE_INLINE clh_is_self_in_queue(void)
{
return big_kernel_lock.node_owners[getCurrentCPUIndex()].node->value == CLHState_Pending;
return big_kernel_lock.node[getCurrentCPUIndex()].myreq->state == CLHState_Pending;
}
#define NODE_LOCK(_irqPath) do { \
clh_lock_acquire(getCurrentCPUIndex(), _irqPath); \
clh_lock_acquire(_irqPath); \
} while(0)
#define NODE_UNLOCK do { \
clh_lock_release(getCurrentCPUIndex()); \
clh_lock_release(); \
} while(0)
#define NODE_LOCK_IF(_cond, _irqPath) do { \

2
src/arch/arm/smp/ipi.c
View file

@ -73,7 +73,7 @@ static void handleRemoteCall(IpiModeRemoteCall_t call, word_t arg0,
break;
}
big_kernel_lock.node_owners[getCurrentCPUIndex()].ipi = 0;
big_kernel_lock.node[getCurrentCPUIndex()].ipi = 0;
ipi_wait(totalCoreBarrier);
}
}

6
src/arch/riscv/smp/ipi.c
View file

@ -50,7 +50,7 @@ static void handleRemoteCall(IpiRemoteCall_t call, word_t arg0,
break;
}
big_kernel_lock.node_owners[getCurrentCPUIndex()].ipi = 0;
big_kernel_lock.node[getCurrentCPUIndex()].ipi = 0;
ipiIrq[getCurrentCPUIndex()] = irqInvalid;
ipi_wait(totalCoreBarrier);
}
@ -64,7 +64,7 @@ void ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking)
irq_t ipi_get_irq(void)
{
assert(!(ipiIrq[getCurrentCPUIndex()] == irqInvalid && big_kernel_lock.node_owners[getCurrentCPUIndex()].ipi == 1));
assert(!(ipiIrq[getCurrentCPUIndex()] == irqInvalid && big_kernel_lock.node[getCurrentCPUIndex()].ipi == 1));
return ipiIrq[getCurrentCPUIndex()];
}
@ -82,7 +82,7 @@ void ipi_send_target(irq_t irq, word_t hart_id)
assert(core_id < CONFIG_MAX_NUM_NODES);
assert((ipiIrq[core_id] == irqInvalid) || (ipiIrq[core_id] == irq_reschedule_ipi) ||
(ipiIrq[core_id] == irq_remote_call_ipi && big_kernel_lock.node_owners[core_id].ipi == 0));
(ipiIrq[core_id] == irq_remote_call_ipi && big_kernel_lock.node[core_id].ipi == 0));
ipiIrq[core_id] = irq;
fence_rw_rw();

6
src/arch/x86/smp/ipi.c
View file

@ -70,7 +70,7 @@ static void handleRemoteCall(IpiModeRemoteCall_t call, word_t arg0,
break;
}
big_kernel_lock.node_owners[getCurrentCPUIndex()].ipi = 0;
big_kernel_lock.node[getCurrentCPUIndex()].ipi = 0;
ipi_wait(totalCoreBarrier);
}
}
@ -92,7 +92,7 @@ static void x86_ipi_send_mask(interrupt_t ipi, word_t mask, bool_t isBlocking)
word_t sub_mask = mask & cpu_mapping.other_indexes_in_cluster[core];
target_clusters[nr_target_clusters] |= cpu_mapping.index_to_logical_id[core];
if (isBlocking) {
big_kernel_lock.node_owners[core].ipi = 1;
big_kernel_lock.node[core].ipi = 1;
}
/* check if there is any other core in this cluster */
@ -100,7 +100,7 @@ static void x86_ipi_send_mask(interrupt_t ipi, word_t mask, bool_t isBlocking)
int index = wordBits - 1 - clzl(sub_mask);
target_clusters[nr_target_clusters] |= cpu_mapping.index_to_logical_id[index];
if (isBlocking) {
big_kernel_lock.node_owners[index].ipi = 1;
big_kernel_lock.node[index].ipi = 1;
}
sub_mask &= ~BIT(index);
}

7
src/smp/ipi.c
View file

@ -41,16 +41,15 @@ void ipiStallCoreCallback(bool_t irqPath)
NODE_STATE(ksSchedulerAction) = SchedulerAction_ResumeCurrentThread;
/* Let the cpu requesting this IPI to continue while we waiting on lock */
big_kernel_lock.node_owners[getCurrentCPUIndex()].ipi = 0;
big_kernel_lock.node[getCurrentCPUIndex()].ipi = 0;
#ifdef CONFIG_ARCH_RISCV
ipi_clear_irq(irq_remote_call_ipi);
#endif
ipi_wait(totalCoreBarrier);
/* Continue waiting on lock */
while (big_kernel_lock.node_owners[getCurrentCPUIndex()].next->value != CLHState_Granted) {
while (big_kernel_lock.node[getCurrentCPUIndex()].watch->state != CLHState_Granted) {
if (clh_is_ipi_pending(getCurrentCPUIndex())) {
/* Multiple calls for similar reason could result in stack overflow */
assert((IpiRemoteCall_t)remoteCall != IpiRemoteCall_Stall);
handleIPI(CORE_IRQ_TO_IRQT(getCurrentCPUIndex(), irq_remote_call_ipi), irqPath);
@ -127,7 +126,7 @@ void generic_ipi_send_mask(irq_t ipi, word_t mask, bool_t isBlocking)
while (mask) {
int index = wordBits - 1 - clzl(mask);
if (isBlocking) {
big_kernel_lock.node_owners[index].ipi = 1;
big_kernel_lock.node[index].ipi = 1;
target_cores[nr_target_cores] = index;
nr_target_cores++;
} else {

8
src/smp/lock.c
View file

@ -14,12 +14,12 @@ clh_lock_t big_kernel_lock ALIGN(L1_CACHE_LINE_SIZE);
BOOT_CODE void clh_lock_init(void)
{
for (int i = 0; i < CONFIG_MAX_NUM_NODES; i++) {
big_kernel_lock.node_owners[i].node = &big_kernel_lock.nodes[i];
big_kernel_lock.node[i].myreq = &big_kernel_lock.request[i];
}
/* Initialize the CLH head */
big_kernel_lock.nodes[CONFIG_MAX_NUM_NODES].value = CLHState_Granted;
big_kernel_lock.head = &big_kernel_lock.nodes[CONFIG_MAX_NUM_NODES];
/* Initialize the CLH tail */
big_kernel_lock.request[CONFIG_MAX_NUM_NODES].state = CLHState_Granted;
big_kernel_lock.tail = &big_kernel_lock.request[CONFIG_MAX_NUM_NODES];
}
#endif /* ENABLE_SMP_SUPPORT */