Re: [PATCH 2/4] locking/osq_lock: Turn to dynamic switch function from osq_lock/unlock.

yongli-os
2024-09-19T09:48:01Z

On 2024/9/15 00:06, Waiman Long wrote:
>
>
> [这封邮件来自外部发件人 谨防风险]
>
> On 9/14/24 04:53, yongli-oc wrote:
>> To support numa-aware osq lock, the struct optimistic_spin_queue
>> is accessed as three members, numa_enable, index, tail16, by union.
>> The size of the struct is the same as before.
>> If dynamic numa-ware lock enable, turns to the crossing, x_osq_lock to
>> check contention level and starts dynamic switch.
>>
>> Signed-off-by: yongli-oc <yongli-oc@zhaoxin.com>
>> ---
>>   include/linux/osq_lock.h  | 33 ++++++++++++++++++++-
>>   kernel/locking/osq_lock.c | 62 +++++++++++++++++++++++++++++++++++++--
>>   2 files changed, 91 insertions(+), 4 deletions(-)
>>
>> diff --git a/include/linux/osq_lock.h b/include/linux/osq_lock.h
>> index ea8fb31379e3..37a7bc4ab530 100644
>> --- a/include/linux/osq_lock.h
>> +++ b/include/linux/osq_lock.h
>> @@ -12,14 +12,42 @@ struct optimistic_spin_queue {
>>        * Stores an encoded value of the CPU # of the tail node in the
>> queue.
>>        * If the queue is empty, then it's set to OSQ_UNLOCKED_VAL.
>>        */
>> +#ifdef CONFIG_LOCK_SPIN_ON_OWNER_NUMA
>> +     union {
>> +             atomic_t tail;
>> +             u32 val;
>> +#ifdef __LITTLE_ENDIAN
>> +             struct {
>> +                     u16 tail16;
>> +                     u8 index;
>> +                     u8 numa_enable;
>> +             };
>> +#else
>> +             struct {
>> +                     u8 numa_enable;
>> +                     u8 index;
>> +                     u16 tail16;
>> +             };
>> +#endif
>> +     };
>> +#else
>>       atomic_t tail;
>> +#endif
>>   };
>>
>>   #define OSQ_UNLOCKED_VAL (0)
>>
>>   /* Init macro and function. */
>> +#ifdef CONFIG_LOCK_SPIN_ON_OWNER_NUMA
>> +
>> +#define OSQ_LOCK_UNLOCKED { .tail = ATOMIC_INIT(OSQ_UNLOCKED_VAL) }
>> +
>> +#else
>> +
>>   #define OSQ_LOCK_UNLOCKED { ATOMIC_INIT(OSQ_UNLOCKED_VAL) }
>>
>> +#endif
>> +
>>   static inline void osq_lock_init(struct optimistic_spin_queue *lock)
>>   {
>>       atomic_set(&lock->tail, OSQ_UNLOCKED_VAL);
>> @@ -28,9 +56,12 @@ static inline void osq_lock_init(struct
>> optimistic_spin_queue *lock)
>>   extern bool osq_lock(struct optimistic_spin_queue *lock);
>>   extern void osq_unlock(struct optimistic_spin_queue *lock);
>>
>> +#ifdef CONFIG_LOCK_SPIN_ON_OWNER_NUMA
>> +extern bool osq_is_locked(struct optimistic_spin_queue *lock);
>> +#else
>>   static inline bool osq_is_locked(struct optimistic_spin_queue *lock)
>>   {
>>       return atomic_read(&lock->tail) != OSQ_UNLOCKED_VAL;
>>   }
>> -
>> +#endif
>>   #endif
>> diff --git a/kernel/locking/osq_lock.c b/kernel/locking/osq_lock.c
>> index 75a6f6133866..a7b516939e00 100644
>> --- a/kernel/locking/osq_lock.c
>> +++ b/kernel/locking/osq_lock.c
>> @@ -2,7 +2,10 @@
>>   #include <linux/percpu.h>
>>   #include <linux/sched.h>
>>   #include <linux/osq_lock.h>
>> -
>> +#ifdef CONFIG_LOCK_SPIN_ON_OWNER_NUMA
>> +#include "numa.h"
>> +#include "numa_osq.h"
>> +#endif
>
> These header files are defined in patch 3. You need to rethink about
> patch ordering in order not to break bisection.
I will move it to the patch 2.
>
>>   /*
>>    * An MCS like lock especially tailored for optimistic spinning for
>> sleeping
>>    * lock implementations (mutex, rwsem, etc).
>> @@ -12,12 +15,34 @@
>>    * spinning.
>>    */
>>
>> +#ifdef CONFIG_LOCK_SPIN_ON_OWNER_NUMA
>> +DEFINE_PER_CPU_SHARED_ALIGNED(struct optimistic_spin_node, osq_node);
>> +/*
>> + * We use the value 0 to represent "no CPU", thus the encoded value
>> + * will be the CPU number incremented by 1.
>> + */
>> +inline int encode_cpu(int cpu_nr)
>> +{
>> +     return cpu_nr + 1;
>> +}
>> +
>> +inline int node_cpu(struct optimistic_spin_node *node)
>> +{
>> +     return node->cpu - 1;
>> +}
>> +
>> +inline struct optimistic_spin_node *decode_cpu(int encoded_cpu_val)
>> +{
>> +     int cpu_nr = encoded_cpu_val - 1;
>> +
>> +     return per_cpu_ptr(&osq_node, cpu_nr);
>> +}
>> +#else
>>   struct optimistic_spin_node {
>>       struct optimistic_spin_node *next, *prev;
>>       int locked; /* 1 if lock acquired */
>>       int cpu; /* encoded CPU # + 1 value */
>>   };
>> -
>>   static DEFINE_PER_CPU_SHARED_ALIGNED(struct optimistic_spin_node,
>> osq_node);
>>
>>   /*
>> @@ -40,6 +65,7 @@ static inline struct optimistic_spin_node
>> *decode_cpu(int encoded_cpu_val)
>>
>>       return per_cpu_ptr(&osq_node, cpu_nr);
>>   }
>> +#endif
>>
>>   /*
>>    * Get a stable @node->next pointer, either for unlock() or
>> unqueue() purposes.
>> @@ -97,6 +123,14 @@ bool osq_lock(struct optimistic_spin_queue *lock)
>>       int curr = encode_cpu(smp_processor_id());
>>       int old;
>>
>> +#ifdef CONFIG_LOCK_SPIN_ON_OWNER_NUMA
>> +     if (unlikely(enable_zx_numa_osq_lock > 1)) {
>> +             node->numa = 1;
>> +             return x_osq_lock(lock);
>> +     }
>> +     node->numa = 0;
>> +#endif
>> +
>>       node->locked = 0;
>>       node->next = NULL;
>>       node->cpu = curr;
>> @@ -108,6 +142,11 @@ bool osq_lock(struct optimistic_spin_queue *lock)
>>        * the lock tail.
>>        */
>>       old = atomic_xchg(&lock->tail, curr);
>> +#ifdef CONFIG_LOCK_SPIN_ON_OWNER_NUMA
>> +     if (enable_zx_numa_osq_lock > 0)
>> +     //enable means all cpu cores are less tan 65534.
>> +             old = old & 0xffff;
>> +#endif
>>       if (old == OSQ_UNLOCKED_VAL)
>>               return true;
>>
>> @@ -212,6 +251,14 @@ void osq_unlock(struct optimistic_spin_queue *lock)
>>       struct optimistic_spin_node *node, *next;
>>       int curr = encode_cpu(smp_processor_id());
>>
>> +     node = this_cpu_ptr(&osq_node);
>> +
>> +#ifdef CONFIG_LOCK_SPIN_ON_OWNER_NUMA
>> +     if (unlikely(enable_zx_numa_osq_lock > 1 &&
>> +             node->numa == 1))
>> +             return x_osq_unlock(lock);
>> +#endif
>> +
>>       /*
>>        * Fast path for the uncontended case.
>>        */
>> @@ -222,7 +269,6 @@ void osq_unlock(struct optimistic_spin_queue *lock)
>>       /*
>>        * Second most likely case.
>>        */
>> -     node = this_cpu_ptr(&osq_node);
>>       next = xchg(&node->next, NULL);
>>       if (next) {
>>               WRITE_ONCE(next->locked, 1);
>> @@ -233,3 +279,13 @@ void osq_unlock(struct optimistic_spin_queue *lock)
>>       if (next)
>>               WRITE_ONCE(next->locked, 1);
>>   }
>> +#ifdef CONFIG_LOCK_SPIN_ON_OWNER_NUMA
>> +bool osq_is_locked(struct optimistic_spin_queue *lock)
>> +{
>> +     if (unlikely(enable_zx_numa_osq_lock > 1))
>> +             return x_osq_is_locked(lock);
>> +     return atomic_read(&lock->tail) != OSQ_UNLOCKED_VAL;
>> +}
>> +#endif
>> +
>> +
>