From: Nicholas Piggin <npiggin@gmail.com>
To: linuxppc-dev@lists.ozlabs.org
Cc: Jordan Niethe <jniethe5@gmail.com>,
Laurent Dufour <ldufour@linux.ibm.com>,
Nicholas Piggin <npiggin@gmail.com>
Subject: [PATCH v3 16/17] powerpc/qspinlock: provide accounting and options for sleepy locks
Date: Sat, 26 Nov 2022 19:59:31 +1000 [thread overview]
Message-ID: <20221126095932.1234527-17-npiggin@gmail.com> (raw)
In-Reply-To: <20221126095932.1234527-1-npiggin@gmail.com>
Finding the owner or a queued waiter on a lock with a preempted vcpu
is indicative of an oversubscribed guest causing the lock to get into
trouble. Provide some options to detect this situation and have new
CPUs avoid queueing for a longer time (more steal iterations) to
minimise the problems caused by vcpu preemption on the queue.
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
arch/powerpc/include/asm/qspinlock_types.h | 7 +-
arch/powerpc/lib/qspinlock.c | 242 +++++++++++++++++++--
2 files changed, 230 insertions(+), 19 deletions(-)
diff --git a/arch/powerpc/include/asm/qspinlock_types.h b/arch/powerpc/include/asm/qspinlock_types.h
index adfeed4aa495..4766a7aa03cb 100644
--- a/arch/powerpc/include/asm/qspinlock_types.h
+++ b/arch/powerpc/include/asm/qspinlock_types.h
@@ -30,7 +30,7 @@ typedef struct qspinlock {
*
* 0: locked bit
* 1-14: lock holder cpu
- * 15: unused bit
+ * 15: lock owner or queuer vcpus observed to be preempted bit
* 16: must queue bit
* 17-31: tail cpu (+1)
*/
@@ -50,6 +50,11 @@ typedef struct qspinlock {
#error "qspinlock does not support such large CONFIG_NR_CPUS"
#endif
+/* 0x00008000 */
+#define _Q_SLEEPY_OFFSET 15
+#define _Q_SLEEPY_BITS 1
+#define _Q_SLEEPY_VAL (1U << _Q_SLEEPY_OFFSET)
+
/* 0x00010000 */
#define _Q_MUST_Q_OFFSET 16
#define _Q_MUST_Q_BITS 1
diff --git a/arch/powerpc/lib/qspinlock.c b/arch/powerpc/lib/qspinlock.c
index 1c9079489b50..9a31b6147a23 100644
--- a/arch/powerpc/lib/qspinlock.c
+++ b/arch/powerpc/lib/qspinlock.c
@@ -5,6 +5,7 @@
#include <linux/percpu.h>
#include <linux/smp.h>
#include <linux/topology.h>
+#include <linux/sched/clock.h>
#include <asm/qspinlock.h>
#include <asm/paravirt.h>
@@ -36,25 +37,56 @@ static int head_spins __read_mostly = (1<<8);
static bool pv_yield_owner __read_mostly = true;
static bool pv_yield_allow_steal __read_mostly = false;
static bool pv_spin_on_preempted_owner __read_mostly = false;
+static bool pv_sleepy_lock __read_mostly = true;
+static bool pv_sleepy_lock_sticky __read_mostly = false;
+static u64 pv_sleepy_lock_interval_ns __read_mostly = 0;
+static int pv_sleepy_lock_factor __read_mostly = 256;
static bool pv_yield_prev __read_mostly = true;
static bool pv_yield_propagate_owner __read_mostly = true;
static bool pv_prod_head __read_mostly = false;
static DEFINE_PER_CPU_ALIGNED(struct qnodes, qnodes);
+static DEFINE_PER_CPU_ALIGNED(u64, sleepy_lock_seen_clock);
-static __always_inline int get_steal_spins(bool paravirt)
+static __always_inline bool recently_sleepy(void)
{
- return steal_spins;
+ /* pv_sleepy_lock is true when this is called */
+ if (pv_sleepy_lock_interval_ns) {
+ u64 seen = this_cpu_read(sleepy_lock_seen_clock);
+
+ if (seen) {
+ u64 delta = sched_clock() - seen;
+ if (delta < pv_sleepy_lock_interval_ns)
+ return true;
+ this_cpu_write(sleepy_lock_seen_clock, 0);
+ }
+ }
+
+ return false;
}
-static __always_inline int get_remote_steal_spins(bool paravirt)
+static __always_inline int get_steal_spins(bool paravirt, bool sleepy)
{
- return remote_steal_spins;
+ if (paravirt && sleepy)
+ return steal_spins * pv_sleepy_lock_factor;
+ else
+ return steal_spins;
}
-static __always_inline int get_head_spins(bool paravirt)
+static __always_inline int get_remote_steal_spins(bool paravirt, bool sleepy)
{
- return head_spins;
+ if (paravirt && sleepy)
+ return remote_steal_spins * pv_sleepy_lock_factor;
+ else
+ return remote_steal_spins;
+}
+
+static __always_inline int get_head_spins(bool paravirt, bool sleepy)
+{
+ if (paravirt && sleepy)
+ return head_spins * pv_sleepy_lock_factor;
+ else
+ return head_spins;
}
static inline u32 encode_tail_cpu(int cpu)
@@ -168,6 +200,56 @@ static __always_inline u32 clear_mustq(struct qspinlock *lock)
return prev;
}
+static __always_inline bool try_set_sleepy(struct qspinlock *lock, u32 old)
+{
+ u32 prev;
+ u32 new = old | _Q_SLEEPY_VAL;
+
+ BUG_ON(!(old & _Q_LOCKED_VAL));
+ BUG_ON(old & _Q_SLEEPY_VAL);
+
+ asm volatile(
+"1: lwarx %0,0,%1 # try_set_sleepy \n"
+" cmpw 0,%0,%2 \n"
+" bne- 2f \n"
+" stwcx. %3,0,%1 \n"
+" bne- 1b \n"
+"2: \n"
+ : "=&r" (prev)
+ : "r" (&lock->val), "r"(old), "r" (new)
+ : "cr0", "memory");
+
+ return likely(prev == old);
+}
+
+static __always_inline void seen_sleepy_owner(struct qspinlock *lock, u32 val)
+{
+ if (pv_sleepy_lock) {
+ if (pv_sleepy_lock_interval_ns)
+ this_cpu_write(sleepy_lock_seen_clock, sched_clock());
+ if (!(val & _Q_SLEEPY_VAL))
+ try_set_sleepy(lock, val);
+ }
+}
+
+static __always_inline void seen_sleepy_lock(void)
+{
+ if (pv_sleepy_lock && pv_sleepy_lock_interval_ns)
+ this_cpu_write(sleepy_lock_seen_clock, sched_clock());
+}
+
+static __always_inline void seen_sleepy_node(struct qspinlock *lock, u32 val)
+{
+ if (pv_sleepy_lock) {
+ if (pv_sleepy_lock_interval_ns)
+ this_cpu_write(sleepy_lock_seen_clock, sched_clock());
+ if (val & _Q_LOCKED_VAL) {
+ if (!(val & _Q_SLEEPY_VAL))
+ try_set_sleepy(lock, val);
+ }
+ }
+}
+
static struct qnode *get_tail_qnode(struct qspinlock *lock, u32 val)
{
int cpu = decode_tail_cpu(val);
@@ -215,6 +297,7 @@ static __always_inline bool __yield_to_locked_owner(struct qspinlock *lock, u32
spin_end();
+ seen_sleepy_owner(lock, val);
preempted = true;
/*
@@ -289,11 +372,12 @@ static __always_inline void propagate_yield_cpu(struct qnode *node, u32 val, int
}
/* Called inside spin_begin() */
-static __always_inline void yield_to_prev(struct qspinlock *lock, struct qnode *node, u32 val, bool paravirt)
+static __always_inline bool yield_to_prev(struct qspinlock *lock, struct qnode *node, u32 val, bool paravirt)
{
int prev_cpu = decode_tail_cpu(val);
u32 yield_count;
int yield_cpu;
+ bool preempted = false;
if (!paravirt)
goto relax;
@@ -315,6 +399,9 @@ static __always_inline void yield_to_prev(struct qspinlock *lock, struct qnode *
spin_end();
+ preempted = true;
+ seen_sleepy_node(lock, val);
+
smp_rmb();
if (yield_cpu == node->yield_cpu) {
@@ -322,7 +409,7 @@ static __always_inline void yield_to_prev(struct qspinlock *lock, struct qnode *
node->next->yield_cpu = yield_cpu;
yield_to_preempted(yield_cpu, yield_count);
spin_begin();
- return;
+ return preempted;
}
spin_begin();
@@ -336,26 +423,31 @@ static __always_inline void yield_to_prev(struct qspinlock *lock, struct qnode *
spin_end();
+ preempted = true;
+ seen_sleepy_node(lock, val);
+
smp_rmb(); /* See __yield_to_locked_owner comment */
if (!node->locked) {
yield_to_preempted(prev_cpu, yield_count);
spin_begin();
- return;
+ return preempted;
}
spin_begin();
relax:
spin_cpu_relax();
+
+ return preempted;
}
-static __always_inline bool steal_break(u32 val, int iters, bool paravirt)
+static __always_inline bool steal_break(u32 val, int iters, bool paravirt, bool sleepy)
{
- if (iters >= get_steal_spins(paravirt))
+ if (iters >= get_steal_spins(paravirt, sleepy))
return true;
if (IS_ENABLED(CONFIG_NUMA) &&
- (iters >= get_remote_steal_spins(paravirt))) {
+ (iters >= get_remote_steal_spins(paravirt, sleepy))) {
int cpu = get_owner_cpu(val);
if (numa_node_id() != cpu_to_node(cpu))
return true;
@@ -365,6 +457,8 @@ static __always_inline bool steal_break(u32 val, int iters, bool paravirt)
static __always_inline bool try_to_steal_lock(struct qspinlock *lock, bool paravirt)
{
+ bool seen_preempted = false;
+ bool sleepy = false;
int iters = 0;
u32 val;
@@ -391,7 +485,25 @@ static __always_inline bool try_to_steal_lock(struct qspinlock *lock, bool parav
preempted = yield_to_locked_owner(lock, val, paravirt);
}
+ if (paravirt && pv_sleepy_lock) {
+ if (!sleepy) {
+ if (val & _Q_SLEEPY_VAL) {
+ seen_sleepy_lock();
+ sleepy = true;
+ } else if (recently_sleepy()) {
+ sleepy = true;
+ }
+ }
+ if (pv_sleepy_lock_sticky && seen_preempted &&
+ !(val & _Q_SLEEPY_VAL)) {
+ if (try_set_sleepy(lock, val))
+ val |= _Q_SLEEPY_VAL;
+ }
+ }
+
if (preempted) {
+ seen_preempted = true;
+ sleepy = true;
if (!pv_spin_on_preempted_owner)
iters++;
/*
@@ -405,7 +517,7 @@ static __always_inline bool try_to_steal_lock(struct qspinlock *lock, bool parav
} else {
iters++;
}
- } while (!steal_break(val, iters, paravirt));
+ } while (!steal_break(val, iters, paravirt, sleepy));
spin_end();
@@ -417,6 +529,8 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
struct qnodes *qnodesp;
struct qnode *next, *node;
u32 val, old, tail;
+ bool seen_preempted = false;
+ bool sleepy = false;
bool mustq = false;
int idx;
int set_yield_cpu = -1;
@@ -461,8 +575,10 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
/* Wait for mcs node lock to be released */
spin_begin();
- while (!node->locked)
- yield_to_prev(lock, node, old, paravirt);
+ while (!node->locked) {
+ if (yield_to_prev(lock, node, old, paravirt))
+ seen_preempted = true;
+ }
spin_end();
/* Clear out stale propagated yield_cpu */
@@ -472,8 +588,8 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
smp_rmb(); /* acquire barrier for the mcs lock */
}
-again:
/* We're at the head of the waitqueue, wait for the lock. */
+again:
spin_begin();
for (;;) {
bool preempted;
@@ -482,18 +598,40 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
if (!(val & _Q_LOCKED_VAL))
break;
+ if (paravirt && pv_sleepy_lock && maybe_stealers) {
+ if (!sleepy) {
+ if (val & _Q_SLEEPY_VAL) {
+ seen_sleepy_lock();
+ sleepy = true;
+ } else if (recently_sleepy()) {
+ sleepy = true;
+ }
+ }
+ if (pv_sleepy_lock_sticky && seen_preempted &&
+ !(val & _Q_SLEEPY_VAL)) {
+ if (try_set_sleepy(lock, val))
+ val |= _Q_SLEEPY_VAL;
+ }
+ }
+
propagate_yield_cpu(node, val, &set_yield_cpu, paravirt);
preempted = yield_head_to_locked_owner(lock, val, paravirt);
if (!maybe_stealers)
continue;
- if (preempted) {
+
+ if (preempted)
+ seen_preempted = true;
+
+ if (paravirt && preempted) {
+ sleepy = true;
+
if (!pv_spin_on_preempted_owner)
iters++;
} else {
iters++;
}
- if (!mustq && iters >= get_head_spins(paravirt)) {
+ if (!mustq && iters >= get_head_spins(paravirt, sleepy)) {
mustq = true;
set_mustq(lock);
val |= _Q_MUST_Q_VAL;
@@ -690,6 +828,70 @@ static int pv_spin_on_preempted_owner_get(void *data, u64 *val)
DEFINE_SIMPLE_ATTRIBUTE(fops_pv_spin_on_preempted_owner, pv_spin_on_preempted_owner_get, pv_spin_on_preempted_owner_set, "%llu\n");
+static int pv_sleepy_lock_set(void *data, u64 val)
+{
+ pv_sleepy_lock = !!val;
+
+ return 0;
+}
+
+static int pv_sleepy_lock_get(void *data, u64 *val)
+{
+ *val = pv_sleepy_lock;
+
+ return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(fops_pv_sleepy_lock, pv_sleepy_lock_get, pv_sleepy_lock_set, "%llu\n");
+
+static int pv_sleepy_lock_sticky_set(void *data, u64 val)
+{
+ pv_sleepy_lock_sticky = !!val;
+
+ return 0;
+}
+
+static int pv_sleepy_lock_sticky_get(void *data, u64 *val)
+{
+ *val = pv_sleepy_lock_sticky;
+
+ return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(fops_pv_sleepy_lock_sticky, pv_sleepy_lock_sticky_get, pv_sleepy_lock_sticky_set, "%llu\n");
+
+static int pv_sleepy_lock_interval_ns_set(void *data, u64 val)
+{
+ pv_sleepy_lock_interval_ns = val;
+
+ return 0;
+}
+
+static int pv_sleepy_lock_interval_ns_get(void *data, u64 *val)
+{
+ *val = pv_sleepy_lock_interval_ns;
+
+ return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(fops_pv_sleepy_lock_interval_ns, pv_sleepy_lock_interval_ns_get, pv_sleepy_lock_interval_ns_set, "%llu\n");
+
+static int pv_sleepy_lock_factor_set(void *data, u64 val)
+{
+ pv_sleepy_lock_factor = val;
+
+ return 0;
+}
+
+static int pv_sleepy_lock_factor_get(void *data, u64 *val)
+{
+ *val = pv_sleepy_lock_factor;
+
+ return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(fops_pv_sleepy_lock_factor, pv_sleepy_lock_factor_get, pv_sleepy_lock_factor_set, "%llu\n");
+
static int pv_yield_prev_set(void *data, u64 val)
{
pv_yield_prev = !!val;
@@ -747,6 +949,10 @@ static __init int spinlock_debugfs_init(void)
debugfs_create_file("qspl_pv_yield_owner", 0600, arch_debugfs_dir, NULL, &fops_pv_yield_owner);
debugfs_create_file("qspl_pv_yield_allow_steal", 0600, arch_debugfs_dir, NULL, &fops_pv_yield_allow_steal);
debugfs_create_file("qspl_pv_spin_on_preempted_owner", 0600, arch_debugfs_dir, NULL, &fops_pv_spin_on_preempted_owner);
+ debugfs_create_file("qspl_pv_sleepy_lock", 0600, arch_debugfs_dir, NULL, &fops_pv_sleepy_lock);
+ debugfs_create_file("qspl_pv_sleepy_lock_sticky", 0600, arch_debugfs_dir, NULL, &fops_pv_sleepy_lock_sticky);
+ debugfs_create_file("qspl_pv_sleepy_lock_interval_ns", 0600, arch_debugfs_dir, NULL, &fops_pv_sleepy_lock_interval_ns);
+ debugfs_create_file("qspl_pv_sleepy_lock_factor", 0600, arch_debugfs_dir, NULL, &fops_pv_sleepy_lock_factor);
debugfs_create_file("qspl_pv_yield_prev", 0600, arch_debugfs_dir, NULL, &fops_pv_yield_prev);
debugfs_create_file("qspl_pv_yield_propagate_owner", 0600, arch_debugfs_dir, NULL, &fops_pv_yield_propagate_owner);
debugfs_create_file("qspl_pv_prod_head", 0600, arch_debugfs_dir, NULL, &fops_pv_prod_head);
--
2.37.2
next prev parent reply other threads:[~2022-11-26 10:15 UTC|newest]
Thread overview: 21+ messages / expand[flat|nested] mbox.gz Atom feed top
2022-11-26 9:59 [PATCH v3 00/17] powerpc: alternate queued spinlock implementation Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 01/17] powerpc/qspinlock: add mcs queueing for contended waiters Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 02/17] powerpc/qspinlock: use a half-word store to unlock to avoid larx/stcx Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 03/17] powerpc/qspinlock: convert atomic operations to assembly Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 04/17] powerpc/qspinlock: allow new waiters to steal the lock before queueing Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 05/17] powerpc/qspinlock: theft prevention to control latency Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 06/17] powerpc/qspinlock: store owner CPU in lock word Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 07/17] powerpc/qspinlock: paravirt yield to lock owner Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 08/17] powerpc/qspinlock: implement option to yield to previous node Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 09/17] powerpc/qspinlock: allow stealing when head of queue yields Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 10/17] powerpc/qspinlock: allow propagation of yield CPU down the queue Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 11/17] powerpc/qspinlock: add ability to prod new queue head CPU Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 12/17] powerpc/qspinlock: allow lock stealing in trylock and lock fastpath Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 13/17] powerpc/qspinlock: use spin_begin/end API Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 14/17] powerpc/qspinlock: reduce remote node steal spins Nicholas Piggin
2022-11-26 9:59 ` [PATCH v3 15/17] powerpc/qspinlock: allow indefinite spinning on a preempted owner Nicholas Piggin
2022-11-26 9:59 ` Nicholas Piggin [this message]
2022-11-26 9:59 ` [PATCH v3 17/17] powerpc/qspinlock: add compile-time tuning adjustments Nicholas Piggin
2022-11-28 3:11 ` [PATCH v3 real 01/17] powerpc/qspinlock: powerpc qspinlock implementation Nicholas Piggin
2022-12-08 12:39 ` (subset) [PATCH v3 00/17] powerpc: alternate queued spinlock implementation Michael Ellerman
2023-04-13 10:58 ` Shrikanth Hegde
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20221126095932.1234527-17-npiggin@gmail.com \
--to=npiggin@gmail.com \
--cc=jniethe5@gmail.com \
--cc=ldufour@linux.ibm.com \
--cc=linuxppc-dev@lists.ozlabs.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.