[RFC PATCH 0/4] powerpc/qspinlock: make slowpath accesses more efficient

[RFC PATCH 0/4] powerpc/qspinlock: make slowpath accesses more efficient

[Nicholas Piggin]

This series (at least the first 3 patches) will get folded into the main
series, but I will to post incremental changes first to make it easier
to see the change.

The 3rd patch is now quite an ugly mess, but once it is folded in then
everything should become simpler because that special case will never be
added in the first place. So don't review it with a fine comb, all it
really does is remove that branch.

There were a couple of cases where I tried to make things a bit simpler
but less efficient in contended paths. It turns out some microbenchmarks
or pathological cases might be limited by this so I'm trying to find
improvements. In one case there is already a better primitive available,
in another case the end result allows a much bigger simplification so it
should be worth the more complicated asm primitive.

Thanks,
Nick

Nicholas Piggin (4):
  powerpc/qspinlock: Avoid cmpxchg pattern in lock stealing
  powerpc/qspinlock: Avoid cmpxchg style patterns in queue head locking
  powerpc/qspinlock: Remove !maybe_waiters special case queue head
    locking
  powerpc/qspinlock: add compile-time tuning adjustments

 arch/powerpc/include/asm/qspinlock.h |  61 ++++++-
 arch/powerpc/lib/qspinlock.c         | 237 ++++++++++++---------------
 2 files changed, 167 insertions(+), 131 deletions(-)

-- 
2.37.2

[RFC PATCH 1/4] powerpc/qspinlock: Avoid cmpxchg pattern in lock stealing

[Nicholas Piggin]

Using a cmpxchg-style trylock in the lock stealing code opens the
possibility for failures due to the lock word changing, even if it
could have been stolen.

Instead, use the stealing trylock which will do the right thing
and succeed unless the lwarx finds it locked or mustq.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
 arch/powerpc/lib/qspinlock.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/arch/powerpc/lib/qspinlock.c b/arch/powerpc/lib/qspinlock.c
index 36afdfde41aa..ff718f27cbc9 100644
--- a/arch/powerpc/lib/qspinlock.c
+++ b/arch/powerpc/lib/qspinlock.c
@@ -497,7 +497,7 @@ static __always_inline bool try_to_steal_lock(struct qspinlock *lock, bool parav
 
 		if (unlikely(!(val & _Q_LOCKED_VAL))) {
 			spin_end();
-			if (trylock_with_tail_cpu(lock, val))
+			if (__queued_spin_trylock_steal(lock))
 				return true;
 			spin_begin();
 		} else {
-- 
2.37.2

[RFC PATCH 2/4] powerpc/qspinlock: Avoid cmpxchg style patterns in queue head locking

[Nicholas Piggin]

Locking by the MCS queue head must clear the tail CPU if there are
no more queue entries left, and it has to deal with concurrent lock
stealing. Implementing these with cmpxchg style updates leaves the
possibility for unnecessary failure when the lock word changes.

Implement this instead within one larx/stcx. critical section that
tests the value and takes the appropriate action: bail out if
it was locked, otherwise lock and clear the tail if we are the tail,
else lock and leave the tail.

With this primitive, there is no longer a significant reason to keep
the large !maybe_stealers special case, so remove it.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
 arch/powerpc/lib/qspinlock.c | 92 ++++++++++++------------------------
 1 file changed, 29 insertions(+), 63 deletions(-)

diff --git a/arch/powerpc/lib/qspinlock.c b/arch/powerpc/lib/qspinlock.c
index ff718f27cbc9..79793b3209ea 100644
--- a/arch/powerpc/lib/qspinlock.c
+++ b/arch/powerpc/lib/qspinlock.c
@@ -104,62 +104,36 @@ static inline int get_owner_cpu(u32 val)
 	return (val & _Q_OWNER_CPU_MASK) >> _Q_OWNER_CPU_OFFSET;
 }
 
-/* Take the lock by setting the lock bit, no other CPUs will touch it. */
-static __always_inline void set_locked(struct qspinlock *lock)
+static __always_inline u32 trylock_clear_my_tail(struct qspinlock *lock, u32 mytail)
 {
-	u32 new = queued_spin_encode_locked_val();
+	u32 newval = queued_spin_encode_locked_val();
 	u32 prev, tmp;
 
 	asm volatile(
-"1:	lwarx	%0,0,%2,%4	# set_locked				\n"
-"	or	%1,%0,%3						\n"
-"	stwcx.	%1,0,%2							\n"
+"1:	lwarx	%0,0,%2,%7	# trylock_clear_my_tail			\n"
+	/* This test is necessary if there could be stealers */
+"	andi.	%1,%0,%5						\n"
+"	bne	3f							\n"
+	/* Test whether the lock tail == mytail */
+"	and	%1,%0,%6						\n"
+"	cmpw	0,%1,%3							\n"
+	/* Merge the new locked value */
+"	or	%1,%1,%4						\n"
+"	bne	2f							\n"
+	/* If the lock tail matched, then clear it, otherwise leave it. */
+"	andc	%1,%1,%6						\n"
+"2:	stwcx.	%1,0,%2							\n"
 "	bne-	1b							\n"
 "\t"	PPC_ACQUIRE_BARRIER "						\n"
+"3:									\n"
 	: "=&r" (prev), "=&r" (tmp)
-	: "r" (&lock->val), "r" (new),
+	: "r" (&lock->val), "r"(mytail), "r" (newval),
+	  "i" (_Q_LOCKED_VAL),
+	  "r" (_Q_TAIL_CPU_MASK),
 	  "i" (IS_ENABLED(CONFIG_PPC64))
 	: "cr0", "memory");
 
-	BUG_ON(prev & _Q_LOCKED_VAL);
-}
-
-static __always_inline u32 __trylock_cmpxchg(struct qspinlock *lock, u32 old, u32 new)
-{
-	u32 prev;
-
-	BUG_ON(old & _Q_LOCKED_VAL);
-
-	asm volatile(
-"1:	lwarx	%0,0,%1,%4	# __trylock_cmpxchg			\n"
-"	cmpw	0,%0,%2							\n"
-"	bne-	2f							\n"
-"	stwcx.	%3,0,%1							\n"
-"	bne-	1b							\n"
-"\t"	PPC_ACQUIRE_BARRIER "						\n"
-"2:									\n"
-	: "=&r" (prev)
-	: "r" (&lock->val), "r"(old), "r" (new),
-	  "i" (IS_ENABLED(CONFIG_PPC64))
-	: "cr0", "memory");
-
-	return likely(prev == old);
-}
-
-/* Take lock, clearing tail, cmpxchg with old (which must not be locked) */
-static __always_inline int trylock_clear_tail_cpu(struct qspinlock *lock, u32 val)
-{
-	u32 newval = queued_spin_encode_locked_val();
-
-	return __trylock_cmpxchg(lock, val, newval);
-}
-
-/* Take lock, preserving tail, cmpxchg with val (which must not be locked) */
-static __always_inline int trylock_with_tail_cpu(struct qspinlock *lock, u32 val)
-{
-	u32 newval = queued_spin_encode_locked_val() | (val & _Q_TAIL_CPU_MASK);
-
-	return __trylock_cmpxchg(lock, val, newval);
+	return prev;
 }
 
 /*
@@ -620,14 +594,11 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 		spin_end();
 
 		/* If we're the last queued, must clean up the tail. */
-		if ((val & _Q_TAIL_CPU_MASK) == tail) {
-			if (trylock_clear_tail_cpu(lock, val))
-				goto release;
-			/* Another waiter must have enqueued. */
-		}
+		old = trylock_clear_my_tail(lock, tail);
+		BUG_ON(old & _Q_LOCKED_VAL);
+		if ((old & _Q_TAIL_CPU_MASK) == tail)
+			goto release;
 
-		/* We must be the owner, just set the lock bit and acquire */
-		set_locked(lock);
 	} else {
 		int set_yield_cpu = -1;
 		int iters = 0;
@@ -682,18 +653,13 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 		spin_end();
 
 		/* If we're the last queued, must clean up the tail. */
-		if ((val & _Q_TAIL_CPU_MASK) == tail) {
-			if (trylock_clear_tail_cpu(lock, val))
-				goto release;
-			/* Another waiter must have enqueued, or lock stolen. */
-		} else {
-			if (trylock_with_tail_cpu(lock, val))
-				goto unlock_next;
-		}
-		goto again;
+		old = trylock_clear_my_tail(lock, tail);
+		if (unlikely(old & _Q_LOCKED_VAL))
+			goto again;
+		if ((old & _Q_TAIL_CPU_MASK) == tail)
+			goto release;
 	}
 
-unlock_next:
 	/* contended path; must wait for next != NULL (MCS protocol) */
 	next = READ_ONCE(node->next);
 	if (!next) {
-- 
2.37.2

[RFC PATCH 3/4] powerpc/qspinlock: Remove !maybe_waiters special case queue head locking

[Nicholas Piggin]

With the update primitive that clears the tail if it matches, and is
tolerant of other queueing activity on the lock, there is no longer a
significant reason to keep the large !maybe_stealers special case, so
remove it.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
 arch/powerpc/lib/qspinlock.c | 124 +++++++++++++++--------------------
 1 file changed, 53 insertions(+), 71 deletions(-)

diff --git a/arch/powerpc/lib/qspinlock.c b/arch/powerpc/lib/qspinlock.c
index 79793b3209ea..457e748b0078 100644
--- a/arch/powerpc/lib/qspinlock.c
+++ b/arch/powerpc/lib/qspinlock.c
@@ -523,7 +523,11 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 	struct qnode *next, *node;
 	u32 val, old, tail;
 	bool seen_preempted = false;
+	bool sleepy = false;
+	bool mustq = false;
 	int idx;
+	int set_yield_cpu = -1;
+	int iters = 0;
 
 	BUILD_BUG_ON(CONFIG_NR_CPUS >= (1U << _Q_TAIL_CPU_BITS));
 
@@ -577,90 +581,68 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 		smp_rmb(); /* acquire barrier for the mcs lock */
 	}
 
-	if (!maybe_stealers) {
-		int set_yield_cpu = -1;
-
-		/* We're at the head of the waitqueue, wait for the lock. */
-		spin_begin();
-		for (;;) {
-			val = READ_ONCE(lock->val);
-			if (!(val & _Q_LOCKED_VAL))
-				break;
-
-			propagate_yield_cpu(node, val, &set_yield_cpu, paravirt);
-			if (yield_head_to_locked_owner(lock, val, paravirt))
-				seen_preempted = true;
-		}
-		spin_end();
-
-		/* If we're the last queued, must clean up the tail. */
-		old = trylock_clear_my_tail(lock, tail);
-		BUG_ON(old & _Q_LOCKED_VAL);
-		if ((old & _Q_TAIL_CPU_MASK) == tail)
-			goto release;
-
-	} else {
-		int set_yield_cpu = -1;
-		int iters = 0;
-		bool sleepy = false;
-		bool mustq = false;
+	/* We're at the head of the waitqueue, wait for the lock. */
+again:
+	spin_begin();
+	for (;;) {
 		bool preempted;
 
-again:
-		/* We're at the head of the waitqueue, wait for the lock. */
-		spin_begin();
-		for (;;) {
-			val = READ_ONCE(lock->val);
-			if (!(val & _Q_LOCKED_VAL))
-				break;
-
-			if (paravirt && pv_sleepy_lock) {
-				if (!sleepy) {
-					if (val & _Q_SLEEPY_VAL) {
-						seen_sleepy_lock();
-						sleepy = true;
-					} else if (recently_sleepy()) {
-						sleepy = true;
-					}
-				}
-				if (pv_sleepy_lock_sticky && seen_preempted &&
-						!(val & _Q_SLEEPY_VAL)) {
-					if (try_set_sleepy(lock, val))
-						val |= _Q_SLEEPY_VAL;
+		val = READ_ONCE(lock->val);
+		if (!(val & _Q_LOCKED_VAL))
+			break;
+
+		if (paravirt && pv_sleepy_lock && maybe_stealers) {
+			if (!sleepy) {
+				if (val & _Q_SLEEPY_VAL) {
+					seen_sleepy_lock();
+					sleepy = true;
+				} else if (recently_sleepy()) {
+					sleepy = true;
 				}
 			}
+			if (pv_sleepy_lock_sticky && seen_preempted &&
+					!(val & _Q_SLEEPY_VAL)) {
+				if (try_set_sleepy(lock, val))
+					val |= _Q_SLEEPY_VAL;
+			}
+		}
 
-			propagate_yield_cpu(node, val, &set_yield_cpu, paravirt);
-			preempted = yield_head_to_locked_owner(lock, val, paravirt);
-			if (preempted)
-				seen_preempted = true;
+		propagate_yield_cpu(node, val, &set_yield_cpu, paravirt);
+		preempted = yield_head_to_locked_owner(lock, val, paravirt);
+		if (!maybe_stealers)
+			continue;
+
+		if (preempted)
+			seen_preempted = true;
 
-			if (paravirt && preempted) {
-				sleepy = true;
+		if (paravirt && preempted) {
+			sleepy = true;
 
-				if (!pv_spin_on_preempted_owner)
-					iters++;
-			} else {
+			if (!pv_spin_on_preempted_owner)
 				iters++;
-			}
+		} else {
+			iters++;
+		}
 
-			if (!mustq && iters >= get_head_spins(paravirt, sleepy)) {
-				mustq = true;
-				set_mustq(lock);
-				val |= _Q_MUST_Q_VAL;
-			}
+		if (!mustq && iters >= get_head_spins(paravirt, sleepy)) {
+			mustq = true;
+			set_mustq(lock);
+			val |= _Q_MUST_Q_VAL;
 		}
-		spin_end();
+	}
+	spin_end();
 
-		/* If we're the last queued, must clean up the tail. */
-		old = trylock_clear_my_tail(lock, tail);
-		if (unlikely(old & _Q_LOCKED_VAL))
-			goto again;
-		if ((old & _Q_TAIL_CPU_MASK) == tail)
-			goto release;
+	/* If we're the last queued, must clean up the tail. */
+	old = trylock_clear_my_tail(lock, tail);
+	if (unlikely(old & _Q_LOCKED_VAL)) {
+		BUG_ON(!maybe_stealers);
+		goto again; /* Can only be true if maybe_stealers. */
 	}
 
-	/* contended path; must wait for next != NULL (MCS protocol) */
+	if ((old & _Q_TAIL_CPU_MASK) == tail)
+		goto release; /* We were the tail, no next. */
+
+	/* There is a next, must wait for node->next != NULL (MCS protocol) */
 	next = READ_ONCE(node->next);
 	if (!next) {
 		spin_begin();
-- 
2.37.2

[RFC PATCH 4/4] powerpc/qspinlock: add compile-time tuning adjustments

[Nicholas Piggin]

This adds compile-time options that allow the EH lock hint bit to be
enabled or disabled, and adds some new options that may or may not
help matters. To help with experimentation and tuning.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
---
 arch/powerpc/include/asm/qspinlock.h | 61 ++++++++++++++++++++++++++--
 arch/powerpc/lib/qspinlock.c         | 39 ++++++++++++++++--
 2 files changed, 94 insertions(+), 6 deletions(-)

diff --git a/arch/powerpc/include/asm/qspinlock.h b/arch/powerpc/include/asm/qspinlock.h
index 56638175e49b..5efd66aeb907 100644
--- a/arch/powerpc/include/asm/qspinlock.h
+++ b/arch/powerpc/include/asm/qspinlock.h
@@ -5,15 +5,68 @@
 #include <linux/compiler.h>
 #include <asm/qspinlock_types.h>
 
+#ifdef CONFIG_PPC64
+/*
+ * Use the EH=1 hint for accesses that result in the lock being acquired.
+ * The hardware is supposed to optimise this pattern by holding the lock
+ * cacheline longer, and releasing when a store to the same memory (the
+ * unlock) is performed.
+ */
+#define _Q_SPIN_EH_HINT 1
+#else
+#define _Q_SPIN_EH_HINT 0
+#endif
+
 /*
  * The trylock itself may steal. This makes trylocks slightly stronger, and
- * might make spin locks slightly more efficient when stealing.
+ * makes locks slightly more efficient when stealing.
  *
  * This is compile-time, so if true then there may always be stealers, so the
  * nosteal paths become unused.
  */
 #define _Q_SPIN_TRY_LOCK_STEAL 1
 
+/*
+ * Put a speculation barrier after testing the lock/node and finding it
+ * busy. Try to prevent pointless speculation in slow paths.
+ *
+ * Slows down the lockstorm microbenchmark with no stealing, where locking
+ * is purely FIFO through the queue. May have more benefit in real workload
+ * where speculating into the wrong place could have a greater cost.
+ */
+#define _Q_SPIN_SPEC_BARRIER 0
+
+#ifdef CONFIG_PPC64
+/*
+ * Execute a miso instruction after passing the MCS lock ownership to the
+ * queue head. Miso is intended to make stores visible to other CPUs sooner.
+ *
+ * This seems to make the lockstorm microbenchmark nospin test go slightly
+ * faster on POWER10, but disable for now.
+ */
+#define _Q_SPIN_MISO 0
+#else
+#define _Q_SPIN_MISO 0
+#endif
+
+#ifdef CONFIG_PPC64
+/*
+ * This executes miso after an unlock of the lock word, having ownership
+ * pass to the next CPU sooner. This will slow the uncontended path to some
+ * degree. Not evidence it helps yet.
+ */
+#define _Q_SPIN_MISO_UNLOCK 0
+#else
+#define _Q_SPIN_MISO_UNLOCK 0
+#endif
+
+/*
+ * Seems to slow down lockstorm microbenchmark, suspect queue node just
+ * has to become shared again right afterwards when its waiter spins on
+ * the lock field.
+ */
+#define _Q_SPIN_PREFETCH_NEXT 0
+
 static __always_inline int queued_spin_is_locked(struct qspinlock *lock)
 {
 	return READ_ONCE(lock->val);
@@ -51,7 +104,7 @@ static __always_inline int __queued_spin_trylock_nosteal(struct qspinlock *lock)
 "2:									\n"
 	: "=&r" (prev)
 	: "r" (&lock->val), "r" (new),
-	  "i" (IS_ENABLED(CONFIG_PPC64))
+	  "i" (_Q_SPIN_EH_HINT)
 	: "cr0", "memory");
 
 	return likely(prev == 0);
@@ -75,7 +128,7 @@ static __always_inline int __queued_spin_trylock_steal(struct qspinlock *lock)
 "2:									\n"
 	: "=&r" (prev), "=&r" (tmp)
 	: "r" (&lock->val), "r" (new), "r" (_Q_TAIL_CPU_MASK),
-	  "i" (IS_ENABLED(CONFIG_PPC64))
+	  "i" (_Q_SPIN_EH_HINT)
 	: "cr0", "memory");
 
 	return likely(!(prev & ~_Q_TAIL_CPU_MASK));
@@ -100,6 +153,8 @@ static __always_inline void queued_spin_lock(struct qspinlock *lock)
 static inline void queued_spin_unlock(struct qspinlock *lock)
 {
 	smp_store_release(&lock->locked, 0);
+	if (_Q_SPIN_MISO_UNLOCK)
+		asm volatile("miso" ::: "memory");
 }
 
 #define arch_spin_is_locked(l)		queued_spin_is_locked(l)
diff --git a/arch/powerpc/lib/qspinlock.c b/arch/powerpc/lib/qspinlock.c
index 457e748b0078..01fbb4209367 100644
--- a/arch/powerpc/lib/qspinlock.c
+++ b/arch/powerpc/lib/qspinlock.c
@@ -48,6 +48,12 @@ static bool pv_prod_head __read_mostly = false;
 static DEFINE_PER_CPU_ALIGNED(struct qnodes, qnodes);
 static DEFINE_PER_CPU_ALIGNED(u64, sleepy_lock_seen_clock);
 
+#if _Q_SPIN_SPEC_BARRIER == 1
+#define spec_barrier() do { asm volatile("ori 31,31,0" ::: "memory"); } while (0)
+#else
+#define spec_barrier() do { } while (0)
+#endif
+
 static __always_inline bool recently_sleepy(void)
 {
 	/* pv_sleepy_lock is true when this is called */
@@ -130,7 +136,7 @@ static __always_inline u32 trylock_clear_my_tail(struct qspinlock *lock, u32 myt
 	: "r" (&lock->val), "r"(mytail), "r" (newval),
 	  "i" (_Q_LOCKED_VAL),
 	  "r" (_Q_TAIL_CPU_MASK),
-	  "i" (IS_ENABLED(CONFIG_PPC64))
+	  "i" (_Q_SPIN_EH_HINT)
 	: "cr0", "memory");
 
 	return prev;
@@ -468,6 +474,7 @@ static __always_inline bool try_to_steal_lock(struct qspinlock *lock, bool parav
 		val = READ_ONCE(lock->val);
 		if (val & _Q_MUST_Q_VAL)
 			break;
+		spec_barrier();
 
 		if (unlikely(!(val & _Q_LOCKED_VAL))) {
 			spin_end();
@@ -533,6 +540,7 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 
 	qnodesp = this_cpu_ptr(&qnodes);
 	if (unlikely(qnodesp->count >= MAX_NODES)) {
+		spec_barrier();
 		while (!queued_spin_trylock(lock))
 			cpu_relax();
 		return;
@@ -569,9 +577,12 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 		/* Wait for mcs node lock to be released */
 		spin_begin();
 		while (!node->locked) {
+			spec_barrier();
+
 			if (yield_to_prev(lock, node, old, paravirt))
 				seen_preempted = true;
 		}
+		spec_barrier();
 		spin_end();
 
 		/* Clear out stale propagated yield_cpu */
@@ -579,6 +590,17 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 			node->yield_cpu = -1;
 
 		smp_rmb(); /* acquire barrier for the mcs lock */
+
+		/*
+		 * Generic qspinlocks have this prefetch here, but it seems
+		 * like it could cause additional line transitions because
+		 * the waiter will keep loading from it.
+		 */
+		if (_Q_SPIN_PREFETCH_NEXT) {
+			next = READ_ONCE(node->next);
+			if (next)
+				prefetchw(next);
+		}
 	}
 
 	/* We're at the head of the waitqueue, wait for the lock. */
@@ -590,6 +612,7 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 		val = READ_ONCE(lock->val);
 		if (!(val & _Q_LOCKED_VAL))
 			break;
+		spec_barrier();
 
 		if (paravirt && pv_sleepy_lock && maybe_stealers) {
 			if (!sleepy) {
@@ -630,6 +653,7 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 			val |= _Q_MUST_Q_VAL;
 		}
 	}
+	spec_barrier();
 	spin_end();
 
 	/* If we're the last queued, must clean up the tail. */
@@ -650,6 +674,7 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 			cpu_relax();
 		spin_end();
 	}
+	spec_barrier();
 
 	/*
 	 * Unlock the next mcs waiter node. Release barrier is not required
@@ -661,10 +686,14 @@ static __always_inline void queued_spin_lock_mcs_queue(struct qspinlock *lock, b
 	if (paravirt && pv_prod_head) {
 		int next_cpu = next->cpu;
 		WRITE_ONCE(next->locked, 1);
+		if (_Q_SPIN_MISO)
+			asm volatile("miso" ::: "memory");
 		if (vcpu_is_preempted(next_cpu))
 			prod_cpu(next_cpu);
 	} else {
 		WRITE_ONCE(next->locked, 1);
+		if (_Q_SPIN_MISO)
+			asm volatile("miso" ::: "memory");
 	}
 
 release:
@@ -679,12 +708,16 @@ void queued_spin_lock_slowpath(struct qspinlock *lock)
 	 * is passed as the paravirt argument to the functions.
 	 */
 	if (IS_ENABLED(CONFIG_PARAVIRT_SPINLOCKS) && is_shared_processor()) {
-		if (try_to_steal_lock(lock, true))
+		if (try_to_steal_lock(lock, true)) {
+			spec_barrier();
 			return;
+		}
 		queued_spin_lock_mcs_queue(lock, true);
 	} else {
-		if (try_to_steal_lock(lock, false))
+		if (try_to_steal_lock(lock, false)) {
+			spec_barrier();
 			return;
+		}
 		queued_spin_lock_mcs_queue(lock, false);
 	}
 }
-- 
2.37.2