[PATCH 28/29] x86, tsx: Add adaptive elision for rwsems

[Andi Kleen <andi@firstfloor.org>]

From: Andi Kleen <ak@linux.intel.com>

Convert the rwsem elision to be adaptive. This requires adding
4/8 bytes to the rwsem for the adaptation state. The algorithm
is the same as for other adaptive lock types. The elision
configuration is split for readers and writers.

Signed-off-by: Andi Kleen <ak@linux.intel.com>
---
 arch/x86/include/asm/rwsem.h |   22 +++++++++++++++++-----
 arch/x86/kernel/rtm-locks.c  |   10 ++++++++++
 include/linux/rwsem.h        |    3 +++
 3 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/arch/x86/include/asm/rwsem.h b/arch/x86/include/asm/rwsem.h
index f9297af..da6437c 100644
--- a/arch/x86/include/asm/rwsem.h
+++ b/arch/x86/include/asm/rwsem.h
@@ -39,6 +39,7 @@
 #ifdef __KERNEL__
 #include <asm/asm.h>
 #include <linux/elide.h>
+#include <linux/jump_label.h>
 
 /*
  * The bias values and the counter type limits the number of
@@ -58,14 +59,18 @@
 #define RWSEM_ACTIVE_READ_BIAS		RWSEM_ACTIVE_BIAS
 #define RWSEM_ACTIVE_WRITE_BIAS		(RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
 
-extern bool rwsem_elision;
+extern struct static_key rwsem_elision;
+extern struct elision_config readsem_elision_config;
+extern struct elision_config writesem_elision_config;
 
 /*
  * lock for reading
  */
 static inline void __down_read(struct rw_semaphore *sem)
 {
-	if (elide_lock(rwsem_elision, sem->count == 0))
+	if (elide_lock_adapt(rwsem_elision, sem->count == 0,
+			     &sem->elision_adapt,
+			     &readsem_elision_config))
 		return;
 	asm volatile("# beginning down_read\n\t"
 		     LOCK_PREFIX _ASM_INC "(%1)\n\t"
@@ -86,7 +91,9 @@ static inline int __down_read_trylock(struct rw_semaphore *sem)
 {
 	long result, tmp;
 
-	if (elide_lock(rwsem_elision, sem->count == 0))
+	if (elide_lock_adapt(rwsem_elision, sem->count == 0,
+			     &sem->elision_adapt,
+			     &readsem_elision_config))
 		return 1;
 	asm volatile("# beginning __down_read_trylock\n\t"
 		     "  mov          %0,%1\n\t"
@@ -111,7 +118,9 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
 {
 	long tmp;
 
-	if (elide_lock(rwsem_elision, sem->count == 0))
+	if (elide_lock_adapt(rwsem_elision, sem->count == 0,
+			     &sem->elision_adapt,
+			     &readsem_elision_config))
 		return;
 	asm volatile("# beginning down_write\n\t"
 		     LOCK_PREFIX "  xadd      %1,(%2)\n\t"
@@ -139,7 +148,9 @@ static inline int __down_write_trylock(struct rw_semaphore *sem)
 {
 	long ret;
 
-	if (elide_lock(rwsem_elision, sem->count == 0))
+	if (elide_lock_adapt(rwsem_elision, sem->count == 0,
+			     &sem->elision_adapt,
+			     &writesem_elision_config))
 		return 1;
 	ret = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
 			   RWSEM_ACTIVE_WRITE_BIAS);
@@ -195,6 +206,7 @@ static inline void __up_write(struct rw_semaphore *sem)
  */
 static inline void __downgrade_write(struct rw_semaphore *sem)
 {
+	/* if (sem->count == 0) return; */
 	asm volatile("# beginning __downgrade_write\n\t"
 		     LOCK_PREFIX _ASM_ADD "%2,(%1)\n\t"
 		     /*
diff --git a/arch/x86/kernel/rtm-locks.c b/arch/x86/kernel/rtm-locks.c
index 40a0e7d..5a33a12 100644
--- a/arch/x86/kernel/rtm-locks.c
+++ b/arch/x86/kernel/rtm-locks.c
@@ -436,6 +436,15 @@ static unsigned rtm_patch(u8 type, u16 clobbers, void *ibuf,
 struct static_key mutex_elision = STATIC_KEY_INIT_FALSE;
 module_param(mutex_elision, static_key, 0644);
 
+struct static_key rwsem_elision = STATIC_KEY_INIT_FALSE;
+module_param(rwsem_elision, static_key, 0644);
+__read_mostly struct elision_config readsem_elision_config =
+	DEFAULT_ELISION_CONFIG;
+TUNE_ELISION_CONFIG(readsem, readsem_elision_config);
+__read_mostly struct elision_config writesem_elision_config =
+	DEFAULT_ELISION_CONFIG;
+TUNE_ELISION_CONFIG(writesem, writesem_elision_config);
+
 void init_rtm_spinlocks(void)
 {
 	if (!boot_cpu_has(X86_FEATURE_RTM))
@@ -464,6 +473,7 @@ void init_rtm_spinlocks(void)
 
 	static_key_slow_inc(&rwlock_elision);
 	static_key_slow_inc(&mutex_elision);
+	static_key_slow_inc(&rwsem_elision);
 	bitlock_elision = true;
 }
 
diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h
index 436e430..fb96c565 100644
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -28,6 +28,9 @@ struct rw_semaphore {
 	long			count;
 	raw_spinlock_t		wait_lock;
 	struct list_head	wait_list;
+#ifdef CONFIG_RTM_LOCKS
+	short			elision_adapt;
+#endif
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 	struct lockdep_map	dep_map;
 #endif
-- 
1.7.7.6