[PATCH 08/29] locking, tsx: Add support for arch_read/write_unlock_irq/flags

[Andi Kleen <andi@firstfloor.org>]

From: Andi Kleen <ak@linux.intel.com>

The TSX RTM lock elision code needs to distinguish unlocks that reenable
interrupts from others. Add arch_read/write_unlock_irq/flags for rwlocks
similar to the ones for spinlocks.  This is opt in by the architecture.

Signed-off-by: Andi Kleen <ak@linux.intel.com>
---
 include/linux/rwlock.h         |   76 ++++++++++++++++++++++++++++++++++++++-
 include/linux/rwlock_api_smp.h |   12 ++----
 2 files changed, 78 insertions(+), 10 deletions(-)

diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h
index bc2994e..82a7f61 100644
--- a/include/linux/rwlock.h
+++ b/include/linux/rwlock.h
@@ -14,6 +14,34 @@
  * Released under the General Public License (GPL).
  */
 
+#if !defined(ARCH_HAS_RWLOCK_UNLOCK_IRQ) && defined(CONFIG_SMP)
+static inline void arch_read_unlock_irqrestore(arch_rwlock_t *lock,
+					       unsigned long flags)
+{
+	arch_read_unlock(lock);
+	local_irq_restore(flags);
+}
+
+static inline void arch_read_unlock_irq(arch_rwlock_t *lock)
+{
+	arch_read_unlock(lock);
+	local_irq_enable();
+}
+
+static inline void arch_write_unlock_irqrestore(arch_rwlock_t *lock,
+						unsigned long flags)
+{
+	arch_write_unlock(lock);
+	local_irq_restore(flags);
+}
+
+static inline void arch_write_unlock_irq(arch_rwlock_t *lock)
+{
+	arch_write_unlock(lock);
+	local_irq_enable();
+}
+#endif
+
 #ifdef CONFIG_DEBUG_SPINLOCK
   extern void __rwlock_init(rwlock_t *lock, const char *name,
 			    struct lock_class_key *key);
@@ -37,17 +65,61 @@ do {								\
 #define do_raw_write_lock_flags(lock, flags) do_raw_write_lock(lock)
  extern int do_raw_write_trylock(rwlock_t *lock);
  extern void do_raw_write_unlock(rwlock_t *lock) __releases(lock);
+
+static inline void do_raw_write_unlock_irq(rwlock_t *lock) __releases(lock)
+{
+	do_raw_write_unlock(lock);
+	local_irq_enable();
+}
+
+static inline void do_raw_read_unlock_irq(rwlock_t *lock) __releases(lock)
+{
+	do_raw_read_unlock(lock);
+	local_irq_enable();
+}
+
+static inline void do_raw_write_unlock_irqrestore(rwlock_t *lock,
+						  unsigned long flags)
+	__releases(lock)
+{
+	do_raw_write_unlock(lock);
+	local_irq_restore(flags);
+}
+
+static inline void do_raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
+	__releases(lock)
+{
+	do_raw_read_unlock(lock);
+	local_irq_restore(flags);
+}
+
 #else
 # define do_raw_read_lock(rwlock)	do {__acquire(lock); arch_read_lock(&(rwlock)->raw_lock); } while (0)
-# define do_raw_read_lock_flags(lock, flags) \
-		do {__acquire(lock); arch_read_lock_flags(&(lock)->raw_lock, *(flags)); } while (0)
+# define do_raw_read_lock_flags(lock, flags)	\
+		do { __acquire(lock);		\
+		     arch_read_lock_flags(&(lock)->raw_lock, *(flags)); } while (0)
 # define do_raw_read_trylock(rwlock)	arch_read_trylock(&(rwlock)->raw_lock)
 # define do_raw_read_unlock(rwlock)	do {arch_read_unlock(&(rwlock)->raw_lock); __release(lock); } while (0)
+# define do_raw_read_unlock_irqrestore(rwlock, flags)			\
+	do {								\
+		arch_read_unlock_irqrestore(&(rwlock)->raw_lock, flags); \
+		__release(lock); \
+	} while (0)
+# define do_raw_read_unlock_irq(rwlock)	\
+	do {arch_read_unlock_irq(&(rwlock)->raw_lock); __release(lock); } while (0)
 # define do_raw_write_lock(rwlock)	do {__acquire(lock); arch_write_lock(&(rwlock)->raw_lock); } while (0)
 # define do_raw_write_lock_flags(lock, flags) \
 		do {__acquire(lock); arch_write_lock_flags(&(lock)->raw_lock, *(flags)); } while (0)
 # define do_raw_write_trylock(rwlock)	arch_write_trylock(&(rwlock)->raw_lock)
 # define do_raw_write_unlock(rwlock)	do {arch_write_unlock(&(rwlock)->raw_lock); __release(lock); } while (0)
+# define do_raw_write_unlock_irqrestore(rwlock, flags)			\
+	do {								\
+		arch_write_unlock_irqrestore(&(rwlock)->raw_lock, flags); \
+		__release(lock); \
+	} while (0)
+# define do_raw_write_unlock_irq(rwlock)		 \
+	do { arch_write_unlock_irq(&(rwlock)->raw_lock); \
+	     __release(lock); } while (0)
 #endif
 
 #define read_can_lock(rwlock)		arch_read_can_lock(&(rwlock)->raw_lock)
diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h
index 9c9f049..8ac4a73 100644
--- a/include/linux/rwlock_api_smp.h
+++ b/include/linux/rwlock_api_smp.h
@@ -233,16 +233,14 @@ static inline void
 __raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
 {
 	rwlock_release(&lock->dep_map, 1, _RET_IP_);
-	do_raw_read_unlock(lock);
-	local_irq_restore(flags);
+	do_raw_read_unlock_irqrestore(lock, flags);
 	preempt_enable();
 }
 
 static inline void __raw_read_unlock_irq(rwlock_t *lock)
 {
 	rwlock_release(&lock->dep_map, 1, _RET_IP_);
-	do_raw_read_unlock(lock);
-	local_irq_enable();
+	do_raw_read_unlock_irq(lock);
 	preempt_enable();
 }
 
@@ -258,16 +256,14 @@ static inline void __raw_write_unlock_irqrestore(rwlock_t *lock,
 					     unsigned long flags)
 {
 	rwlock_release(&lock->dep_map, 1, _RET_IP_);
-	do_raw_write_unlock(lock);
-	local_irq_restore(flags);
+	do_raw_write_unlock_irqrestore(lock, flags);
 	preempt_enable();
 }
 
 static inline void __raw_write_unlock_irq(rwlock_t *lock)
 {
 	rwlock_release(&lock->dep_map, 1, _RET_IP_);
-	do_raw_write_unlock(lock);
-	local_irq_enable();
+	do_raw_write_unlock_irq(lock);
 	preempt_enable();
 }
 
-- 
1.7.7.6