From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1753208AbbF1X6Q (ORCPT ); Sun, 28 Jun 2015 19:58:16 -0400 Received: from mx1.redhat.com ([209.132.183.28]:34979 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752834AbbF1X6I (ORCPT ); Sun, 28 Jun 2015 19:58:08 -0400 Date: Mon, 29 Jun 2015 01:56:45 +0200 From: Oleg Nesterov To: Peter Zijlstra Cc: paulmck@linux.vnet.ibm.com, tj@kernel.org, mingo@redhat.com, der.herr@hofr.at, dave@stgolabs.net, riel@redhat.com, viro@ZenIV.linux.org.uk, torvalds@linux-foundation.org, linux-kernel@vger.kernel.org Subject: [PATCH 3/3] percpu-rwsem: introduce percpu_rw_semaphore->recursive mode Message-ID: <20150628235645.GA25180@redhat.com> MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline In-Reply-To: <20150628235614.GA24454@redhat.com> User-Agent: Mutt/1.5.18 (2008-05-17) Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Add percpu_rw_semaphore->recursive boolean. If it is true then the recursive percpu_down_read() is safe, percpu_down_write() doesn't exclude the new readers, like cpu_hotplug_begin(). Signed-off-by: Oleg Nesterov --- include/linux/percpu-rwsem.h | 15 ++++++++++----- kernel/events/uprobes.c | 2 +- kernel/locking/percpu-rwsem.c | 15 +++++++++++---- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h index 9202e73..9441abd 100644 --- a/include/linux/percpu-rwsem.h +++ b/include/linux/percpu-rwsem.h @@ -13,16 +13,18 @@ struct percpu_rw_semaphore { int state; struct rcu_sync_struct rss; wait_queue_head_t writer; + bool recursive; struct rw_semaphore rw_sem; }; -#define DEFINE_STATIC_PERCPU_RWSEM(name) \ +#define DEFINE_STATIC_PERCPU_RWSEM(name, rec) \ static DEFINE_PER_CPU(unsigned int, __percpu_rwsem_refcount_##name); \ static struct percpu_rw_semaphore name = { \ .refcount = &__percpu_rwsem_refcount_##name, \ .state = 0, \ .rss = __RCU_SYNC_INITIALIZER(name.rss, RCU_SCHED_SYNC, 1), \ .writer = __WAIT_QUEUE_HEAD_INITIALIZER(name.writer), \ + .recursive = rec, \ .rw_sem = __RWSEM_INITIALIZER(name.rw_sem), \ } @@ -37,7 +39,10 @@ static inline void percpu_down_read(struct percpu_rw_semaphore *sem) { might_sleep(); - rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 0, _RET_IP_); + if (sem->recursive) + rwlock_acquire_read(&sem->rw_sem.dep_map, 0, 0, _RET_IP_); + else + rwsem_acquire_read(&sem->rw_sem.dep_map, 0, 0, _RET_IP_); preempt_disable(); /* @@ -97,14 +102,14 @@ static inline void percpu_up_read(struct percpu_rw_semaphore *sem) extern void percpu_down_write(struct percpu_rw_semaphore *); extern void percpu_up_write(struct percpu_rw_semaphore *); -extern int __percpu_init_rwsem(struct percpu_rw_semaphore *, +extern int __percpu_init_rwsem(struct percpu_rw_semaphore *, bool, const char *, struct lock_class_key *); extern void percpu_free_rwsem(struct percpu_rw_semaphore *); -#define percpu_init_rwsem(sem) \ +#define percpu_init_rwsem(sem, recursive) \ ({ \ static struct lock_class_key rwsem_key; \ - __percpu_init_rwsem(sem, #sem, &rwsem_key); \ + __percpu_init_rwsem(sem, recursive, #sem, &rwsem_key); \ }) #endif diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index cb346f2..a4813a1 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -1985,7 +1985,7 @@ static int __init init_uprobes(void) for (i = 0; i < UPROBES_HASH_SZ; i++) mutex_init(&uprobes_mmap_mutex[i]); - if (percpu_init_rwsem(&dup_mmap_sem)) + if (percpu_init_rwsem(&dup_mmap_sem, false)) return -ENOMEM; return register_die_notifier(&uprobe_exception_nb); diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c index 609c13b..3db7c45 100644 --- a/kernel/locking/percpu-rwsem.c +++ b/kernel/locking/percpu-rwsem.c @@ -10,7 +10,7 @@ enum { readers_slow, readers_block }; -int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, +int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, bool recursive, const char *name, struct lock_class_key *rwsem_key) { sem->refcount = alloc_percpu(unsigned int); @@ -20,6 +20,7 @@ int __percpu_init_rwsem(struct percpu_rw_semaphore *sem, sem->state = readers_slow; rcu_sync_init(&sem->rss, RCU_SCHED_SYNC, true); init_waitqueue_head(&sem->writer); + sem->recursive = recursive; __init_rwsem(&sem->rw_sem, name, rwsem_key); return 0; @@ -124,9 +125,15 @@ void __percpu_up_read(struct percpu_rw_semaphore *sem) */ static bool readers_active_check(struct percpu_rw_semaphore *sem) { - if (per_cpu_sum(*sem->refcount) != 0) + if (sem->recursive && !down_write_trylock(&sem->rw_sem)) return false; + if (per_cpu_sum(*sem->refcount) != 0) { + if (sem->recursive) + up_write(&sem->rw_sem); + return false; + } + /* * If we observed the decrement; ensure we see the entire critical * section. @@ -155,8 +162,8 @@ void percpu_down_write(struct percpu_rw_semaphore *sem) * then we are guaranteed to see their sem->refcount increment, and * therefore will wait for them. */ - - down_write(&sem->rw_sem); + if (!sem->recursive) + down_write(&sem->rw_sem); /* Wait for all now active readers to complete. */ wait_event(sem->writer, readers_active_check(sem)); } -- 1.5.5.1