[PATCH 0/4] rbd: client list locking improvements

[PATCH 0/4] rbd: client list locking improvements

[Alex Elder]

This series reduces the window during which the client list
lock is held, gives it a more meaningful name, and moves the
locking calls closer to the places they're really needed.

					-Alex

[PATCH 1/4] rbd: release client list lock sooner

[Alex Elder]

In rbd_get_client(), if a client is reused, a number of things
get done while still holding the list lock unnecessarily.

This just moves a few things that need no lock protection outside
the lock.

Signed-off-by: Alex Elder <elder@dreamhost.com>
---
  drivers/block/rbd.c |   10 ++++++----
  1 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index 4c5bb39..4bf8300 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -383,13 +383,15 @@ static int rbd_get_client(struct rbd_device 
*rbd_dev, const char *mon_addr,
  	spin_lock(&node_lock);
  	rbdc = __rbd_client_find(opt);
  	if (rbdc) {
-		ceph_destroy_options(opt);
-		kfree(rbd_opts);
-
  		/* using an existing client */
  		kref_get(&rbdc->kref);
-		rbd_dev->rbd_client = rbdc;
  		spin_unlock(&node_lock);
+
+		rbd_dev->rbd_client = rbdc;
+
+		ceph_destroy_options(opt);
+		kfree(rbd_opts);
+
  		return 0;
  	}
  	spin_unlock(&node_lock);
-- 
1.7.5.4

[PATCH 2/4] rbd: move ctl_mutex lock inside rbd_get_client()

[Alex Elder]

Since rbd_get_client() is only called in one place, move the
acquisition of the mutex around that call inside that function.

Furthermore, within rbd_get_client(), it appears the mutex only
needs to be held while calling rbd_client_create().  (Moving
the lock inside that function will wait for the next patch.)

Signed-off-by: Alex Elder <elder@dreamhost.com>
---
  drivers/block/rbd.c |    6 +++---
  1 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index 4bf8300..99d5920 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -396,7 +396,10 @@ static int rbd_get_client(struct rbd_device 
*rbd_dev, const char *mon_addr,
  	}
  	spin_unlock(&node_lock);

+	mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
  	rbdc = rbd_client_create(opt, rbd_opts);
+	mutex_unlock(&ctl_mutex);
+
  	if (IS_ERR(rbdc)) {
  		ret = PTR_ERR(rbdc);
  		goto done_err;
@@ -2273,10 +2276,7 @@ static ssize_t rbd_add(struct bus_type *bus,
  	/* initialize rest of new object */
  	snprintf(rbd_dev->name, DEV_NAME_LEN, DRV_NAME "%d", rbd_dev->id);

-	mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
  	rc = rbd_get_client(rbd_dev, mon_dev_name, options);
-	mutex_unlock(&ctl_mutex);
-
  	if (rc < 0)
  		goto err_out_slot;

-- 
1.7.5.4

[PATCH 3/4] rbd: move ctl_mutex lock inside rbd_client_create()

[Alex Elder]

Since rbd_client_create() is only called in one place, move the
acquisition of the mutex around that call inside that function.

Signed-off-by: Alex Elder <elder@dreamhost.com>
---
  drivers/block/rbd.c |   11 +++++++----
  1 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index 99d5920..b3c4140 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -257,9 +257,11 @@ static struct rbd_client *rbd_client_create(struct 
ceph_options *opt,
  	kref_init(&rbdc->kref);
  	INIT_LIST_HEAD(&rbdc->node);

+	mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
+
  	rbdc->client = ceph_create_client(opt, rbdc, 0, 0);
  	if (IS_ERR(rbdc->client))
-		goto out_rbdc;
+		goto out_mutex;
  	opt = NULL; /* Now rbdc->client is responsible for opt */

  	ret = ceph_open_session(rbdc->client);
@@ -272,12 +274,15 @@ static struct rbd_client *rbd_client_create(struct 
ceph_options *opt,
  	list_add_tail(&rbdc->node, &rbd_client_list);
  	spin_unlock(&node_lock);

+	mutex_unlock(&ctl_mutex);
+
  	dout("rbd_client_create created %p\n", rbdc);
  	return rbdc;

  out_err:
  	ceph_destroy_client(rbdc->client);
-out_rbdc:
+out_mutex:
+	mutex_unlock(&ctl_mutex);
  	kfree(rbdc);
  out_opt:
  	if (opt)
@@ -396,9 +401,7 @@ static int rbd_get_client(struct rbd_device 
*rbd_dev, const char *mon_addr,
  	}
  	spin_unlock(&node_lock);

-	mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
  	rbdc = rbd_client_create(opt, rbd_opts);
-	mutex_unlock(&ctl_mutex);

  	if (IS_ERR(rbdc)) {
  		ret = PTR_ERR(rbdc);
-- 
1.7.5.4

[PATCH 4/4] rbd: rename "node_lock"

[Alex Elder]

The spinlock used to protect rbd_client_list is named "node_lock".
Rename it to "rbd_client_list_lock" to make it more obvious what
it's for.

Signed-off-by: Alex Elder <elder@dreamhost.com>
---
  drivers/block/rbd.c |   20 ++++++++++----------
  1 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index b3c4140..f6dba36 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -179,8 +179,8 @@ static DEFINE_MUTEX(ctl_mutex);	  /* Serialize 
open/close/setup/teardown */
  static LIST_HEAD(rbd_dev_list);    /* devices */
  static DEFINE_SPINLOCK(rbd_dev_list_lock);

-static LIST_HEAD(rbd_client_list);      /* clients */
-static DEFINE_SPINLOCK(node_lock);      /* protects client get/put */
+static LIST_HEAD(rbd_client_list);      	/* clients */
+static DEFINE_SPINLOCK(rbd_client_list_lock);

  static int __rbd_init_snaps_header(struct rbd_device *rbd_dev);
  static void rbd_dev_release(struct device *dev);
@@ -270,9 +270,9 @@ static struct rbd_client *rbd_client_create(struct 
ceph_options *opt,

  	rbdc->rbd_opts = rbd_opts;

-	spin_lock(&node_lock);
+	spin_lock(&rbd_client_list_lock);
  	list_add_tail(&rbdc->node, &rbd_client_list);
-	spin_unlock(&node_lock);
+	spin_unlock(&rbd_client_list_lock);

  	mutex_unlock(&ctl_mutex);

@@ -385,12 +385,12 @@ static int rbd_get_client(struct rbd_device 
*rbd_dev, const char *mon_addr,
  		goto done_err;
  	}

-	spin_lock(&node_lock);
+	spin_lock(&rbd_client_list_lock);
  	rbdc = __rbd_client_find(opt);
  	if (rbdc) {
  		/* using an existing client */
  		kref_get(&rbdc->kref);
-		spin_unlock(&node_lock);
+		spin_unlock(&rbd_client_list_lock);

  		rbd_dev->rbd_client = rbdc;

@@ -399,7 +399,7 @@ static int rbd_get_client(struct rbd_device 
*rbd_dev, const char *mon_addr,

  		return 0;
  	}
-	spin_unlock(&node_lock);
+	spin_unlock(&rbd_client_list_lock);

  	rbdc = rbd_client_create(opt, rbd_opts);

@@ -418,7 +418,7 @@ done_err:
  /*
   * Destroy ceph client
   *
- * Caller must hold node_lock.
+ * Caller must hold rbd_client_list_lock.
   */
  static void rbd_client_release(struct kref *kref)
  {
@@ -438,9 +438,9 @@ static void rbd_client_release(struct kref *kref)
   */
  static void rbd_put_client(struct rbd_device *rbd_dev)
  {
-	spin_lock(&node_lock);
+	spin_lock(&rbd_client_list_lock);
  	kref_put(&rbd_dev->rbd_client->kref, rbd_client_release);
-	spin_unlock(&node_lock);
+	spin_unlock(&rbd_client_list_lock);
  	rbd_dev->rbd_client = NULL;
  }

-- 
1.7.5.4

Re: [PATCH 0/4] rbd: client list locking improvements

[Sage Weil]

Looks good,

Reviewed-by: Sage Weil <sage@newdream.net>

On Tue, 28 Feb 2012, Alex Elder wrote:

> This series reduces the window during which the client list
> lock is held, gives it a more meaningful name, and moves the
> locking calls closer to the places they're really needed.
> 
> 					-Alex
> --
> To unsubscribe from this list: send the line "unsubscribe ceph-devel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> 
>