[PATCH] rbd: move snap_rwsem to the device, rename to header_rwsem

[Alex Elder <elder@dreamhost.com>]

(Re-posting Josh's prescient patch--it fixes a newly identified issue.)

A new temporary header is allocated each time the header changes, but
only the changed properties are copied over. We don't need a new
semaphore for each header update.

This addresses http://tracker.newdream.net/issues/2174

Signed-off-by: Josh Durgin <josh.durgin@dreamhost.com>
Reviewed-by: Alex Elder <elder@dreamhost.com>
---
  drivers/block/rbd.c |   20 ++++++++++----------
  1 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index bc75d82..50117dd 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -82,7 +82,6 @@ struct rbd_image_header {
  	__u8 obj_order;
  	__u8 crypt_type;
  	__u8 comp_type;
-	struct rw_semaphore snap_rwsem;
  	struct ceph_snap_context *snapc;
  	size_t snap_names_len;
  	u64 snap_seq;
@@ -173,6 +172,8 @@ struct rbd_device {
  	struct ceph_osd_event   *watch_event;
  	struct ceph_osd_request *watch_request;

+	/* protects updating the header */
+	struct rw_semaphore     header_rwsem;
  	char                    snap_name[RBD_MAX_SNAP_NAME_LEN];
  	u32 cur_snap;	/* index+1 of current snapshot within snap context
  			   0 - for the head */
@@ -502,7 +503,6 @@ static int rbd_header_from_disk(struct 
rbd_image_header *header,
  	if (!header->snapc)
  		return -ENOMEM;

-	init_rwsem(&header->snap_rwsem);
  	header->snap_names_len = le64_to_cpu(ondisk->snap_names_len);
  	if (snap_count) {
  		header->snap_names = kmalloc(header->snap_names_len,
@@ -597,7 +597,7 @@ static int rbd_header_set_snap(struct rbd_device 
*dev, u64 *size)

  	BUILD_BUG_ON(sizeof (dev->snap_name) < sizeof (RBD_SNAP_HEAD_NAME));

-	down_write(&header->snap_rwsem);
+	down_write(&dev->header_rwsem);

  	if (!memcmp(dev->snap_name, RBD_SNAP_HEAD_NAME,
  		    sizeof (RBD_SNAP_HEAD_NAME))) {
@@ -620,7 +620,7 @@ static int rbd_header_set_snap(struct rbd_device 
*dev, u64 *size)

  	ret = 0;
  done:
-	up_write(&header->snap_rwsem);
+	up_write(&dev->header_rwsem);
  	return ret;
  }

@@ -887,7 +887,6 @@ static int rbd_do_request(struct request *rq,
  	struct timespec mtime = CURRENT_TIME;
  	struct rbd_request *req_data;
  	struct ceph_osd_request_head *reqhead;
-	struct rbd_image_header *header = &dev->header;
  	struct ceph_osd_client *osdc;

  	req_data = kzalloc(sizeof(*req_data), GFP_NOIO);
@@ -905,13 +904,13 @@ static int rbd_do_request(struct request *rq,

  	dout("rbd_do_request obj=%s ofs=%lld len=%lld\n", obj, len, ofs);

-	down_read(&header->snap_rwsem);
+	down_read(&dev->header_rwsem);

  	osdc = &dev->rbd_client->client->osdc;
  	req = ceph_osdc_alloc_request(osdc, flags, snapc, ops,
  					false, GFP_NOIO, pages, bio);
  	if (!req) {
-		up_read(&header->snap_rwsem);
+		up_read(&dev->header_rwsem);
  		ret = -ENOMEM;
  		goto done_pages;
  	}
@@ -946,7 +945,7 @@ static int rbd_do_request(struct request *rq,
  				snapc,
  				&mtime,
  				req->r_oid, req->r_oid_len);
-	up_read(&header->snap_rwsem);
+	up_read(&dev->header_rwsem);

  	if (linger_req) {
  		ceph_osdc_set_request_linger(osdc, req);
@@ -1718,7 +1717,7 @@ static int __rbd_update_snaps(struct rbd_device 
*rbd_dev)
  	/* resized? */
  	set_capacity(rbd_dev->disk, h.image_size / SECTOR_SIZE);

-	down_write(&rbd_dev->header.snap_rwsem);
+	down_write(&rbd_dev->header_rwsem);

  	snap_seq = rbd_dev->header.snapc->seq;
  	if (rbd_dev->header.total_snaps &&
@@ -1743,7 +1742,7 @@ static int __rbd_update_snaps(struct rbd_device 
*rbd_dev)

  	ret = __rbd_init_snaps_header(rbd_dev);

-	up_write(&rbd_dev->header.snap_rwsem);
+	up_write(&rbd_dev->header_rwsem);

  	return ret;
  }
@@ -2380,6 +2379,7 @@ static ssize_t rbd_add(struct bus_type *bus,
  	spin_lock_init(&rbd_dev->lock);
  	INIT_LIST_HEAD(&rbd_dev->node);
  	INIT_LIST_HEAD(&rbd_dev->snaps);
+	init_rwsem(&rbd_dev->header_rwsem);

  	/* generate unique id: find highest unique id, add one */
  	rbd_id_get(rbd_dev);
-- 
1.7.5.4