[PATCH 1/3] md-cluster: append some actions when change bitmap from clustered to none

[PATCH 1/3] md-cluster: append some actions when change bitmap from clustered to none

[Guoqing Jiang]

For clustered raid, we need to do extra actions when change
bitmap to none.

1. check if all the bitmap lock could be get or not, if yes then
   we can continue the change since cluster raid is only active
   in current node. Otherwise return fail and unlock the related
   bitmap locks
2. set nodes to 0 and then leave cluster environment.
3. release other nodes's bitmap lock.

Signed-off-by: Guoqing Jiang <gqjiang@suse.com>
---
 drivers/md/md-cluster.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++++
 drivers/md/md-cluster.h |  2 ++
 drivers/md/md.c         | 14 ++++++++++++
 3 files changed, 73 insertions(+)

diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 430b139..148cd85 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -69,6 +69,7 @@ struct md_cluster_info {
 	struct completion completion;
 	struct mutex recv_mutex;
 	struct dlm_lock_resource *bitmap_lockres;
+	struct dlm_lock_resource **other_bitmap_lockres;
 	struct dlm_lock_resource *resync_lockres;
 	struct list_head suspend_list;
 	spinlock_t suspend_lock;
@@ -835,6 +836,7 @@ static void resync_bitmap(struct mddev *mddev)
 			__func__, __LINE__, err);
 }
 
+static void unlock_all_bitmaps(struct mddev *mddev);
 static int leave(struct mddev *mddev)
 {
 	struct md_cluster_info *cinfo = mddev->cluster_info;
@@ -855,6 +857,7 @@ static int leave(struct mddev *mddev)
 	lockres_free(cinfo->ack_lockres);
 	lockres_free(cinfo->no_new_dev_lockres);
 	lockres_free(cinfo->bitmap_lockres);
+	unlock_all_bitmaps(mddev);
 	dlm_release_lockspace(cinfo->lockspace, 2);
 	return 0;
 }
@@ -1054,6 +1057,58 @@ static int remove_disk(struct mddev *mddev, struct md_rdev *rdev)
 	return sendmsg(cinfo, &cmsg);
 }
 
+static int lock_all_bitmaps(struct mddev *mddev)
+{
+	int slot, my_slot, ret, held = 1, i = 0;
+	char str[64];
+	struct md_cluster_info *cinfo = mddev->cluster_info;
+
+	cinfo->other_bitmap_lockres = kzalloc((mddev->bitmap_info.nodes - 1) *
+					     sizeof(struct dlm_lock_resource *),
+					     GFP_KERNEL);
+	if (!cinfo->other_bitmap_lockres) {
+		pr_err("%s:%d: can't alloc mem for other bitmap locks\n", __func__, __LINE__);
+		return 0;
+	}
+
+	my_slot = slot_number(mddev);
+	for (slot = 0; slot < mddev->bitmap_info.nodes; slot++) {
+		if (slot == my_slot)
+			continue;
+
+		memset(str, '\0', 64);
+		snprintf(str, 64, "bitmap%04d", slot);
+		cinfo->other_bitmap_lockres[i] = lockres_init(mddev, str, NULL, 1);
+		if (!cinfo->other_bitmap_lockres[i])
+			return -ENOMEM;
+
+		cinfo->other_bitmap_lockres[i]->flags |= DLM_LKF_NOQUEUE;
+		ret = dlm_lock_sync(cinfo->other_bitmap_lockres[i], DLM_LOCK_PW);
+		if (ret)
+			held = -1;
+		i++;
+	}
+
+	return held;
+}
+
+static void unlock_all_bitmaps(struct mddev *mddev)
+{
+	struct md_cluster_info *cinfo = mddev->cluster_info;
+	int i;
+
+	/* release other node's bitmap lock if they are existed */
+	if (cinfo->other_bitmap_lockres) {
+		for (i = 0; i < mddev->bitmap_info.nodes - 1; i++) {
+			if (cinfo->other_bitmap_lockres[i]) {
+				dlm_unlock_sync(cinfo->other_bitmap_lockres[i]);
+				lockres_free(cinfo->other_bitmap_lockres[i]);
+			}
+		}
+		kfree(cinfo->other_bitmap_lockres);
+	}
+}
+
 static int gather_bitmaps(struct md_rdev *rdev)
 {
 	int sn, err;
@@ -1099,6 +1154,8 @@ static struct md_cluster_operations cluster_ops = {
 	.new_disk_ack = new_disk_ack,
 	.remove_disk = remove_disk,
 	.gather_bitmaps = gather_bitmaps,
+	.lock_all_bitmaps = lock_all_bitmaps,
+	.unlock_all_bitmaps = unlock_all_bitmaps,
 };
 
 static int __init cluster_init(void)
diff --git a/drivers/md/md-cluster.h b/drivers/md/md-cluster.h
index e75ea26..45ce6c9 100644
--- a/drivers/md/md-cluster.h
+++ b/drivers/md/md-cluster.h
@@ -24,6 +24,8 @@ struct md_cluster_operations {
 	int (*new_disk_ack)(struct mddev *mddev, bool ack);
 	int (*remove_disk)(struct mddev *mddev, struct md_rdev *rdev);
 	int (*gather_bitmaps)(struct md_rdev *rdev);
+	int (*lock_all_bitmaps)(struct mddev *mddev);
+	void (*unlock_all_bitmaps)(struct mddev *mddev);
 };
 
 #endif /* _MD_CLUSTER_H */
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 06066e67..47ee8ae 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -6586,6 +6586,20 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info)
 				rv = -EINVAL;
 				goto err;
 			}
+			if (mddev->bitmap_info.nodes) {
+				/* hold PW on all the bitmap lock */
+				if (md_cluster_ops->lock_all_bitmaps(mddev) <= 0) {
+					printk("%s:%d can't change bitmap to none since the"
+						" array is using by more than one node\n",
+						__func__, __LINE__);
+					rv = -EPERM;
+					md_cluster_ops->unlock_all_bitmaps(mddev);
+					goto err;
+				}
+
+				mddev->bitmap_info.nodes = 0;
+				md_cluster_ops->leave(mddev);
+			}
 			mddev->pers->quiesce(mddev, 1);
 			bitmap_destroy(mddev);
 			mddev->pers->quiesce(mddev, 0);
-- 
2.1.4

[PATCH 2/3] md-cluster: introduce clear_clusterinfo_from_sb

[Guoqing Jiang]

To change the bitmap from clustered to none, we also
need to clean related info about cluster from sb, such
as version, nodes and cluster_name.

Signed-off-by: Guoqing Jiang <gqjiang@suse.com>
---
 drivers/md/bitmap.c | 25 +++++++++++++++++++++++++
 drivers/md/bitmap.h |  1 +
 drivers/md/md.c     |  1 +
 3 files changed, 27 insertions(+)

diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c
index 4f22e91..c11390f 100644
--- a/drivers/md/bitmap.c
+++ b/drivers/md/bitmap.c
@@ -417,6 +417,31 @@ out:
 /*
  * bitmap file superblock operations
  */
+void clear_clusterinfo_from_sb(struct bitmap *bitmap)
+{
+	bitmap_super_t *sb;
+
+	if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
+		return;
+	if (bitmap->mddev->bitmap_info.external)
+		return;
+	if (!bitmap->storage.sb_page) /* no superblock */
+		return;
+	sb = kmap_atomic(bitmap->storage.sb_page);
+	sb->nodes = 0;
+	sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
+	memset(sb->cluster_name, NULL, 64);
+
+	kunmap_atomic(sb);
+	/* Don't write until any other writes have completed */
+	if (bitmap->storage.file)
+		wait_event(bitmap->write_wait,
+			   atomic_read(&bitmap->pending_writes)==0);
+	else
+		md_super_wait(bitmap->mddev);
+
+	write_page(bitmap, bitmap->storage.sb_page, 1);
+}
 
 /* update the event counter and sync the superblock to disk */
 void bitmap_update_sb(struct bitmap *bitmap)
diff --git a/drivers/md/bitmap.h b/drivers/md/bitmap.h
index 7d5c3a6..1d0975b 100644
--- a/drivers/md/bitmap.h
+++ b/drivers/md/bitmap.h
@@ -241,6 +241,7 @@ void bitmap_flush(struct mddev *mddev);
 void bitmap_destroy(struct mddev *mddev);
 
 void bitmap_print_sb(struct bitmap *bitmap);
+void clear_clusterinfo_from_sb(struct bitmap *bitmap);
 void bitmap_update_sb(struct bitmap *bitmap);
 void bitmap_status(struct seq_file *seq, struct bitmap *bitmap);
 
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 47ee8ae..f06dc43 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -6598,6 +6598,7 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info)
 				}
 
 				mddev->bitmap_info.nodes = 0;
+				clear_clusterinfo_from_sb(mddev->bitmap);
 				md_cluster_ops->leave(mddev);
 			}
 			mddev->pers->quiesce(mddev, 1);
-- 
2.1.4

[PATCH 3/3] md-cluster: update the documentation

[Guoqing Jiang]

Update design documentation based on recent development.
original version comes from Neil.

Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
Signed-off-by: Guoqing Jiang <gqjiang@suse.com>
---
 Documentation/md-cluster.txt | 314 +++++++++++++++++++++++++++++++------------
 1 file changed, 228 insertions(+), 86 deletions(-)

diff --git a/Documentation/md-cluster.txt b/Documentation/md-cluster.txt
index 1b79436..c100c71 100644
--- a/Documentation/md-cluster.txt
+++ b/Documentation/md-cluster.txt
@@ -3,7 +3,7 @@ The cluster MD is a shared-device RAID for a cluster.
 
 1. On-disk format
 
-Separate write-intent-bitmap are used for each cluster node.
+Separate write-intent-bitmaps are used for each cluster node.
 The bitmaps record all writes that may have been started on that node,
 and may not yet have finished. The on-disk layout is:
 
@@ -14,117 +14,161 @@ and may not yet have finished. The on-disk layout is:
 | bm super[2] + bits  | bm bits [2, contd]  | bm super[3] + bits  |
 | bm bits [3, contd]  |                     |                     |
 
-During "normal" functioning we assume the filesystem ensures that only one
-node writes to any given block at a time, so a write
-request will
+During "normal" functioning we assume the filesystem ensures that only
+one node writes to any given block at a time, so a write request will
+
  - set the appropriate bit (if not already set)
  - commit the write to all mirrors
  - schedule the bit to be cleared after a timeout.
 
-Reads are just handled normally.  It is up to the filesystem to
-ensure one node doesn't read from a location where another node (or the same
+Reads are just handled normally. It is up to the filesystem to ensure
+one node doesn't read from a location where another node (or the same
 node) is writing.
 
 
 2. DLM Locks for management
 
-There are two locks for managing the device:
+There are three groups of locks for managing the device:
 
 2.1 Bitmap lock resource (bm_lockres)
 
- The bm_lockres protects individual node bitmaps. They are named in the
- form bitmap001 for node 1, bitmap002 for node and so on. When a node
- joins the cluster, it acquires the lock in PW mode and it stays so
- during the lifetime the node is part of the cluster. The lock resource
- number is based on the slot number returned by the DLM subsystem. Since
- DLM starts node count from one and bitmap slots start from zero, one is
- subtracted from the DLM slot number to arrive at the bitmap slot number.
+ The bm_lockres protects individual node bitmaps. They are named in
+ the form bitmap000 for node 1, bitmap001 for node 2 and so on. When a
+ node joins the cluster, it acquires the lock in PW mode and it stays
+ so during the lifetime the node is part of the cluster. The lock
+ resource number is based on the slot number returned by the DLM
+ subsystem. Since DLM starts node count from one and bitmap slots
+ start from zero, one is subtracted from the DLM slot number to arrive
+ at the bitmap slot number.
+
+ The LVB of the bitmap lock for a particular node records the range
+ of sectors that are being re-synced by that node.  No other
+ node may write to those sectors.  This is used when a new nodes
+ joins the cluster.
+
+2.2 Message passing locks
+
+ Each node has to communicate with other nodes when starting or ending
+ resync, and for metadata superblock updates.  This communication is
+ managed through three locks: "token", "message", and "ack", together
+ with the Lock Value Block (LVB) of one of the "message" lock.
+
+2.3 new-device management
+
+ A single lock: "no-new-dev" is used to co-ordinate the addition of
+ new devices - this must be synchronized across the array.
+ Normally all nodes hold a concurrent-read lock on this device.
 
 3. Communication
 
-Each node has to communicate with other nodes when starting or ending
-resync, and metadata superblock updates.
+ Messages can be broadcast to all nodes, and the sender waits for all
+ other nodes to acknowledge the message before proceeding.  Only one
+ message can be processed at a time.
 
 3.1 Message Types
 
- There are 3 types, of messages which are passed
+ There are six types of messages which are passed:
 
- 3.1.1 METADATA_UPDATED: informs other nodes that the metadata has been
-   updated, and the node must re-read the md superblock. This is performed
-   synchronously.
+ 3.1.1 METADATA_UPDATED: informs other nodes that the metadata has
+   been updated, and the node must re-read the md superblock. This is
+   performed synchronously. It is primarily used to signal device
+   failure.
 
- 3.1.2 RESYNC: informs other nodes that a resync is initiated or ended
-   so that each node may suspend or resume the region.
+ 3.1.2 RESYNCING: informs other nodes that a resync is initiated or
+   ended so that each node may suspend or resume the region.  Each
+   RESYNCING message identifies a range of the devices that the
+   sending node is about to resync. This over-rides any pervious
+   notification from that node: only one ranged can be resynced at a
+   time per-node.
+
+ 3.1.3 NEWDISK: informs other nodes that a device is being added to
+   the array. Message contains an identifier for that device.  See
+   below for further details.
+
+ 3.1.4 REMOVE: A failed or spare device is being removed from the
+   array. The slot-number of the device is included in the message.
+
+ 3.1.5 RE_ADD: A failed device is being re-activated - the assumption
+   is that it has been determined to be working again.
+
+ 3.1.6 BITMAP_NEEDS_SYNC: if a node is stopped locally but the bitmap
+   isn't clean, then another node is informed to take the ownership of
+   resync.
 
 3.2 Communication mechanism
 
  The DLM LVB is used to communicate within nodes of the cluster. There
  are three resources used for the purpose:
 
-  3.2.1 Token: The resource which protects the entire communication
+  3.2.1 token: The resource which protects the entire communication
    system. The node having the token resource is allowed to
    communicate.
 
-  3.2.2 Message: The lock resource which carries the data to
+  3.2.2 message: The lock resource which carries the data to
    communicate.
 
-  3.2.3 Ack: The resource, acquiring which means the message has been
+  3.2.3 ack: The resource, acquiring which means the message has been
    acknowledged by all nodes in the cluster. The BAST of the resource
-   is used to inform the receive node that a node wants to communicate.
+   is used to inform the receiving node that a node wants to
+   communicate.
 
 The algorithm is:
 
- 1. receive status
+ 1. receive status - all nodes have concurrent-reader lock on "ack".
 
-   sender                         receiver                   receiver
-   ACK:CR                          ACK:CR                     ACK:CR
+   sender                         receiver                 receiver
+   "ack":CR                       "ack":CR                 "ack":CR
 
- 2. sender get EX of TOKEN
-    sender get EX of MESSAGE
+ 2. sender get EX on "token"
+    sender get EX on "message"
     sender                        receiver                 receiver
-    TOKEN:EX                       ACK:CR                   ACK:CR
-    MESSAGE:EX
-    ACK:CR
+    "token":EX                    "ack":CR                 "ack":CR
+    "message":EX
+    "ack":CR
 
-    Sender checks that it still needs to send a message. Messages received
-    or other events that happened while waiting for the TOKEN may have made
-    this message inappropriate or redundant.
+    Sender checks that it still needs to send a message. Messages
+    received or other events that happened while waiting for the
+    "token" may have made this message inappropriate or redundant.
 
- 3. sender write LVB.
-    sender down-convert MESSAGE from EX to CW
-    sender try to get EX of ACK
-    [ wait until all receiver has *processed* the MESSAGE ]
+ 3. sender writes LVB.
+    sender down-convert "message" from EX to CW
+    sender try to get EX of "ack"
+    [ wait until all receivers have *processed* the "message" ]
 
-                                     [ triggered by bast of ACK ]
-                                     receiver get CR of MESSAGE
+                                     [ triggered by bast of "ack" ]
+                                     receiver get CR on "message"
                                      receiver read LVB
                                      receiver processes the message
                                      [ wait finish ]
-                                     receiver release ACK
-
-   sender                         receiver                   receiver
-   TOKEN:EX                       MESSAGE:CR                 MESSAGE:CR
-   MESSAGE:CR
-   ACK:EX
-
- 4. triggered by grant of EX on ACK (indicating all receivers have processed
-    message)
-    sender down-convert ACK from EX to CR
-    sender release MESSAGE
-    sender release TOKEN
-                               receiver upconvert to PR of MESSAGE
-                               receiver get CR of ACK
-                               receiver release MESSAGE
+                                     receiver releases "ack"
+                                     receiver tries to get PR on "message"
+
+   sender                         receiver                  receiver
+   "token":EX                     "message":CR              "message":CR
+   "message":CW
+   "ack":EX
+
+ 4. triggered by grant of EX on "ack" (indicating all receivers
+    have processed message)
+    sender down-converts "ack" from EX to CR
+    sender releases "message"
+    sender releases "token"
+                               receiver upconvert to PR on "message"
+                               receiver get CR of "ack"
+                               receiver release "message"
 
    sender                      receiver                   receiver
-   ACK:CR                       ACK:CR                     ACK:CR
+   "ack":CR                    "ack":CR                   "ack":CR
 
 
 4. Handling Failures
 
 4.1 Node Failure
- When a node fails, the DLM informs the cluster with the slot. The node
- starts a cluster recovery thread. The cluster recovery thread:
+
+ When a node fails, the DLM informs the cluster with the slot
+ number. The node starts a cluster recovery thread. The cluster
+ recovery thread:
+
 	- acquires the bitmap<number> lock of the failed node
 	- opens the bitmap
 	- reads the bitmap of the failed node
@@ -132,45 +176,143 @@ The algorithm is:
 	- cleans the bitmap of the failed node
 	- releases bitmap<number> lock of the failed node
 	- initiates resync of the bitmap on the current node
+		md_check_recovery is invoked within recover_bitmaps,
+		then md_check_recovery -> metadata_update_start/finish,
+		it will lock the communication by lock_comm.
+		Which means when one node is resyncing it blocks all
+		other nodes from writing anywhere on the array.
 
- The resync process, is the regular md resync. However, in a clustered
+ The resync process is the regular md resync. However, in a clustered
  environment when a resync is performed, it needs to tell other nodes
  of the areas which are suspended. Before a resync starts, the node
- send out RESYNC_START with the (lo,hi) range of the area which needs
- to be suspended. Each node maintains a suspend_list, which contains
- the list  of ranges which are currently suspended. On receiving
- RESYNC_START, the node adds the range to the suspend_list. Similarly,
- when the node performing resync finishes, it send RESYNC_FINISHED
- to other nodes and other nodes remove the corresponding entry from
- the suspend_list.
+ send out RESYNCING with the (lo,hi) range of the area which needs to
+ be suspended. Each node maintains a suspend_list, which contains the
+ list of ranges which are currently suspended. On receiving RESYNCING,
+ the node adds the range to the suspend_list. Similarly, when the node
+ performing resync finishes, it sends RESYNCING with an empty range to
+ other nodes and other nodes remove the corresponding entry from the
+ suspend_list.
 
- A helper function, should_suspend() can be used to check if a particular
- I/O range should be suspended or not.
+ A helper function, ->area_resyncing() can be used to check if a
+ particular I/O range should be suspended or not.
 
 4.2 Device Failure
+
  Device failures are handled and communicated with the metadata update
- routine.
+ routine.  When a node detects a device failure it does not allow
+ any further writes to that device until the failure has been
+ acknowledged by all other nodes.
 
 5. Adding a new Device
-For adding a new device, it is necessary that all nodes "see" the new device
-to be added. For this, the following algorithm is used:
+
+ For adding a new device, it is necessary that all nodes "see" the new
+ device to be added. For this, the following algorithm is used:
 
     1. Node 1 issues mdadm --manage /dev/mdX --add /dev/sdYY which issues
-       ioctl(ADD_NEW_DISC with disc.state set to MD_DISK_CLUSTER_ADD)
-    2. Node 1 sends NEWDISK with uuid and slot number
+       ioctl(ADD_NEW_DISK with disc.state set to MD_DISK_CLUSTER_ADD)
+    2. Node 1 sends a NEWDISK message with uuid and slot number
     3. Other nodes issue kobject_uevent_env with uuid and slot number
        (Steps 4,5 could be a udev rule)
     4. In userspace, the node searches for the disk, perhaps
        using blkid -t SUB_UUID=""
-    5. Other nodes issue either of the following depending on whether the disk
-       was found:
+    5. Other nodes issue either of the following depending on whether
+       the disk was found:
        ioctl(ADD_NEW_DISK with disc.state set to MD_DISK_CANDIDATE and
-                disc.number set to slot number)
+             disc.number set to slot number)
        ioctl(CLUSTERED_DISK_NACK)
-    6. Other nodes drop lock on no-new-devs (CR) if device is found
-    7. Node 1 attempts EX lock on no-new-devs
-    8. If node 1 gets the lock, it sends METADATA_UPDATED after unmarking the disk
-       as SpareLocal
-    9. If not (get no-new-dev lock), it fails the operation and sends METADATA_UPDATED
-    10. Other nodes get the information whether a disk is added or not
-	by the following METADATA_UPDATED.
+    6. Other nodes drop lock on "no-new-devs" (CR) if device is found
+    7. Node 1 attempts EX lock on "no-new-dev"
+    8. If node 1 gets the lock, it sends METADATA_UPDATED after
+       unmarking the disk as SpareLocal
+    9. If not (get "no-new-dev" lock), it fails the operation and sends
+       METADATA_UPDATED.
+   10. Other nodes get the information whether a disk is added or not
+       by the following METADATA_UPDATED.
+
+6. Module interface.
+
+ There are 17 call-backs which the md core can make to the cluster
+ module.  Understanding these can give a good overview of the whole
+ process.
+
+6.1 join(nodes) and leave()
+
+ These are called when an array is started with a clustered bitmap,
+ and when the array is stopped.  join() ensures the cluster is
+ available and initializes the various resources.
+ Only the first 'nodes' nodes in the cluster can use the array.
+
+6.2 slot_number()
+
+ Reports the slot number advised by the cluster infrastructure.
+ Range is from 0 to nodes-1.
+
+6.3 resync_info_update()
+
+ This updates the resync range that is stored in the bitmap lock.
+ The starting point is updated as the resync progresses.  The
+ end point is always the end of the array.
+ It does *not* send a RESYNCING message.
+
+6.4 resync_start(), resync_finish()
+
+ These are called when resync/recovery/reshape starts or stops.
+ They update the resyncing range in the bitmap lock and also
+ send a RESYNCING message.  resync_start reports the whole
+ array as resyncing, resync_finish reports none of it.
+
+ resync_finish() also sends a BITMAP_NEEDS_SYNC message which
+ allows some other node to take over.
+
+6.5 metadata_update_start(), metadata_update_finish(),
+    metadata_update_cancel().
+
+ metadata_update_start is used to get exclusive access to
+ the metadata.  If a change is still needed once that access is
+ gained, metadata_update_finish() will send a METADATA_UPDATE
+ message to all other nodes, otherwise metadata_update_cancel()
+ can be used to release the lock.
+
+6.6 area_resyncing()
+
+ This combines two elements of functionality.
+
+ Firstly, it will check if any node is currently resyncing
+ anything in a given range of sectors.  If any resync is found,
+ then the caller will avoid writing or read-balancing in that
+ range.
+
+ Secondly, while node recovery is happening it reports that
+ all areas are resyncing for READ requests.  This avoids races
+ between the cluster-filesystem and the cluster-RAID handling
+ a node failure.
+
+6.7 add_new_disk_start(), add_new_disk_finish(), new_disk_ack()
+
+ These are used to manage the new-disk protocol described above.
+ When a new device is added, add_new_disk_start() is called before
+ it is bound to the array and, if that succeeds, add_new_disk_finish()
+ is called the device is fully added.
+
+ When a device is added in acknowledgement to a previous
+ request, or when the device is declared "unavailable",
+ new_disk_ack() is called.
+
+6.8 remove_disk()
+
+ This is called when a spare or failed device is removed from
+ the array.  It causes a REMOVE message to be send to other nodes.
+
+6.9 gather_bitmaps()
+
+ This sends a RE_ADD message to all other nodes and then
+ gathers bitmap information from all bitmaps.  This combined
+ bitmap is then used to recovery the re-added device.
+
+6.10 lock_all_bitmaps() and unlock_all_bitmaps()
+
+ These are called when change bitmap to none. If a node plans
+ to clear the cluster raid's bitmap, it need to make sure no other
+ nodes are using the raid which is achieved by lock all bitmap
+ locks within the cluster, and also those locks are unlocked
+ accordingly.
-- 
2.1.4

Re: [PATCH 2/3] md-cluster: introduce clear_clusterinfo_from_sb

[NeilBrown]

[-- Attachment #1: Type: text/plain, Size: 3013 bytes --]

On Fri, Nov 20 2015, Guoqing Jiang <gqjiang@suse.com> wrote:

> To change the bitmap from clustered to none, we also
> need to clean related info about cluster from sb, such
> as version, nodes and cluster_name.

I've applied the other two (though I removed the __func__ and __LINE__
From error messages - they are just noise: the source of the message can
easily be found with out them - I should remove all the others).

However I haven't applied this because there doesn't seem to be any
point.
Once your told that main md superblock that there is no bitmap, it
doesn't really matter what data is in the superblock of that
non-existent bitmap - it will never be looked at.  Will it?

Thanks,
NeilBrown

>
> Signed-off-by: Guoqing Jiang <gqjiang@suse.com>
> ---
>  drivers/md/bitmap.c | 25 +++++++++++++++++++++++++
>  drivers/md/bitmap.h |  1 +
>  drivers/md/md.c     |  1 +
>  3 files changed, 27 insertions(+)
>
> diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c
> index 4f22e91..c11390f 100644
> --- a/drivers/md/bitmap.c
> +++ b/drivers/md/bitmap.c
> @@ -417,6 +417,31 @@ out:
>  /*
>   * bitmap file superblock operations
>   */
> +void clear_clusterinfo_from_sb(struct bitmap *bitmap)
> +{
> +	bitmap_super_t *sb;
> +
> +	if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
> +		return;
> +	if (bitmap->mddev->bitmap_info.external)
> +		return;
> +	if (!bitmap->storage.sb_page) /* no superblock */
> +		return;
> +	sb = kmap_atomic(bitmap->storage.sb_page);
> +	sb->nodes = 0;
> +	sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
> +	memset(sb->cluster_name, NULL, 64);
> +
> +	kunmap_atomic(sb);
> +	/* Don't write until any other writes have completed */
> +	if (bitmap->storage.file)
> +		wait_event(bitmap->write_wait,
> +			   atomic_read(&bitmap->pending_writes)==0);
> +	else
> +		md_super_wait(bitmap->mddev);
> +
> +	write_page(bitmap, bitmap->storage.sb_page, 1);
> +}
>  
>  /* update the event counter and sync the superblock to disk */
>  void bitmap_update_sb(struct bitmap *bitmap)
> diff --git a/drivers/md/bitmap.h b/drivers/md/bitmap.h
> index 7d5c3a6..1d0975b 100644
> --- a/drivers/md/bitmap.h
> +++ b/drivers/md/bitmap.h
> @@ -241,6 +241,7 @@ void bitmap_flush(struct mddev *mddev);
>  void bitmap_destroy(struct mddev *mddev);
>  
>  void bitmap_print_sb(struct bitmap *bitmap);
> +void clear_clusterinfo_from_sb(struct bitmap *bitmap);
>  void bitmap_update_sb(struct bitmap *bitmap);
>  void bitmap_status(struct seq_file *seq, struct bitmap *bitmap);
>  
> diff --git a/drivers/md/md.c b/drivers/md/md.c
> index 47ee8ae..f06dc43 100644
> --- a/drivers/md/md.c
> +++ b/drivers/md/md.c
> @@ -6598,6 +6598,7 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info)
>  				}
>  
>  				mddev->bitmap_info.nodes = 0;
> +				clear_clusterinfo_from_sb(mddev->bitmap);
>  				md_cluster_ops->leave(mddev);
>  			}
>  			mddev->pers->quiesce(mddev, 1);
> -- 
> 2.1.4

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 818 bytes --]

Re: [PATCH 2/3] md-cluster: introduce clear_clusterinfo_from_sb

[Guoqing Jiang]

Hi Neil,

On 11/27/2015 01:42 PM, NeilBrown wrote:
> On Fri, Nov 20 2015, Guoqing Jiang <gqjiang@suse.com> wrote:
>
>> To change the bitmap from clustered to none, we also
>> need to clean related info about cluster from sb, such
>> as version, nodes and cluster_name.
> I've applied the other two (though I removed the __func__ and __LINE__
>  From error messages - they are just noise: the source of the message can
> easily be found with out them - I should remove all the others).
>
> However I haven't applied this because there doesn't seem to be any
> point.
> Once your told that main md superblock that there is no bitmap, it
> doesn't really matter what data is in the superblock of that
> non-existent bitmap - it will never be looked at.  Will it?
mdadm could still get nodes and cluster_name after change
bitmap to none, so some cmds like "mdadm -X /dev/sda" could
still see clustered info, it seems a little confused for user.

Thanks,
Guoqing

Re: [PATCH 2/3] md-cluster: introduce clear_clusterinfo_from_sb

[NeilBrown]

[-- Attachment #1: Type: text/plain, Size: 1235 bytes --]

On Fri, Nov 27 2015, Guoqing Jiang wrote:

> Hi Neil,
>
> On 11/27/2015 01:42 PM, NeilBrown wrote:
>> On Fri, Nov 20 2015, Guoqing Jiang <gqjiang@suse.com> wrote:
>>
>>> To change the bitmap from clustered to none, we also
>>> need to clean related info about cluster from sb, such
>>> as version, nodes and cluster_name.
>> I've applied the other two (though I removed the __func__ and __LINE__
>>  From error messages - they are just noise: the source of the message can
>> easily be found with out them - I should remove all the others).
>>
>> However I haven't applied this because there doesn't seem to be any
>> point.
>> Once your told that main md superblock that there is no bitmap, it
>> doesn't really matter what data is in the superblock of that
>> non-existent bitmap - it will never be looked at.  Will it?
> mdadm could still get nodes and cluster_name after change
> bitmap to none, so some cmds like "mdadm -X /dev/sda" could
> still see clustered info, it seems a little confused for user.

In that case we should fix mdadm to not report a bitmap if
MD_FEATURE_BITMAP_OFFSET isn't set (though maybe still report it if
--force is given).

i.e. if this is an issue, it is not specific to clustered-bitmaps.

NeilBrown

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 818 bytes --]

Re: [PATCH 2/3] md-cluster: introduce clear_clusterinfo_from_sb

[Guoqing Jiang]

NeilBrown wrote:
> On Fri, Nov 27 2015, Guoqing Jiang wrote:
>
>   
>> Hi Neil,
>>
>> On 11/27/2015 01:42 PM, NeilBrown wrote:
>>     
>>> On Fri, Nov 20 2015, Guoqing Jiang <gqjiang@suse.com> wrote:
>>>
>>>       
>>>> To change the bitmap from clustered to none, we also
>>>> need to clean related info about cluster from sb, such
>>>> as version, nodes and cluster_name.
>>>>         
>>> I've applied the other two (though I removed the __func__ and __LINE__
>>>  From error messages - they are just noise: the source of the message can
>>> easily be found with out them - I should remove all the others).
>>>
>>> However I haven't applied this because there doesn't seem to be any
>>> point.
>>> Once your told that main md superblock that there is no bitmap, it
>>> doesn't really matter what data is in the superblock of that
>>> non-existent bitmap - it will never be looked at.  Will it?
>>>       
>> mdadm could still get nodes and cluster_name after change
>> bitmap to none, so some cmds like "mdadm -X /dev/sda" could
>> still see clustered info, it seems a little confused for user.
>>     
>
> In that case we should fix mdadm to not report a bitmap if
> MD_FEATURE_BITMAP_OFFSET isn't set (though maybe still report it if
> --force is given).
>
> i.e. if this is an issue, it is not specific to clustered-bitmaps.
>
>   
Thanks for the comment, I will modify mdadm for the case.

Regards,
Guoqing