[RFC][PATCH] md: force full sync when adding the wrong device to a mirror

["Nate Dailey" <nate.dailey@stratus.com>]

I've got a patch (against kernel 2.6.36) to address the following scenario: the "wrong" disk with the correct UUID is added to a degraded mirror, and a fast resync is done instead of a full resync.

I posted about a similar situation a few months ago (http://www.spinics.net/lists/raid/msg29324.html). In that case I was concerned with two disks in a mirror, each disk having been assembled on its own without the other. It was suggested that I look at each superblock, and see if each device thinks that the other is failed/removed. This did indeed work.

Now, I've got something a bit different:

- one disk from a raid1 (with internal bitmap) is set aside as a backup
- remaining disk is re-mirrored with a new partner
- at some later time, the system is booted from the backup disk
- one of the more-recent disks is then paired with the backup disk
- we get an incomplete resync, using the bitmap on the backup disk

In this case, neither disk thinks that the other is failed.

Another possible scenario is cloning a mirror to create a boot disk for a different system. Now we have two different mirrors in two different systems, each with the same MD UUID. Moving a disk from one system to the other (to replace a failed disk, for example) leads to an incorrect bitmap resync.

To deal with this, I've added a resync signature to the superblock. A new signature is generated when resync begins. If a disk with the wrong signature is added to an array, a full sync is performed.

Comments?

Thanks,

Nate Dailey
Stratus Technologies

Signed-off-by: Nate Dailey <Nate.Dailey@stratus.com>

diff -uprN -X linux-2.6.36-vanilla/Documentation/dontdiff linux-2.6.36-vanilla/drivers/md/md.c linux-2.6.36/drivers/md/md.c
--- linux-2.6.36-vanilla/drivers/md/md.c	2010-11-15 10:47:58.000000000 -0500
+++ linux-2.6.36/drivers/md/md.c	2010-11-15 12:47:41.000000000 -0500
@@ -653,6 +653,23 @@ static inline sector_t calc_dev_sboffset
 	return MD_NEW_SIZE_SECTORS(num_sectors);
 }
 
+#define MD_ZERO_SIGNATURE "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
+
+static int md_has_zero_signature(mddev_t *mddev)
+{
+	return !memcmp(mddev->signature, MD_ZERO_SIGNATURE, MD_SIGNATURE_LEN);
+}
+
+static void md_new_signature(mddev_t *mddev)
+{
+	do {
+		get_random_bytes(mddev->signature, MD_SIGNATURE_LEN);
+	} while (md_has_zero_signature(mddev));
+
+	/* Make sure the new signature is written to all disks. */
+	set_bit(MD_CHANGE_CLEAN, &mddev->flags);
+}
+
 static int alloc_disk_sb(mdk_rdev_t * rdev)
 {
 	if (rdev->sb_page)
@@ -1125,6 +1142,8 @@ static int super_90_validate(mddev_t *md
 			mddev->bitmap_info.offset =
 				mddev->bitmap_info.default_offset;
 
+		memcpy(mddev->signature, sb->signature, MD_SIGNATURE_LEN);
+
 	} else if (mddev->pers == NULL) {
 		/* Insist on good event counter while assembling, except
 		 * for spares (which don't need an event count) */
@@ -1145,6 +1164,14 @@ static int super_90_validate(mddev_t *md
 			return 0;
 	}
 
+	/* Full sync for mismatched signatures. */
+	if (memcmp(mddev->signature, sb->signature, MD_SIGNATURE_LEN)) {
+		char b[BDEVNAME_SIZE];
+		printk(KERN_WARNING "md: %s mismatched signature on %s\n",
+		       mdname(mddev), bdevname(rdev->bdev, b));
+		return 0;
+	}
+
 	if (mddev->level != LEVEL_MULTIPATH) {
 		desc = sb->disks + rdev->desc_nr;
 
@@ -1310,6 +1337,9 @@ static void super_90_sync(mddev_t *mddev
 	sb->spare_disks = spare;
 
 	sb->this_disk = sb->disks[rdev->desc_nr];
+
+	memcpy(sb->signature, mddev->signature, MD_SB_SIGNATURE_LEN);
+
 	sb->sb_csum = calc_sb_csum(sb);
 }
 
@@ -1527,6 +1557,8 @@ static int super_1_validate(mddev_t *mdd
 			mddev->new_chunk_sectors = mddev->chunk_sectors;
 		}
 
+		memcpy(mddev->signature, sb->signature, MD_SIGNATURE_LEN);
+
 	} else if (mddev->pers == NULL) {
 		/* Insist of good event counter while assembling, except for
 		 * spares (which don't need an event count) */
@@ -1547,6 +1579,15 @@ static int super_1_validate(mddev_t *mdd
 			/* just a hot-add of a new device, leave raid_disk at -1 */
 			return 0;
 	}
+
+	/* Full sync for mismatched signatures. */
+	if (memcmp(mddev->signature, sb->signature, MD_SIGNATURE_LEN)) {
+		char b[BDEVNAME_SIZE];
+		printk(KERN_WARNING "md: %s mismatched signature on %s\n",
+		       mdname(mddev), bdevname(rdev->bdev, b));
+		return 0;
+	}
+
 	if (mddev->level != LEVEL_MULTIPATH) {
 		int role;
 		if (rdev->desc_nr < 0 ||
@@ -1661,6 +1702,8 @@ static void super_1_sync(mddev_t *mddev,
 			sb->dev_roles[i] = cpu_to_le16(0xffff);
 	}
 
+	memcpy(sb->signature, mddev->signature, MD_SB_SIGNATURE_LEN);
+
 	sb->sb_csum = calc_sb_1_csum(sb);
 }
 
@@ -4403,6 +4446,12 @@ int md_run(mddev_t *mddev)
 		analyze_sbs(mddev);
 	}
 
+	/* Generate a new signature for a zero-signature array, which means
+	   the array was last assembled on a non-signature-aware kernel. */
+	if (md_has_zero_signature(mddev)) {
+		md_new_signature(mddev);
+	}
+
 	if (mddev->level != LEVEL_NONE)
 		request_module("md-level-%d", mddev->level);
 	else if (mddev->clevel[0])
@@ -6804,6 +6853,12 @@ void md_do_sync(mddev_t *mddev)
 	}
 	mddev->curr_resync_completed = mddev->curr_resync;
 
+	/* Generate a new signature at the start of resync; after this point
+	   we don't want to allow a different disk to be added to the array. */
+	if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
+		md_new_signature(mddev);
+	}
+
 	while (j < max_sectors) {
 		sector_t sectors;
 
diff -uprN -X linux-2.6.36-vanilla/Documentation/dontdiff linux-2.6.36-vanilla/drivers/md/md.h linux-2.6.36/drivers/md/md.h
--- linux-2.6.36-vanilla/drivers/md/md.h	2010-11-15 10:47:58.000000000 -0500
+++ linux-2.6.36/drivers/md/md.h	2010-11-15 12:01:08.000000000 -0500
@@ -350,6 +350,9 @@ struct mddev_s
 	atomic_t flush_pending;
 	struct work_struct barrier_work;
 	struct work_struct event_work;	/* used by dm to report failure event */
+
+#define MD_SIGNATURE_LEN 16
+	char				signature[MD_SIGNATURE_LEN];
 };
 
 
diff -uprN -X linux-2.6.36-vanilla/Documentation/dontdiff linux-2.6.36-vanilla/include/linux/raid/md_p.h linux-2.6.36/include/linux/raid/md_p.h
--- linux-2.6.36-vanilla/include/linux/raid/md_p.h	2010-11-15 10:47:58.000000000 -0500
+++ linux-2.6.36/include/linux/raid/md_p.h	2010-11-15 12:29:08.000000000 -0500
@@ -61,6 +61,7 @@
 #define MD_SB_DESCRIPTOR_OFFSET		992
 
 #define MD_SB_GENERIC_CONSTANT_WORDS	32
+#define MD_SB_SIGNATURE_LEN		16
 #define MD_SB_GENERIC_STATE_WORDS	32
 #define MD_SB_GENERIC_WORDS		(MD_SB_GENERIC_CONSTANT_WORDS + MD_SB_GENERIC_STATE_WORDS)
 #define MD_SB_PERSONALITY_WORDS		64
@@ -163,7 +164,8 @@ typedef struct mdp_superblock_s {
 	__u32 delta_disks;	/* 15 change in number of raid_disks	      */
 	__u32 new_layout;	/* 16 new layout			      */
 	__u32 new_chunk;	/* 17 new chunk size (bytes)		      */
-	__u32 gstate_sreserved[MD_SB_GENERIC_STATE_WORDS - 18];
+	__u8 signature[MD_SB_SIGNATURE_LEN];	/* 18-21 sync signature	      */
+	__u32 gstate_sreserved[MD_SB_GENERIC_STATE_WORDS - 22];
 
 	/*
 	 * Personality information
@@ -253,7 +255,8 @@ struct mdp_superblock_1 {
 	__le64	resync_offset;	/* data before this offset (from data_offset) known to be in sync */
 	__le32	sb_csum;	/* checksum upto devs[max_dev] */
 	__le32	max_dev;	/* size of devs[] array to consider */
-	__u8	pad3[64-32];	/* set to 0 when writing */
+	__u8	signature[MD_SB_SIGNATURE_LEN];	/* sync signature */
+	__u8	pad3[64-48];	/* set to 0 when writing */
 
 	/* device state information. Indexed by dev_number.
 	 * 2 bytes per device