Linux RAID subsystem development
 help / color / mirror / Atom feed

* [PATCH 4/7] Assemble array with writecache
From: Song Liu @ 2015-05-14  6:43 UTC (permalink / raw)
  To: linux-raid; +Cc: shli, dan.j.williams, neilb, hch, Song Liu
In-Reply-To: <1431585836-4103033-1-git-send-email-songliubraving@fb.com>

Example output:

./mdadm --assemble /dev/md0 /dev/sd[c-f] /dev/sdb1
mdadm: /dev/md0 has been started with 4 drives and 1 cache.

mdadm checks superblock for cache devices. If the
array appears to have a cache device, but it is not given,
it will complain as

./mdadm --assemble /dev/md0 /dev/sd[c-f]
mdadm: Not safe to assemble with cache device missing, consider --force.

This can be overwritten with --force

./mdadm --assemble /dev/md0 /dev/sd[c-f] --force
mdadm: Force start with missing cache device...
mdadm: /dev/md0 has been started with 4 drives.

Signed-off-by: Song Liu <songliubraving@fb.com>
---
 Assemble.c | 52 ++++++++++++++++++++++++++++++++++++++++++----------
 mdadm.h    |  2 ++
 super1.c   | 22 ++++++++++++++++++++++
 util.c     |  3 ++-
 4 files changed, 68 insertions(+), 11 deletions(-)

diff --git a/Assemble.c b/Assemble.c
index 25a103d..ccf142e 100644
--- a/Assemble.c
+++ b/Assemble.c
@@ -718,7 +718,8 @@ static int load_devices(struct devs *devices, char *devmap,
 			i = devcnt;
 		else
 			i = devices[devcnt].i.disk.raid_disk;
-		if (i+1 == 0) {
+
+		if (i+1 == 0 || i == 0xfffd) {
 			if (nextspare < content->array.raid_disks*2)
 				nextspare = content->array.raid_disks*2;
 			i = nextspare++;
@@ -772,8 +773,9 @@ static int load_devices(struct devs *devices, char *devmap,
 			}
 			if (best[i] == -1
 			    || (devices[best[i]].i.events
-				< devices[devcnt].i.events))
+				< devices[devcnt].i.events)) {
 				best[i] = devcnt;
+			}
 		}
 		devcnt++;
 	}
@@ -927,6 +929,7 @@ static int start_array(int mdfd,
 		       unsigned int okcnt,
 		       unsigned int sparecnt,
 		       unsigned int rebuilding_cnt,
+		       unsigned int cachecnt,
 		       struct context *c,
 		       int clean, char *avail,
 		       int start_partial_ok,
@@ -938,6 +941,22 @@ static int start_array(int mdfd,
 	int i;
 	unsigned int req_cnt;
 
+	if (st->ss->require_cache) {
+		rv = st->ss->require_cache(st);
+		if (rv == 2) {
+			pr_err("BUG: Superblock not loaded in Assemble.c:start_array\n");
+			return 1;
+		}
+
+		if (cachecnt == 0 && rv == 1) {
+			if (!(c->force)) {
+				pr_err("Not safe to assemble with cache device missing, consider --force.\n");
+				return 1;
+			} else
+				pr_err("Force start with missing cache device...\n");
+		}
+	}
+
 	rv = set_array_info(mdfd, st, content);
 	if (rv && !err_ok) {
 		pr_err("failed to set array info for %s: %s\n",
@@ -1015,7 +1034,8 @@ static int start_array(int mdfd,
 	if (content->array.level == LEVEL_CONTAINER) {
 		if (c->verbose >= 0) {
 			pr_err("Container %s has been assembled with %d drive%s",
-			       mddev, okcnt+sparecnt, okcnt+sparecnt==1?"":"s");
+			       mddev, okcnt+sparecnt+cachecnt,
+			       okcnt+sparecnt+cachecnt==1?"":"s");
 			if (okcnt < (unsigned)content->array.raid_disks)
 				fprintf(stderr, " (out of %d)",
 					content->array.raid_disks);
@@ -1101,6 +1121,8 @@ static int start_array(int mdfd,
 					fprintf(stderr, "%s %d rebuilding", sparecnt?",":" and", rebuilding_cnt);
 				if (sparecnt)
 					fprintf(stderr, " and %d spare%s", sparecnt, sparecnt==1?"":"s");
+				if (cachecnt == 1)
+					fprintf(stderr, " and 1 cache");
 				fprintf(stderr, ".\n");
 			}
 			if (content->reshape_active &&
@@ -1268,7 +1290,7 @@ int Assemble(struct supertype *st, char *mddev,
 	int *best = NULL; /* indexed by raid_disk */
 	int bestcnt = 0;
 	int devcnt;
-	unsigned int okcnt, sparecnt, rebuilding_cnt, replcnt;
+	unsigned int okcnt, sparecnt, rebuilding_cnt, replcnt, cachecnt;
 	int i;
 	int was_forced = 0;
 	int most_recent = 0;
@@ -1479,6 +1501,7 @@ try_again:
 	devcnt = load_devices(devices, devmap, ident, &st, devlist,
 			      c, content, mdfd, mddev,
 			      &most_recent, &bestcnt, &best, inargv);
+
 	if (devcnt < 0)
 		return 1;
 
@@ -1506,7 +1529,9 @@ try_again:
 	okcnt = 0;
 	replcnt = 0;
 	sparecnt=0;
+	cachecnt=0;
 	rebuilding_cnt=0;
+
 	for (i=0; i< bestcnt; i++) {
 		int j = best[i];
 		int event_margin = 1; /* always allow a difference of '1'
@@ -1516,8 +1541,10 @@ try_again:
 		/* note: we ignore error flags in multipath arrays
 		 * as they don't make sense
 		 */
-		if (content->array.level != LEVEL_MULTIPATH)
-			if (!(devices[j].i.disk.state & (1<<MD_DISK_ACTIVE))) {
+		if (content->array.level != LEVEL_MULTIPATH) {
+			if (devices[j].i.disk.state & (1<<MD_DISK_WRITECACHE)) {
+				cachecnt++;
+			} else if (!(devices[j].i.disk.state & (1<<MD_DISK_ACTIVE))) {
 				if (!(devices[j].i.disk.state
 				      & (1<<MD_DISK_FAULTY))) {
 					devices[j].uptodate = 1;
@@ -1525,6 +1552,7 @@ try_again:
 				}
 				continue;
 			}
+		}
 		/* If this device thinks that 'most_recent' has failed, then
 		 * we must reject this device.
 		 */
@@ -1559,10 +1587,11 @@ try_again:
 						replcnt++;
 				} else
 					rebuilding_cnt++;
-			} else
+			} else if (devices[j].i.disk.raid_disk != 0xfffd)
 				sparecnt++;
 		}
 	}
+
 	free(devmap);
 	if (c->force) {
 		int force_ok = force_array(content, devices, best, bestcnt,
@@ -1583,8 +1612,9 @@ try_again:
 		int j = best[i];
 		int fd;
 
-		if (j<0)
+		if (j<0) {
 			continue;
+		}
 		if (!devices[j].uptodate)
 			continue;
 		if (devices[j].i.events < devices[most_recent].i.events)
@@ -1623,7 +1653,9 @@ try_again:
 		int j = best[i];
 		unsigned int desired_state;
 
-		if (i >= content->array.raid_disks * 2)
+		if (devices[j].i.disk.raid_disk == 0xfffd)
+			desired_state = (1<<MD_DISK_WRITECACHE);
+		else if (i >= content->array.raid_disks * 2)
 			desired_state = 0;
 		else if (i & 1)
 			desired_state = (1<<MD_DISK_ACTIVE) | (1<<MD_DISK_REPLACEMENT);
@@ -1770,7 +1802,7 @@ try_again:
 	rv = start_array(mdfd, mddev, content,
 			 st, ident, best, bestcnt,
 			 chosen_drive, devices, okcnt, sparecnt,
-			 rebuilding_cnt,
+			 rebuilding_cnt, cachecnt,
 			 c,
 			 clean, avail, start_partial_ok,
 			 pre_exist != NULL,
diff --git a/mdadm.h b/mdadm.h
index d7a205c..62a0293 100644
--- a/mdadm.h
+++ b/mdadm.h
@@ -971,6 +971,8 @@ extern struct superswitch {
 
 	/* write super block of raid5-cache*/
 	int (*write_r5l_super)(struct supertype *st, int fd);
+	/* whether the array require a cache device */
+	int (*require_cache)(struct supertype *st);
 	int swapuuid; /* true if uuid is bigending rather than hostendian */
 	int external;
 	const char *name; /* canonical metadata name */
diff --git a/super1.c b/super1.c
index f2697a8..c345a40 100644
--- a/super1.c
+++ b/super1.c
@@ -135,6 +135,27 @@ struct misc_dev_info {
 					)
 static int write_r5l_super1(struct supertype *st, int fd);
 
+/* return value:
+ *    0, cache not required
+ *    1, cache required
+ *    2, no superblock loated (st->sb == NULL)
+ */
+static int require_cache1(struct supertype *st)
+{
+	struct mdp_superblock_1 *sb = st->sb;
+	int i;
+	if (sb)
+		for (i=0; i<MAX_DEVS; i++) {
+			if (0xFFFD == sb->dev_roles[i])
+				return 1;
+		}
+	else
+		return 2;  /* no sb loaded */
+	return 0;
+}
+
+static int write_r5l_super1(struct supertype *st, int fd);
+
 static int role_from_sb(struct mdp_superblock_1 *sb)
 {
 	unsigned int d;
@@ -2556,6 +2577,7 @@ struct superswitch super1 = {
 	.write_bitmap = write_bitmap1,
 	.free_super = free_super1,
 	.write_r5l_super = write_r5l_super1,
+	.require_cache = require_cache1,
 #if __BYTE_ORDER == BIG_ENDIAN
 	.swapuuid = 0,
 #else
diff --git a/util.c b/util.c
index cc98d3b..f314748 100644
--- a/util.c
+++ b/util.c
@@ -334,8 +334,9 @@ int enough(int level, int raid_disks, int layout, int clean, char *avail)
 	int i;
 	int avail_disks = 0;
 
-	for (i = 0; i < raid_disks; i++)
+	for (i = 0; i < raid_disks; i++) {
 		avail_disks += !!avail[i];
+	}
 
 	switch (level) {
 	case 10:
-- 
1.8.1


^ permalink raw reply related

* [PATCH 3/7] Create write-cache superblock in mdadm --create
From: Song Liu @ 2015-05-14  6:43 UTC (permalink / raw)
  To: linux-raid; +Cc: shli, dan.j.williams, neilb, hch, Song Liu
In-Reply-To: <1431585836-4103033-1-git-send-email-songliubraving@fb.com>


Signed-off-by: Song Liu <songliubraving@fb.com>
---
 md_p.h   |  72 +++++++++++++++++++++++++++++++
 mdadm.h  |   7 ++-
 super1.c | 148 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---
 3 files changed, 221 insertions(+), 6 deletions(-)

diff --git a/md_p.h b/md_p.h
index f45c4e0..8a30f7b 100644
--- a/md_p.h
+++ b/md_p.h
@@ -197,4 +197,76 @@ static inline __u64 md_event(mdp_super_t *sb) {
 	return (ev<<32)| sb->events_lo;
 }
 
+/* all disk position of below struct start from rdev->start_offset */
+struct r5l_meta_header {
+	__u32 magic;
+	__u32 type;
+	__u32 checksum; /* checksum(metadata block + uuid) */
+	__u32 meta_size;
+	__u64 seq;
+	__u64 position; /* block number the meta is written */
+} __attribute__ ((__packed__));
+
+#define R5LOG_VERSION 0x1
+#define R5LOG_MAGIC 0x6433c509
+
+enum {
+	R5LOG_TYPE_META = 0,
+	R5LOG_TYPE_SUPER = 1,
+	R5LOG_TYPE_FLUSH_START = 2,
+	R5LOG_TYPE_FLUSH_END = 3,
+};
+
+struct r5l_super_block {
+	struct r5l_meta_header header;
+	__u32 version;
+	__u32 stripe_cache_size; /* bytes */
+	__u32 block_size; /* bytes */
+	__u32 stripe_data_size; /* bytes */
+	__u32 chunk_size; /* bytes */
+	__u32 stripe_size; /* bytes */
+	__u32 parity_disks;
+	__u32 zero_padding;
+	__u64 total_blocks; /* block */
+	__u64 first_block; /* block */
+	__u64 last_checkpoint; /* block */
+	__u64 update_time_sec;
+	__u64 update_time_nsec;
+	__u8 meta_checksum_type;
+	__u8 data_checksum_type;
+	__u8 uuid[16];
+	/* fill with 0 */
+} __attribute__ ((__packed__));
+
+enum {
+	R5LOG_CHECKSUM_CRC32 = 0,
+	R5LOG_CHECKSUM_NR = 1,
+};
+
+struct r5l_meta_payload {
+	__u16 payload_type;
+	__u16 payload_flags;
+	__u32 blocks; /* block. For parity, should be 1 or 2 pages */
+	__u64 location; /* sector. For data, it's raid sector.
+			    For parity, it's stripe sector */
+	__u32 data_checksum[]; /* checksum(data + uuid) */
+} __attribute__ ((__packed__));
+
+enum {
+	/* type */
+	R5LOG_PAYLOAD_DATA = 0,
+	R5LOG_PAYLOAD_PARITY = 1,
+	/* flags */
+	R5LOG_PAYLOAD_DISCARD = 1,
+};
+
+struct r5l_meta_block {
+	struct r5l_meta_header header;
+	struct r5l_meta_payload payloads[];
+} __attribute__ ((__packed__));
+
+struct r5l_flush_block {
+	struct r5l_meta_header header;
+	__u64 flush_stripes[]; /* stripe sector */
+} __attribute__ ((__packed__));
 #endif
diff --git a/mdadm.h b/mdadm.h
index d28caa0..d7a205c 100644
--- a/mdadm.h
+++ b/mdadm.h
@@ -969,6 +969,8 @@ extern struct superswitch {
 	/* validate container after assemble */
 	int (*validate_container)(struct mdinfo *info);
 
+	/* write super block of raid5-cache*/
+	int (*write_r5l_super)(struct supertype *st, int fd);
 	int swapuuid; /* true if uuid is bigending rather than hostendian */
 	int external;
 	const char *name; /* canonical metadata name */
@@ -1033,7 +1035,7 @@ struct supertype {
 	int retry_soon;
 
 	struct mdinfo *devs;
-
+	struct r5l_super_block *r5l_sb;
 };
 
 extern struct supertype *super_by_fd(int fd, char **subarray);
@@ -1585,3 +1587,6 @@ char *xstrdup(const char *str);
 #define INVALID_SECTORS 1
 /* And another special number needed for --data_offset=variable */
 #define VARIABLE_OFFSET 3
+
+#define LOG_BLOCK_SIZE 4096
+extern int ExamineR5LSuper(struct r5l_super_block *sb_blk);
diff --git a/super1.c b/super1.c
index f8a55c6..f2697a8 100644
--- a/super1.c
+++ b/super1.c
@@ -133,6 +133,20 @@ struct misc_dev_info {
 					|MD_FEATURE_RESHAPE_BACKWARDS	\
 					|MD_FEATURE_NEW_OFFSET		\
 					)
+static int write_r5l_super1(struct supertype *st, int fd);
+
+static int role_from_sb(struct mdp_superblock_1 *sb)
+{
+	unsigned int d;
+	int role;
+
+	d = __le32_to_cpu(sb->dev_number);
+	if (d < __le32_to_cpu(sb->max_dev))
+		role = __le16_to_cpu(sb->dev_roles[d]);
+	else
+		role = 0xFFFF;
+	return role;
+}
 
 static unsigned int calc_sb_1_csum(struct mdp_superblock_1 * sb)
 {
@@ -458,11 +472,7 @@ static void examine_super1(struct supertype *st, char *homehost)
 	printf(")\n");
 #endif
 	printf("   Device Role : ");
-	d = __le32_to_cpu(sb->dev_number);
-	if (d < __le32_to_cpu(sb->max_dev))
-		role = __le16_to_cpu(sb->dev_roles[d]);
-	else
-		role = 0xFFFF;
+	role = role_from_sb(sb);
 	if (role >= 0xFFFE)
 		printf("spare\n");
 	else if (role == 0xFFFD)
@@ -1559,8 +1569,11 @@ static int write_init_super1(struct supertype *st)
 	unsigned long long dsize, array_size;
 	unsigned long long sb_offset;
 	unsigned long long data_offset;
+	struct devinfo *cache_di = NULL;
 
 	for (di = st->info; di; di = di->next) {
+		if (di->disk.state & (1 << MD_DISK_WRITECACHE))
+		    cache_di = di;
 		if (di->disk.state & (1 << MD_DISK_FAULTY))
 			continue;
 		if (di->fd < 0)
@@ -1700,11 +1713,17 @@ static int write_init_super1(struct supertype *st)
 		rv = store_super1(st, di->fd);
 		if (rv == 0 && (__le32_to_cpu(sb->feature_map) & 1))
 			rv = st->ss->write_bitmap(st, di->fd);
+
+		if (cache_di == di) {
+			rv = write_r5l_super1(st, cache_di->fd);
+		}
+
 		close(di->fd);
 		di->fd = -1;
 		if (rv)
 			goto error_out;
 	}
+
 error_out:
 	if (rv)
 		pr_err("Failed to write metadata to %s\n",
@@ -1758,6 +1777,7 @@ static int load_super1(struct supertype *st, int fd, char *devname)
 	unsigned long long dsize;
 	unsigned long long sb_offset;
 	struct mdp_superblock_1 *super;
+	struct r5l_super_block *r5l_super;
 	int uuid[4];
 	struct bitmap_super_s *bsb;
 	struct misc_dev_info *misc;
@@ -1848,11 +1868,18 @@ static int load_super1(struct supertype *st, int fd, char *devname)
 		return 1;
 	}
 
+	if (posix_memalign((void**)&r5l_super, 4096, LOG_BLOCK_SIZE) != 0) {
+		pr_err("could not allocate superblock\n");
+		free(super);
+		return 1;
+	}
+
 	if (aread(&afd, super, MAX_SB_SIZE) != MAX_SB_SIZE) {
 		if (devname)
 			pr_err("Cannot read superblock on %s\n",
 				devname);
 		free(super);
+		free(r5l_super);
 		return 1;
 	}
 
@@ -1861,6 +1888,7 @@ static int load_super1(struct supertype *st, int fd, char *devname)
 			pr_err("No super block found on %s (Expected magic %08x, got %08x)\n",
 				devname, MD_SB_MAGIC, __le32_to_cpu(super->magic));
 		free(super);
+		free(r5l_super);
 		return 2;
 	}
 
@@ -1869,6 +1897,7 @@ static int load_super1(struct supertype *st, int fd, char *devname)
 			pr_err("Cannot interpret superblock on %s - version is %d\n",
 				devname, __le32_to_cpu(super->major_version));
 		free(super);
+		free(r5l_super);
 		return 2;
 	}
 	if (__le64_to_cpu(super->super_offset) != sb_offset) {
@@ -1876,9 +1905,27 @@ static int load_super1(struct supertype *st, int fd, char *devname)
 			pr_err("No superblock found on %s (super_offset is wrong)\n",
 				devname);
 		free(super);
+		free(r5l_super);
 		return 2;
 	}
 	st->sb = super;
+	if (0xFFFD == role_from_sb(super)) {
+		if (lseek64(fd, 512 * (super->data_offset), 0) < 0LL) {
+			if (devname)
+				pr_err("Cannot seek to r5l superblock on %s: %s\n",
+				       devname, strerror(errno));
+			free(r5l_super);
+		} else {
+			if (aread(&afd, r5l_super, LOG_BLOCK_SIZE) != LOG_BLOCK_SIZE) {
+				if (devname)
+					pr_err("Cannot read r5l superblock on %s: %s\n",
+					       devname, strerror(errno));
+				free(r5l_super);
+			} else
+				st->r5l_sb = r5l_super;
+		}
+	} else
+		free(r5l_super);
 
 	bsb = (struct bitmap_super_s *)(((char*)super)+MAX_SB_SIZE);
 
@@ -2223,6 +2270,8 @@ static int write_bitmap1(struct supertype *st, int fd)
 
 static void free_super1(struct supertype *st)
 {
+	if (st->r5l_sb)
+		free(st->r5l_sb);
 	if (st->sb)
 		free(st->sb);
 	while (st->info) {
@@ -2233,6 +2282,7 @@ static void free_super1(struct supertype *st)
 		free(di);
 	}
 	st->sb = NULL;
+	st->r5l_sb = NULL;
 }
 
 #ifndef MDASSEMBLE
@@ -2389,6 +2439,93 @@ void *super1_make_v0(struct supertype *st, struct mdinfo *info, mdp_super_t *sb0
 	return ret;
 }
 
+unsigned long crc32(
+	unsigned long crc,
+	const unsigned char *buf,
+	unsigned len);
+
+static int write_r5l_super1(struct supertype *st, int fd)
+{
+	struct r5l_super_block *sb_blk;
+	struct mdp_superblock_1 *sb = st->sb;
+	unsigned long long dsize;
+	int parity_disks;
+	struct align_fd afd;
+	__u32 crc;
+
+	init_afd(&afd, fd);
+	if (sb->level == 5 || sb->level == 4)
+		parity_disks = 1;
+	else if (sb->level == 6)
+		parity_disks = 2;
+	else {
+		pr_err("Write cache is only applicable to RAID 4/5/6.\n");
+		return 1;
+	}
+
+	if (!get_dev_size(fd, NULL, &dsize)) {
+		pr_err("Cannot get size of cache device.\n");
+		return 1;
+	}
+
+	if (st->r5l_sb)
+		free(st->r5l_sb);
+
+	if (posix_memalign((void**)&sb_blk, 4096, LOG_BLOCK_SIZE) != 0) {
+		pr_err("Could not allocate memory for writecache superblock.\n");
+		return 1;
+	}
+
+	memset(sb_blk, 0, LOG_BLOCK_SIZE);
+
+	sb_blk->header.magic = __cpu_to_le32(R5LOG_MAGIC);
+	sb_blk->header.type = __cpu_to_le32(R5LOG_TYPE_SUPER);
+	sb_blk->header.seq = __cpu_to_le64(random32());
+	sb_blk->header.meta_size = __cpu_to_le32(sizeof(*sb_blk));
+	sb_blk->version = __cpu_to_le32(R5LOG_VERSION);
+	sb_blk->stripe_cache_size = __cpu_to_le32(LOG_BLOCK_SIZE);
+	sb_blk->block_size = __cpu_to_le32(LOG_BLOCK_SIZE);
+	sb_blk->total_blocks = __cpu_to_le64(((dsize - (sb->data_offset << 9)) / LOG_BLOCK_SIZE) - 1);
+	sb_blk->stripe_data_size = __cpu_to_le32((sb->chunksize << 9) *
+		(sb->raid_disks - parity_disks));
+	sb_blk->chunk_size = __cpu_to_le32(sb->chunksize << 9);
+
+
+	sb_blk->stripe_size = __cpu_to_le32(sb_blk->chunk_size * sb->raid_disks);
+	sb_blk->parity_disks = __cpu_to_le32(parity_disks);
+
+	sb_blk->first_block = __cpu_to_le64(1);
+	sb_blk->last_checkpoint = __cpu_to_le64(1);
+	sb_blk->update_time_sec = __cpu_to_le64((unsigned long long)time(0));
+	sb_blk->update_time_nsec = 0;
+	sb_blk->meta_checksum_type = R5LOG_CHECKSUM_CRC32;
+	sb_blk->data_checksum_type = R5LOG_CHECKSUM_CRC32;
+	memcpy(sb_blk->uuid, sb->set_uuid, 16);
+
+	crc = __cpu_to_le32(crc32(0xffffffff, (unsigned char *)(sb_blk->uuid), sizeof(sb_blk->uuid)));
+	crc = __cpu_to_le32(crc32(crc, (unsigned char *)sb_blk, LOG_BLOCK_SIZE));
+	sb_blk->header.checksum = crc;
+
+	if (lseek64(fd, (sb->data_offset) * 512, 0) < 0LL) {
+		pr_err("cannot seek to offset of write cache superblock\n");
+		goto fail_to_write;
+	}
+	if (awrite(&afd, sb_blk, sizeof(*sb_blk)) != sizeof(*sb_blk)) {
+		pr_err("failed to store write cache superblock \n");
+		goto fail_to_write;
+	}
+	fsync(fd);
+
+	st->r5l_sb = sb_blk;
+	return 0;
+
+fail_to_write:
+	free(sb_blk);
+	st->r5l_sb = NULL;
+	return 1;
+}
+
+
 struct superswitch super1 = {
 #ifndef MDASSEMBLE
 	.examine_super = examine_super1,
@@ -2418,6 +2555,7 @@ struct superswitch super1 = {
 	.locate_bitmap = locate_bitmap1,
 	.write_bitmap = write_bitmap1,
 	.free_super = free_super1,
+	.write_r5l_super = write_r5l_super1,
 #if __BYTE_ORDER == BIG_ENDIAN
 	.swapuuid = 0,
 #else
-- 
1.8.1


^ permalink raw reply related

* [PATCH 2/7] Enable create array with write cache (--write-cache DEVICE).
From: Song Liu @ 2015-05-14  6:43 UTC (permalink / raw)
  To: linux-raid; +Cc: shli, dan.j.williams, neilb, hch, Song Liu
In-Reply-To: <1431585836-4103033-1-git-send-email-songliubraving@fb.com>

Specify the write cache device with --write-cache DEVICE

./mdadm --create -f /dev/md0 --assume-clean -c 32 --raid-devices=4 --level=5 /dev/sd[c-f] --write-cache /dev/sdb1
mdadm: Defaulting to version 1.2 metadata
mdadm: array /dev/md0 started.

Only one cache device is allowed. If multiple --write-cache
are given, mdadm will use the first and ignore others

./mdadm --create -f /dev/md0 --assume-clean -c 32 --raid-devices=4 --level=5 /dev/sd[c-f] --write-cache /dev/sdb1 --write-cache /dev/sdx
mdadm: Please specify only one cache device for the array.
mdadm: Ignoring --write-cache /dev/sdx...
mdadm: Defaulting to version 1.2 metadata
mdadm: array /dev/md0 started.

Signed-off-by: Song Liu <songliubraving@fb.com>
---
 Create.c | 21 +++++++++++++++------
 ReadMe.c |  1 +
 mdadm.c  | 19 +++++++++++++++++++
 mdadm.h  |  2 ++
 super1.c |  8 ++++++++
 5 files changed, 45 insertions(+), 6 deletions(-)

diff --git a/Create.c b/Create.c
index ef28da0..0d2a591 100644
--- a/Create.c
+++ b/Create.c
@@ -87,7 +87,7 @@ int Create(struct supertype *st, char *mddev,
 	unsigned long long minsize=0, maxsize=0;
 	char *mindisc = NULL;
 	char *maxdisc = NULL;
-	int dnum;
+	int dnum, raid_disk_num;
 	struct mddev_dev *dv;
 	int fail=0, warn=0;
 	struct stat stb;
@@ -180,11 +180,11 @@ int Create(struct supertype *st, char *mddev,
 		pr_err("This metadata type does not support spare disks at create time\n");
 		return 1;
 	}
-	if (subdevs > s->raiddisks+s->sparedisks) {
+	if (subdevs > s->raiddisks+s->sparedisks+s->cachedisks) {
 		pr_err("You have listed more devices (%d) than are in the array(%d)!\n", subdevs, s->raiddisks+s->sparedisks);
 		return 1;
 	}
-	if (!have_container && subdevs < s->raiddisks+s->sparedisks) {
+	if (!have_container && subdevs < s->raiddisks+s->sparedisks+s->cachedisks) {
 		pr_err("You haven't given enough devices (real or missing) to create this array\n");
 		return 1;
 	}
@@ -397,6 +397,9 @@ int Create(struct supertype *st, char *mddev,
 			}
 		}
 
+		if (dv->disposition == 'c')
+			continue;  /* skip write cache for size check */
+
 		freesize /= 2; /* convert to K */
 		if (s->chunk && s->chunk != UnSet) {
 			/* round to chunk size */
@@ -834,7 +837,7 @@ int Create(struct supertype *st, char *mddev,
 	for (pass=1; pass <=2 ; pass++) {
 		struct mddev_dev *moved_disk = NULL; /* the disk that was moved out of the insert point */
 
-		for (dnum=0, dv = devlist ; dv ;
+		for (dnum=0, raid_disk_num=0, dv = devlist ; dv ;
 		     dv=(dv->next)?(dv->next):moved_disk, dnum++) {
 			int fd;
 			struct stat stb;
@@ -859,8 +862,13 @@ int Create(struct supertype *st, char *mddev,
 				*inf = info;
 
 				inf->disk.number = dnum;
-				inf->disk.raid_disk = dnum;
-				if (inf->disk.raid_disk < s->raiddisks)
+				inf->disk.raid_disk = raid_disk_num++;
+
+				if (dv->disposition == 'c') {
+					inf->disk.raid_disk = 0xfffd;
+					inf->disk.state = (1<<MD_DISK_WRITECACHE);
+					raid_disk_num--;
+				} else if (inf->disk.raid_disk < s->raiddisks)
 					inf->disk.state = (1<<MD_DISK_ACTIVE) |
 						(1<<MD_DISK_SYNC);
 				else
@@ -909,6 +917,7 @@ int Create(struct supertype *st, char *mddev,
 					inf->disk.major = major(stb.st_rdev);
 					inf->disk.minor = minor(stb.st_rdev);
 				}
+
 				break;
 			case 2:
 				inf->errors = 0;
diff --git a/ReadMe.c b/ReadMe.c
index 87a4916..74f7162 100644
--- a/ReadMe.c
+++ b/ReadMe.c
@@ -140,6 +140,7 @@ struct option long_options[] = {
     {"homehost",  1, 0,  HomeHost},
     {"symlinks",  1, 0,  Symlinks},
     {"data-offset",1, 0, DataOffset},
+    {"write-cache",1, 0, WriteCache},
 
     /* For assemble */
     {"uuid",      1, 0, 'u'},
diff --git a/mdadm.c b/mdadm.c
index 3e8c49b..9a7048b 100644
--- a/mdadm.c
+++ b/mdadm.c
@@ -74,6 +74,7 @@ int main(int argc, char *argv[])
 		.require_homehost = 1,
 	};
 	struct shape s = {
+		.cachedisks	= 0,
 		.level		= UnSet,
 		.layout		= UnSet,
 		.bitmap_chunk	= UnSet,
@@ -1137,6 +1138,24 @@ int main(int argc, char *argv[])
 		case O(INCREMENTAL, IncrementalPath):
 			remove_path = optarg;
 			continue;
+		case O(CREATE, WriteCache):
+			if (s.cachedisks) {
+				pr_err("Please specify only one cache device for the array.\n");
+				pr_err("Ignoring --write-cache %s...\n", optarg);
+				continue;
+			}
+			dv = xmalloc(sizeof(*dv));
+			dv->devname = optarg;
+			dv->disposition = 'c';  /* WriteCache */
+			dv->writemostly = writemostly;
+			dv->used = 0;
+			dv->next = NULL;
+			*devlistend = dv;
+			devlistend = &dv->next;
+			devs_found++;
+
+			s.cachedisks = 1;
+			continue;
 		}
 		/* We have now processed all the valid options. Anything else is
 		 * an error
diff --git a/mdadm.h b/mdadm.h
index 141f963..d28caa0 100644
--- a/mdadm.h
+++ b/mdadm.h
@@ -344,6 +344,7 @@ enum special_options {
 	Dump,
 	Restore,
 	Action,
+	WriteCache,
 };
 
 enum prefix_standard {
@@ -423,6 +424,7 @@ struct context {
 struct shape {
 	int	raiddisks;
 	int	sparedisks;
+	int	cachedisks;
 	int	level;
 	int	layout;
 	char	*layout_str;
diff --git a/super1.c b/super1.c
index 7af0fd5..f8a55c6 100644
--- a/super1.c
+++ b/super1.c
@@ -977,6 +977,10 @@ static void getinfo_super1(struct supertype *st, struct mdinfo *info, char *map)
 	case 0xFFFE:
 		info->disk.state = 1; /* faulty */
 		break;
+	case 0xFFFD:
+		info->disk.state = (1 << MD_DISK_WRITECACHE); /* faulty */
+		info->disk.raid_disk = role;
+		break;
 	default:
 		info->disk.state = 6; /* active and in sync */
 		info->disk.raid_disk = role;
@@ -1096,6 +1100,8 @@ static int update_super1(struct supertype *st, struct mdinfo *info,
 		int want;
 		if (info->disk.state & (1<<MD_DISK_ACTIVE))
 			want = info->disk.raid_disk;
+		else if (info->disk.state & (1<<MD_DISK_WRITECACHE))
+			want = 0xFFFD;
 		else
 			want = 0xFFFF;
 		if (sb->dev_roles[d] != __cpu_to_le16(want)) {
@@ -1422,6 +1428,8 @@ static int add_to_super1(struct supertype *st, mdu_disk_info_t *dk,
 		*rp = __cpu_to_le16(dk->raid_disk);
 	else if ((dk->state & ~2) == 0) /* active or idle -> spare */
 		*rp = 0xffff;
+	else if (dk->state & (1<<MD_DISK_WRITECACHE))
+		*rp = 0xfffd;
 	else
 		*rp = 0xfffe;
 
-- 
1.8.1


^ permalink raw reply related

* [PATCH 1/7] Show device as cache in --detail
From: Song Liu @ 2015-05-14  6:43 UTC (permalink / raw)
  To: linux-raid; +Cc: shli, dan.j.williams, neilb, hch, Song Liu
In-Reply-To: <1431585836-4103033-1-git-send-email-songliubraving@fb.com>

Example output:

./mdadm --detail /dev/md127
/dev/md127:
        Version : 1.2
  Creation Time : Wed May 13 17:01:12 2015
     Raid Level : raid5
     Array Size : 11720662464 (11177.69 GiB 12001.96 GB)
  Used Dev Size : 3906887488 (3725.90 GiB 4000.65 GB)
   Raid Devices : 4
  Total Devices : 5
    Persistence : Superblock is persistent

  Intent Bitmap : Internal

    Update Time : Wed May 13 17:01:12 2015
          State : clean
 Active Devices : 4
Working Devices : 5
 Failed Devices : 0
  Spare Devices : 1

         Layout : left-symmetric
     Chunk Size : 32K

           Name : 0
           UUID : 8fb9ee05:3831d52f:e5c23825:28cd6881
         Events : 0

    Number   Major   Minor   RaidDevice State
       0       8       32        0      active sync   /dev/sdc
       1       8       48        1      active sync   /dev/sdd
       2       8       64        2      active sync   /dev/sde
       3       8       80        3      active sync   /dev/sdf

       4       8       17        -      cache   /dev/sdb1

Signed-off-by: Song Liu <songliubraving@fb.com>
---
 Detail.c | 3 ++-
 md_p.h   | 2 ++
 super1.c | 2 ++
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/Detail.c b/Detail.c
index dd72ede..c93a962 100644
--- a/Detail.c
+++ b/Detail.c
@@ -650,9 +650,10 @@ This is pretty boring
 			}
 			if (disk.state & (1<<MD_DISK_REMOVED)) printf(" removed");
 			if (disk.state & (1<<MD_DISK_WRITEMOSTLY)) printf(" writemostly");
+			if (disk.state & (1<<MD_DISK_WRITECACHE)) printf(" cache");
 			if ((disk.state &
 			     ((1<<MD_DISK_ACTIVE)|(1<<MD_DISK_SYNC)
-			      |(1<<MD_DISK_REMOVED)|(1<<MD_DISK_FAULTY)))
+			      |(1<<MD_DISK_REMOVED)|(1<<MD_DISK_FAULTY)|(1<<MD_DISK_WRITECACHE)))
 			    == 0) {
 				printf(" spare");
 				if (is_26) {
diff --git a/md_p.h b/md_p.h
index c4846ba..f45c4e0 100644
--- a/md_p.h
+++ b/md_p.h
@@ -85,6 +85,8 @@
 				   */
 
 #define MD_DISK_REPLACEMENT	17
+#define MD_DISK_WRITECACHE      18 /* disk is used as the write cache in RAID-5/6 */
+
 
 typedef struct mdp_device_descriptor_s {
 	__u32 number;		/* 0 Device number in the entire set	      */
diff --git a/super1.c b/super1.c
index f0508fe..7af0fd5 100644
--- a/super1.c
+++ b/super1.c
@@ -465,6 +465,8 @@ static void examine_super1(struct supertype *st, char *homehost)
 		role = 0xFFFF;
 	if (role >= 0xFFFE)
 		printf("spare\n");
+	else if (role == 0xFFFD)
+		printf("cache\n");
 	else if (sb->feature_map & __cpu_to_le32(MD_FEATURE_REPLACEMENT))
 		printf("Replacement device %d\n", role);
 	else
-- 
1.8.1


^ permalink raw reply related

* [PATCH 0/7] mdadm support for caching layer in raid 5/6
From: Song Liu @ 2015-05-14  6:43 UTC (permalink / raw)
  To: linux-raid; +Cc: shli, dan.j.williams, neilb, hch, Song Liu

Hi, 

These are mdadm patches to support cache layer in raid 5/6. Shaohua
has sent the kernel patch earlier with subject "a caching layer 
for raid 5/6". 

These patches add write cache support for the following commands:

  mdadm --detail
  mdadm --create
  mdadm --assemble
  mdadm --incremental
  mdadm --examine
  mdadm --zero-super

Cache device is assigned with dev_role 0xFFFD (where 0xFFFF is for 
spare and 0xFFFE is for failed). Note that there is compatibility
issue that older mdadm will show cache device as spare in --detail:

    Number   Major   Minor   RaidDevice State
       0       8       32        0      active sync   /dev/sdc
       1       8       48        1      active sync   /dev/sdd
       2       8       64        2      active sync   /dev/sde
       3       8       80        3      active sync   /dev/sdf

       4       8       17        -      spare   /dev/sdb1

Also, older mdadm will show cache device as "Active device 65533" 
in --examine: 

   Device Role : Active device 65533
   Array State : AAAA ('A' == active, '.' == missing, 'R' == replacing)


Song Liu (7):
  Show device as cache in --detail
  Enable create array with write cache (--write-cache DEVICE).
  Create write-cache superblock in mdadm --create
  Assemble array with writecache
  Check write cache in incremental
  Zero write-cache superblock in --zero-super
  Add information about write-cache superblock to --examine

 Assemble.c    |  52 +++++++++++++----
 Create.c      |  21 +++++--
 Detail.c      |   3 +-
 Examine.c     |  29 +++++++++-
 Incremental.c |  37 ++++++++++--
 Kill.c        |  19 ++++++
 ReadMe.c      |   1 +
 md_p.h        |  74 ++++++++++++++++++++++++
 mdadm.c       |  19 ++++++
 mdadm.h       |  11 +++-
 super1.c      | 182 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
 util.c        |   3 +-
 12 files changed, 422 insertions(+), 29 deletions(-)

-- 
1.8.1


^ permalink raw reply

* Re: data corruption after rebuid
From: d tbsky @ 2015-05-14  6:36 UTC (permalink / raw)
  To: Adam Goryachev; +Cc: linux-raid
In-Reply-To: <555439F4.4020200@websitemanagers.com.au>

2015-05-14 14:00 GMT+08:00 Adam Goryachev <adam@websitemanagers.com.au>:
> On 14/05/15 14:54, d tbsky wrote:
> My guess is this is where you went wrong.
> Instead use a tool that will do this intelligently, quick google shows this:
> sgdisk -R=/dev/sdb /dev/sda
> sgdisk -G /dev/sdb
> Taken from:
> http://unix.stackexchange.com/questions/12986/how-to-copy-the-partition-layout-of-a-whole-disk-using-standard-tools
>
> My guess is you copied some data as well as the partition table, and
> that you are using a mdadm label which is stored at the beginning of the
> disk, and possibly are using bitmap on one server and not the other.

    this step did look suspicious. but I had use it many times on
blank hard disks. and it seems works..

> Maybe if you could show some details from each of the raid arrays then
> more people can make more informed comments.

 sure. below is the raid structure of  hostA & hostB:
/dev/md1:
        Version : 1.1
  Creation Time : Wed Feb 19 09:34:03 2014
     Raid Level : raid1
     Array Size : 1953177408 (1862.70 GiB 2000.05 GB)
  Used Dev Size : 1953177408 (1862.70 GiB 2000.05 GB)
   Raid Devices : 2
  Total Devices : 2
    Persistence : Superblock is persistent

  Intent Bitmap : Internal


  and below is the raid structure of my first test:
/dev/md1:
        Version : 1.1
  Creation Time : Mon Oct 20 18:51:47 2014
     Raid Level : raid5
     Array Size : 3906354176 (3725.39 GiB 4000.11 GB)
  Used Dev Size : 1953177088 (1862.69 GiB 2000.05 GB)
   Raid Devices : 3
  Total Devices : 3
    Persistence : Superblock is persistent

  Intent Bitmap : Internal

    Update Time : Thu May 14 14:32:25 2015
          State : clean
 Active Devices : 3
Working Devices : 3
 Failed Devices : 0
  Spare Devices : 0

         Layout : left-symmetric
     Chunk Size : 512K



    and the raid structure of my second and thrid test:
/dev/md1:
        Version : 1.1
  Creation Time : Sun Sep 28 19:20:59 2014
     Raid Level : raid10
     Array Size : 1953033216 (1862.56 GiB 1999.91 GB)
  Used Dev Size : 976516608 (931.28 GiB 999.95 GB)
   Raid Devices : 4
  Total Devices : 4
    Persistence : Superblock is persistent

  Intent Bitmap : Internal

    Update Time : Thu May 14 14:33:14 2015
          State : active
 Active Devices : 4
Working Devices : 4
 Failed Devices : 0
  Spare Devices : 0

         Layout : near=2
     Chunk Size : 512K

^ permalink raw reply

* Re: [PATCH 3/3] md/raid5: per hash value and exclusive wait_for_stripe
From: NeilBrown @ 2015-05-14  6:09 UTC (permalink / raw)
  To: Yuanhan Liu; +Cc: linux-raid, linux-kernel
In-Reply-To: <20150514055508.GJ3124@yliu-dev.sh.intel.com>

[-- Attachment #1: Type: text/plain, Size: 1823 bytes --]

On Thu, 14 May 2015 13:55:08 +0800 Yuanhan Liu <yuanhan.liu@linux.intel.com>
wrote:

> On Thu, May 14, 2015 at 03:45:11PM +1000, NeilBrown wrote:
> > On Wed, 29 Apr 2015 10:48:55 +0800 Yuanhan Liu <yuanhan.liu@linux.intel.com>
> > wrote:
> > 
> > > diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
> > > index 64d5bea..697d77a 100644
> > > --- a/drivers/md/raid5.c
> > > +++ b/drivers/md/raid5.c
> > > @@ -344,7 +344,8 @@ static void release_inactive_stripe_list(struct r5conf *conf,
> > >  					 int hash)
> > >  {
> > >  	int size;
> > > -	bool do_wakeup = false;
> > > +	unsigned long do_wakeup = 0;
> > > +	int i = 0;
> > >  	unsigned long flags;
> > >  
> > >  	if (hash == NR_STRIPE_HASH_LOCKS) {
> > > @@ -365,15 +366,19 @@ static void release_inactive_stripe_list(struct r5conf *conf,
> > >  			    !list_empty(list))
> > >  				atomic_dec(&conf->empty_inactive_list_nr);
> > >  			list_splice_tail_init(list, conf->inactive_list + hash);
> > > -			do_wakeup = true;
> > > +			do_wakeup |= 1 << (size - 1);
> > >  			spin_unlock_irqrestore(conf->hash_locks + hash, flags);
> > >  		}
> > >  		size--;
> > >  		hash--;
> > >  	}
> > >  
> > > +	for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) {
> > > +		if (do_wakeup & (1 << i))
> > > +			wake_up(&conf->wait_for_stripe[i]);
> > > +	}
> > > +
> > 
> > hi,
> >  I've been doing some testing and got a lock-up in resize_stripes, waiting
> >  on wait_for_stripe[].
> > 
> >  Looking at the above code,  I think
> >       do_wakeup |= 1 << (size - 1);
> >  should be
> >       do_wakeup |= 1 << hash;
> > 
> >  do you agree?  Or am I missing something?
> 
> Right. Sorry for the careless mistake.

Thanks for confirming.
Mistakes happen - that's why we test :-)

I've merged that fix into your patch.

NeilBrown

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 811 bytes --]

^ permalink raw reply

* Re: data corruption after rebuid
From: Adam Goryachev @ 2015-05-14  6:00 UTC (permalink / raw)
  To: d tbsky, linux-raid
In-Reply-To: <CAC6SzHKo8kUOJoC+eaZU1mQCzW+WdB6ACRkrM30yUCA7CodLCQ@mail.gmail.com>

On 14/05/15 14:54, d tbsky wrote:
> Hi:
>      I think I did something wrong and cause mdadm data corruption.
> but I am curious which steps brings me down. hope someone can told me
> about it.
>
>        I have two hosts forms a vm HA cluster. each one is a 2TB * 2
> mdadm raid 1. I need to test something but lack of disks, so I pull
> one disk of each hosts.  after testing, I put the disk back and let
> mdadm rebuild. host B took several hours to rebuild and looks fine.
> host A took only 5 min to rebuild. after completing rebuild, the
> virtual machines stand  above the raid crashed one by one. since I
> know I have write many data  bytes during the testing, host A should
> not took only 5 min to recover. I must do something wrong to confuse
> mdadm.  below is what I done:
>
> 1. I test a 3 disk mdadm raid 5: sda (hostA sdb), sdb (hostB sdb), sdc
> (new disk). I wrote about 5G data for system restore testing.
>
> 2. then I test a 4 disk mdadm raid10: sda (hostA sdb), sdb(hostB
> sdb),sdc(new disk),sdd (new disk). I wrote about 5G data for system
> restore testing.
>
> 3. then I test again a  4 disk mdadm rai10, but wrote about 120G data
> for system restore testing.
>
> then I put back the disk to hostA and hostB (hot plugin, HOST A and B
> are still running). at hostA I issue command blow:
>    mdadm --stop /dev/md126; mdadm --stop /dev/md127 (the plugged disk
> has raid data on it and udev seems found it)
>    dd if=/dev/sda of=/dev/sdb bs=1k count=1000  (to recreate mbr
> partition table).

My guess is this is where you went wrong.
Instead use a tool that will do this intelligently, quick google shows this:
sgdisk -R=/dev/sdb /dev/sda
sgdisk -G /dev/sdb
Taken from:
http://unix.stackexchange.com/questions/12986/how-to-copy-the-partition-layout-of-a-whole-disk-using-standard-tools

My guess is you copied some data as well as the partition table, and
that you are using a mdadm label which is stored at the beginning of the
disk, and possibly are using bitmap on one server and not the other.

Maybe if you could show some details from each of the raid arrays then
more people can make more informed comments.

My suggestion is to create *just* the partitions, and then add them to
the array.

Hope that helps.

Regards,
Adam

-- 
Adam Goryachev
Website Managers
Ph: +61 2 8304 0000                            adam@websitemanagers.com.au
Fax: +61 2 8304 0001                            www.websitemanagers.com.au


^ permalink raw reply

* Re: [PATCH 3/3] md/raid5: per hash value and exclusive wait_for_stripe
From: Yuanhan Liu @ 2015-05-14  5:55 UTC (permalink / raw)
  To: NeilBrown; +Cc: linux-raid, linux-kernel
In-Reply-To: <20150514154511.26b74f5b@notabene.brown>

On Thu, May 14, 2015 at 03:45:11PM +1000, NeilBrown wrote:
> On Wed, 29 Apr 2015 10:48:55 +0800 Yuanhan Liu <yuanhan.liu@linux.intel.com>
> wrote:
> 
> > diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
> > index 64d5bea..697d77a 100644
> > --- a/drivers/md/raid5.c
> > +++ b/drivers/md/raid5.c
> > @@ -344,7 +344,8 @@ static void release_inactive_stripe_list(struct r5conf *conf,
> >  					 int hash)
> >  {
> >  	int size;
> > -	bool do_wakeup = false;
> > +	unsigned long do_wakeup = 0;
> > +	int i = 0;
> >  	unsigned long flags;
> >  
> >  	if (hash == NR_STRIPE_HASH_LOCKS) {
> > @@ -365,15 +366,19 @@ static void release_inactive_stripe_list(struct r5conf *conf,
> >  			    !list_empty(list))
> >  				atomic_dec(&conf->empty_inactive_list_nr);
> >  			list_splice_tail_init(list, conf->inactive_list + hash);
> > -			do_wakeup = true;
> > +			do_wakeup |= 1 << (size - 1);
> >  			spin_unlock_irqrestore(conf->hash_locks + hash, flags);
> >  		}
> >  		size--;
> >  		hash--;
> >  	}
> >  
> > +	for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) {
> > +		if (do_wakeup & (1 << i))
> > +			wake_up(&conf->wait_for_stripe[i]);
> > +	}
> > +
> 
> hi,
>  I've been doing some testing and got a lock-up in resize_stripes, waiting
>  on wait_for_stripe[].
> 
>  Looking at the above code,  I think
>       do_wakeup |= 1 << (size - 1);
>  should be
>       do_wakeup |= 1 << hash;
> 
>  do you agree?  Or am I missing something?

Right. Sorry for the careless mistake.

	--yliu

^ permalink raw reply

* Re: [PATCH 3/3] md/raid5: per hash value and exclusive wait_for_stripe
From: NeilBrown @ 2015-05-14  5:45 UTC (permalink / raw)
  To: Yuanhan Liu; +Cc: linux-raid, linux-kernel
In-Reply-To: <1430275735-20290-3-git-send-email-yuanhan.liu@linux.intel.com>

[-- Attachment #1: Type: text/plain, Size: 1349 bytes --]

On Wed, 29 Apr 2015 10:48:55 +0800 Yuanhan Liu <yuanhan.liu@linux.intel.com>
wrote:

> diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
> index 64d5bea..697d77a 100644
> --- a/drivers/md/raid5.c
> +++ b/drivers/md/raid5.c
> @@ -344,7 +344,8 @@ static void release_inactive_stripe_list(struct r5conf *conf,
>  					 int hash)
>  {
>  	int size;
> -	bool do_wakeup = false;
> +	unsigned long do_wakeup = 0;
> +	int i = 0;
>  	unsigned long flags;
>  
>  	if (hash == NR_STRIPE_HASH_LOCKS) {
> @@ -365,15 +366,19 @@ static void release_inactive_stripe_list(struct r5conf *conf,
>  			    !list_empty(list))
>  				atomic_dec(&conf->empty_inactive_list_nr);
>  			list_splice_tail_init(list, conf->inactive_list + hash);
> -			do_wakeup = true;
> +			do_wakeup |= 1 << (size - 1);
>  			spin_unlock_irqrestore(conf->hash_locks + hash, flags);
>  		}
>  		size--;
>  		hash--;
>  	}
>  
> +	for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) {
> +		if (do_wakeup & (1 << i))
> +			wake_up(&conf->wait_for_stripe[i]);
> +	}
> +

hi,
 I've been doing some testing and got a lock-up in resize_stripes, waiting
 on wait_for_stripe[].

 Looking at the above code,  I think
      do_wakeup |= 1 << (size - 1);
 should be
      do_wakeup |= 1 << hash;

 do you agree?  Or am I missing something?

Thanks,
NeilBrown


[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 811 bytes --]

^ permalink raw reply

* data corruption after rebuid
From: d tbsky @ 2015-05-14  4:54 UTC (permalink / raw)
  To: linux-raid

Hi:
     I think I did something wrong and cause mdadm data corruption.
but I am curious which steps brings me down. hope someone can told me
about it.

       I have two hosts forms a vm HA cluster. each one is a 2TB * 2
mdadm raid 1. I need to test something but lack of disks, so I pull
one disk of each hosts.  after testing, I put the disk back and let
mdadm rebuild. host B took several hours to rebuild and looks fine.
host A took only 5 min to rebuild. after completing rebuild, the
virtual machines stand  above the raid crashed one by one. since I
know I have write many data  bytes during the testing, host A should
not took only 5 min to recover. I must do something wrong to confuse
mdadm.  below is what I done:

1. I test a 3 disk mdadm raid 5: sda (hostA sdb), sdb (hostB sdb), sdc
(new disk). I wrote about 5G data for system restore testing.

2. then I test a 4 disk mdadm raid10: sda (hostA sdb), sdb(hostB
sdb),sdc(new disk),sdd (new disk). I wrote about 5G data for system
restore testing.

3. then I test again a  4 disk mdadm rai10, but wrote about 120G data
for system restore testing.

then I put back the disk to hostA and hostB (hot plugin, HOST A and B
are still running). at hostA I issue command blow:
   mdadm --stop /dev/md126; mdadm --stop /dev/md127 (the plugged disk
has raid data on it and udev seems found it)
   dd if=/dev/sda of=/dev/sdb bs=1k count=1000  (to recreate mbr
partition table).
   partprobe /dev/sdb
   mdadm --add /dev/md0 /dev/sdb1 (this is a small 500MB raid for /boot).
   mdadm --add /dev/md1 /dev/sdb2 (this is about 2TB raid).

   mdadm seems confused it only took 5 min to recover. I did the same
at host B and it took several hours to recover.

   so I did something wrong to confuse the mdadm superblock? should I
use "mdadm --zero-superblock /dev/sdb2" before I add it back to mdadm?

  thanks a lot for advice!!

Regards,
tbskyd

^ permalink raw reply

* [PATCH] md-cluster: use %pU to print UUIDs
From: gqjiang @ 2015-05-14  1:39 UTC (permalink / raw)
  To: neilb; +Cc: linux-raid, rgoldwyn, Guoqing Jiang

From: Guoqing Jiang <gqjiang@suse.com>

Signed-off-by: Guoqing Jiang <gqjiang@suse.com>
---
 drivers/md/md-cluster.c | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index fcfc4b9..86d8316 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -176,18 +176,6 @@ static void lockres_free(struct dlm_lock_resource *res)
 	kfree(res);
 }
 
-static char *pretty_uuid(char *dest, char *src)
-{
-	int i, len = 0;
-
-	for (i = 0; i < 16; i++) {
-		if (i == 4 || i == 6 || i == 8 || i == 10)
-			len += sprintf(dest + len, "-");
-		len += sprintf(dest + len, "%02x", (__u8)src[i]);
-	}
-	return dest;
-}
-
 static void add_resync_info(struct mddev *mddev, struct dlm_lock_resource *lockres,
 		sector_t lo, sector_t hi)
 {
@@ -383,7 +371,7 @@ static void process_add_new_disk(struct mddev *mddev, struct cluster_msg *cmsg)
 	int len;
 
 	len = snprintf(disk_uuid, 64, "DEVICE_UUID=");
-	pretty_uuid(disk_uuid + len, cmsg->uuid);
+	sprintf(disk_uuid + len, "%pU", cmsg->uuid);
 	snprintf(raid_slot, 16, "RAID_DISK=%d", cmsg->raid_slot);
 	pr_info("%s:%d Sending kobject change with %s and %s\n", __func__, __LINE__, disk_uuid, raid_slot);
 	init_completion(&cinfo->newdisk_completion);
@@ -641,7 +629,7 @@ static int join(struct mddev *mddev, int nodes)
 	mddev->cluster_info = cinfo;
 
 	memset(str, 0, 64);
-	pretty_uuid(str, mddev->uuid);
+	sprintf(str, "%pU", mddev->uuid);
 	ret = dlm_new_lockspace(str, mddev->bitmap_info.cluster_name,
 				DLM_LSFL_FS, LVB_SIZE,
 				&md_ls_ops, mddev, &ops_rv, &cinfo->lockspace);
-- 
1.7.12.4


^ permalink raw reply related

* [PATCH] raid5: fix broken async operation chain
From: Shaohua Li @ 2015-05-13 16:30 UTC (permalink / raw)
  To: linux-raid; +Cc: neilb, maxime.ripard

ops_run_reconstruct6() doesn't correctly chain asyn operations. The tx returned
by async_gen_syndrome should be added as the dependent tx of next stripe.

The issue is introduced by commit 59fc630b8b5f9f21c8ce3ba153341c107dce1b0c
    RAID5: batch adjacent full stripe write

Reported-and-tested-by: Maxime Ripard <maxime.ripard@free-electrons.com>
Signed-off-by: Shaohua Li <shli@fb.com>
---
 drivers/md/raid5.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 1ba97fd..b9f2b9c 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -1822,7 +1822,7 @@ ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
 	} else
 		init_async_submit(&submit, 0, tx, NULL, NULL,
 				  to_addr_conv(sh, percpu, j));
-	async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
+	tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
 	if (!last_stripe) {
 		j++;
 		sh = list_first_entry(&sh->batch_list, struct stripe_head,
-- 
2.1.0


^ permalink raw reply related

* Re: Breaks LSI RAID on C600 chipset
From: David F. @ 2015-05-13 14:26 UTC (permalink / raw)
  To: NeilBrown; +Cc: John Stoffel, linux-raid@vger.kernel.org
In-Reply-To: <20150513103557.1df6036d@notabene.brown>

Thanks.

1) The update did fix the broken RAID.  I can reboot and the RAID
mirror still exists after reboots.

2) I do want to report that on the first test, the reboot was clean.
On the second reboot there was a warning message about the BIOS
detecting "Inconsistent Timestamps" on one of the drives and something
about which CONFIG the BIOS would use.  I didn't have time to get it
all jotted down.  Whatever it used was still okay since the RAID
mirror was still there after that message.


On Tue, May 12, 2015 at 5:35 PM, NeilBrown <neilb@suse.de> wrote:
> On Mon, 11 May 2015 09:31:26 -0700 "David F." <df7729@gmail.com> wrote:
>
>> any progress or more information needed on this?
>>
>> On Mon, Apr 20, 2015 at 10:39 PM, David F. <df7729@gmail.com> wrote:
>> > If you need access to the system, a network kvm is available - or if
>> > you have a C600 based system, you should see the same problem there.
>> >
>> > On Fri, Apr 10, 2015 at 1:07 PM, David F. <df7729@gmail.com> wrote:
>> >> Okay, and to confirm, this is happening on other C600 based systems.
>> >> Boot to Linux with MDADM raid support and the raid is gone after
>> >> reboot.
>> >>
>> >> On Thu, Apr 9, 2015 at 5:50 PM, NeilBrown <neilb@suse.de> wrote:
>> >>> On Thu, 9 Apr 2015 14:13:05 -0700 "David F." <df7729@gmail.com> wrote:
>> >>>
>> >>>> Hello,
>> >>>>
>> >>>> I built a new system, installed new drives (no partitioning, just raw
>> >>>> new drives), configured RAID 1, boot to linux, created reports
>> >>>> attached.  Rebooted and the system doesn't see any configured raid
>> >>>> drives (just the two raw drives).
>> >>>
>> >>> Thanks.  I might take a look, but I would greatly prefer it if you kept the
>> >>> linux-raid list on the Cc.....
>> >>>
>> >>> NeilBrown
>> >>>
>
> I must confess that I haven't even looked at it.
> But I just saw and email on linux-raid from Martin Wilck:
>
>  Subject: [PATCH] DDF: _write_super_to_disk: fix anchor header type
>
> which very likely fixes your problem.  I've applied it and pushed out to
>    git://neil.brown.name/mdadm/
>
> NeilBrown

^ permalink raw reply

* RAID-5: The following arrays have missing required members and cannot be configured
From: Gravity IT Solutions @ 2015-05-13 11:28 UTC (permalink / raw)
  To: linux-raid

I am developer so my System admin knowledge is basic. 

I have 3 Arrays in my system 1 for OS and 2 for data both data arrays are
Raid-5 and each was showing 1 bad drive, so i shutdown the system and
replaced both drives with a larger drives I checked in Adaptec Bios Array
and it was showing both arrays rebuilding when i was exiting the bios setup,
Array -1 was 2% and Array-2 was showing 1%, so i booted the system and after
few hours i checked the process was complete and i restarted to check the
status in the bois setup, there i was it was showing only 2 Arrays the 3rd
was missing, when i tried rebooting the system it gave me this message

The following arrays have missing required members and cannot be configured
:Array#2-RAID-5

Press <Enter> to accept the current configuration Press< Ctrl-A> to enter
Adaptec RAID Configuration Utility Press <Ctrl-H> to Pause Configuration
Messages (Default is not to accept if no valid key pressed in 30 seconds)
i selected the default to go on and boot, but system did not boot as it
could not find the array and ask me to fix it, so just commented the mount
in the fstab and booted
i can not see the drives in the /dev folder
can you please guide me on how to fix it?

 

Thanks


^ permalink raw reply

* Re: Possible RAID6 regression with ASYNC_TX_DMA enabled in 4.1
From: Maxime Ripard @ 2015-05-13  7:46 UTC (permalink / raw)
  To: Shaohua Li
  Cc: Neil Brown, linux-raid, linux-kernel, Lior Amsalem,
	Thomas Petazzoni, Gregory Clement, Boris Brezillon
In-Reply-To: <20150512105907.GA53461@kernel.org>

[-- Attachment #1: Type: text/plain, Size: 2902 bytes --]

Hi,

On Tue, May 12, 2015 at 03:59:07AM -0700, Shaohua Li wrote:
> On Tue, May 12, 2015 at 02:55:46PM +0200, Maxime Ripard wrote:
> > Hi Shaohua,
> > 
> > On Sun, May 10, 2015 at 11:26:38PM -0700, Shaohua Li wrote:
> > > On Thu, May 07, 2015 at 02:57:02PM +0200, Maxime Ripard wrote:
> > > > Hi,
> > > > 
> > > > I'm currently trying to add support for the PQ operations on the
> > > > marvell XOR engine, in dmaengine, obviously to be able to use async_tx
> > > > to offload these operations.
> > > > 
> > > > I'm testing these patches with a RAID6 array with 4 disks.
> > > > 
> > > > However, since the commit 59fc630b8b5f ("RAID5: batch adjacent full
> > > > stripe write", every write to that array fails with the following
> > > > stacktrace.
> > > > 
> > > > http://code.bulix.org/eh8iew-88342?raw
> > > > 
> > > > It seems to be generated by that warning here:
> > > > 
> > > > http://lxr.free-electrons.com/source/crypto/async_tx/async_tx.c#L173
> > > > 
> > > > And indeed, if we dump the status of depend_tx here, it's already been
> > > > acked.
> > > > 
> > > > That doesn't happen if ASYNC_TX_DMA is disabled, hence using the
> > > > software version of it, instead of relying on our XOR engine. It
> > > > doesn't happen on any commit prior to the one mentionned above, with
> > > > the exact same changes applied. These changes are meant to be
> > > > contributed, so I can definitely push them somewhere if needed.
> > > > 
> > > > I don't really know where to look for though, the change that is
> > > > causing this is probably the change in ops_run_reconstruct6, but I'm
> > > > not sure that this partial revert alone would work with regard to the
> > > > rest of the patch.
> > > 
> > > I don't have a machine with dmaengine, it's likely there is error in this side.
> > > Could you please make stripe_can_batch() returns false always and check if the
> > > error disappear? This should narrow down if it's related to batch issue.
> > 
> > The error indeed disappears if stripe_can_batch always returns false.
> 
> Does this fix it?
> 
> 
> diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
> index 77dfd72..5e820fc 100644
> --- a/drivers/md/raid5.c
> +++ b/drivers/md/raid5.c
> @@ -1825,7 +1825,7 @@ ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
>  	} else
>  		init_async_submit(&submit, 0, tx, NULL, NULL,
>  				  to_addr_conv(sh, percpu, j));
> -	async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
> +	tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
>  	if (!last_stripe) {
>  		j++;
>  		sh = list_first_entry(&sh->batch_list, struct stripe_head,

It does, thanks!

Feel free to add my Tested-by if you submit this patch.

Maxime

-- 
Maxime Ripard, Free Electrons
Embedded Linux, Kernel and Android engineering
http://free-electrons.com

[-- Attachment #2: Digital signature --]
[-- Type: application/pgp-signature, Size: 819 bytes --]

^ permalink raw reply

* Re: unexpected speed difference at RAID initialisation
From: NeilBrown @ 2015-05-13  3:34 UTC (permalink / raw)
  To: Christoph Anton Mitterer; +Cc: linux-raid
In-Reply-To: <1431125239.6206.11.camel@scientia.net>

[-- Attachment #1: Type: text/plain, Size: 4129 bytes --]

On Sat, 09 May 2015 00:47:19 +0200 Christoph Anton Mitterer
<calestyo@scientia.net> wrote:

> Hey.
> 
> I'm just deploying some new servers at the faculty, where I made the
> following strange observation, which I cannot explain.
> 
> All nodes have exactly the same hardware (all recent stuff, some 15k€
> Dell servers with 16 disks á 6 TB, plenty of CPU), the same
> BIOS/firmware config, the same OS (Debian jessie, except the kernel
> 4.0.0 from experimental and btrfs-tools 4.0 from sid) with identical
> config.
> 
> The discs are connected via some Dell PERC RAID controller but for
> testing they're exported as JBODs.
> 
> Nothing except some standard daemons (haveged, irqbalance and that like)
> are running on these nodes.
> 
> 
> I created an MD RAID6 over all disks via:
> mdadm --create /dev/md/data-test-raid --verbose --metadata=1.2
> --size=max --chunk=512K --level=raid6 --bitmap=internal
> --name=data-test-raid --raid-devices=16
> --spare-devices=0 /dev/sda /dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh /dev/sdi /dev/sdj /dev/sdk /dev/sdl /dev/sdm /dev/sdn /dev/sdo /dev/sdp
> basically at the same time (few seconds difference) on both nodes.
> 
> 
> But looking at the initial rebuild on two nodes, one can see substantial
> speed differences:
> node A: 
> # cat /proc/mdstat 
> Personalities : [raid6] [raid5] [raid4] 
> md127 : active raid6 sdp[15] sdo[14] sdn[13] sdm[12] sdl[11] sdk[10] sdj[9] sdi[8] sdh[7] sdg[6] sdf[5] sde[4] sdd[3] sdc[2] sdb[1] sda[0]
>       82045479936 blocks super 1.2 level 6, 512k chunk, algorithm 2 [16/16] [UUUUUUUUUUUUUUUU]
>       [=====>...............]  resync = 28.8% (1691996416/5860391424) finish=841.8min speed=82526K/sec
>       bitmap: 32/44 pages [128KB], 65536KB chunk
> 
> unused devices: <none>
> 
> node B: 
> # cat /proc/mdstat 
> Personalities : [raid6] [raid5] [raid4] 
> md127 : active raid6 sdp[15] sdo[14] sdn[13] sdm[12] sdl[11] sdk[10] sdj[9] sdi[8] sdh[7] sdg[6] sdf[5] sde[4] sdd[3] sdc[2] sdb[1] sda[0]
>       82045479936 blocks super 1.2 level 6, 512k chunk, algorithm 2 [16/16] [UUUUUUUUUUUUUUUU]
>       [====>................]  resync = 20.1% (1180137984/5860391424) finish=1496.2min speed=52132K/sec
>       bitmap: 36/44 pages [144KB], 65536KB chunk
> 
> unused devices: <none>
> 
> (again taken with only few seconds in between).
> 
> As you can see it already shows different speed (~80000K/s for node A
> and ~50000K/s for node B).

This could be quite normal.

When resyncing an array, md will read a full stripe and check if it is in
sync. If it is, it just continues.
If not, it schedules a write with the correct parity.

So if the array happens to be fully in-sync, this goes at maximum see
(sequential reads only).
If it is totally out of sync, it goes very slowly (read/write/read/write...).

What you are probably seeing is that all the drives in one array are (almost)
completely zero, so it seems in-sync.  In the other array, one drive might
have old data on it so syncing is needed.

(snip)

> 
> There's a bigger difference at:
> [ 8334.341904] raid6: using algorithm avx2x4 (24764 MB/s)
> vs.
> [ 8269.712632] raid6: using algorithm avx2x4 (25242 MB/s)
> 
> How are these numbers determined?

Holding a finger up to the wind and seeing how cold it gets.

Well, it really runs a loop performing the calculation over and over and
times how long that takes.
It doesn't take cache effects into account properly so it isn't completely
reliable, but it is reasonably indicative.


> 
> 
> Further:
> node A:
> # ps ax  | grep md127
>   7550 ?        S    273:47 [md127_raid6]
>   7552 ?        D     79:43 [md127_resync]
> 
> 
> node B:
> # ps ax  | grep md127
>   7494 ?        R    251:30 [md127_raid6]
>   7495 ?        D     63:48 [md127_resync]
> 
> 
> 
> Any ideas where this performance difference could come from?

Probably just different initial content of devices.

If it is still going, you could look at the IO stats.  One array might see a
lot more writes than the other.

NeilBrown

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 811 bytes --]

^ permalink raw reply

* Re: [PATCH] md-cluster: avoid deadlock on MESSAGE lock resource
From: Lidong Zhong @ 2015-05-13  2:05 UTC (permalink / raw)
  To: Abhijit Bhopatkar, Goldwyn Rodrigues, linux-raid
  Cc: Reese Faucette (rfaucett)
In-Reply-To: <554CB6B1.3030206@cisco.com>

>>> On 5/8/2015 at 09:14 PM, in message <554CB6B1.3030206@cisco.com>, Abhijit
Bhopatkar <abhopatk@cisco.com> wrote: 
> On 08/05/15 6:40 pm, Abhijit Bhopatkar wrote: 
> >  
> > Every receiver has CR lock on MESSAGE while processing the message. When 
> > every receiver releases ACK lock and for some reason fails to grab EX on 
> > MESSAGE resource in time, a waiting sender could queue an EX on MESSAGE 
> > instead. Now when receiver queues its up convert request on MESSAGE it 
> > will end up in a deadlock situation. 
> >  
> > Setting NOQUEUE flag on MESSAGE lock resource while grabbing the EX on 
> > MESSAGE on sender will avoid this deadlock. If sender can not grab 
> > MESSAGE lock immediately it should retry until the lock is granted. 
> >  
> > Signed-off-by: Abhijit Bhopatkar <abhopatk@cisco.com> 
> > --- 
> > This has been minimally tested on a three node cluster.  
> >  
>  
> I have tested standard mdadm operations (create, assemble etc).  
> What more testing would you want me to do on this before its considered  
> ready? 
>  

The patch seems fine to me.  Let's see what Goldwyn's suggestion is here.

Regards,
Lidong
> Regards, 
> Abhijit 
>  
> >  drivers/md/md-cluster.c | 14 ++++++++++++-- 
> >  1 file changed, 12 insertions(+), 2 deletions(-) 
> >  
> > diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c 
> > index fcfc4b9..04ac309 100644 
> > --- a/drivers/md/md-cluster.c 
> > +++ b/drivers/md/md-cluster.c 
> > @@ -512,7 +512,10 @@ static void unlock_comm(struct md_cluster_info *cinfo) 
> >   * This function performs the actual sending of the message. This function  
> is 
> >   * usually called after performing the encompassing operation 
> >   * The function: 
> > - * 1. Grabs the message lockresource in EX mode 
> > + * 1. Grabs the message lockresource in EX. Do not queue the request if  
> not granted 
> > +      immediately. This avoids deadlock with receivers when receivers try  
> to 
> > +      upconvert CR to EX of message lockresource. The thread will retry  
> until the 
> > +      request is granted. 
> >   * 2. Copies the message to the message LVB 
> >   * 3. Downconverts message lockresource to CR 
> >   * 4. Upconverts ack lock resource from CR to EX. This forces the BAST on  
> other nodes 
> > @@ -526,12 +529,19 @@ static int __sendmsg(struct md_cluster_info *cinfo,  
> struct cluster_msg *cmsg) 
> >  	int slot = cinfo->slot_number - 1; 
> >   
> >  	cmsg->slot = cpu_to_le32(slot); 
> > -	/*get EX on Message*/ 
> > + 
> > +	/* get EX on Message with noqueue flag */ 
> > +	cinfo->message_lockres->flags |= DLM_LKF_NOQUEUE; 
> > + 
> > +retry: 
> >  	error = dlm_lock_sync(cinfo->message_lockres, DLM_LOCK_EX); 
> >  	if (error) { 
> > +		if (error == -EAGAIN) 
> > +			goto retry; 
> >  		pr_err("md-cluster: failed to get EX on MESSAGE (%d)\n", error); 
> >  		goto failed_message; 
> >  	} 
> > +	cinfo->message_lockres->flags &= ~DLM_LKF_NOQUEUE; 
> >   
> >  	memcpy(cinfo->message_lockres->lksb.sb_lvbptr, (void *)cmsg, 
> >  			sizeof(struct cluster_msg)); 
> > -- 2.1.0  
> > -- 
> > To unsubscribe from this list: send the line "unsubscribe linux-raid" in 
> > the body of a message to majordomo@vger.kernel.org 
> > More majordomo info at  http://vger.kernel.org/majordomo-info.html 
> >  
>  
> -- 
> To unsubscribe from this list: send the line "unsubscribe linux-raid" in 
> the body of a message to majordomo@vger.kernel.org 
> More majordomo info at  http://vger.kernel.org/majordomo-info.html 
>  
>  


^ permalink raw reply

* Re: [PATCH 7/7] md/raid5: fix handling of degraded stripes in batches.
From: NeilBrown @ 2015-05-13  0:56 UTC (permalink / raw)
  To: Shaohua Li; +Cc: linux-raid, linux-nfs
In-Reply-To: <20150508191223.GA116778@kernel.org>

[-- Attachment #1: Type: text/plain, Size: 2327 bytes --]

On Fri, 8 May 2015 12:12:23 -0700 Shaohua Li <shli@kernel.org> wrote:

> On Fri, May 08, 2015 at 06:56:12PM +1000, NeilBrown wrote:
> > There is no need for special handling of stripe-batches when the array
> > is degraded.
> > 
> > There may be if there is a failure in the batch, but STRIPE_DEGRADED
> > does not imply an error.
> > 
> > So don't set STRIPE_BATCH_ERR in ops_run_io just because the array is
> > degraded.
> > This actually causes a bug: the STRIPE_DEGRADED flag gets cleared in
> > check_break_stripe_batch_list() and so the bitmap bit gets cleared
> > when it shouldn't.
> > 
> > So in check_break_stripe_batch_list(), split the batch up completely -
> > again STRIPE_DEGRADED isn't meaningful.
> > 
> > Also don't set STRIPE_BATCH_ERR when there is a write error to a
> > replacement device.  This simply removes the replacement device and
> > requires no extra handling.
> > 
> > Signed-off-by: NeilBrown <neilb@suse.de>
> > ---
> >  drivers/md/raid5.c |   17 +++--------------
> >  1 file changed, 3 insertions(+), 14 deletions(-)
> > 
> > diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
> > index 3873eaa6fa2e..1ba97fdc6df1 100644
> > --- a/drivers/md/raid5.c
> > +++ b/drivers/md/raid5.c
> > @@ -1078,9 +1078,6 @@ again:
> >  			pr_debug("skip op %ld on disc %d for sector %llu\n",
> >  				bi->bi_rw, i, (unsigned long long)sh->sector);
> >  			clear_bit(R5_LOCKED, &sh->dev[i].flags);
> > -			if (sh->batch_head)
> > -				set_bit(STRIPE_BATCH_ERR,
> > -					&sh->batch_head->state);
> >  			set_bit(STRIPE_HANDLE, &sh->state);
> >  		}
> 
> Patches look good to me. I had a question here. Is it possible some stripes in
> a batch become degraded here but some not? Seems possible, then the batch
> should be splitted too.

Why?

I don't really understand the purpose of splitting up the batch.
The only possible error handling on a full-stripe write is:
 - fail a device, or
 - record a bad-block.

The first case affects all stripes in a batch equally so there is no need to
split it up.
The second case it is probably best to record the bad blocks while iterating
through the batch in handle_stripe_clean_event().

What exactly do you expect to happen after the stripes in a batch after they
have been split up?

Thanks,
NeilBrown

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 811 bytes --]

^ permalink raw reply

* Re: Breaks LSI RAID on C600 chipset
From: NeilBrown @ 2015-05-13  0:35 UTC (permalink / raw)
  To: David F.; +Cc: John Stoffel, linux-raid@vger.kernel.org
In-Reply-To: <CAGRSmLvdP8y1=KzG0j7Q9oc9HDGDgNGkSnhNLRUU1YXmLq3KDg@mail.gmail.com>

[-- Attachment #1: Type: text/plain, Size: 1503 bytes --]

On Mon, 11 May 2015 09:31:26 -0700 "David F." <df7729@gmail.com> wrote:

> any progress or more information needed on this?
> 
> On Mon, Apr 20, 2015 at 10:39 PM, David F. <df7729@gmail.com> wrote:
> > If you need access to the system, a network kvm is available - or if
> > you have a C600 based system, you should see the same problem there.
> >
> > On Fri, Apr 10, 2015 at 1:07 PM, David F. <df7729@gmail.com> wrote:
> >> Okay, and to confirm, this is happening on other C600 based systems.
> >> Boot to Linux with MDADM raid support and the raid is gone after
> >> reboot.
> >>
> >> On Thu, Apr 9, 2015 at 5:50 PM, NeilBrown <neilb@suse.de> wrote:
> >>> On Thu, 9 Apr 2015 14:13:05 -0700 "David F." <df7729@gmail.com> wrote:
> >>>
> >>>> Hello,
> >>>>
> >>>> I built a new system, installed new drives (no partitioning, just raw
> >>>> new drives), configured RAID 1, boot to linux, created reports
> >>>> attached.  Rebooted and the system doesn't see any configured raid
> >>>> drives (just the two raw drives).
> >>>
> >>> Thanks.  I might take a look, but I would greatly prefer it if you kept the
> >>> linux-raid list on the Cc.....
> >>>
> >>> NeilBrown
> >>>

I must confess that I haven't even looked at it.
But I just saw and email on linux-raid from Martin Wilck:

 Subject: [PATCH] DDF: _write_super_to_disk: fix anchor header type

which very likely fixes your problem.  I've applied it and pushed out to
   git://neil.brown.name/mdadm/

NeilBrown

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 811 bytes --]

^ permalink raw reply

* Re: [PATCH] DDF: _write_super_to_disk: fix anchor header type
From: NeilBrown @ 2015-05-13  0:34 UTC (permalink / raw)
  To: Martin Wilck; +Cc: linux-raid
In-Reply-To: <1431353384-8637-1-git-send-email-martin.wilck@ts.fujitsu.com>

[-- Attachment #1: Type: text/plain, Size: 1157 bytes --]

On Mon, 11 May 2015 16:09:44 +0200 Martin Wilck <martin.wilck@ts.fujitsu.com>
wrote:

> Since commit 30bee0201, the anchor is updated from the active
> DDF header. This requires fixing the header type before the
> anchor is written.
> 
> The LSI Software RAID code will reject DDF meta data with wrong
> anchor type and will erase all meta data when it encounters
> such a broken anchor. Thus starting Linux md once on a system
> with LSI RAID BIOS may cause the meta data to get destroyed.
> ---
>  super-ddf.c | 1 +
>  1 file changed, 1 insertion(+)
> 
> diff --git a/super-ddf.c b/super-ddf.c
> index d0b2ac7..faaf0a7 100644
> --- a/super-ddf.c
> +++ b/super-ddf.c
> @@ -3158,6 +3158,7 @@ static int _write_super_to_disk(struct ddf_super *ddf, struct dl *d)
>  	memcpy(&ddf->primary, &ddf->anchor, 512);
>  	memcpy(&ddf->secondary, &ddf->anchor, 512);
>  
> +	ddf->anchor.type = DDF_HEADER_ANCHOR;
>  	ddf->anchor.openflag = 0xFF; /* 'open' means nothing */
>  	ddf->anchor.seq = cpu_to_be32(0xFFFFFFFF); /* no sequencing in anchor */
>  	ddf->anchor.crc = calc_crc(&ddf->anchor, 512);


Thanks Martin!
Applied,
NeilBrown

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 811 bytes --]

^ permalink raw reply

* Re: Installing Linux directly onto RAID6 Array...........
From: Adam Goryachev @ 2015-05-13  0:02 UTC (permalink / raw)
  To: Another Sillyname, linux-raid
In-Reply-To: <CAOS+5GFwGJouUfZ_RWhtRVXw7wnoP1Ry4doHTJmkruFoccmqRg@mail.gmail.com>

On 12/05/15 20:08, Another Sillyname wrote:
> I've tried to do some research on this but the information out there
> seems a bit contradictory (mainly because some is so old).
>
> I want to install Fedora directly onto a RAID array (no separate boot disk).
>
> My plan is to 'pre configure' the 6 drives as a clean RAID6 array,
> effectively sd[a-f] without partitions and then attempt to install
> Fedora 21, from sources it looks like Grub2 should recognise the array
> and then allow the Kernel to boot thereby 'enabling' the array to
> become visible and active.
>
> However I have not been able to find an actual example of someone
> trying this......thoughts?
>
> The reason to do this is I'm intending to use a Mini ITX board with 6
> sata ports and want to use 8TB drives in Raid6 to give me a very high
> density data resilient small form factor storage box.
>
> Ideas/Suggestions?

Create a small RAID1 partition at the beginning of every disk, use this 
for /boot and install grub. Use the rest of each disk as your RAID6 array.

This means as long as the bios can find any one of your disks, then the 
system will boot. Personally, I'd be inclined to put the complete OS on 
the RAID1, and then only use the RAID6 for the "data" mountpoint. This 
would allow the system to come up fully, even without the RAID6 (eg, 4 
out of 6 drives has died), and allow remote debug/diagnostics/etc 
without needing to try and boot from a rescue disk/etc.

This doesn't help you with what you asked, but just an idea/suggestion.

Regards,
Adam

-- 
Adam Goryachev Website Managers www.websitemanagers.com.au

^ permalink raw reply

* MD RAID hot-replace wants to rewrite to the source! (and fails, and kicks)
From: James J @ 2015-05-12 16:14 UTC (permalink / raw)
  To: linux-raid

Dear Neil and other RAID developers,

I came across a problem of want_replacement not working as intended. Or 
at least not working as I think was intended.

The kernel is this, quite old
     3.4.34 (vanilla)
apologies if the behaviour has been fixed, but I don't recall to having 
seen this problem mentioned in patches or on the ML so I suppose this 
report is still valid.

Disk sdl in our storage server had shown one read error in smart check 
long and was apparently dying rapidly.
I issued want_replacement so to replicate sdl to the spare sdm.
It started correctly, like this:

     md54 : active raid5 sde1[0] sdh1[2] sdm1[5](R) sdf1[1] sdl1[6] sdi1[3]
             11603730432 blocks super 1.2 level 5, 1024k chunk, 
algorithm 2 [5/5] [UUUUU]
             [>....................]  recovery =  0.2% 
(6609916/2900932608) finish=598.6min speed=80573K/sec
             bitmap: 1/173 pages [4KB], 8192KB chunk

but then it met read errors in sdl (source) and instead of skipping 
those or rewriting those on just sdm, it tried to rewrite those (only or 
also) on sdl, i.e. the source.
Sdl was dying and was not capable of accepting the rewrites, so it was 
kicked by MD, the array went to degraded state, and the process 
restarted (or maybe continued) with reconstruction with one failed disk.

So it went from this:

     md54 : active raid5 sde1[0] sdh1[2] sdm1[5](R) sdf1[1] sdl1[6] sdi1[3]
             11603730432 blocks super 1.2 level 5, 1024k chunk, 
algorithm 2 [5/5] [UUUUU]
             [>....................]  recovery =  0.3% 
(11584992/2900932608) finish=1279.9min speed=37621K/sec
             bitmap: 1/173 pages [4KB], 8192KB chunk

To this:

     md54 : active raid5 sde1[0] sdh1[2] sdm1[5](R) sdf1[1] sdl1[6](F) 
sdi1[3]
             11603730432 blocks super 1.2 level 5, 1024k chunk, 
algorithm 2 [5/4] [UUUU_]
             [>....................]  recovery =  1.4% 
(41691236/2900932608) finish=578.6min speed=82352K/sec
             bitmap: 1/173 pages [4KB], 8192KB chunk

 From dmesg one cannot detect that MD performed a rewrite on sdl, but I 
can tell that from smartctl -x /dev/sdl, for which the last recorded 
error is:

     Error 17 [16] occurred at disk power-on lifetime: 15114 hours (629 
days + 18 hours)
     When the command that caused the error occurred, the device was 
active or idle.

     After command completion occurred, registers were:
     ER -- ST COUNT  LBA_48  LH LM LL DV DC
     -- -- -- == -- == == == -- -- -- -- --
     40 -- 51 00 00 00 00 01 62 a8 a0 40 00  Error: WP at LBA = 
0x0162a8a0 = 23242912

     Commands leading to the command that caused the error were:
     CR FEATR COUNT  LBA_48  LH LM LL DV DC  Powered_Up_Time 
Command/Feature_Name
     -- == -- == -- == == == -- -- -- -- --  --------------- 
--------------------
     61 04 00 00 40 00 00 01 62 ab c0 40 00 10d+00:06:33.287  WRITE 
FPDMA QUEUED
     61 04 00 00 38 00 00 01 62 9f c0 40 00 10d+00:06:33.287  WRITE 
FPDMA QUEUED
     61 04 00 00 30 00 00 01 62 c3 c0 40 00 10d+00:06:33.285  WRITE 
FPDMA QUEUED
     61 04 00 00 28 00 00 01 62 c7 c0 40 00 10d+00:06:33.276  WRITE 
FPDMA QUEUED
     60 04 00 00 20 00 00 01 62 5f c0 40 00 10d+00:06:33.276  READ FPDMA 
QUEUED

which is a write error, and was not present before the replacement 
operation.

I thought the whole point of want_replacement was to NOT attempt to 
correct read errors onto the source. There is no point in doing that, 
doing that would just risk to make the source fail.

During a replacement read errors should IMHO remain read errors on the 
source. Even if it happens afterwards that normal array operation wants 
to read from exactly such sector which has not been rewritten, it will 
still receive a read error from the source. No problem because after 
this it has other 2 places to read such data from: sdm (replacement 
drive, if it has already passed that point), or reconstruction from the 
other members of the array. After this, again MD would try to perform a 
rewrite: also such rewrite (not coming from the reconstruction process) 
should again go just to the replacement drive and never to the source. 
Read errors on the source should remain read errors.

Here is an extract of the dmesg:

         ( LOTS of "Unhandled error code" and "read error corrected" for 
sdl1 before this )
     [865020.041576] end_request: I/O error, dev sdl, sector 23240640
     [865020.041831] sd 4:0:11:0: [sdl] Unhandled error code
     [865020.041834] sd 4:0:11:0: [sdl]  Result: hostbyte=DID_SOFT_ERROR 
driverbyte=DRIVER_OK
     [865020.041837] sd 4:0:11:0: [sdl] CDB: Read(10): 28 00 01 62 af c0 
00 04 00 00
     [865020.041842] end_request: I/O error, dev sdl, sector 23244736
     [865020.041994] sd 4:0:11:0: [sdl] Unhandled error code
     [865020.041996] sd 4:0:11:0: [sdl]  Result: hostbyte=DID_SOFT_ERROR 
driverbyte=DRIVER_OK
     [865020.041998] sd 4:0:11:0: [sdl] CDB: Read(10): 28 00 01 62 b3 c0 
00 04 00 00
     [865020.042004] end_request: I/O error, dev sdl, sector 23245760
     [865020.042153] sd 4:0:11:0: [sdl] Unhandled error code
     [865020.042155] sd 4:0:11:0: [sdl]  Result: hostbyte=DID_SOFT_ERROR 
driverbyte=DRIVER_OK
     [865020.042158] sd 4:0:11:0: [sdl] CDB: Read(10): 28 00 01 62 b7 c0 
00 04 00 00
     [865020.042163] end_request: I/O error, dev sdl, sector 23246784
     [865020.042315] sd 4:0:11:0: [sdl] Unhandled error code
     [865020.042317] sd 4:0:11:0: [sdl]  Result: hostbyte=DID_SOFT_ERROR 
driverbyte=DRIVER_OK
     [865020.042320] sd 4:0:11:0: [sdl] CDB: Read(10): 28 00 01 62 bb c0 
00 04 00 00
     [865020.042325] end_request: I/O error, dev sdl, sector 23247808
     [865020.042476] sd 4:0:11:0: [sdl] Unhandled error code
     [865020.042478] sd 4:0:11:0: [sdl]  Result: hostbyte=DID_SOFT_ERROR 
driverbyte=DRIVER_OK
     [865020.042481] sd 4:0:11:0: [sdl] CDB: Read(10): 28 00 01 62 bf c0 
00 04 00 00
     [865020.042486] end_request: I/O error, dev sdl, sector 23248832
     [865020.042636] sd 4:0:11:0: [sdl] Unhandled error code
     [865020.042638] sd 4:0:11:0: [sdl]  Result: hostbyte=DID_SOFT_ERROR 
driverbyte=DRIVER_OK
     [865020.042641] sd 4:0:11:0: [sdl] CDB: Read(10): 28 00 01 62 c7 c0 
00 04 00 00
     [865020.042647] end_request: I/O error, dev sdl, sector 23250880
     [865020.042797] sd 4:0:11:0: [sdl] Unhandled error code
     [865020.042799] sd 4:0:11:0: [sdl]  Result: hostbyte=DID_SOFT_ERROR 
driverbyte=DRIVER_OK
     [865020.042801] sd 4:0:11:0: [sdl] CDB: Read(10): 28 00 01 62 cf c0 
00 04 00 00
     [865020.042807] end_request: I/O error, dev sdl, sector 23252928
     [865020.042956] sd 4:0:11:0: [sdl] Unhandled error code
     [865020.042958] sd 4:0:11:0: [sdl]  Result: hostbyte=DID_SOFT_ERROR 
driverbyte=DRIVER_OK
     [865020.042961] sd 4:0:11:0: [sdl] CDB: Read(10): 28 00 01 62 d3 c0 
00 04 00 00
     [865020.042966] end_request: I/O error, dev sdl, sector 23253952
     [865020.100097] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865020.100124] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865020.100133] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865020.100142] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865020.100152] sd 4:0:11:0: [sdl] Unhandled error code
     [865020.100157] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865020.100169] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865020.100174] sd 4:0:11:0: [sdl]  Result: hostbyte=DID_SOFT_ERROR 
driverbyte=DRIVER_OK
     [865020.100181] sd 4:0:11:0: [sdl] CDB: Read(10): 28 00 01 62 d7 c0 
00 04 00 00
     [865020.100202] end_request: I/O error, dev sdl, sector 23254976
     [865020.106716] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865020.106725] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865020.106734] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865020.106743] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865020.106752] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865020.106761] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865020.106771] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865020.106780] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865020.106902] sd 4:0:11:0: [sdl] Unhandled sense code
     [865020.106906] sd 4:0:11:0: [sdl]  Result: hostbyte=DID_OK 
driverbyte=DRIVER_SENSE
     [865020.106910] sd 4:0:11:0: [sdl]  Sense Key : Medium Error [current]
     [865020.106915] Info fld=0x162abc0
     [865020.106917] sd 4:0:11:0: [sdl]  Add. Sense: Unrecovered read error
     [865020.106923] sd 4:0:11:0: [sdl] CDB: Read(10): 28 00 01 62 ab c0 
00 04 00 00
     [865020.106932] end_request: critical target error, dev sdl, sector 
23243712
     [865022.912537] sd 4:0:11:0: [sdl] Unhandled sense code
     [865022.912543] sd 4:0:11:0: [sdl]  Result: hostbyte=DID_OK 
driverbyte=DRIVER_SENSE
     [865022.912548] sd 4:0:11:0: [sdl]  Sense Key : Medium Error [current]
     [865022.912554] Info fld=0x162dfc0
     [865022.912556] sd 4:0:11:0: [sdl]  Add. Sense: Unrecovered read error
     [865022.912561] sd 4:0:11:0: [sdl] CDB: Read(10): 28 00 01 62 df c0 
00 00 08 00
     [865022.912571] end_request: critical target error, dev sdl, sector 
23257024
     [865022.951721] raid5_end_read_request: 118 callbacks suppressed
     [865022.951726] md/raid:md54: read error corrected (8 sectors at 
23177152 on sdl1)
     [865022.951733] md/raid:md54: read error corrected (8 sectors at 
23177160 on sdl1)
     [865022.951737] md/raid:md54: read error corrected (8 sectors at 
23177168 on sdl1)
     [865022.951741] md/raid:md54: read error corrected (8 sectors at 
23177176 on sdl1)
     [865022.951744] md/raid:md54: read error corrected (8 sectors at 
23177184 on sdl1)
     [865022.951748] md/raid:md54: read error corrected (8 sectors at 
23177192 on sdl1)
     [865022.951751] md/raid:md54: read error corrected (8 sectors at 
23177200 on sdl1)
     [865022.951755] md/raid:md54: read error corrected (8 sectors at 
23177208 on sdl1)
     [865022.951758] md/raid:md54: read error corrected (8 sectors at 
23177216 on sdl1)
     [865022.951762] md/raid:md54: read error corrected (8 sectors at 
23177224 on sdl1)
     [865029.982668] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982685] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982694] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982722] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982731] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982740] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982764] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982804] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982812] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982821] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982830] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982839] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982848] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982857] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982866] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982875] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982884] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982893] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982903] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982912] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982921] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982930] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982939] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982948] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.982957] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865029.983003] sd 4:0:11:0: [sdl] Unhandled sense code
     [865029.983008] sd 4:0:11:0: [sdl]  Result: hostbyte=DID_OK 
driverbyte=DRIVER_SENSE
     [865029.983013] sd 4:0:11:0: [sdl]  Sense Key : Medium Error [current]
     [865029.983018] Info fld=0x1625c27
     [865029.983021] sd 4:0:11:0: [sdl]  Add. Sense: Unrecovered read error
     [865029.983026] sd 4:0:11:0: [sdl] CDB: Read(10): 28 00 01 62 5b c0 
00 04 00 00
     [865029.983036] end_request: critical target error, dev sdl, sector 
23223232
     [865029.990723] md/raid:md54: read error NOT corrected!! (sector 
23190464 on sdl1).
     [865029.990728] md/raid:md54: Disk failure on sdl1, disabling device.
     [865029.990729] md/raid:md54: Operation continuing on 4 devices.
     [865030.004221] md/raid:md54: read error not correctable (sector 
23190472 on sdl1).
     [865030.004224] md/raid:md54: read error not correctable (sector 
23190480 on sdl1).
     [865030.004227] md/raid:md54: read error not correctable (sector 
23190488 on sdl1).
     [865030.004231] md/raid:md54: read error not correctable (sector 
23190496 on sdl1).
     [865030.004234] md/raid:md54: read error not correctable (sector 
23190504 on sdl1).
     [865030.004237] md/raid:md54: read error not correctable (sector 
23190512 on sdl1).
     [865030.004240] md/raid:md54: read error not correctable (sector 
23190520 on sdl1).
     [865030.004243] md/raid:md54: read error not correctable (sector 
23190528 on sdl1).
     [865030.004246] md/raid:md54: read error not correctable (sector 
23190536 on sdl1).
     [865030.004250] md/raid:md54: read error not correctable (sector 
23190544 on sdl1).
     [865030.063169] raid5_end_read_request: 1142 callbacks suppressed
     [865030.063174] md/raid:md54: read error corrected (8 sectors at 
23200704 on sdl1)
     [865030.063181] md/raid:md54: read error corrected (8 sectors at 
23200712 on sdl1)
     [865030.063185] md/raid:md54: read error corrected (8 sectors at 
23200720 on sdl1)
     [865030.063189] md/raid:md54: read error corrected (8 sectors at 
23200728 on sdl1)
     [865030.063192] md/raid:md54: read error corrected (8 sectors at 
23200736 on sdl1)
     [865030.063196] md/raid:md54: read error corrected (8 sectors at 
23200744 on sdl1)
     [865030.063199] md/raid:md54: read error corrected (8 sectors at 
23200752 on sdl1)
     [865030.063203] md/raid:md54: read error corrected (8 sectors at 
23200760 on sdl1)
     [865030.063207] md/raid:md54: read error corrected (8 sectors at 
23200768 on sdl1)
     [865030.063210] md/raid:md54: read error corrected (8 sectors at 
23200776 on sdl1)
     [865031.472160] md: md54: recovery done.
     [865031.586603] md: recovery of RAID array md54
     [865031.586629] md: minimum _guaranteed_  speed: 20000 KB/sec/disk.
     [865031.586633] md: using maximum available idle IO bandwidth (but 
not more than 80000 KB/sec) for recovery.
     [865031.586647] md: using 128k window, over a total of 2900932608k.
     [865031.586650] md: resuming recovery of md54 from checkpoint.
     [865038.511922] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865038.511934] mpt2sas0: log_info(0x31080000): originator(PL), 
code(0x08), sub_code(0x0000)
     [865038.512018] sd 4:0:11:0: [sdl] Unhandled sense code
     [865038.512032] sd 4:0:11:0: [sdl]
     [865038.512037] sd 4:0:11:0: attempting task abort! 
scmd(ffff880063d16500)
     [865038.512044] sd 4:0:11:0: [sdl] CDB: Result: hostbyte=DID_OK 
driverbyte=DRIVER_SENSE
     [865038.512050] sd 4:0:11:0: [sdl]  Sense Key : Medium Error [current]
     [865038.512057] ATA command pass through(16)Info fld=0x162a8a0
     [865038.512062] :
     [865038.512064] sd 4:0:11:0: [sdl]   85Add. Sense: Unrecovered read 
error 08 2e 00 d0 00 01 00 00 00 4f 00 c2 00 b0 00
     [865038.512082]
     [865038.512085] scsi target4:0:11: handle(0x001a), 
sas_address(0x443322110a000000), phy(10)
     [865038.512091] scsi target4:0:11: 
enclosure_logical_id(0x500062b2000c2f80), slot(10)
     [865038.512096] sd 4:0:11:0: [sdl] CDB: Read(10): 28 00 01 62 a7 c0 
00 04 00 00
     [865038.512106] end_request: critical target error, dev sdl, sector 
23242688
     [865038.519787] raid5_end_read_request: 117 callbacks suppressed
     [865038.519790] md/raid:md54: read error not correctable (sector 
23209920 on sdl1).
     [865038.519796] md/raid:md54: read error not correctable (sector 
23209928 on sdl1).
     [865038.519799] md/raid:md54: read error not correctable (sector 
23209936 on sdl1).
     [865038.519803] md/raid:md54: read error not correctable (sector 
23209944 on sdl1).
     [865038.519806] md/raid:md54: read error not correctable (sector 
23209952 on sdl1).
     [865038.519809] md/raid:md54: read error not correctable (sector 
23209960 on sdl1).
     [865038.519813] md/raid:md54: read error not correctable (sector 
23209968 on sdl1).
     [865038.519816] md/raid:md54: read error not correctable (sector 
23209976 on sdl1).
     [865038.519819] md/raid:md54: read error not correctable (sector 
23209984 on sdl1).
     [865038.519822] md/raid:md54: read error not correctable (sector 
23209992 on sdl1).
     [865042.322285] sd 4:0:11:0: task abort: SUCCESS scmd(ffff880063d16500)
     [865043.156822] raid5_end_read_request: 2294 callbacks suppressed
     [865043.156828] md/raid:md54: read error corrected (8 sectors at 
23211968 on sdm1)
     [865043.156835] md/raid:md54: read error corrected (8 sectors at 
23211976 on sdm1)
     [865043.156838] md/raid:md54: read error corrected (8 sectors at 
23211984 on sdm1)
     [865043.156842] md/raid:md54: read error corrected (8 sectors at 
23211992 on sdm1)
     [865043.156845] md/raid:md54: read error corrected (8 sectors at 
23212000 on sdm1)
     [865043.156848] md/raid:md54: read error corrected (8 sectors at 
23212008 on sdm1)
     [865043.156852] md/raid:md54: read error corrected (8 sectors at 
23212016 on sdm1)
     [865043.156855] md/raid:md54: read error corrected (8 sectors at 
23212024 on sdm1)
     [865043.156858] md/raid:md54: read error corrected (8 sectors at 
23212032 on sdm1)
     [865043.156861] md/raid:md54: read error corrected (8 sectors at 
23212040 on sdm1)
     -- End of dmesg.---

(The recovery is still running. Let's hope I don't have ANY bad sector 
on the other 4 disks... )

Also you can notice some strange dmesg logs at 865043 which are a bit 
confusing. I am guessing it is a leftover of the reconstruction process 
which saw sdm1 suddenly take the place of sdl1 while there were still 
ongoing rewrites. Note that smartctl -x /dev/sdm does not report any 
read or write error during operation, so I am quite sure it cannot 
really be a read error from sdm1.

Thank you
JJ


^ permalink raw reply

* Re: [PATCH 0/5] a caching layer for raid 5/6
From: Shaohua Li @ 2015-05-12 15:23 UTC (permalink / raw)
  To: Christoph Hellwig
  Cc: linux-raid, Kernel-team, songliubraving, dan.j.williams, neilb
In-Reply-To: <20150512071854.GA16731@infradead.org>

On Tue, May 12, 2015 at 12:18:54AM -0700, Christoph Hellwig wrote:
> On Mon, May 11, 2015 at 09:03:51AM -0700, Shaohua Li wrote:
> > >   - What is the reason for retry_bio_list?  If a driver returns an
> > >     I/O error to the higher levels it already has retried and came
> > >     to the conclusion this is a permanent error.
> > 
> > The retry_bio_list is to handle io to cache disk. If IO to cache disk
> > has error, it's not a permanent error here. The cache disk is a cache,
> > We can still dispatch the IO to its final destination, the raid disks.
> 
> How does this work in practice?  We've filled our cache disk with
> dirty data, and it now returns non-correctable write errors.  At this
> point we had claimed to caller that data is on stable disk, but our
> cache disk is toast now.  Is it really a good idea to now start a large
> window where we do not actually have the cache data on stable storage
> we can get back at but pretent business as usual?
> 
> IMHO the only sane way is to shut down the array when write to the cache
> disk fail.  Hopefully the disk will still allow reading from it.  Note
> that to be on the safe side you'll need a mirrored cache disk anyway.

We have a memory pool here. All data which aren't flushed to raid disks
are in the pool. So if there is io error in cache disk, we still can
flush the data from memory pool to raid disks. The pool is a limited
resource, so can reduce IO aggregation effect though, but we flush full
stripe data almost immediately to raid disks, which can mitigate a
little. This will make the caching layer like hardware raid card very
much.

Thanks,
Shaohua

^ permalink raw reply

* Re: Installing Linux directly onto RAID6 Array...........
From: Another Sillyname @ 2015-05-12 14:05 UTC (permalink / raw)
  To: Wilson, Jonathan; +Cc: linux-raid
In-Reply-To: <BLU436-SMTP20942BFA85744B69593179D98DA0@phx.gbl>

It's a new z97 board so EFI rules apply.

I have some spare 3TB drives sitting around that I may try to practice
on.......see what will and won't work.

Still open to all suggestions and especially from anyone who's
actually done this.

On 12 May 2015 at 14:27, Wilson, Jonathan <piercing_male@hotmail.com> wrote:
> On Tue, 2015-05-12 at 11:08 +0100, Another Sillyname wrote:
>> I've tried to do some research on this but the information out there
>> seems a bit contradictory (mainly because some is so old).
>>
>> I want to install Fedora directly onto a RAID array (no separate boot disk).
>>
>> My plan is to 'pre configure' the 6 drives as a clean RAID6 array,
>> effectively sd[a-f] without partitions and then attempt to install
>> Fedora 21, from sources it looks like Grub2 should recognise the array
>> and then allow the Kernel to boot thereby 'enabling' the array to
>> become visible and active.
>>
>> However I have not been able to find an actual example of someone
>> trying this......thoughts?
>>
>> The reason to do this is I'm intending to use a Mini ITX board with 6
>> sata ports and want to use 8TB drives in Raid6 to give me a very high
>> density data resilient small form factor storage box.
>
> Grub2 can handle booting into raid6, but some while ago the support was
> sketchy if the array was degraded; this may have improved.
>
> Your problem would be that it would require a biosboot (GPT, type:EF02,
> size:1 MiB) partition to hold part of the loader as it will not fit
> entirely into the "mbr."
>
> A second problem might be that while drives larger than 2 GB can be used
> on (most?) older boards they might not be able to be accessed/read
> correctly/bootable by older non-EFI bios's. My old MB was quite happy to
> boot from a 1 TB drive and linux could see and use my 3TB drives but the
> bios only saw the 3TB drives as 700MB (approx, I recall)
>
> If you are using an EFI system in EFI mode, you will need an EFI
> partition(s) somewhere. On my new system I have all 5 of my 3TB drives
> contain an EFI-dos partition of about 200-500MB and the the rest as one
> large partition for the raid6 containing everything else (/, /home,
> etc.). The only pain in the neck is remembering to copy everything from
> the "live EFI" (default loaded as "/boot") to the backup EFI's when ever
> I change/update stuff in it.
>>
>> Ideas/Suggestions?
>>
>> Thanks
>>
>> Tony
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-raid" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>>
>
>

^ permalink raw reply

page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]

This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox