* [PATCH 07/24] Return MD_SB_CLUSTERED if mddev is clustered
From: Goldwyn Rodrigues @ 2014-12-18 16:16 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md.c | 3 +++
include/uapi/linux/raid/md_p.h | 1 +
2 files changed, 4 insertions(+)
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 795d925..5f45951 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -5630,6 +5630,9 @@ static int get_array_info(struct mddev * mddev, void __user * arg)
info.state = (1<<MD_SB_CLEAN);
if (mddev->bitmap && mddev->bitmap_info.offset)
info.state = (1<<MD_SB_BITMAP_PRESENT);
+ if (mddev_is_clustered(mddev))
+ info.state |= (1<<MD_SB_CLUSTERED);
+
info.active_disks = insync;
info.working_disks = working;
info.failed_disks = failed;
diff --git a/include/uapi/linux/raid/md_p.h b/include/uapi/linux/raid/md_p.h
index 49f4210..643489d 100644
--- a/include/uapi/linux/raid/md_p.h
+++ b/include/uapi/linux/raid/md_p.h
@@ -101,6 +101,7 @@ typedef struct mdp_device_descriptor_s {
#define MD_SB_CLEAN 0
#define MD_SB_ERRORS 1
+#define MD_SB_CLUSTERED 5 /* MD is clustered */
#define MD_SB_BITMAP_PRESENT 8 /* bitmap may be present nearby */
/*
--
2.1.2
^ permalink raw reply related
* [PATCH 08/24] Add node recovery callbacks
From: Goldwyn Rodrigues @ 2014-12-18 16:16 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
DLM offers callbacks when a node fails and the lock remastery
is performed:
1. recover_prep: called when DLM discovers a node is down
2. recover_slot: called when DLM identifies the node and recovery
can start
3. recover_done: called when all nodes have completed recover_slot
recover_slot() and recover_done() are also called when the node joins
initially in order to inform the node with its slot number. These slot
numbers start from one, so we deduct one to make it start with zero
which the cluster-md code uses.
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/bitmap.c | 1 +
drivers/md/bitmap.h | 4 ++--
drivers/md/md-cluster.c | 60 ++++++++++++++++++++++++++++++++++++++++++++-----
drivers/md/md-cluster.h | 4 +++-
drivers/md/md.c | 2 +-
drivers/md/md.h | 3 ++-
6 files changed, 64 insertions(+), 10 deletions(-)
diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c
index 33374de..f4df37c 100644
--- a/drivers/md/bitmap.c
+++ b/drivers/md/bitmap.c
@@ -624,6 +624,7 @@ static int bitmap_read_sb(struct bitmap *bitmap)
if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
+ strlcpy(bitmap->mddev->bitmap_info.cluster_name, sb->cluster_name, 64);
err = 0;
out:
kunmap_atomic(sb);
diff --git a/drivers/md/bitmap.h b/drivers/md/bitmap.h
index 6872945..49163b6 100644
--- a/drivers/md/bitmap.h
+++ b/drivers/md/bitmap.h
@@ -131,8 +131,8 @@ typedef struct bitmap_super_s {
__le32 nodes; /* 64 the maximum number of nodes in cluster. */
__le32 sectors_reserved; /* 68 number of 512-byte sectors that are
* reserved for the bitmap. */
-
- __u8 pad[256 - 72]; /* set to zero */
+ __u8 cluster_name[64]; /* 72 cluster name to which this md belongs */
+ __u8 pad[256 - 136]; /* set to zero */
} bitmap_super_t;
/* notes:
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index fa28aed..98e45e1 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -33,6 +33,8 @@ struct dlm_lock_resource {
struct md_cluster_info {
/* dlm lock space and resources for clustered raid. */
dlm_lockspace_t *lockspace;
+ int slot_number;
+ struct completion completion;
struct dlm_lock_resource *sb_lock;
struct mutex sb_mutex;
};
@@ -140,10 +142,41 @@ static char *pretty_uuid(char *dest, char *src)
return dest;
}
+static void recover_prep(void *arg)
+{
+}
+
+static void recover_slot(void *arg, struct dlm_slot *slot)
+{
+ struct mddev *mddev = arg;
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ pr_info("md-cluster: %s Node %d/%d down. My slot: %d. "
+ "Initiating recovery.\n",
+ mddev->bitmap_info.cluster_name,
+ slot->nodeid, slot->slot,
+ cinfo->slot_number);
+}
+
+static void recover_done(void *arg, struct dlm_slot *slots,
+ int num_slots, int our_slot,
+ uint32_t generation)
+{
+ struct mddev *mddev = arg;
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ cinfo->slot_number = our_slot;
+ complete(&cinfo->completion);
+}
+
+static const struct dlm_lockspace_ops md_ls_ops = {
+ .recover_prep = recover_prep,
+ .recover_slot = recover_slot,
+ .recover_done = recover_done,
+};
+
static int join(struct mddev *mddev, int nodes)
{
struct md_cluster_info *cinfo;
- int ret;
+ int ret, ops_rv;
char str[64];
if (!try_module_get(THIS_MODULE))
@@ -153,24 +186,30 @@ static int join(struct mddev *mddev, int nodes)
if (!cinfo)
return -ENOMEM;
+ init_completion(&cinfo->completion);
+
+ mutex_init(&cinfo->sb_mutex);
+ mddev->cluster_info = cinfo;
+
memset(str, 0, 64);
pretty_uuid(str, mddev->uuid);
- ret = dlm_new_lockspace(str, NULL, DLM_LSFL_FS, LVB_SIZE,
- NULL, NULL, NULL, &cinfo->lockspace);
+ ret = dlm_new_lockspace(str, mddev->bitmap_info.cluster_name,
+ DLM_LSFL_FS, LVB_SIZE,
+ &md_ls_ops, mddev, &ops_rv, &cinfo->lockspace);
if (ret)
goto err;
+ wait_for_completion(&cinfo->completion);
cinfo->sb_lock = lockres_init(mddev, "cmd-super",
NULL, 0);
if (!cinfo->sb_lock) {
ret = -ENOMEM;
goto err;
}
- mutex_init(&cinfo->sb_mutex);
- mddev->cluster_info = cinfo;
return 0;
err:
if (cinfo->lockspace)
dlm_release_lockspace(cinfo->lockspace, 2);
+ mddev->cluster_info = NULL;
kfree(cinfo);
module_put(THIS_MODULE);
return ret;
@@ -186,9 +225,20 @@ static int leave(struct mddev *mddev)
return 0;
}
+/* slot_number(): Returns the MD slot number to use
+ * DLM starts the slot numbers from 1, wheras cluster-md
+ * wants the number to be from zero, so we deduct one
+ */
+static int slot_number(struct mddev *mddev)
+{
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ return cinfo->slot_number - 1;
+}
+
static struct md_cluster_operations cluster_ops = {
.join = join,
.leave = leave,
+ .slot_number = slot_number
};
static int __init cluster_init(void)
diff --git a/drivers/md/md-cluster.h b/drivers/md/md-cluster.h
index aa9f07b..d9a1d88 100644
--- a/drivers/md/md-cluster.h
+++ b/drivers/md/md-cluster.h
@@ -8,8 +8,10 @@
struct mddev;
struct md_cluster_operations {
- int (*join)(struct mddev *mddev);
+ int (*join)(struct mddev *mddev, int nodes);
int (*leave)(struct mddev *mddev);
+ int (*slot_number)(struct mddev *mddev);
+
};
#endif /* _MD_CLUSTER_H */
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 5f45951..db28f21 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -7273,7 +7273,7 @@ int md_setup_cluster(struct mddev *mddev, int nodes)
}
spin_unlock(&pers_lock);
- return md_cluster_ops->join(mddev);
+ return md_cluster_ops->join(mddev, nodes);
}
void md_cluster_stop(struct mddev *mddev)
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 8017e18..6c6e992 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -422,7 +422,8 @@ struct mddev {
unsigned long daemon_sleep; /* how many jiffies between updates? */
unsigned long max_write_behind; /* write-behind mode */
int external;
- int nodes;
+ int nodes; /* Maximum unmber of nodes in the cluster */
+ char cluster_name[64]; /* Name of the cluster */
} bitmap_info;
atomic_t max_corr_read_errors; /* max read retries */
--
2.1.2
^ permalink raw reply related
* [PATCH 09/24] Use separate bitmaps for each nodes in the cluster
From: Goldwyn Rodrigues @ 2014-12-18 16:16 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
On-disk format:
0 4k 8k 12k
-------------------------------------------------------------------
| idle | md super | bm super [0] + bits |
| bm bits[0, contd] | bm super[1] + bits | bm bits[1, contd] |
| bm super[2] + bits | bm bits [2, contd] | bm super[3] + bits |
| bm bits [3, contd] | | |
Bitmap super has a field nodes, which defines the maximum number
of nodes the device can use. While reading the bitmap super, if
the cluster finds out that the number of nodes is > 0:
1. Requests the md-cluster module.
2. Calls md_cluster_ops->join(), which sets up clustering such as
joining DLM lockspace.
Since the first time, the first bitmap is read. After the call
to the cluster_setup, the bitmap offset is adjusted and the
superblock is re-read. This also ensures the bitmap is read
the bitmap lock (when bitmap lock is introduced in later patches)
Questions:
1. cluster name is repeated in all bitmap supers. Is that okay?
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/bitmap.c | 68 ++++++++++++++++++++++++++++++++++++++++---------
drivers/md/bitmap.h | 1 +
drivers/md/md-cluster.c | 7 +++++
drivers/md/md-cluster.h | 1 -
4 files changed, 64 insertions(+), 13 deletions(-)
diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c
index f4df37c..23a71a7 100644
--- a/drivers/md/bitmap.c
+++ b/drivers/md/bitmap.c
@@ -192,6 +192,10 @@ static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
struct block_device *bdev;
struct mddev *mddev = bitmap->mddev;
struct bitmap_storage *store = &bitmap->storage;
+ int node_offset = 0;
+
+ if (mddev_is_clustered(bitmap->mddev))
+ node_offset = bitmap->cluster_slot * store->file_pages;
while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
int size = PAGE_SIZE;
@@ -536,6 +540,7 @@ static int bitmap_read_sb(struct bitmap *bitmap)
unsigned long sectors_reserved = 0;
int err = -EINVAL;
struct page *sb_page;
+ int cluster_setup_done = 0;
if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
chunksize = 128 * 1024 * 1024;
@@ -551,6 +556,7 @@ static int bitmap_read_sb(struct bitmap *bitmap)
return -ENOMEM;
bitmap->storage.sb_page = sb_page;
+re_read:
if (bitmap->storage.file) {
loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
@@ -566,6 +572,7 @@ static int bitmap_read_sb(struct bitmap *bitmap)
if (err)
return err;
+ err = -EINVAL;
sb = kmap_atomic(sb_page);
chunksize = le32_to_cpu(sb->chunksize);
@@ -573,6 +580,7 @@ static int bitmap_read_sb(struct bitmap *bitmap)
write_behind = le32_to_cpu(sb->write_behind);
sectors_reserved = le32_to_cpu(sb->sectors_reserved);
nodes = le32_to_cpu(sb->nodes);
+ strlcpy(bitmap->mddev->bitmap_info.cluster_name, sb->cluster_name, 64);
/* verify that the bitmap-specific fields are valid */
if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
@@ -609,7 +617,7 @@ static int bitmap_read_sb(struct bitmap *bitmap)
goto out;
}
events = le64_to_cpu(sb->events);
- if (events < bitmap->mddev->events) {
+ if (!nodes && (events < bitmap->mddev->events)) {
printk(KERN_INFO
"%s: bitmap file is out of date (%llu < %llu) "
"-- forcing full recovery\n",
@@ -626,8 +634,32 @@ static int bitmap_read_sb(struct bitmap *bitmap)
bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
strlcpy(bitmap->mddev->bitmap_info.cluster_name, sb->cluster_name, 64);
err = 0;
+
out:
kunmap_atomic(sb);
+ if (nodes && !cluster_setup_done) {
+ sector_t bm_blocks = sector_div(bitmap->mddev->resync_max_sectors,(chunksize >> 9));
+ bm_blocks = bm_blocks << 3;
+ /* We have bitmap supers at 4k boundaries, hence this
+ * is hardcoded */
+ bm_blocks = DIV_ROUND_UP(bm_blocks, 4096);
+ err = md_setup_cluster(bitmap->mddev, nodes);
+ if (err) {
+ pr_err("%s: Could not setup cluster service (%d)\n",
+ bmname(bitmap), err);
+ goto out_no_sb;
+ }
+ bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
+ bitmap->mddev->bitmap_info.offset +=
+ bitmap->cluster_slot * (bm_blocks << 3);
+ printk("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
+ bitmap->cluster_slot,
+ (unsigned long long)bitmap->mddev->bitmap_info.offset);
+ cluster_setup_done = 1;
+ goto re_read;
+ }
+
+
out_no_sb:
if (test_bit(BITMAP_STALE, &bitmap->flags))
bitmap->events_cleared = bitmap->mddev->events;
@@ -638,8 +670,11 @@ out_no_sb:
if (bitmap->mddev->bitmap_info.space == 0 ||
bitmap->mddev->bitmap_info.space > sectors_reserved)
bitmap->mddev->bitmap_info.space = sectors_reserved;
- if (err)
+ if (err) {
bitmap_print_sb(bitmap);
+ if (cluster_setup_done)
+ md_cluster_stop(bitmap->mddev);
+ }
return err;
}
@@ -683,14 +718,14 @@ static inline struct page *filemap_get_page(struct bitmap_storage *store,
{
if (file_page_index(store, chunk) >= store->file_pages)
return NULL;
- return store->filemap[file_page_index(store, chunk)
- - file_page_index(store, 0)];
+ return store->filemap[file_page_index(store, chunk)];
}
static int bitmap_storage_alloc(struct bitmap_storage *store,
- unsigned long chunks, int with_super)
+ unsigned long chunks, int with_super,
+ int slot_number)
{
- int pnum;
+ int pnum, offset = 0;
unsigned long num_pages;
unsigned long bytes;
@@ -699,6 +734,7 @@ static int bitmap_storage_alloc(struct bitmap_storage *store,
bytes += sizeof(bitmap_super_t);
num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
+ offset = slot_number * (num_pages - 1);
store->filemap = kmalloc(sizeof(struct page *)
* num_pages, GFP_KERNEL);
@@ -709,20 +745,22 @@ static int bitmap_storage_alloc(struct bitmap_storage *store,
store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
if (store->sb_page == NULL)
return -ENOMEM;
- store->sb_page->index = 0;
}
+
pnum = 0;
if (store->sb_page) {
store->filemap[0] = store->sb_page;
pnum = 1;
+ store->sb_page->index = offset;
}
+
for ( ; pnum < num_pages; pnum++) {
store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
if (!store->filemap[pnum]) {
store->file_pages = pnum;
return -ENOMEM;
}
- store->filemap[pnum]->index = pnum;
+ store->filemap[pnum]->index = pnum + offset;
}
store->file_pages = pnum;
@@ -935,7 +973,7 @@ static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int n
*/
static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
{
- unsigned long i, chunks, index, oldindex, bit;
+ unsigned long i, chunks, index, oldindex, bit, node_offset = 0;
struct page *page = NULL;
unsigned long bit_cnt = 0;
struct file *file;
@@ -981,6 +1019,9 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
if (!bitmap->mddev->bitmap_info.external)
offset = sizeof(bitmap_super_t);
+ if (mddev_is_clustered(bitmap->mddev))
+ node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE));
+
for (i = 0; i < chunks; i++) {
int b;
index = file_page_index(&bitmap->storage, i);
@@ -1001,7 +1042,7 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
bitmap->mddev,
bitmap->mddev->bitmap_info.offset,
page,
- index, count);
+ index + node_offset, count);
if (ret)
goto err;
@@ -1207,7 +1248,6 @@ void bitmap_daemon_work(struct mddev *mddev)
j < bitmap->storage.file_pages
&& !test_bit(BITMAP_STALE, &bitmap->flags);
j++) {
-
if (test_page_attr(bitmap, j,
BITMAP_PAGE_DIRTY))
/* bitmap_unplug will handle the rest */
@@ -1591,6 +1631,9 @@ static void bitmap_free(struct bitmap *bitmap)
if (!bitmap) /* there was no bitmap */
return;
+ if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info)
+ md_cluster_stop(bitmap->mddev);
+
/* Shouldn't be needed - but just in case.... */
wait_event(bitmap->write_wait,
atomic_read(&bitmap->pending_writes) == 0);
@@ -1847,7 +1890,8 @@ int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
memset(&store, 0, sizeof(store));
if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
ret = bitmap_storage_alloc(&store, chunks,
- !bitmap->mddev->bitmap_info.external);
+ !bitmap->mddev->bitmap_info.external,
+ bitmap->cluster_slot);
if (ret)
goto err;
diff --git a/drivers/md/bitmap.h b/drivers/md/bitmap.h
index 49163b6..2e0f540 100644
--- a/drivers/md/bitmap.h
+++ b/drivers/md/bitmap.h
@@ -227,6 +227,7 @@ struct bitmap {
wait_queue_head_t behind_wait;
struct kernfs_node *sysfs_can_clear;
+ int cluster_slot; /* Slot offset for clustered env */
};
/* the bitmap API */
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 98e45e1..43b3cda 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -199,6 +199,13 @@ static int join(struct mddev *mddev, int nodes)
if (ret)
goto err;
wait_for_completion(&cinfo->completion);
+ if (nodes <= cinfo->slot_number) {
+ pr_err("md-cluster: Slot allotted(%d) greater than"
+ "available slots(%d)", cinfo->slot_number - 1,
+ nodes);
+ ret = -ERANGE;
+ goto err;
+ }
cinfo->sb_lock = lockres_init(mddev, "cmd-super",
NULL, 0);
if (!cinfo->sb_lock) {
diff --git a/drivers/md/md-cluster.h b/drivers/md/md-cluster.h
index d9a1d88..52a21e0 100644
--- a/drivers/md/md-cluster.h
+++ b/drivers/md/md-cluster.h
@@ -11,7 +11,6 @@ struct md_cluster_operations {
int (*join)(struct mddev *mddev, int nodes);
int (*leave)(struct mddev *mddev);
int (*slot_number)(struct mddev *mddev);
-
};
#endif /* _MD_CLUSTER_H */
--
2.1.2
^ permalink raw reply related
* [PATCH 10/24] Lock bitmap while joining the cluster
From: Goldwyn Rodrigues @ 2014-12-18 16:17 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md-cluster.c | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 43b3cda..4c3ad2c 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -37,6 +37,7 @@ struct md_cluster_info {
struct completion completion;
struct dlm_lock_resource *sb_lock;
struct mutex sb_mutex;
+ struct dlm_lock_resource *bitmap_lockres;
};
static void sync_ast(void *arg)
@@ -212,6 +213,20 @@ static int join(struct mddev *mddev, int nodes)
ret = -ENOMEM;
goto err;
}
+
+ pr_info("Joined cluster %s slot %d\n", str, cinfo->slot_number);
+
+ memset(str, '\0', 64);
+ snprintf(str, 64, "bitmap%04d", cinfo->slot_number - 1);
+ cinfo->bitmap_lockres = lockres_init(mddev, str, NULL, 1);
+ if (!cinfo->bitmap_lockres)
+ goto err;
+ if (dlm_lock_sync(cinfo->bitmap_lockres, DLM_LOCK_PW)) {
+ pr_err("Failed to get bitmap lock\n");
+ ret = -EINVAL;
+ goto err;
+ }
+
return 0;
err:
if (cinfo->lockspace)
@@ -228,6 +243,7 @@ static int leave(struct mddev *mddev)
if (!cinfo)
return 0;
lockres_free(cinfo->sb_lock);
+ lockres_free(cinfo->bitmap_lockres);
dlm_release_lockspace(cinfo->lockspace, 2);
return 0;
}
--
2.1.2
^ permalink raw reply related
* [PATCH 11/24] Gather on-going resync information of other nodes
From: Goldwyn Rodrigues @ 2014-12-18 16:17 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
When a node joins, it does not know of other nodes performing resync.
So, each node keeps the resync information in it's LVB. When a new
node joins, it reads the LVB of each "online" bitmap.
[TODO] The new node attempts to get the PW lock on other bitmap, if
it is successful, it reads the bitmap and performs the resync (if
required) on it's behalf.
If the node does not get the PW, it requests CR and reads the LVB
for the resync information.
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md-cluster.c | 111 +++++++++++++++++++++++++++++++++++++++++++++++-
drivers/md/md-cluster.h | 1 +
drivers/md/md.c | 8 ++++
3 files changed, 119 insertions(+), 1 deletion(-)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 4c3ad2c..7d57f3f 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -30,6 +30,18 @@ struct dlm_lock_resource {
void (*bast)(void *arg, int mode); /* blocking AST function pointer*/
};
+struct suspend_info {
+ int slot;
+ sector_t lo;
+ sector_t hi;
+ struct list_head list;
+};
+
+struct resync_info {
+ __le64 lo;
+ __le64 hi;
+};
+
struct md_cluster_info {
/* dlm lock space and resources for clustered raid. */
dlm_lockspace_t *lockspace;
@@ -38,6 +50,8 @@ struct md_cluster_info {
struct dlm_lock_resource *sb_lock;
struct mutex sb_mutex;
struct dlm_lock_resource *bitmap_lockres;
+ struct list_head suspend_list;
+ spinlock_t suspend_lock;
};
static void sync_ast(void *arg)
@@ -143,6 +157,36 @@ static char *pretty_uuid(char *dest, char *src)
return dest;
}
+static void add_resync_info(struct mddev *mddev, struct dlm_lock_resource *lockres,
+ sector_t lo, sector_t hi)
+{
+ struct resync_info *ri;
+ ri = (struct resync_info *)lockres->lksb.sb_lvbptr;
+ ri->lo = cpu_to_le64(lo);
+ ri->hi = cpu_to_le64(hi);
+}
+
+static struct suspend_info *read_resync_info(struct mddev *mddev, struct dlm_lock_resource *lockres)
+{
+ struct resync_info ri;
+ struct suspend_info *s = NULL;
+ sector_t hi = 0;
+
+ dlm_lock_sync(lockres, DLM_LOCK_CR);
+ memcpy(&ri, lockres->lksb.sb_lvbptr, sizeof(struct resync_info));
+ hi = le64_to_cpu(ri.hi);
+ if (ri.hi > 0) {
+ s = kzalloc(sizeof(struct suspend_info), GFP_KERNEL);
+ if (!s)
+ goto out;
+ s->hi = hi;
+ s->lo = le64_to_cpu(ri.lo);
+ }
+ dlm_unlock_sync(lockres);
+out:
+ return s;
+}
+
static void recover_prep(void *arg)
{
}
@@ -174,6 +218,53 @@ static const struct dlm_lockspace_ops md_ls_ops = {
.recover_done = recover_done,
};
+static int gather_all_resync_info(struct mddev *mddev, int total_slots)
+{
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ int i, ret = 0;
+ struct dlm_lock_resource *bm_lockres;
+ struct suspend_info *s;
+ char str[64];
+
+
+ for (i = 0; i < total_slots; i++) {
+ memset(str, '\0', 64);
+ snprintf(str, 64, "bitmap%04d", i);
+ bm_lockres = lockres_init(mddev, str, NULL, 1);
+ if (!bm_lockres)
+ return -ENOMEM;
+ if (i == (cinfo->slot_number - 1))
+ continue;
+
+ bm_lockres->flags |= DLM_LKF_NOQUEUE;
+ ret = dlm_lock_sync(bm_lockres, DLM_LOCK_PW);
+ if (ret == -EAGAIN) {
+ memset(bm_lockres->lksb.sb_lvbptr, '\0', LVB_SIZE);
+ s = read_resync_info(mddev, bm_lockres);
+ if (s) {
+ pr_info("%s:%d Resync[%llu..%llu] in progress on %d\n",
+ __func__, __LINE__,
+ (unsigned long long) s->lo,
+ (unsigned long long) s->hi, i);
+ spin_lock_irq(&cinfo->suspend_lock);
+ s->slot = i;
+ list_add(&s->list, &cinfo->suspend_list);
+ spin_unlock_irq(&cinfo->suspend_lock);
+ }
+ ret = 0;
+ lockres_free(bm_lockres);
+ continue;
+ }
+ if (ret)
+ goto out;
+ /* TODO: Read the disk bitmap sb and check if it needs recovery */
+ dlm_unlock_sync(bm_lockres);
+ lockres_free(bm_lockres);
+ }
+out:
+ return ret;
+}
+
static int join(struct mddev *mddev, int nodes)
{
struct md_cluster_info *cinfo;
@@ -227,8 +318,17 @@ static int join(struct mddev *mddev, int nodes)
goto err;
}
+ INIT_LIST_HEAD(&cinfo->suspend_list);
+ spin_lock_init(&cinfo->suspend_lock);
+
+ ret = gather_all_resync_info(mddev, nodes);
+ if (ret)
+ goto err;
+
return 0;
err:
+ lockres_free(cinfo->bitmap_lockres);
+ lockres_free(cinfo->sb_lock);
if (cinfo->lockspace)
dlm_release_lockspace(cinfo->lockspace, 2);
mddev->cluster_info = NULL;
@@ -258,10 +358,19 @@ static int slot_number(struct mddev *mddev)
return cinfo->slot_number - 1;
}
+static void resync_info_update(struct mddev *mddev, sector_t lo, sector_t hi)
+{
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ add_resync_info(mddev, cinfo->bitmap_lockres, lo, hi);
+ /* Re-acquire the lock to refresh LVB */
+ dlm_lock_sync(cinfo->bitmap_lockres, DLM_LOCK_PW);
+}
+
static struct md_cluster_operations cluster_ops = {
.join = join,
.leave = leave,
- .slot_number = slot_number
+ .slot_number = slot_number,
+ .resync_info_update = resync_info_update
};
static int __init cluster_init(void)
diff --git a/drivers/md/md-cluster.h b/drivers/md/md-cluster.h
index 52a21e0..51a24df 100644
--- a/drivers/md/md-cluster.h
+++ b/drivers/md/md-cluster.h
@@ -11,6 +11,7 @@ struct md_cluster_operations {
int (*join)(struct mddev *mddev, int nodes);
int (*leave)(struct mddev *mddev);
int (*slot_number)(struct mddev *mddev);
+ void (*resync_info_update)(struct mddev *mddev, sector_t lo, sector_t hi);
};
#endif /* _MD_CLUSTER_H */
diff --git a/drivers/md/md.c b/drivers/md/md.c
index db28f21..a2c1cac 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -7618,6 +7618,9 @@ void md_do_sync(struct md_thread *thread)
md_new_event(mddev);
update_time = jiffies;
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->resync_info_update(mddev, j, max_sectors);
+
blk_start_plug(&plug);
while (j < max_sectors) {
sector_t sectors;
@@ -7678,6 +7681,8 @@ void md_do_sync(struct md_thread *thread)
j += sectors;
if (j > 2)
mddev->curr_resync = j;
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->resync_info_update(mddev, j, max_sectors);
mddev->curr_mark_cnt = io_sectors;
if (last_check == 0)
/* this is the earliest that rebuild will be
@@ -7737,6 +7742,9 @@ void md_do_sync(struct md_thread *thread)
/* tell personality that we are finished */
mddev->pers->sync_request(mddev, max_sectors, &skipped, 1);
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->resync_info_update(mddev, 0, 0);
+
if (!test_bit(MD_RECOVERY_CHECK, &mddev->recovery) &&
mddev->curr_resync > 2) {
if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
--
2.1.2
^ permalink raw reply related
* [PATCH 12/24] bitmap_create returns bitmap pointer
From: Goldwyn Rodrigues @ 2014-12-18 16:17 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
This is done to have multiple bitmaps open at the same time.
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/bitmap.c | 59 ++++++++++++++++++++++++++++++-----------------------
drivers/md/bitmap.h | 2 +-
drivers/md/md.c | 22 +++++++++++++-------
3 files changed, 50 insertions(+), 33 deletions(-)
diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c
index 23a71a7..49b7e33 100644
--- a/drivers/md/bitmap.c
+++ b/drivers/md/bitmap.c
@@ -540,7 +540,6 @@ static int bitmap_read_sb(struct bitmap *bitmap)
unsigned long sectors_reserved = 0;
int err = -EINVAL;
struct page *sb_page;
- int cluster_setup_done = 0;
if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
chunksize = 128 * 1024 * 1024;
@@ -557,6 +556,16 @@ static int bitmap_read_sb(struct bitmap *bitmap)
bitmap->storage.sb_page = sb_page;
re_read:
+ /* If cluster_slot is set, the cluster is setup */
+ if (bitmap->cluster_slot >= 0) {
+ long long bm_blocks = bitmap->mddev->resync_max_sectors / (bitmap->mddev->bitmap_info.chunksize >> 9);
+ bm_blocks = bm_blocks << 3;
+ bm_blocks = DIV_ROUND_UP(bm_blocks, 4096);
+ bitmap->mddev->bitmap_info.offset += bitmap->cluster_slot * (bm_blocks << 3);
+ pr_info("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
+ bitmap->cluster_slot, (unsigned long long)bitmap->mddev->bitmap_info.offset);
+ }
+
if (bitmap->storage.file) {
loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
@@ -637,12 +646,9 @@ re_read:
out:
kunmap_atomic(sb);
- if (nodes && !cluster_setup_done) {
- sector_t bm_blocks = sector_div(bitmap->mddev->resync_max_sectors,(chunksize >> 9));
- bm_blocks = bm_blocks << 3;
- /* We have bitmap supers at 4k boundaries, hence this
- * is hardcoded */
- bm_blocks = DIV_ROUND_UP(bm_blocks, 4096);
+ /* Assiging chunksize is required for "re_read" */
+ bitmap->mddev->bitmap_info.chunksize = chunksize;
+ if (nodes && (bitmap->cluster_slot < 0)) {
err = md_setup_cluster(bitmap->mddev, nodes);
if (err) {
pr_err("%s: Could not setup cluster service (%d)\n",
@@ -650,12 +656,6 @@ out:
goto out_no_sb;
}
bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
- bitmap->mddev->bitmap_info.offset +=
- bitmap->cluster_slot * (bm_blocks << 3);
- printk("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
- bitmap->cluster_slot,
- (unsigned long long)bitmap->mddev->bitmap_info.offset);
- cluster_setup_done = 1;
goto re_read;
}
@@ -672,7 +672,7 @@ out_no_sb:
bitmap->mddev->bitmap_info.space = sectors_reserved;
if (err) {
bitmap_print_sb(bitmap);
- if (cluster_setup_done)
+ if (bitmap->cluster_slot < 0)
md_cluster_stop(bitmap->mddev);
}
return err;
@@ -1631,7 +1631,8 @@ static void bitmap_free(struct bitmap *bitmap)
if (!bitmap) /* there was no bitmap */
return;
- if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info)
+ if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info &&
+ bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev))
md_cluster_stop(bitmap->mddev);
/* Shouldn't be needed - but just in case.... */
@@ -1677,7 +1678,7 @@ void bitmap_destroy(struct mddev *mddev)
* initialize the bitmap structure
* if this returns an error, bitmap_destroy must be called to do clean up
*/
-int bitmap_create(struct mddev *mddev)
+struct bitmap *bitmap_create(struct mddev *mddev, int slot)
{
struct bitmap *bitmap;
sector_t blocks = mddev->resync_max_sectors;
@@ -1691,7 +1692,7 @@ int bitmap_create(struct mddev *mddev)
bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
if (!bitmap)
- return -ENOMEM;
+ return ERR_PTR(-ENOMEM);
spin_lock_init(&bitmap->counts.lock);
atomic_set(&bitmap->pending_writes, 0);
@@ -1700,6 +1701,7 @@ int bitmap_create(struct mddev *mddev)
init_waitqueue_head(&bitmap->behind_wait);
bitmap->mddev = mddev;
+ bitmap->cluster_slot = slot;
if (mddev->kobj.sd)
bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
@@ -1747,12 +1749,14 @@ int bitmap_create(struct mddev *mddev)
printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
bitmap->counts.pages, bmname(bitmap));
- mddev->bitmap = bitmap;
- return test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
+ err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
+ if (err)
+ goto error;
+ return bitmap;
error:
bitmap_free(bitmap);
- return err;
+ return ERR_PTR(err);
}
int bitmap_load(struct mddev *mddev)
@@ -2064,13 +2068,18 @@ location_store(struct mddev *mddev, const char *buf, size_t len)
return -EINVAL;
mddev->bitmap_info.offset = offset;
if (mddev->pers) {
+ struct bitmap *bitmap;
mddev->pers->quiesce(mddev, 1);
- rv = bitmap_create(mddev);
- if (!rv)
+ bitmap = bitmap_create(mddev, -1);
+ if (IS_ERR(bitmap))
+ rv = PTR_ERR(bitmap);
+ else {
+ mddev->bitmap = bitmap;
rv = bitmap_load(mddev);
- if (rv) {
- bitmap_destroy(mddev);
- mddev->bitmap_info.offset = 0;
+ if (rv) {
+ bitmap_destroy(mddev);
+ mddev->bitmap_info.offset = 0;
+ }
}
mddev->pers->quiesce(mddev, 0);
if (rv)
diff --git a/drivers/md/bitmap.h b/drivers/md/bitmap.h
index 2e0f540..ac3fc40 100644
--- a/drivers/md/bitmap.h
+++ b/drivers/md/bitmap.h
@@ -233,7 +233,7 @@ struct bitmap {
/* the bitmap API */
/* these are used only by md/bitmap */
-int bitmap_create(struct mddev *mddev);
+struct bitmap *bitmap_create(struct mddev *mddev, int slot);
int bitmap_load(struct mddev *mddev);
void bitmap_flush(struct mddev *mddev);
void bitmap_destroy(struct mddev *mddev);
diff --git a/drivers/md/md.c b/drivers/md/md.c
index a2c1cac..6ac6609 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -5087,12 +5087,15 @@ int md_run(struct mddev *mddev)
}
if (err == 0 && mddev->pers->sync_request &&
(mddev->bitmap_info.file || mddev->bitmap_info.offset)) {
- err = bitmap_create(mddev);
- if (err) {
+ struct bitmap *bitmap = bitmap_create(mddev, -1);
+ if (IS_ERR(bitmap)) {
+ err = PTR_ERR(bitmap);
printk(KERN_ERR "%s: failed to create bitmap (%d)\n",
mdname(mddev), err);
mddev->pers->stop(mddev);
- }
+ } else
+ mddev->bitmap = bitmap;
+
}
if (err) {
module_put(mddev->pers->owner);
@@ -6028,9 +6031,11 @@ static int set_bitmap_file(struct mddev *mddev, int fd)
if (mddev->pers) {
mddev->pers->quiesce(mddev, 1);
if (fd >= 0) {
- err = bitmap_create(mddev);
- if (!err)
+ struct bitmap *bitmap = bitmap_create(mddev, -1);
+ if (!IS_ERR(bitmap)) {
+ mddev->bitmap = bitmap;
err = bitmap_load(mddev);
+ }
}
if (fd < 0 || err) {
bitmap_destroy(mddev);
@@ -6287,6 +6292,7 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info)
if (mddev->recovery || mddev->sync_thread)
return -EBUSY;
if (info->state & (1<<MD_SB_BITMAP_PRESENT)) {
+ struct bitmap *bitmap;
/* add the bitmap */
if (mddev->bitmap)
return -EEXIST;
@@ -6297,9 +6303,11 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info)
mddev->bitmap_info.space =
mddev->bitmap_info.default_space;
mddev->pers->quiesce(mddev, 1);
- rv = bitmap_create(mddev);
- if (!rv)
+ bitmap = bitmap_create(mddev, -1);
+ if (!IS_ERR(bitmap)) {
+ mddev->bitmap = bitmap;
rv = bitmap_load(mddev);
+ }
if (rv)
bitmap_destroy(mddev);
mddev->pers->quiesce(mddev, 0);
--
2.1.2
^ permalink raw reply related
* [PATCH 13/24] Copy set bits from another slot
From: Goldwyn Rodrigues @ 2014-12-18 16:17 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
bitmap_copy_from_slot reads the bitmap from the slot mentioned.
It then copies the set bits to the node local bitmap.
This is helper function for the resync operation on node failure.
bitmap_set_memory_bits() currently assumes it is only run at startup and that
they bitmap is currently empty. So if it finds that a region is already
marked as dirty, it won't mark it dirty again. Change bitmap_set_memory_bits()
to always set the NEEDED_MASK bit if 'needed' is set.
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/bitmap.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++-
drivers/md/bitmap.h | 2 ++
2 files changed, 78 insertions(+), 1 deletion(-)
diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c
index 49b7e33..6f7e178 100644
--- a/drivers/md/bitmap.c
+++ b/drivers/md/bitmap.c
@@ -919,6 +919,28 @@ static void bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
}
}
+static int bitmap_file_test_bit(struct bitmap *bitmap, sector_t block)
+{
+ unsigned long bit;
+ struct page *page;
+ void *paddr;
+ unsigned long chunk = block >> bitmap->counts.chunkshift;
+ int set = 0;
+
+ page = filemap_get_page(&bitmap->storage, chunk);
+ if (!page)
+ return -EINVAL;
+ bit = file_page_offset(&bitmap->storage, chunk);
+ paddr = kmap_atomic(page);
+ if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
+ set = test_bit(bit, paddr);
+ else
+ set = test_bit_le(bit, paddr);
+ kunmap_atomic(paddr);
+ return set;
+}
+
+
/* this gets called when the md device is ready to unplug its underlying
* (slave) device queues -- before we let any writes go down, we need to
* sync the dirty pages of the bitmap file to disk */
@@ -1570,11 +1592,13 @@ static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int n
return;
}
if (!*bmc) {
- *bmc = 2 | (needed ? NEEDED_MASK : 0);
+ *bmc = 2;
bitmap_count_page(&bitmap->counts, offset, 1);
bitmap_set_pending(&bitmap->counts, offset);
bitmap->allclean = 0;
}
+ if (needed)
+ *bmc |= NEEDED_MASK;
spin_unlock_irq(&bitmap->counts.lock);
}
@@ -1810,6 +1834,57 @@ out:
}
EXPORT_SYMBOL_GPL(bitmap_load);
+/* Loads the bitmap associated with slot and copies the resync information
+ * to our bitmap
+ */
+int bitmap_copy_from_slot(struct mddev *mddev, int slot,
+ sector_t *low, sector_t *high)
+{
+ int rv = 0, i, j;
+ sector_t block, lo = 0, hi = 0;
+ struct bitmap_counts *counts;
+ struct bitmap *bitmap = bitmap_create(mddev, slot);
+ if (IS_ERR(bitmap))
+ return PTR_ERR(bitmap);
+ else
+ rv = bitmap_read_sb(bitmap);
+ if (rv)
+ goto err;
+
+ rv = bitmap_init_from_disk(bitmap, 0);
+ if (rv)
+ goto err;
+
+ counts = &bitmap->counts;
+ for (j = 0; j < counts->chunks; j++) {
+ block = (sector_t)j << counts->chunkshift;
+ if (bitmap_file_test_bit(bitmap, block)) {
+ if (!lo)
+ lo = block;
+ hi = block;
+ bitmap_file_clear_bit(bitmap, block);
+ bitmap_set_memory_bits(mddev->bitmap, block, 1);
+ bitmap_file_set_bit(mddev->bitmap, block);
+ }
+ }
+
+ bitmap_update_sb(bitmap);
+ /* Setting this for the ev_page should be enough.
+ * And we do not require both write_all and PAGE_DIRT either
+ */
+ for (i = 0; i < bitmap->storage.file_pages; i++)
+ set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
+ bitmap_write_all(bitmap);
+ bitmap_unplug(bitmap);
+ *low = lo;
+ *high = hi;
+err:
+ bitmap_free(bitmap);
+ return rv;
+}
+EXPORT_SYMBOL_GPL(bitmap_copy_from_slot);
+
+
void bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
{
unsigned long chunk_kb;
diff --git a/drivers/md/bitmap.h b/drivers/md/bitmap.h
index ac3fc40..123ef8c 100644
--- a/drivers/md/bitmap.h
+++ b/drivers/md/bitmap.h
@@ -262,6 +262,8 @@ void bitmap_daemon_work(struct mddev *mddev);
int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
int chunksize, int init);
+int bitmap_copy_from_slot(struct mddev *mddev, int slot,
+ sector_t *lo, sector_t *hi);
#endif
#endif
--
2.1.2
^ permalink raw reply related
* [PATCH 14/24] Initiate recovery on node failure
From: Goldwyn Rodrigues @ 2014-12-18 16:17 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
The DLM informs us in case of node failure with the DLM slot number.
cluster_info->recovery_map sets the bit corresponding to the slot number
and wakes up the recovery thread.
The recovery thread:
1. Derives the slot number from the recovery_map
2. Locks the bitmap corresponding to the slot
3. Copies the set bits to the node-local bitmap
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md-cluster.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 59 insertions(+)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 7d57f3f..4f1ea5f 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -16,6 +16,7 @@
#include <linux/dlm.h>
#include <linux/sched.h>
#include "md.h"
+#include "bitmap.h"
#include "md-cluster.h"
#define LVB_SIZE 64
@@ -52,6 +53,8 @@ struct md_cluster_info {
struct dlm_lock_resource *bitmap_lockres;
struct list_head suspend_list;
spinlock_t suspend_lock;
+ struct md_thread *recovery_thread;
+ unsigned long recovery_map;
};
static void sync_ast(void *arg)
@@ -187,6 +190,51 @@ out:
return s;
}
+void recover_bitmaps(struct md_thread *thread)
+{
+ struct mddev *mddev = thread->mddev;
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ struct dlm_lock_resource *bm_lockres;
+ char str[64];
+ int slot, ret;
+ struct suspend_info *s, *tmp;
+ sector_t lo, hi;
+
+ while (cinfo->recovery_map) {
+ slot = fls64((u64)cinfo->recovery_map) - 1;
+
+ /* Clear suspend_area associated with the bitmap */
+ spin_lock_irq(&cinfo->suspend_lock);
+ list_for_each_entry_safe(s, tmp, &cinfo->suspend_list, list)
+ if (slot == s->slot) {
+ list_del(&s->list);
+ kfree(s);
+ }
+ spin_unlock_irq(&cinfo->suspend_lock);
+
+ snprintf(str, 64, "bitmap%04d", slot);
+ bm_lockres = lockres_init(mddev, str, NULL, 1);
+ if (!bm_lockres) {
+ pr_err("md-cluster: Cannot initialize bitmaps\n");
+ goto clear_bit;
+ }
+
+ ret = dlm_lock_sync(bm_lockres, DLM_LOCK_PW);
+ if (ret) {
+ pr_err("md-cluster: Could not DLM lock %s: %d\n",
+ str, ret);
+ goto clear_bit;
+ }
+ ret = bitmap_copy_from_slot(mddev, slot, &lo, &hi);
+ if (ret)
+ pr_err("md-cluster: Could not copy data "
+ "from bitmap %d\n", slot);
+ dlm_unlock_sync(bm_lockres);
+clear_bit:
+ clear_bit(slot, &cinfo->recovery_map);
+ }
+}
+
static void recover_prep(void *arg)
{
}
@@ -200,6 +248,16 @@ static void recover_slot(void *arg, struct dlm_slot *slot)
mddev->bitmap_info.cluster_name,
slot->nodeid, slot->slot,
cinfo->slot_number);
+ set_bit(slot->slot - 1, &cinfo->recovery_map);
+ if (!cinfo->recovery_thread) {
+ cinfo->recovery_thread = md_register_thread(recover_bitmaps,
+ mddev, "recover");
+ if (!cinfo->recovery_thread) {
+ printk("md-cluster: Could not create recovery thread\n");
+ return;
+ }
+ }
+ md_wakeup_thread(cinfo->recovery_thread);
}
static void recover_done(void *arg, struct dlm_slot *slots,
@@ -342,6 +400,7 @@ static int leave(struct mddev *mddev)
struct md_cluster_info *cinfo = mddev->cluster_info;
if (!cinfo)
return 0;
+ md_unregister_thread(&cinfo->recovery_thread);
lockres_free(cinfo->sb_lock);
lockres_free(cinfo->bitmap_lockres);
dlm_release_lockspace(cinfo->lockspace, 2);
--
2.1.2
^ permalink raw reply related
* [PATCH 15/24] Perform resync for cluster node failure
From: Goldwyn Rodrigues @ 2014-12-18 16:17 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
If bitmap_copy_slot returns hi>0, we need to perform resync.
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md-cluster.c | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 4f1ea5f..617ffee 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -226,9 +226,19 @@ void recover_bitmaps(struct md_thread *thread)
goto clear_bit;
}
ret = bitmap_copy_from_slot(mddev, slot, &lo, &hi);
- if (ret)
+ if (ret) {
pr_err("md-cluster: Could not copy data "
"from bitmap %d\n", slot);
+ goto dlm_unlock;
+ }
+ if (hi > 0) {
+ /* TODO:Wait for current resync to get over */
+ set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
+ if (lo < mddev->recovery_cp)
+ mddev->recovery_cp = lo;
+ md_check_recovery(mddev);
+ }
+dlm_unlock:
dlm_unlock_sync(bm_lockres);
clear_bit:
clear_bit(slot, &cinfo->recovery_map);
--
2.1.2
^ permalink raw reply related
* [PATCH 16/24] Communication Framework: Receiving
From: Goldwyn Rodrigues @ 2014-12-18 16:18 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
1. receive status
sender receiver receiver
ACK:CR ACK:CR ACK:CR
2. sender get EX of TOKEN
sender get EX of MESSAGE
sender receiver receiver
TOKEN:EX ACK:CR ACK:CR
MESSAGE:EX
ACK:CR
3. sender write LVB.
sender down-convert MESSAGE from EX to CR
sender try to get EX of ACK
[ wait until all receiver has *processed* the MESSAGE ]
[ triggered by bast of ACK ]
receiver get CR of MESSAGE
receiver read LVB
receiver processes the message
[ wait finish ]
receiver release ACK
sender receiver receiver
TOKEN:EX MESSAGE:CR MESSAGE:CR
MESSAGE:CR
ACK:EX
4. sender down-convert ACK from EX to CR
sender release MESSAGE
sender release TOKEN
receiver upconvert to EX of MESSAGE
receiver get CR of ACK
receiver release MESSAGE
sender receiver receiver
ACK:CR ACK:CR ACK:CR
Signed-off-by: Lidong Zhong <lzhong@suse.com>
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md-cluster.c | 101 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 101 insertions(+)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 617ffee..259d4c2 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -55,6 +55,23 @@ struct md_cluster_info {
spinlock_t suspend_lock;
struct md_thread *recovery_thread;
unsigned long recovery_map;
+ /* communication loc resources */
+ struct dlm_lock_resource *ack_lockres;
+ struct dlm_lock_resource *message_lockres;
+ struct dlm_lock_resource *token_lockres;
+ struct md_thread *recv_thread;
+};
+
+enum msg_type {
+ METADATA_UPDATED = 0,
+ RESYNCING
+};
+
+struct cluster_msg {
+ int type;
+ int slot;
+ sector_t low;
+ sector_t high;
};
static void sync_ast(void *arg)
@@ -286,6 +303,63 @@ static const struct dlm_lockspace_ops md_ls_ops = {
.recover_done = recover_done,
};
+/*
+ * The BAST function for the ack lock resource
+ * This function wakes up the receive thread in
+ * order to receive and process the message.
+ */
+static void ack_bast(void *arg, int mode)
+{
+ struct dlm_lock_resource *res = (struct dlm_lock_resource *)arg;
+ struct md_cluster_info *cinfo = res->mddev->cluster_info;
+ if (mode == DLM_LOCK_EX)
+ md_wakeup_thread(cinfo->recv_thread);
+}
+
+static void process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg)
+{
+ switch (msg->type) {
+ case METADATA_UPDATED:
+ pr_info("%s: %d Received message: METADATA_UPDATE from %d\n",
+ __func__, __LINE__, msg->slot);
+ break;
+ case RESYNCING:
+ pr_info("%s: %d Received message: RESYNCING from %d\n",
+ __func__, __LINE__, msg->slot);
+ break;
+ };
+}
+
+/*
+ * thread for receiving message
+ */
+static void recv_daemon(struct md_thread *thread)
+{
+ struct md_cluster_info *cinfo = thread->mddev->cluster_info;
+ struct dlm_lock_resource *ack_lockres = cinfo->ack_lockres;
+ struct dlm_lock_resource *message_lockres = cinfo->message_lockres;
+ struct cluster_msg msg;
+
+ /*get CR on Message*/
+ if (dlm_lock_sync(message_lockres, DLM_LOCK_CR)) {
+ pr_err("md/raid1:failed to get CR on MESSAGE\n");
+ return;
+ }
+
+ /* read lvb and wake up thread to process this message_lockres */
+ memcpy(&msg, message_lockres->lksb.sb_lvbptr, sizeof(struct cluster_msg));
+ process_recvd_msg(thread->mddev, &msg);
+
+ /*release CR on ack_lockres*/
+ dlm_unlock_sync(ack_lockres);
+ /*up-convert to EX on message_lockres*/
+ dlm_lock_sync(message_lockres, DLM_LOCK_EX);
+ /*get CR on ack_lockres again*/
+ dlm_lock_sync(ack_lockres, DLM_LOCK_CR);
+ /*release CR on message_lockres*/
+ dlm_unlock_sync(message_lockres);
+}
+
static int gather_all_resync_info(struct mddev *mddev, int total_slots)
{
struct md_cluster_info *cinfo = mddev->cluster_info;
@@ -372,6 +446,26 @@ static int join(struct mddev *mddev, int nodes)
ret = -ENOMEM;
goto err;
}
+ /* Initiate the communication resources */
+ ret = -ENOMEM;
+ cinfo->recv_thread = md_register_thread(recv_daemon, mddev, "cluster_recv");
+ if (!cinfo->recv_thread) {
+ pr_err("md-cluster: cannot allocate memory for recv_thread!\n");
+ goto err;
+ }
+ cinfo->message_lockres = lockres_init(mddev, "message", NULL, 1);
+ if (!cinfo->message_lockres)
+ goto err;
+ cinfo->token_lockres = lockres_init(mddev, "token", NULL, 0);
+ if (!cinfo->token_lockres)
+ goto err;
+ cinfo->ack_lockres = lockres_init(mddev, "ack", ack_bast, 0);
+ if (!cinfo->ack_lockres)
+ goto err;
+ /* get sync CR lock on ACK. */
+ if (dlm_lock_sync(cinfo->ack_lockres, DLM_LOCK_CR))
+ pr_err("md-cluster: failed to get a sync CR lock on ACK!(%d)\n",
+ ret);
pr_info("Joined cluster %s slot %d\n", str, cinfo->slot_number);
@@ -395,6 +489,9 @@ static int join(struct mddev *mddev, int nodes)
return 0;
err:
+ lockres_free(cinfo->message_lockres);
+ lockres_free(cinfo->token_lockres);
+ lockres_free(cinfo->ack_lockres);
lockres_free(cinfo->bitmap_lockres);
lockres_free(cinfo->sb_lock);
if (cinfo->lockspace)
@@ -411,6 +508,10 @@ static int leave(struct mddev *mddev)
if (!cinfo)
return 0;
md_unregister_thread(&cinfo->recovery_thread);
+ md_unregister_thread(&cinfo->recv_thread);
+ lockres_free(cinfo->message_lockres);
+ lockres_free(cinfo->token_lockres);
+ lockres_free(cinfo->ack_lockres);
lockres_free(cinfo->sb_lock);
lockres_free(cinfo->bitmap_lockres);
dlm_release_lockspace(cinfo->lockspace, 2);
--
2.1.2
^ permalink raw reply related
* [PATCH 17/24] Communication Framework: Sending functions
From: Goldwyn Rodrigues @ 2014-12-18 16:18 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
The sending part is split in two functions to make sure
atomicity of the operations, such as the MD superblock update.
Signed-off-by: Lidong Zhong <lzhong@suse.com>
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md-cluster.c | 84 +++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 84 insertions(+)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 259d4c2..1e11a9b 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -360,6 +360,90 @@ static void recv_daemon(struct md_thread *thread)
dlm_unlock_sync(message_lockres);
}
+/* lock_comm()
+ * Takes the lock on the TOKEN lock resource so no other
+ * node can communicate while the operation is underway.
+ */
+static int lock_comm(struct md_cluster_info *cinfo)
+{
+ int error;
+ error = dlm_lock_sync(cinfo->token_lockres, DLM_LOCK_EX);
+ if (error)
+ pr_err("md-cluster(%s:%d): failed to get EX on "
+ "TOKEN (%d)\n", __func__, __LINE__, error);
+ return error;
+}
+
+static void unlock_comm(struct md_cluster_info *cinfo)
+{
+ dlm_unlock_sync(cinfo->token_lockres);
+}
+
+/* __sendmsg()
+ * This function performs the actual sending of the message. This function is
+ * usually called after performing the encompassing operation
+ * The function:
+ * 1. Grabs the message lockresource in EX mode
+ * 2. Copies the message to the message LVB
+ * 3. Downconverts message lockresource to CR
+ * 4. Upconverts ack lock resource from CR to EX. This forces the BAST on other nodes
+ * and the other nodes read the message. The thread will wait here until all other
+ * nodes have released ack lock resource.
+ * 5. Downconvert ack lockresource to CR
+ */
+static int __sendmsg(struct md_cluster_info *cinfo, struct cluster_msg *cmsg)
+{
+ int error;
+ int slot = cinfo->slot_number - 1;
+ cmsg->slot = cpu_to_le32(slot);
+ /*get EX on Message*/
+ error = dlm_lock_sync(cinfo->message_lockres, DLM_LOCK_EX);
+ if (error) {
+ pr_err("md-cluster: failed to get EX on MESSAGE (%d)\n", error);
+ goto failed_message;
+ }
+
+ memcpy(cinfo->message_lockres->lksb.sb_lvbptr, (void *)cmsg,
+ sizeof(struct cluster_msg));
+ /*down-convert EX to CR on Message*/
+ error = dlm_lock_sync(cinfo->message_lockres, DLM_LOCK_CR);
+ if (error) {
+ pr_err("md-cluster: failed to convert EX to CR "
+ "on MESSAGE(%d)\n", error);
+ goto failed_message;
+ }
+
+ /*up-convert CR to EX on Ack*/
+ error = dlm_lock_sync(cinfo->ack_lockres, DLM_LOCK_EX);
+ if (error) {
+ pr_err("md-cluster: failed to convert CR to EX "
+ "on ACK(%d)\n", error);
+ goto failed_ack;
+ }
+
+ /*down-convert EX to CR on Ack*/
+ error = dlm_lock_sync(cinfo->ack_lockres, DLM_LOCK_CR);
+ if (error) {
+ pr_err("md-cluster: failed to convert EX to CR "
+ "on ACK(%d)\n", error);
+ goto failed_ack;
+ }
+
+failed_ack:
+ dlm_unlock_sync(cinfo->message_lockres);
+failed_message:
+ return error;
+}
+
+static int sendmsg(struct md_cluster_info *cinfo, struct cluster_msg *cmsg)
+{
+ int ret;
+ lock_comm(cinfo);
+ ret = __sendmsg(cinfo, cmsg);
+ unlock_comm(cinfo);
+ return ret;
+}
+
static int gather_all_resync_info(struct mddev *mddev, int total_slots)
{
struct md_cluster_info *cinfo = mddev->cluster_info;
--
2.1.2
^ permalink raw reply related
* [PATCH 18/24] metadata_update sends message to other nodes
From: Goldwyn Rodrigues @ 2014-12-18 16:18 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
- request to send a message
- make changes to superblock
- send messages telling everyone that the superblock has changed
- other nodes all read the superblock
- other nodes all ack the messages
- updating node release the "I'm sending a message" resource.
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md-cluster.c | 28 ++++++++++++++-
drivers/md/md-cluster.h | 3 ++
drivers/md/md.c | 91 +++++++++++++++++++++++++++++++++++++++++--------
3 files changed, 106 insertions(+), 16 deletions(-)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 1e11a9b..db81ef1 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -620,11 +620,37 @@ static void resync_info_update(struct mddev *mddev, sector_t lo, sector_t hi)
dlm_lock_sync(cinfo->bitmap_lockres, DLM_LOCK_PW);
}
+static int metadata_update_start(struct mddev *mddev)
+{
+ return lock_comm(mddev->cluster_info);
+}
+
+static int metadata_update_finish(struct mddev *mddev)
+{
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ struct cluster_msg cmsg;
+ int ret;
+ memset(&cmsg, 0, sizeof(cmsg));
+ cmsg.type = cpu_to_le32(METADATA_UPDATED);
+ ret = __sendmsg(cinfo, &cmsg);
+ unlock_comm(cinfo);
+ return ret;
+}
+
+static int metadata_update_cancel(struct mddev *mddev)
+{
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ return dlm_unlock_sync(cinfo->token_lockres);
+}
+
static struct md_cluster_operations cluster_ops = {
.join = join,
.leave = leave,
.slot_number = slot_number,
- .resync_info_update = resync_info_update
+ .resync_info_update = resync_info_update,
+ .metadata_update_start = metadata_update_start,
+ .metadata_update_finish = metadata_update_finish,
+ .metadata_update_cancel = metadata_update_cancel
};
static int __init cluster_init(void)
diff --git a/drivers/md/md-cluster.h b/drivers/md/md-cluster.h
index 51a24df..658982a 100644
--- a/drivers/md/md-cluster.h
+++ b/drivers/md/md-cluster.h
@@ -12,6 +12,9 @@ struct md_cluster_operations {
int (*leave)(struct mddev *mddev);
int (*slot_number)(struct mddev *mddev);
void (*resync_info_update)(struct mddev *mddev, sector_t lo, sector_t hi);
+ int (*metadata_update_start)(struct mddev *mddev);
+ int (*metadata_update_finish)(struct mddev *mddev);
+ int (*metadata_update_cancel)(struct mddev *mddev);
};
#endif /* _MD_CLUSTER_H */
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 6ac6609..2288137 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -2639,10 +2639,14 @@ state_store(struct md_rdev *rdev, const char *buf, size_t len)
err = -EBUSY;
else {
struct mddev *mddev = rdev->mddev;
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_start(mddev);
kick_rdev_from_array(rdev);
if (mddev->pers)
md_update_sb(mddev, 1);
md_new_event(mddev);
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_finish(mddev);
err = 0;
}
} else if (cmd_match(buf, "writemostly")) {
@@ -4092,8 +4096,12 @@ size_store(struct mddev *mddev, const char *buf, size_t len)
if (err < 0)
return err;
if (mddev->pers) {
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_start(mddev);
err = update_size(mddev, sectors);
md_update_sb(mddev, 1);
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_finish(mddev);
} else {
if (mddev->dev_sectors == 0 ||
mddev->dev_sectors > sectors)
@@ -5263,6 +5271,8 @@ static void md_clean(struct mddev *mddev)
static void __md_stop_writes(struct mddev *mddev)
{
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_start(mddev);
set_bit(MD_RECOVERY_FROZEN, &mddev->recovery);
if (mddev->sync_thread) {
set_bit(MD_RECOVERY_INTR, &mddev->recovery);
@@ -5280,6 +5290,8 @@ static void __md_stop_writes(struct mddev *mddev)
mddev->in_sync = 1;
md_update_sb(mddev, 1);
}
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_finish(mddev);
}
void md_stop_writes(struct mddev *mddev)
@@ -5901,6 +5913,9 @@ static int hot_remove_disk(struct mddev * mddev, dev_t dev)
if (!rdev)
return -ENXIO;
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_start(mddev);
+
clear_bit(Blocked, &rdev->flags);
remove_and_add_spares(mddev, rdev);
@@ -5911,8 +5926,13 @@ static int hot_remove_disk(struct mddev * mddev, dev_t dev)
md_update_sb(mddev, 1);
md_new_event(mddev);
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_finish(mddev);
+
return 0;
busy:
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_cancel(mddev);
printk(KERN_WARNING "md: cannot remove active disk %s from %s ...\n",
bdevname(rdev->bdev,b), mdname(mddev));
return -EBUSY;
@@ -5962,12 +5982,15 @@ static int hot_add_disk(struct mddev * mddev, dev_t dev)
err = -EINVAL;
goto abort_export;
}
+
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_start(mddev);
clear_bit(In_sync, &rdev->flags);
rdev->desc_nr = -1;
rdev->saved_raid_disk = -1;
err = bind_rdev_to_array(rdev, mddev);
if (err)
- goto abort_export;
+ goto abort_clustered;
/*
* The rest should better be atomic, we can have disk failures
@@ -5978,6 +6001,8 @@ static int hot_add_disk(struct mddev * mddev, dev_t dev)
md_update_sb(mddev, 1);
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_finish(mddev);
/*
* Kick recovery, maybe this spare has to be added to the
* array immediately.
@@ -5987,6 +6012,9 @@ static int hot_add_disk(struct mddev * mddev, dev_t dev)
md_new_event(mddev);
return 0;
+abort_clustered:
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_cancel(mddev);
abort_export:
export_rdev(rdev);
return err;
@@ -6231,7 +6259,7 @@ static int update_raid_disks(struct mddev *mddev, int raid_disks)
*/
static int update_array_info(struct mddev *mddev, mdu_array_info_t *info)
{
- int rv = 0;
+ int rv = -EINVAL;
int cnt = 0;
int state = 0;
@@ -6280,6 +6308,8 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info)
return rv;
}
}
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_start(mddev);
if (info->size >= 0 && mddev->dev_sectors / 2 != info->size)
rv = update_size(mddev, (sector_t)info->size * 2);
@@ -6287,17 +6317,25 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info)
rv = update_raid_disks(mddev, info->raid_disks);
if ((state ^ info->state) & (1<<MD_SB_BITMAP_PRESENT)) {
- if (mddev->pers->quiesce == NULL)
- return -EINVAL;
- if (mddev->recovery || mddev->sync_thread)
- return -EBUSY;
+ if (mddev->pers->quiesce == NULL) {
+ rv = -EINVAL;
+ goto err;
+ }
+ if (mddev->recovery || mddev->sync_thread) {
+ rv = -EBUSY;
+ goto err;
+ }
if (info->state & (1<<MD_SB_BITMAP_PRESENT)) {
struct bitmap *bitmap;
/* add the bitmap */
- if (mddev->bitmap)
- return -EEXIST;
- if (mddev->bitmap_info.default_offset == 0)
- return -EINVAL;
+ if (mddev->bitmap) {
+ rv = -EEXIST;
+ goto err;
+ }
+ if (mddev->bitmap_info.default_offset == 0) {
+ rv = -EINVAL;
+ goto err;
+ }
mddev->bitmap_info.offset =
mddev->bitmap_info.default_offset;
mddev->bitmap_info.space =
@@ -6313,10 +6351,14 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info)
mddev->pers->quiesce(mddev, 0);
} else {
/* remove the bitmap */
- if (!mddev->bitmap)
- return -ENOENT;
- if (mddev->bitmap->storage.file)
- return -EINVAL;
+ if (!mddev->bitmap) {
+ rv = -ENOENT;
+ goto err;
+ }
+ if (mddev->bitmap->storage.file) {
+ rv = -EINVAL;
+ goto err;
+ }
mddev->pers->quiesce(mddev, 1);
bitmap_destroy(mddev);
mddev->pers->quiesce(mddev, 0);
@@ -6324,6 +6366,12 @@ static int update_array_info(struct mddev *mddev, mdu_array_info_t *info)
}
}
md_update_sb(mddev, 1);
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_finish(mddev);
+ return rv;
+err:
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_cancel(mddev);
return rv;
}
@@ -7425,7 +7473,11 @@ int md_allow_write(struct mddev *mddev)
mddev->safemode == 0)
mddev->safemode = 1;
spin_unlock_irq(&mddev->write_lock);
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_start(mddev);
md_update_sb(mddev, 0);
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_finish(mddev);
sysfs_notify_dirent_safe(mddev->sysfs_state);
} else
spin_unlock_irq(&mddev->write_lock);
@@ -7951,8 +8003,13 @@ void md_check_recovery(struct mddev *mddev)
sysfs_notify_dirent_safe(mddev->sysfs_state);
}
- if (mddev->flags & MD_UPDATE_SB_FLAGS)
+ if (mddev->flags & MD_UPDATE_SB_FLAGS) {
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_start(mddev);
md_update_sb(mddev, 0);
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_finish(mddev);
+ }
if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) &&
!test_bit(MD_RECOVERY_DONE, &mddev->recovery)) {
@@ -8061,6 +8118,8 @@ void md_reap_sync_thread(struct mddev *mddev)
set_bit(MD_CHANGE_DEVS, &mddev->flags);
}
}
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_start(mddev);
if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
mddev->pers->finish_reshape)
mddev->pers->finish_reshape(mddev);
@@ -8073,6 +8132,8 @@ void md_reap_sync_thread(struct mddev *mddev)
rdev->saved_raid_disk = -1;
md_update_sb(mddev, 1);
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->metadata_update_finish(mddev);
clear_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
clear_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
--
2.1.2
^ permalink raw reply related
* [PATCH 19/24] Reload superblock if METADATA_UPDATED is received
From: Goldwyn Rodrigues @ 2014-12-18 16:18 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
Re-reads the devices by invalidating the cache.
Since we don't write to faulty devices, this is detected using
events recorded in the devices. If it is old as compared to the mddev
mark it is faulty.
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md-cluster.c | 1 +
drivers/md/md.c | 21 +++++++++++++++++++++
drivers/md/md.h | 1 +
3 files changed, 23 insertions(+)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index db81ef1..084834d 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -322,6 +322,7 @@ static void process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg)
case METADATA_UPDATED:
pr_info("%s: %d Received message: METADATA_UPDATE from %d\n",
__func__, __LINE__, msg->slot);
+ md_reload_sb(mddev);
break;
case RESYNCING:
pr_info("%s: %d Received message: RESYNCING from %d\n",
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 2288137..829086c 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -8752,6 +8752,27 @@ err_wq:
return ret;
}
+void md_reload_sb(struct mddev *mddev)
+{
+ struct md_rdev *rdev, *tmp;
+ rdev_for_each_safe(rdev, tmp, mddev) {
+ rdev->sb_loaded = 0;
+ ClearPageUptodate(rdev->sb_page);
+ }
+ mddev->raid_disks = 0;
+ analyze_sbs(mddev);
+ rdev_for_each_safe(rdev, tmp, mddev) {
+ struct mdp_superblock_1 *sb = page_address(rdev->sb_page);
+ /* since we don't write to faulty devices, we figure out if the
+ * disk is faulty by comparing events
+ */
+ if (mddev->events > sb->events)
+ set_bit(Faulty, &rdev->flags);
+ }
+
+}
+EXPORT_SYMBOL(md_reload_sb);
+
#ifndef MODULE
/*
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 6c6e992..620384a 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -633,6 +633,7 @@ extern struct bio *bio_alloc_mddev(gfp_t gfp_mask, int nr_iovecs,
struct mddev *mddev);
extern void md_unplug(struct blk_plug_cb *cb, bool from_schedule);
+extern void md_reload_sb(struct mddev *mddev);
static inline int mddev_check_plugged(struct mddev *mddev)
{
return !!blk_check_plugged(md_unplug, mddev,
--
2.1.2
^ permalink raw reply related
* [PATCH 20/24] Send RESYNCING while performing resync start/stop
From: Goldwyn Rodrigues @ 2014-12-18 16:18 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
When a resync is initiated, RESYNCING message is sent to all active
nodes with the range (lo,hi). When the resync is over, a RESYNCING
message is sent with (0,0). A high sector value of zero indicates
that the resync is over.
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md-cluster.c | 31 +++++++++++++++++++++++++++++++
drivers/md/md-cluster.h | 2 ++
drivers/md/md.c | 4 ++--
3 files changed, 35 insertions(+), 2 deletions(-)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 084834d..fff40a0 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -644,11 +644,42 @@ static int metadata_update_cancel(struct mddev *mddev)
return dlm_unlock_sync(cinfo->token_lockres);
}
+static int resync_send(struct mddev *mddev, enum msg_type type,
+ sector_t lo, sector_t hi)
+{
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ struct cluster_msg cmsg;
+ int slot = cinfo->slot_number - 1;
+ pr_info("%s:%d lo: %llu hi: %llu\n", __func__, __LINE__,
+ (unsigned long long)lo,
+ (unsigned long long)hi);
+ resync_info_update(mddev, lo, hi);
+ cmsg.type = cpu_to_le32(type);
+ cmsg.slot = cpu_to_le32(slot);
+ cmsg.low = cpu_to_le64(lo);
+ cmsg.high = cpu_to_le64(hi);
+ return sendmsg(cinfo, &cmsg);
+}
+
+static int resync_start(struct mddev *mddev, sector_t lo, sector_t hi)
+{
+ pr_info("%s:%d\n", __func__, __LINE__);
+ return resync_send(mddev, RESYNCING, lo, hi);
+}
+
+static void resync_finish(struct mddev *mddev)
+{
+ pr_info("%s:%d\n", __func__, __LINE__);
+ resync_send(mddev, RESYNCING, 0, 0);
+}
+
static struct md_cluster_operations cluster_ops = {
.join = join,
.leave = leave,
.slot_number = slot_number,
.resync_info_update = resync_info_update,
+ .resync_start = resync_start,
+ .resync_finish = resync_finish,
.metadata_update_start = metadata_update_start,
.metadata_update_finish = metadata_update_finish,
.metadata_update_cancel = metadata_update_cancel
diff --git a/drivers/md/md-cluster.h b/drivers/md/md-cluster.h
index 658982a..054f9ea 100644
--- a/drivers/md/md-cluster.h
+++ b/drivers/md/md-cluster.h
@@ -12,6 +12,8 @@ struct md_cluster_operations {
int (*leave)(struct mddev *mddev);
int (*slot_number)(struct mddev *mddev);
void (*resync_info_update)(struct mddev *mddev, sector_t lo, sector_t hi);
+ int (*resync_start)(struct mddev *mddev, sector_t lo, sector_t hi);
+ void (*resync_finish)(struct mddev *mddev);
int (*metadata_update_start)(struct mddev *mddev);
int (*metadata_update_finish)(struct mddev *mddev);
int (*metadata_update_cancel)(struct mddev *mddev);
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 829086c..1532b36 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -7679,7 +7679,7 @@ void md_do_sync(struct md_thread *thread)
update_time = jiffies;
if (mddev_is_clustered(mddev))
- md_cluster_ops->resync_info_update(mddev, j, max_sectors);
+ md_cluster_ops->resync_start(mddev, j, max_sectors);
blk_start_plug(&plug);
while (j < max_sectors) {
@@ -7803,7 +7803,7 @@ void md_do_sync(struct md_thread *thread)
mddev->pers->sync_request(mddev, max_sectors, &skipped, 1);
if (mddev_is_clustered(mddev))
- md_cluster_ops->resync_info_update(mddev, 0, 0);
+ md_cluster_ops->resync_finish(mddev);
if (!test_bit(MD_RECOVERY_CHECK, &mddev->recovery) &&
mddev->curr_resync > 2) {
--
2.1.2
^ permalink raw reply related
* [PATCH 21/24] Resync start/Finish actions
From: Goldwyn Rodrigues @ 2014-12-18 16:18 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
When a RESYNC_START message arrives, the node removes the entry
with the current slot number and adds the range to the
suspend_list.
Simlarly, when a RESYNC_FINISHED message is received, node clears
entry with respect to the bitmap number.
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md-cluster.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 46 insertions(+)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index fff40a0..6f156a8 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -316,6 +316,50 @@ static void ack_bast(void *arg, int mode)
md_wakeup_thread(cinfo->recv_thread);
}
+static void __remove_suspend_info(struct md_cluster_info *cinfo, int slot)
+{
+ struct suspend_info *s, *tmp;
+ list_for_each_entry_safe(s, tmp, &cinfo->suspend_list, list)
+ if (slot == s->slot) {
+ printk("%s:%d Deleting suspend_info: %d\n",
+ __func__, __LINE__, slot);
+ list_del(&s->list);
+ kfree(s);
+ break;
+ }
+}
+
+static void remove_suspend_info(struct md_cluster_info *cinfo, int slot)
+{
+ spin_lock_irq(&cinfo->suspend_lock);
+ __remove_suspend_info(cinfo, slot);
+ spin_unlock_irq(&cinfo->suspend_lock);
+}
+
+
+static void process_suspend_info(struct md_cluster_info *cinfo,
+ int slot, sector_t lo, sector_t hi)
+{
+ struct suspend_info *s;
+ if (!hi) {
+ remove_suspend_info(cinfo, slot);
+ return;
+ }
+ s = kzalloc(sizeof(struct suspend_info), GFP_KERNEL);
+ if (!s) {
+ pr_err("md-cluster: Could not allocate suspend_area\n");
+ return;
+ }
+ s->slot = slot;
+ s->lo = lo;
+ s->hi = hi;
+ spin_lock_irq(&cinfo->suspend_lock);
+ /* Remove existing entry (if exists) before adding */
+ __remove_suspend_info(cinfo, slot);
+ list_add(&s->list, &cinfo->suspend_list);
+ spin_unlock_irq(&cinfo->suspend_lock);
+}
+
static void process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg)
{
switch (msg->type) {
@@ -327,6 +371,8 @@ static void process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg)
case RESYNCING:
pr_info("%s: %d Received message: RESYNCING from %d\n",
__func__, __LINE__, msg->slot);
+ process_suspend_info(mddev->cluster_info, msg->slot,
+ msg->low, msg->high);
break;
};
}
--
2.1.2
^ permalink raw reply related
* [PATCH 22/24] Suspend writes in RAID1 if within range
From: Goldwyn Rodrigues @ 2014-12-18 16:18 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
If there is a resync going on, all nodes must suspend writes to the
range. This is recorded in the suspend_info/suspend_list.
If there is an I/O within the ranges of any of the suspend_info,
should_suspend will return 1.
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md-cluster.c | 22 +++++++++++++++++++++-
drivers/md/md-cluster.h | 1 +
drivers/md/md.c | 1 +
drivers/md/raid1.c | 11 ++++++++---
4 files changed, 31 insertions(+), 4 deletions(-)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 6f156a8..9feb6ff2 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -719,6 +719,25 @@ static void resync_finish(struct mddev *mddev)
resync_send(mddev, RESYNCING, 0, 0);
}
+static int area_resyncing(struct mddev *mddev, sector_t lo, sector_t hi)
+{
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ int ret = 0;
+ struct suspend_info *s;
+
+ spin_lock_irq(&cinfo->suspend_lock);
+ if (list_empty(&cinfo->suspend_list))
+ goto out;
+ list_for_each_entry(s, &cinfo->suspend_list, list)
+ if (hi > s->lo && lo < s->hi) {
+ ret = 1;
+ break;
+ }
+out:
+ spin_unlock_irq(&cinfo->suspend_lock);
+ return ret;
+}
+
static struct md_cluster_operations cluster_ops = {
.join = join,
.leave = leave,
@@ -728,7 +747,8 @@ static struct md_cluster_operations cluster_ops = {
.resync_finish = resync_finish,
.metadata_update_start = metadata_update_start,
.metadata_update_finish = metadata_update_finish,
- .metadata_update_cancel = metadata_update_cancel
+ .metadata_update_cancel = metadata_update_cancel,
+ .area_resyncing = area_resyncing
};
static int __init cluster_init(void)
diff --git a/drivers/md/md-cluster.h b/drivers/md/md-cluster.h
index 054f9ea..0378540 100644
--- a/drivers/md/md-cluster.h
+++ b/drivers/md/md-cluster.h
@@ -17,6 +17,7 @@ struct md_cluster_operations {
int (*metadata_update_start)(struct mddev *mddev);
int (*metadata_update_finish)(struct mddev *mddev);
int (*metadata_update_cancel)(struct mddev *mddev);
+ int (*area_resyncing)(struct mddev *mddev, sector_t lo, sector_t hi);
};
#endif /* _MD_CLUSTER_H */
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 1532b36..5be7719 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -68,6 +68,7 @@ static LIST_HEAD(pers_list);
static DEFINE_SPINLOCK(pers_lock);
struct md_cluster_operations *md_cluster_ops;
+EXPORT_SYMBOL(md_cluster_ops);
struct module *md_cluster_mod;
EXPORT_SYMBOL(md_cluster_mod);
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index 55de4f6..eb8fd2e 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -1113,8 +1113,10 @@ static void make_request(struct mddev *mddev, struct bio * bio)
md_write_start(mddev, bio); /* wait on superblock update early */
if (bio_data_dir(bio) == WRITE &&
- bio_end_sector(bio) > mddev->suspend_lo &&
- bio->bi_iter.bi_sector < mddev->suspend_hi) {
+ ((bio_end_sector(bio) > mddev->suspend_lo &&
+ bio->bi_iter.bi_sector < mddev->suspend_hi) ||
+ (mddev_is_clustered(mddev) &&
+ md_cluster_ops->area_resyncing(mddev, bio->bi_iter.bi_sector, bio_end_sector(bio))))) {
/* As the suspend_* range is controlled by
* userspace, we want an interruptible
* wait.
@@ -1125,7 +1127,10 @@ static void make_request(struct mddev *mddev, struct bio * bio)
prepare_to_wait(&conf->wait_barrier,
&w, TASK_INTERRUPTIBLE);
if (bio_end_sector(bio) <= mddev->suspend_lo ||
- bio->bi_iter.bi_sector >= mddev->suspend_hi)
+ bio->bi_iter.bi_sector >= mddev->suspend_hi ||
+ (mddev_is_clustered(mddev) &&
+ !md_cluster_ops->area_resyncing(mddev,
+ bio->bi_iter.bi_sector, bio_end_sector(bio))))
break;
schedule();
}
--
2.1.2
^ permalink raw reply related
* [PATCH 23/24] Read from the first device when an area is resyncing
From: Goldwyn Rodrigues @ 2014-12-18 16:19 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
set choose_first true for cluster read in read balance when the area
is resyncing.
Signed-off-by: Lidong Zhong <lzhong@suse.com>
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/raid1.c | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index eb8fd2e..4dcbaae 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -540,7 +540,13 @@ static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, int *max_sect
has_nonrot_disk = 0;
choose_next_idle = 0;
- choose_first = (conf->mddev->recovery_cp < this_sector + sectors);
+ if ((conf->mddev->recovery_cp < this_sector + sectors) ||
+ (mddev_is_clustered(conf->mddev) &&
+ md_cluster_ops->area_resyncing(conf->mddev, this_sector,
+ this_sector + sectors)))
+ choose_first = 1;
+ else
+ choose_first = 0;
for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) {
sector_t dist;
--
2.1.2
^ permalink raw reply related
* [PATCH 24/24] Add new disk to clustered array
From: Goldwyn Rodrigues @ 2014-12-18 16:19 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
Algorithm:
1. Node 1 issues mdadm --manage /dev/mdX --add /dev/sdYY which issues
ioctl(ADD_NEW_DISC with disc.state set to MD_DISK_CLUSTER_ADD)
2. Node 1 sends NEWDISK with uuid and slot number
3. Other nodes issue kobject_uevent_env with uuid and slot number
(Steps 4,5 could be a udev rule)
4. In userspace, the node searches for the disk, perhaps
using blkid -t SUB_UUID=""
5. Other nodes issue either of the following depending on whether the disk
was found:
ioctl(ADD_NEW_DISK with disc.state set to MD_DISK_CANDIDATE and
disc.number set to slot number)
ioctl(CLUSTERED_DISK_NACK)
6. Other nodes drop lock on no-new-devs (CR) if device is found
7. Node 1 attempts EX lock on no-new-devs
8. If node 1 gets the lock, it sends METADATA_UPDATED after unmarking the disk
as SpareLocal
9. If not (get no-new-dev lock), it fails the operation and sends METADATA_UPDATED
10. Other nodes understand if the device is added or not by reading the superblock again after receiving the METADATA_UPDATED message.
Signed-off-by: Lidong Zhong <lzhong@suse.com>
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md-cluster.c | 107 +++++++++++++++++++++++++++++++++++++++--
drivers/md/md-cluster.h | 4 ++
drivers/md/md.c | 54 +++++++++++++++++++--
drivers/md/md.h | 5 ++
drivers/md/raid1.c | 1 +
include/uapi/linux/raid/md_p.h | 8 ++-
include/uapi/linux/raid/md_u.h | 1 +
7 files changed, 171 insertions(+), 9 deletions(-)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 9feb6ff2..37158f3 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -15,11 +15,13 @@
#include <linux/module.h>
#include <linux/dlm.h>
#include <linux/sched.h>
+#include <linux/raid/md_p.h>
#include "md.h"
#include "bitmap.h"
#include "md-cluster.h"
#define LVB_SIZE 64
+#define NEW_DEV_TIMEOUT 5000
struct dlm_lock_resource {
dlm_lockspace_t *ls;
@@ -59,19 +61,25 @@ struct md_cluster_info {
struct dlm_lock_resource *ack_lockres;
struct dlm_lock_resource *message_lockres;
struct dlm_lock_resource *token_lockres;
+ struct dlm_lock_resource *no_new_dev_lockres;
struct md_thread *recv_thread;
+ struct completion newdisk_completion;
};
enum msg_type {
METADATA_UPDATED = 0,
- RESYNCING
+ RESYNCING,
+ NEWDISK,
};
struct cluster_msg {
int type;
int slot;
+ /* TODO: Unionize this for smaller footprint */
sector_t low;
sector_t high;
+ char uuid[16];
+ int raid_slot;
};
static void sync_ast(void *arg)
@@ -360,13 +368,40 @@ static void process_suspend_info(struct md_cluster_info *cinfo,
spin_unlock_irq(&cinfo->suspend_lock);
}
+static void process_add_new_disk(struct mddev *mddev, struct cluster_msg *cmsg)
+{
+ char disk_uuid[64];
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ char event_name[]="EVENT=ADD_DEVICE";
+ char raid_slot[16];
+ char *envp[] = {event_name, disk_uuid, raid_slot, NULL};
+ int len;
+
+ len = snprintf(disk_uuid, 64, "DEVICE_UUID=");
+ pretty_uuid(disk_uuid + len, cmsg->uuid);
+ snprintf(raid_slot, 16, "RAID_DISK=%d", cmsg->raid_slot);
+ printk("%s:%d Sending kobject change with %s and %s\n", __func__, __LINE__, disk_uuid, raid_slot);
+ init_completion(&cinfo->newdisk_completion);
+ kobject_uevent_env(&disk_to_dev(mddev->gendisk)->kobj, KOBJ_CHANGE, envp);
+ wait_for_completion_timeout(&cinfo->newdisk_completion,
+ NEW_DEV_TIMEOUT);
+}
+
+
+static void process_metadata_update(struct mddev *mddev, struct cluster_msg *msg)
+{
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ md_reload_sb(mddev);
+ dlm_lock_sync(cinfo->no_new_dev_lockres, DLM_LOCK_CR);
+}
+
static void process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg)
{
switch (msg->type) {
case METADATA_UPDATED:
pr_info("%s: %d Received message: METADATA_UPDATE from %d\n",
__func__, __LINE__, msg->slot);
- md_reload_sb(mddev);
+ process_metadata_update(mddev, msg);
break;
case RESYNCING:
pr_info("%s: %d Received message: RESYNCING from %d\n",
@@ -374,6 +409,10 @@ static void process_recvd_msg(struct mddev *mddev, struct cluster_msg *msg)
process_suspend_info(mddev->cluster_info, msg->slot,
msg->low, msg->high);
break;
+ case NEWDISK:
+ pr_info("%s: %d Received message: NEWDISK from %d\n",
+ __func__, __LINE__, msg->slot);
+ process_add_new_disk(mddev, msg);
};
}
@@ -593,14 +632,22 @@ static int join(struct mddev *mddev, int nodes)
cinfo->ack_lockres = lockres_init(mddev, "ack", ack_bast, 0);
if (!cinfo->ack_lockres)
goto err;
+ cinfo->no_new_dev_lockres = lockres_init(mddev, "no-new-dev", NULL, 0);
+ if (!cinfo->no_new_dev_lockres)
+ goto err;
+
/* get sync CR lock on ACK. */
if (dlm_lock_sync(cinfo->ack_lockres, DLM_LOCK_CR))
pr_err("md-cluster: failed to get a sync CR lock on ACK!(%d)\n",
ret);
+ /* get sync CR lock on no-new-dev. */
+ if (dlm_lock_sync(cinfo->no_new_dev_lockres, DLM_LOCK_CR))
+ pr_err("md-cluster: failed to get a sync CR lock on "
+ "no-new-dev!(%d)\n", ret);
+
pr_info("Joined cluster %s slot %d\n", str, cinfo->slot_number);
- memset(str, '\0', 64);
snprintf(str, 64, "bitmap%04d", cinfo->slot_number - 1);
cinfo->bitmap_lockres = lockres_init(mddev, str, NULL, 1);
if (!cinfo->bitmap_lockres)
@@ -623,6 +670,7 @@ err:
lockres_free(cinfo->message_lockres);
lockres_free(cinfo->token_lockres);
lockres_free(cinfo->ack_lockres);
+ lockres_free(cinfo->no_new_dev_lockres);
lockres_free(cinfo->bitmap_lockres);
lockres_free(cinfo->sb_lock);
if (cinfo->lockspace)
@@ -643,6 +691,7 @@ static int leave(struct mddev *mddev)
lockres_free(cinfo->message_lockres);
lockres_free(cinfo->token_lockres);
lockres_free(cinfo->ack_lockres);
+ lockres_free(cinfo->no_new_dev_lockres);
lockres_free(cinfo->sb_lock);
lockres_free(cinfo->bitmap_lockres);
dlm_release_lockspace(cinfo->lockspace, 2);
@@ -738,6 +787,53 @@ out:
return ret;
}
+static int add_new_disk_start(struct mddev *mddev, struct md_rdev *rdev)
+{
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ struct cluster_msg cmsg;
+ int ret = 0;
+ struct mdp_superblock_1 *sb = page_address(rdev->sb_page);
+ char *uuid = sb->device_uuid;
+ memset(&cmsg, 0, sizeof(cmsg));
+ cmsg.type = cpu_to_le32(NEWDISK);
+ memcpy(cmsg.uuid, uuid, 16);
+ cmsg.raid_slot = rdev->desc_nr;
+ lock_comm(cinfo);
+ ret = __sendmsg(cinfo, &cmsg);
+ if (ret)
+ return ret;
+ cinfo->no_new_dev_lockres->flags |= DLM_LKF_NOQUEUE;
+ ret = dlm_lock_sync(cinfo->no_new_dev_lockres, DLM_LOCK_EX);
+ cinfo->no_new_dev_lockres->flags &= ~DLM_LKF_NOQUEUE;
+ /* Some node does not "see" the device */
+ if (ret == -EAGAIN)
+ ret = -ENOENT;
+ else
+ dlm_lock_sync(cinfo->no_new_dev_lockres, DLM_LOCK_CR);
+ return ret;
+}
+
+static int add_new_disk_finish(struct mddev *mddev)
+{
+ struct cluster_msg cmsg;
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ int ret;
+ /* Write sb and inform others */
+ md_update_sb(mddev, 1);
+ cmsg.type = METADATA_UPDATED;
+ ret = __sendmsg(cinfo, &cmsg);
+ unlock_comm(cinfo);
+ return ret;
+}
+
+static void new_disk_ack(struct mddev *mddev, bool ack)
+{
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ if (ack)
+ dlm_unlock_sync(cinfo->no_new_dev_lockres);
+ complete(&cinfo->newdisk_completion);
+}
+
static struct md_cluster_operations cluster_ops = {
.join = join,
.leave = leave,
@@ -748,7 +844,10 @@ static struct md_cluster_operations cluster_ops = {
.metadata_update_start = metadata_update_start,
.metadata_update_finish = metadata_update_finish,
.metadata_update_cancel = metadata_update_cancel,
- .area_resyncing = area_resyncing
+ .area_resyncing = area_resyncing,
+ .add_new_disk_start = add_new_disk_start,
+ .add_new_disk_finish = add_new_disk_finish,
+ .new_disk_ack = new_disk_ack,
};
static int __init cluster_init(void)
diff --git a/drivers/md/md-cluster.h b/drivers/md/md-cluster.h
index 0378540..60d7e58 100644
--- a/drivers/md/md-cluster.h
+++ b/drivers/md/md-cluster.h
@@ -6,6 +6,7 @@
#include "md.h"
struct mddev;
+struct md_rdev;
struct md_cluster_operations {
int (*join)(struct mddev *mddev, int nodes);
@@ -18,6 +19,9 @@ struct md_cluster_operations {
int (*metadata_update_finish)(struct mddev *mddev);
int (*metadata_update_cancel)(struct mddev *mddev);
int (*area_resyncing)(struct mddev *mddev, sector_t lo, sector_t hi);
+ int (*add_new_disk_start)(struct mddev *mddev, struct md_rdev *rdev);
+ int (*add_new_disk_finish)(struct mddev *mddev);
+ void (*new_disk_ack)(struct mddev *mddev, bool ack);
};
#endif /* _MD_CLUSTER_H */
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 5be7719..6de96a3 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -2375,7 +2375,7 @@ static void sync_sbs(struct mddev * mddev, int nospares)
}
}
-static void md_update_sb(struct mddev * mddev, int force_change)
+void md_update_sb(struct mddev * mddev, int force_change)
{
struct md_rdev *rdev;
int sync_req;
@@ -2539,6 +2539,7 @@ repeat:
wake_up(&rdev->blocked_wait);
}
}
+EXPORT_SYMBOL(md_update_sb);
/* words written to sysfs files may, or may not, be \n terminated.
* We want to accept with case. For this we use cmd_match.
@@ -3331,7 +3332,7 @@ static void analyze_sbs(struct mddev * mddev)
kick_rdev_from_array(rdev);
continue;
}
- if (rdev != freshest)
+ if (rdev != freshest) {
if (super_types[mddev->major_version].
validate_super(mddev, rdev)) {
printk(KERN_WARNING "md: kicking non-fresh %s"
@@ -3340,6 +3341,16 @@ static void analyze_sbs(struct mddev * mddev)
kick_rdev_from_array(rdev);
continue;
}
+ /* No device should have a Candidate flag
+ * when reading devices
+ */
+ if (test_bit(Candidate, &rdev->flags)) {
+ pr_info("md: kicking Cluster Candidate %s"
+ " from array!\n",
+ bdevname(rdev->bdev,b));
+ kick_rdev_from_array(rdev);
+ }
+ }
if (mddev->level == LEVEL_MULTIPATH) {
rdev->desc_nr = i++;
rdev->raid_disk = rdev->desc_nr;
@@ -5648,7 +5659,6 @@ static int get_array_info(struct mddev * mddev, void __user * arg)
info.state = (1<<MD_SB_BITMAP_PRESENT);
if (mddev_is_clustered(mddev))
info.state |= (1<<MD_SB_CLUSTERED);
-
info.active_disks = insync;
info.working_disks = working;
info.failed_disks = failed;
@@ -5743,6 +5753,12 @@ static int add_new_disk(struct mddev * mddev, mdu_disk_info_t *info)
struct md_rdev *rdev;
dev_t dev = MKDEV(info->major,info->minor);
+ if (mddev_is_clustered(mddev) &&
+ !(info->state & ((1 << MD_DISK_CLUSTER_ADD) | (1 << MD_DISK_CANDIDATE)))) {
+ pr_err("%s: Cannot add to clustered mddev. Try --cluster-add\n", mdname(mddev));
+ return -EINVAL;
+ }
+
if (info->major != MAJOR(dev) || info->minor != MINOR(dev))
return -EOVERFLOW;
@@ -5829,6 +5845,25 @@ static int add_new_disk(struct mddev * mddev, mdu_disk_info_t *info)
else
clear_bit(WriteMostly, &rdev->flags);
+ /*
+ * check whether the device shows up in other nodes
+ */
+ if (mddev_is_clustered(mddev)) {
+ if (info->state & (1 << MD_DISK_CANDIDATE)) {
+ /* Through --cluster-confirm */
+ set_bit(Candidate, &rdev->flags);
+ md_cluster_ops->new_disk_ack(mddev, true);
+ } else if (info->state & (1 << MD_DISK_CLUSTER_ADD)) {
+ /* --add initiated by this node */
+ err = md_cluster_ops->add_new_disk_start(mddev, rdev);
+ if (err) {
+ md_cluster_ops->add_new_disk_finish(mddev);
+ export_rdev(rdev);
+ return err;
+ }
+ }
+ }
+
rdev->raid_disk = -1;
err = bind_rdev_to_array(rdev, mddev);
if (!err && !mddev->pers->hot_remove_disk) {
@@ -5854,6 +5889,9 @@ static int add_new_disk(struct mddev * mddev, mdu_disk_info_t *info)
if (!err)
md_new_event(mddev);
md_wakeup_thread(mddev->thread);
+ if (mddev_is_clustered(mddev) &&
+ (info->state & (1 << MD_DISK_CLUSTER_ADD)))
+ md_cluster_ops->add_new_disk_finish(mddev);
return err;
}
@@ -6433,6 +6471,7 @@ static inline bool md_ioctl_valid(unsigned int cmd)
case SET_DISK_FAULTY:
case STOP_ARRAY:
case STOP_ARRAY_RO:
+ case CLUSTERED_DISK_NACK:
return true;
default:
return false;
@@ -6638,6 +6677,7 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode,
break;
else
err = add_new_disk(mddev, &info);
+
goto done_unlock;
}
break;
@@ -6712,6 +6752,13 @@ static int md_ioctl(struct block_device *bdev, fmode_t mode,
goto done_unlock;
}
+ case CLUSTERED_DISK_NACK:
+ if (mddev_is_clustered(mddev))
+ md_cluster_ops->new_disk_ack(mddev, false);
+ else
+ err = -EINVAL;
+ goto done_unlock;
+
case HOT_ADD_DISK:
err = hot_add_disk(mddev, new_decode_dev(arg));
goto done_unlock;
@@ -6735,7 +6782,6 @@ abort_unlock:
err != -EINVAL)
mddev->hold_active = 0;
mddev_unlock(mddev);
-
return err;
done:
if (err)
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 620384a..1395a93 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -171,6 +171,10 @@ enum flag_bits {
* a want_replacement device with same
* raid_disk number.
*/
+ Candidate, /* For clustered environments only:
+ * This device is seen locally but not
+ * by the whole cluster
+ */
};
#define BB_LEN_MASK (0x00000000000001FFULL)
@@ -634,6 +638,7 @@ extern struct bio *bio_alloc_mddev(gfp_t gfp_mask, int nr_iovecs,
extern void md_unplug(struct blk_plug_cb *cb, bool from_schedule);
extern void md_reload_sb(struct mddev *mddev);
+extern void md_update_sb(struct mddev *mddev, int force);
static inline int mddev_check_plugged(struct mddev *mddev)
{
return !!blk_check_plugged(md_unplug, mddev,
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index 4dcbaae..cb2e378 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -1584,6 +1584,7 @@ static int raid1_spare_active(struct mddev *mddev)
struct md_rdev *rdev = conf->mirrors[i].rdev;
struct md_rdev *repl = conf->mirrors[conf->raid_disks + i].rdev;
if (repl
+ && !test_bit(Candidate, &repl->flags)
&& repl->recovery_offset == MaxSector
&& !test_bit(Faulty, &repl->flags)
&& !test_and_set_bit(In_sync, &repl->flags)) {
diff --git a/include/uapi/linux/raid/md_p.h b/include/uapi/linux/raid/md_p.h
index 643489d..5d0e4c0 100644
--- a/include/uapi/linux/raid/md_p.h
+++ b/include/uapi/linux/raid/md_p.h
@@ -78,6 +78,12 @@
#define MD_DISK_ACTIVE 1 /* disk is running or spare disk */
#define MD_DISK_SYNC 2 /* disk is in sync with the raid set */
#define MD_DISK_REMOVED 3 /* disk is in sync with the raid set */
+#define MD_DISK_CLUSTER_ADD 4 /* Initiate a disk add across the cluster
+ * For clustered enviroments only.
+ */
+#define MD_DISK_CANDIDATE 5 /* disk is added as spare (local) until confirmed
+ * For clustered enviroments only.
+ */
#define MD_DISK_WRITEMOSTLY 9 /* disk is "write-mostly" is RAID1 config.
* read requests will only be sent here in
@@ -101,7 +107,7 @@ typedef struct mdp_device_descriptor_s {
#define MD_SB_CLEAN 0
#define MD_SB_ERRORS 1
-#define MD_SB_CLUSTERED 5 /* MD is clustered */
+#define MD_SB_CLUSTERED 5 /* MD is clustered */
#define MD_SB_BITMAP_PRESENT 8 /* bitmap may be present nearby */
/*
diff --git a/include/uapi/linux/raid/md_u.h b/include/uapi/linux/raid/md_u.h
index 4133e74..5a7029c 100644
--- a/include/uapi/linux/raid/md_u.h
+++ b/include/uapi/linux/raid/md_u.h
@@ -63,6 +63,7 @@
#define STOP_ARRAY _IO (MD_MAJOR, 0x32)
#define STOP_ARRAY_RO _IO (MD_MAJOR, 0x33)
#define RESTART_ARRAY_RW _IO (MD_MAJOR, 0x34)
+#define CLUSTERED_DISK_NACK _IO (MD_MAJOR, 0x35)
/* 63 partitions with the alternate major number (mdp) */
#define MdpMinorShift 6
--
2.1.2
^ permalink raw reply related
* Re: Panic doing BLKDISCARD on a raid 5 array on linux 3.17.3
From: Chris Murphy @ 2014-12-18 18:05 UTC (permalink / raw)
Cc: linux-raid
In-Reply-To: <5492B359.6060804@overnetdata.com>
On Thu, Dec 18, 2014 at 3:58 AM, Anthony Wright <anthony@overnetdata.com> wrote:
> On 18/12/2014 05:28, NeilBrown wrote:
>> I suspect md/raid5 is sending down a discard request in some way that the
>> scsi/sata layer or driver doesn't like, but without the full oops, I really
>> cannot guess what it might be.
> We've tried 4 times to reproduce the panic we got originally, but
> unforunately with no luck. Below are the outputs from all four crashes
> as captured by netconsole, in case they are any help.
>
> Crash #1
> [63207.177400] BUG: unable to handle kernel paging request at
> 0000001e00008000
>
> Crash #2
> [ 531.210340] BUG: unable to handle kernel paging request at
> 0000000100000000
> [ 531.210514] IP:[ 531.210340] BUG: unable to handle kernel
> [<ffffffff8128788e>] __blk_segment_map_sg+0x5e/0x1b0
> paging request[ 531.210632] PGD 20187f067 PUD 0 at 0000000100000000
>
> [ 531.210514] IP: [<ffffffff8128788e>] __blk_segment_map_sg+0x5e/0x1b0
> [ 531.210632] PGD 20187f067 PUD 0
> [ 531.210783] Oops: 0000 [#1] SMP
> [ 531.210932] Modules linked in: eql netconsole configfs raid456[
> 531.210783] Oops: 0000 [#1] SMP
> [ 531.210932] Modules linked in: eql netconsole configfs raid456
> async_raid6_recov async_memcpy async_pq async_xor xor async_tx raid6_pq
> xt_multiport async_raid6_recov async_memcpy async_pq async_xor xor
> async_tx raid6_pq xt_multiport xt_tcpudp iptable_filter ip_tables
> x_tables aesni_intel aes_x86_64 glue_helper lrw xt_tcpudp iptable_filter
> ip_tables x_tables aesni_intel aes_x86_64 glue_helper lrw gf128mul
> ablk_helper cryptd ppdev gf128mul ablk_helper cryptd ppdev
>
> Crash #3
> [ 268.115094] general protection fault: 0000 [#1] SMP
> [ 268.115263] Modules linked in:[ 268.115094] general protection
> fault: 0000 [#1] SMP
> [ 268.115263] Modules linked in:
>
> Crash #4
> [ 276.325157] general protection fault: 0000 [#1] SMP
>
Are you trimming these? Or do they literally end with nothing else
reported? In any case you must be trimming what comes before what
you've posted and I don't think that's a good idea, often there's
something wrong happening well before an oops gets reported.
--
Chris Murphy
^ permalink raw reply
* Re: indefinite hang when growing/reshaping
From: Xiao Ni @ 2014-12-19 8:14 UTC (permalink / raw)
To: Chris Murphy; +Cc: linux-raid, eguan, Jes Sorensen
In-Reply-To: <CAJCQCtQWSVbx08xafh6MDikiJOZJ6MVJ4Jx=qt=UZnX3t47A7A@mail.gmail.com>
Hi all
There is the same problem. I can reproduce with the same steps. And
I can reproduce this in a different way too. I don't know whether it's
the same problem, but the result is the same.
The different steps are as follows:
mdadm -CR /dev/md0 -l5 -n7 /dev/sda1 /dev/sdb1 /dev/sdc1 /dev/sdd1 /dev/sde1 /dev/sdf1 /dev/sdh1 -x1 /dev/sdi1 --assume-clean --bitmap=internal
mdadm --grow -l0 /dev/md0 --backup-file=tmp0
cat /proc/mdstat
Personalities : [raid6] [raid5] [raid4] [raid1] [raid0]
md0 : active raid5 loop3[4] loop2[3] loop1[1] loop0[0]
2045952 blocks super 1.2 level 5, 512k chunk, algorithm 2 [4/4] [UUUU]
[>....................] reshape = 0.0% (0/1022976) finish=5.8min speed=2818K/sec
unused devices: <none>
There is speed 2818K/sec, but if you look by iostat, there is no speed about the disks.
And there is a workaround:
mdadm -CR /dev/md0 -l5 -n7 /dev/sda1 /dev/sdb1 /dev/sdc1 /dev/sdd1 /dev/sde1 /dev/sdf1 /dev/sdh1 -x1 /dev/sdi1 --assume-clean --bitmap=internal
mdadm --grow -l0 /dev/md0
If we don't specify the backup file. The reshape can finish correctly.
Best Regards
Xiao
----- Original Message -----
> From: "Chris Murphy" <chris@colorremedies.com>
> To: linux-raid@vger.kernel.org
> Sent: Thursday, December 18, 2014 6:20:50 AM
> Subject: indefinite hang when growing/reshaping
>
> If this is user error I'd expect a discreet error message, but I don't
> get one, just a hang during the reshape following successful --grow
> command (going from 3x device raid5 to 4x device raid5).
>
> # cat /proc/mdstat
> Personalities : [raid6] [raid5] [raid4]
> md127 : active raid5 loop3[4] loop2[3] loop1[1] loop0[0]
> 2095104 blocks super 1.2 level 5, 512k chunk, algorithm 2 [4/4] [UUUU]
> [>....................] reshape = 0.0% (0/1047552)
> finish=7.0min speed=2402K/sec
>
> Reproduces with:
> kernel-3.17.6-300.fc21.x86_64
> mdadm-3.3.2-1.fc21.x86_64
> and
> 3.18.0-2.fc22.i686+debug
> mdadm-3.3.2-1.fc21.i686
>
> Details are here:
> https://bugzilla.kernel.org/show_bug.cgi?id=89851
>
> --
> Chris Murphy
> --
> To unsubscribe from this list: send the line "unsubscribe linux-raid" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
^ permalink raw reply
* Re: indefinite hang when growing/reshaping
From: Xiao Ni @ 2014-12-19 8:17 UTC (permalink / raw)
To: Chris Murphy; +Cc: linux-raid, eguan, Jes Sorensen
In-Reply-To: <997573837.677557.1418976887168.JavaMail.zimbra@redhat.com>
Sorry, forget to say kernel version and mdadm version
[root@storageqe-19 home]# uname -r
3.18.1
[root@storageqe-19 home]# mdadm --version
mdadm - v3.3.2 - 21st August 2014
----- Original Message -----
> From: "Xiao Ni" <xni@redhat.com>
> To: "Chris Murphy" <chris@colorremedies.com>
> Cc: linux-raid@vger.kernel.org, eguan@redhat.com, "Jes Sorensen" <jes.sorensen@redhat.com>
> Sent: Friday, December 19, 2014 4:14:47 PM
> Subject: Re: indefinite hang when growing/reshaping
>
> Hi all
>
> There is the same problem. I can reproduce with the same steps. And
> I can reproduce this in a different way too. I don't know whether it's
> the same problem, but the result is the same.
>
> The different steps are as follows:
>
> mdadm -CR /dev/md0 -l5 -n7 /dev/sda1 /dev/sdb1 /dev/sdc1 /dev/sdd1 /dev/sde1
> /dev/sdf1 /dev/sdh1 -x1 /dev/sdi1 --assume-clean --bitmap=internal
> mdadm --grow -l0 /dev/md0 --backup-file=tmp0
> cat /proc/mdstat
> Personalities : [raid6] [raid5] [raid4] [raid1] [raid0]
> md0 : active raid5 loop3[4] loop2[3] loop1[1] loop0[0]
> 2045952 blocks super 1.2 level 5, 512k chunk, algorithm 2 [4/4] [UUUU]
> [>....................] reshape = 0.0% (0/1022976) finish=5.8min
> speed=2818K/sec
>
> unused devices: <none>
>
> There is speed 2818K/sec, but if you look by iostat, there is no speed
> about the disks.
>
>
> And there is a workaround:
> mdadm -CR /dev/md0 -l5 -n7 /dev/sda1 /dev/sdb1 /dev/sdc1 /dev/sdd1
> /dev/sde1 /dev/sdf1 /dev/sdh1 -x1 /dev/sdi1 --assume-clean
> --bitmap=internal
> mdadm --grow -l0 /dev/md0
>
> If we don't specify the backup file. The reshape can finish correctly.
>
> Best Regards
> Xiao
>
> ----- Original Message -----
> > From: "Chris Murphy" <chris@colorremedies.com>
> > To: linux-raid@vger.kernel.org
> > Sent: Thursday, December 18, 2014 6:20:50 AM
> > Subject: indefinite hang when growing/reshaping
> >
> > If this is user error I'd expect a discreet error message, but I don't
> > get one, just a hang during the reshape following successful --grow
> > command (going from 3x device raid5 to 4x device raid5).
> >
> > # cat /proc/mdstat
> > Personalities : [raid6] [raid5] [raid4]
> > md127 : active raid5 loop3[4] loop2[3] loop1[1] loop0[0]
> > 2095104 blocks super 1.2 level 5, 512k chunk, algorithm 2 [4/4]
> > [UUUU]
> > [>....................] reshape = 0.0% (0/1047552)
> > finish=7.0min speed=2402K/sec
> >
> > Reproduces with:
> > kernel-3.17.6-300.fc21.x86_64
> > mdadm-3.3.2-1.fc21.x86_64
> > and
> > 3.18.0-2.fc22.i686+debug
> > mdadm-3.3.2-1.fc21.i686
> >
> > Details are here:
> > https://bugzilla.kernel.org/show_bug.cgi?id=89851
> >
> > --
> > Chris Murphy
> > --
> > To unsubscribe from this list: send the line "unsubscribe linux-raid" in
> > the body of a message to majordomo@vger.kernel.org
> > More majordomo info at http://vger.kernel.org/majordomo-info.html
> >
> --
> To unsubscribe from this list: send the line "unsubscribe linux-raid" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
^ permalink raw reply
* Re: [PATCH 01/24] md-cluster: Design Documentation
From: John Stoffel @ 2014-12-19 15:38 UTC (permalink / raw)
To: Goldwyn Rodrigues; +Cc: neilb, lzhong, linux-raid
In-Reply-To: <20141218161510.GA29543@shrek.lan>
>>>>> "Goldwyn" == Goldwyn Rodrigues <rgoldwyn@suse.de> writes:
This is an interesting concept, but I think you're glossing over the
details here way too much. You're so close to the trees, that you're
missing the forest. You need to spell out the requirements in terms
of software, configuration, etc ahead of time.
Showing how people can configure this for testing would be good as
well. Right now though, I wouldn't touch this with a ten foot pole.
Goldwyn> Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
Goldwyn> ---
Goldwyn> Documentation/md-cluster.txt | 178 +++++++++++++++++++++++++++++++++++++++++++
Goldwyn> 1 file changed, 178 insertions(+)
Goldwyn> create mode 100644 Documentation/md-cluster.txt
Goldwyn> diff --git a/Documentation/md-cluster.txt b/Documentation/md-cluster.txt
Goldwyn> new file mode 100644
Goldwyn> index 0000000..038d0f0
Goldwyn> --- /dev/null
Goldwyn> +++ b/Documentation/md-cluster.txt
Goldwyn> @@ -0,0 +1,178 @@
Goldwyn> +The cluster MD is a shared-device RAID for a cluster.
How is this cluster setup? What are the restrictions? You just
straight into the ondisk format, without any introduction to the
problem and how you solve it.
Goldwyn> +
Goldwyn> +
Goldwyn> +1. On-disk format
Goldwyn> +
Goldwyn> +Separate write-intent-bitmap are used for each cluster node.
Goldwyn> +The bitmaps record all writes that may have been started on that node,
Goldwyn> +and may not yet have finished. The on-disk layout is:
Goldwyn> +
Goldwyn> +0 4k 8k 12k
Goldwyn> +-------------------------------------------------------------------
Goldwyn> +| idle | md super | bm super [0] + bits |
Goldwyn> +| bm bits[0, contd] | bm super[1] + bits | bm bits[1, contd] |
Goldwyn> +| bm super[2] + bits | bm bits [2, contd] | bm super[3] + bits |
Goldwyn> +| bm bits [3, contd] | | |
Goldwyn> +
Goldwyn> +During "normal" functioning we assume the filesystem ensures that only one
Goldwyn> +node writes to any given block at a time, so a write
Goldwyn> +request will
Goldwyn> + - set the appropriate bit (if not already set)
Goldwyn> + - commit the write to all mirrors
Goldwyn> + - schedule the bit to be cleared after a timeout.
Goldwyn> +
Goldwyn> +Reads are just handled normally. It is up to the filesystem to
Goldwyn> +ensure one node doesn't read from a location where another node (or the same
Goldwyn> +node) is writing.
GAH! So what filesystem(s) are supported and known to work? Why this
this information not in the introduction? You just toss off this
statement without any context.
And you also seem to imply that I can't just put LVM volumes ontop of
this mirror either, which to me is a huge layering violation. If I'm
using MD to build RAID1 devices, I don't care how MD handles
reads/writes being out of sync. My filesystem or volumes on top get
consistent storage without having to know anything special.
Right there, this is a huge fail for me.
Goldwyn> +2. DLM Locks for management
Goldwyn> +
Goldwyn> +There are two locks for managing the device:
Goldwyn> +
Goldwyn> +2.1 Bitmap lock resource (bm_lockres)
Goldwyn> +
Goldwyn> + The bm_lockres protects individual node bitmaps. They are named in the
Goldwyn> + form bitmap001 for node 1, bitmap002 for node and so on. When a node
Goldwyn> + joins the cluster, it acquires the lock in PW mode and it stays so
PW is what? Make sure you expand all your acronyms the first time you
use them so we can confirm we all understand them please.
Goldwyn> + during the lifetime the node is part of the cluster. The lock resource
Goldwyn> + number is based on the slot number returned by the DLM subsystem. Since
Goldwyn> + DLM starts node count from one and bitmap slots start from zero, one is
Goldwyn> + subtracted from the DLM slot number to arrive at the bitmap slot number.
Why do you bother? Why not just make the bitmap slots start at 1 and
reserve zero for a special case? Say that the bitmap is setup but not
initialized?
Goldwyn> +
Goldwyn> +3. Communication
Goldwyn> +
Goldwyn> +Each node has to communicate with other nodes when starting or ending
Goldwyn> +resync, and metadata superblock updates.
HOW!!!! Does this all depend on DRDB being installed? Or some other
HA software?
Goldwyn> +
Goldwyn> +3.1 Message Types
Goldwyn> +
Goldwyn> + There are 3 types, of messages which are passed
Goldwyn> +
Goldwyn> + 3.1.1 METADATA_UPDATED: informs other nodes that the metadata has been
Goldwyn> + updated, and the node must re-read the md superblock. This is performed
Goldwyn> + synchronously.
Goldwyn> +
Goldwyn> + 3.1.2 RESYNC: informs other nodes that a resync is initiated or ended
Goldwyn> + so that each node may suspend or resume the region.
Goldwyn> +
Goldwyn> +3.2 Communication mechanism
Goldwyn> +
Goldwyn> + The DLM LVB is used to communicate within nodes of the cluster. There
Goldwyn> + are three resources used for the purpose:
Goldwyn> +
Goldwyn> + 3.2.1 Token: The resource which protects the entire communication
Goldwyn> + system. The node having the token resource is allowed to
Goldwyn> + communicate.
Goldwyn> +
Goldwyn> + 3.2.2 Message: The lock resource which carries the data to
Goldwyn> + communicate.
Goldwyn> +
Goldwyn> + 3.2.3 Ack: The resource, acquiring which means the message has been
Goldwyn> + acknowledged by all nodes in the cluster. The BAST of the resource
Goldwyn> + is used to inform the receive node that a node wants to communicate.
Goldwyn> +
Goldwyn> +The algorithm is:
Goldwyn> +
Goldwyn> + 1. receive status
Goldwyn> +
Goldwyn> + sender receiver receiver
Goldwyn> + ACK:CR ACK:CR ACK:CR
Goldwyn> +
Goldwyn> + 2. sender get EX of TOKEN
Goldwyn> + sender get EX of MESSAGE
Goldwyn> + sender receiver receiver
Goldwyn> + TOKEN:EX ACK:CR ACK:CR
Goldwyn> + MESSAGE:EX
Goldwyn> + ACK:CR
Goldwyn> +
Goldwyn> + Sender checks that it still needs to send a message. Messages received
Goldwyn> + or other events that happened while waiting for the TOKEN may have made
Goldwyn> + this message inappropriate or redundant.
Goldwyn> +
Goldwyn> + 3. sender write LVB.
Goldwyn> + sender down-convert MESSAGE from EX to CR
Goldwyn> + sender try to get EX of ACK
Goldwyn> + [ wait until all receiver has *processed* the MESSAGE ]
Goldwyn> +
Goldwyn> + [ triggered by bast of ACK ]
Goldwyn> + receiver get CR of MESSAGE
Goldwyn> + receiver read LVB
Goldwyn> + receiver processes the message
Goldwyn> + [ wait finish ]
Goldwyn> + receiver release ACK
Goldwyn> +
Goldwyn> + sender receiver receiver
Goldwyn> + TOKEN:EX MESSAGE:CR MESSAGE:CR
Goldwyn> + MESSAGE:CR
Goldwyn> + ACK:EX
Goldwyn> +
Goldwyn> + 4. triggered by grant of EX on ACK (indicating all receivers have processed
Goldwyn> + message)
Goldwyn> + sender down-convert ACK from EX to CR
Goldwyn> + sender release MESSAGE
Goldwyn> + sender release TOKEN
Goldwyn> + receiver upconvert to EX of MESSAGE
Goldwyn> + receiver get CR of ACK
Goldwyn> + receiver release MESSAGE
Goldwyn> +
Goldwyn> + sender receiver receiver
Goldwyn> + ACK:CR ACK:CR ACK:CR
Goldwyn> +
Goldwyn> +
Goldwyn> +4. Handling Failures
Goldwyn> +
Goldwyn> +4.1 Node Failure
Goldwyn> + When a node fails, the DLM informs the cluster with the slot. The node
This needs to be re-worded. The cluster is the entire group of
machines, I think you mean:
The DLM informs the node with the slot.
And is a node failure as simple as a reboot? How about if the entire
cluster crashes, how to do you know which node is the more upto date
and should be the master?
Goldwyn> + starts a cluster recovery thread. The cluster recovery thread:
Goldwyn> + - acquires the bitmap<number> lock of the failed node
Goldwyn> + - opens the bitmap
Goldwyn> + - reads the bitmap of the failed node
Goldwyn> + - copies the set bitmap to local node
Goldwyn> + - cleans the bitmap of the failed node
Goldwyn> + - releases bitmap<number> lock of the failed node
Goldwyn> + - initiates resync of the bitmap on the current node
Goldwyn> +
Goldwyn> + The resync process, is the regular md resync. However, in a clustered
Goldwyn> + environment when a resync is performed, it needs to tell other nodes
Goldwyn> + of the areas which are suspended. Before a resync starts, the node
Goldwyn> + send out RESYNC_START with the (lo,hi) range of the area which needs
Goldwyn> + to be suspended. Each node maintains a suspend_list, which contains
Goldwyn> + the list of ranges which are currently suspended. On receiving
Goldwyn> + RESYNC_START, the node adds the range to the suspend_list. Similarly,
Goldwyn> + when the node performing resync finishes, it send RESYNC_FINISHED
Goldwyn> + to other nodes and other nodes remove the corresponding entry from
Goldwyn> + the suspend_list.
Goldwyn> +
Goldwyn> + A helper function, should_suspend() can be used to check if a particular
Goldwyn> + I/O range should be suspended or not.
Goldwyn> +
Goldwyn> +4.2 Device Failure
Goldwyn> + Device failures are handled and communicated with the metadata update
Goldwyn> + routine.
Goldwyn> +
Goldwyn> +5. Adding a new Device
Goldwyn> +For adding a new device, it is necessary that all nodes "see" the new device
Goldwyn> +to be added. For this, the following algorithm is used:
Goldwyn> +
Goldwyn> + 1. Node 1 issues mdadm --manage /dev/mdX --add /dev/sdYY which issues
Goldwyn> + ioctl(ADD_NEW_DISC with disc.state set to MD_DISK_CLUSTER_ADD)
Goldwyn> + 2. Node 1 sends NEWDISK with uuid and slot number
Goldwyn> + 3. Other nodes issue kobject_uevent_env with uuid and slot number
Goldwyn> + (Steps 4,5 could be a udev rule)
Goldwyn> + 4. In userspace, the node searches for the disk, perhaps
Goldwyn> + using blkid -t SUB_UUID=""
Goldwyn> + 5. Other nodes issue either of the following depending on whether the disk
Goldwyn> + was found:
Goldwyn> + ioctl(ADD_NEW_DISK with disc.state set to MD_DISK_CANDIDATE and
Goldwyn> + disc.number set to slot number)
Goldwyn> + ioctl(CLUSTERED_DISK_NACK)
Goldwyn> + 6. Other nodes drop lock on no-new-devs (CR) if device is found
Goldwyn> + 7. Node 1 attempts EX lock on no-new-devs
Goldwyn> + 8. If node 1 gets the lock, it sends METADATA_UPDATED after unmarking the disk
Goldwyn> + as SpareLocal
Goldwyn> + 9. If not (get no-new-dev lock), it fails the operation and sends METADATA_UPDATED
Goldwyn> + 10. Other nodes get the information whether a disk is added or not
Goldwyn> + by the following METADATA_UPDATED.
Goldwyn> +
Goldwyn> +
Goldwyn> --
Goldwyn> 2.1.2
Goldwyn> --
Goldwyn> To unsubscribe from this list: send the line "unsubscribe linux-raid" in
Goldwyn> the body of a message to majordomo@vger.kernel.org
Goldwyn> More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply
* Re: [PATCH 01/24] md-cluster: Design Documentation
From: Goldwyn Rodrigues @ 2014-12-19 22:38 UTC (permalink / raw)
To: John Stoffel; +Cc: neilb, lzhong, linux-raid
In-Reply-To: <21652.18047.978617.730122@quad.stoffel.home>
Hi John,
Thanks for the review.
On 12/19/2014 09:38 AM, John Stoffel wrote:
>>>>>> "Goldwyn" == Goldwyn Rodrigues <rgoldwyn@suse.de> writes:
>
> This is an interesting concept, but I think you're glossing over the
> details here way too much. You're so close to the trees, that you're
> missing the forest. You need to spell out the requirements in terms
> of software, configuration, etc ahead of time.
>
> Showing how people can configure this for testing would be good as
> well. Right now though, I wouldn't touch this with a ten foot pole.
I mentioned a quick howto in patch zero. However, putting it in the
design document will not hurt. Currently, it is known to work with
corosync 2.3.x and pacemaker 1.1 on Kernels 3.14.x
>
> Goldwyn> Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
> Goldwyn> ---
> Goldwyn> Documentation/md-cluster.txt | 178 +++++++++++++++++++++++++++++++++++++++++++
> Goldwyn> 1 file changed, 178 insertions(+)
> Goldwyn> create mode 100644 Documentation/md-cluster.txt
>
> Goldwyn> diff --git a/Documentation/md-cluster.txt b/Documentation/md-cluster.txt
> Goldwyn> new file mode 100644
> Goldwyn> index 0000000..038d0f0
> Goldwyn> --- /dev/null
> Goldwyn> +++ b/Documentation/md-cluster.txt
> Goldwyn> @@ -0,0 +1,178 @@
> Goldwyn> +The cluster MD is a shared-device RAID for a cluster.
>
>
> How is this cluster setup? What are the restrictions? You just
> straight into the ondisk format, without any introduction to the
> problem and how you solve it.
The cluster is a regular corosync/pacemaker cluster with DLM setup. I
mentioned this in patch zero as well. However, I assumed configuring a
cluster is not in the scope of the design document. This is the design
of cluster-md. I agree it could use a foreword though.
>
> Goldwyn> +
> Goldwyn> +
> Goldwyn> +1. On-disk format
> Goldwyn> +
> Goldwyn> +Separate write-intent-bitmap are used for each cluster node.
> Goldwyn> +The bitmaps record all writes that may have been started on that node,
> Goldwyn> +and may not yet have finished. The on-disk layout is:
> Goldwyn> +
> Goldwyn> +0 4k 8k 12k
> Goldwyn> +-------------------------------------------------------------------
> Goldwyn> +| idle | md super | bm super [0] + bits |
> Goldwyn> +| bm bits[0, contd] | bm super[1] + bits | bm bits[1, contd] |
> Goldwyn> +| bm super[2] + bits | bm bits [2, contd] | bm super[3] + bits |
> Goldwyn> +| bm bits [3, contd] | | |
> Goldwyn> +
> Goldwyn> +During "normal" functioning we assume the filesystem ensures that only one
> Goldwyn> +node writes to any given block at a time, so a write
> Goldwyn> +request will
> Goldwyn> + - set the appropriate bit (if not already set)
> Goldwyn> + - commit the write to all mirrors
> Goldwyn> + - schedule the bit to be cleared after a timeout.
> Goldwyn> +
> Goldwyn> +Reads are just handled normally. It is up to the filesystem to
> Goldwyn> +ensure one node doesn't read from a location where another node (or the same
> Goldwyn> +node) is writing.
>
>
> GAH! So what filesystem(s) are supported and known to work? Why this
> this information not in the introduction? You just toss off this
> statement without any context.
The point here is data integrity is the responsibility of the
filesystem. The cluster-md just ensures that all it has confirmed as
written is stable and mirrored (RAID1). As for filesystem support, all
device based filesystems are supported. However, we are targeting
cluster based filesystems such as ocfs2. Yes, it could be moved in the
Introduction.
>
> And you also seem to imply that I can't just put LVM volumes ontop of
> this mirror either, which to me is a huge layering violation. If I'm
No, I am not implying LVM cannot be used. LVM can be used in conjunction
with cluster-md.
> using MD to build RAID1 devices, I don't care how MD handles
> reads/writes being out of sync. My filesystem or volumes on top get
> consistent storage without having to know anything special.
If you are reading the design document of cluster-md. I think you should
be concerned on how out of sync data is handled in order to understand
the design better. Filesystem just treat this as a normal block device
and do not need to know anything special.
>
>
> Goldwyn> +2. DLM Locks for management
> Goldwyn> +
> Goldwyn> +There are two locks for managing the device:
> Goldwyn> +
> Goldwyn> +2.1 Bitmap lock resource (bm_lockres)
> Goldwyn> +
> Goldwyn> + The bm_lockres protects individual node bitmaps. They are named in the
> Goldwyn> + form bitmap001 for node 1, bitmap002 for node and so on. When a node
> Goldwyn> + joins the cluster, it acquires the lock in PW mode and it stays so
>
> PW is what? Make sure you expand all your acronyms the first time you
> use them so we can confirm we all understand them please.
PW is Protected Write. I will add that.
>
> Goldwyn> + during the lifetime the node is part of the cluster. The lock resource
> Goldwyn> + number is based on the slot number returned by the DLM subsystem. Since
> Goldwyn> + DLM starts node count from one and bitmap slots start from zero, one is
> Goldwyn> + subtracted from the DLM slot number to arrive at the bitmap slot number.
>
> Why do you bother? Why not just make the bitmap slots start at 1 and
> reserve zero for a special case? Say that the bitmap is setup but not
> initialized?
What would that special case be? The bitmap setup is not a two-step
process. If it is setup, it is also initialized.
>
> Goldwyn> +
> Goldwyn> +3. Communication
> Goldwyn> +
> Goldwyn> +Each node has to communicate with other nodes when starting or ending
> Goldwyn> +resync, and metadata superblock updates.
>
> HOW!!!! Does this all depend on DRDB being installed? Or some other
> HA software?
DLM. Mentioned later in the design. Yes, I will add that as well.
>
> Goldwyn> +
> Goldwyn> +3.1 Message Types
> Goldwyn> +
> Goldwyn> + There are 3 types, of messages which are passed
> Goldwyn> +
> Goldwyn> + 3.1.1 METADATA_UPDATED: informs other nodes that the metadata has been
> Goldwyn> + updated, and the node must re-read the md superblock. This is performed
> Goldwyn> + synchronously.
> Goldwyn> +
> Goldwyn> + 3.1.2 RESYNC: informs other nodes that a resync is initiated or ended
> Goldwyn> + so that each node may suspend or resume the region.
> Goldwyn> +
> Goldwyn> +3.2 Communication mechanism
> Goldwyn> +
> Goldwyn> + The DLM LVB is used to communicate within nodes of the cluster. There
> Goldwyn> + are three resources used for the purpose:
> Goldwyn> +
> Goldwyn> + 3.2.1 Token: The resource which protects the entire communication
> Goldwyn> + system. The node having the token resource is allowed to
> Goldwyn> + communicate.
> Goldwyn> +
> Goldwyn> + 3.2.2 Message: The lock resource which carries the data to
> Goldwyn> + communicate.
> Goldwyn> +
> Goldwyn> + 3.2.3 Ack: The resource, acquiring which means the message has been
> Goldwyn> + acknowledged by all nodes in the cluster. The BAST of the resource
> Goldwyn> + is used to inform the receive node that a node wants to communicate.
> Goldwyn> +
> Goldwyn> +The algorithm is:
> Goldwyn> +
> Goldwyn> + 1. receive status
> Goldwyn> +
> Goldwyn> + sender receiver receiver
> Goldwyn> + ACK:CR ACK:CR ACK:CR
> Goldwyn> +
> Goldwyn> + 2. sender get EX of TOKEN
> Goldwyn> + sender get EX of MESSAGE
> Goldwyn> + sender receiver receiver
> Goldwyn> + TOKEN:EX ACK:CR ACK:CR
> Goldwyn> + MESSAGE:EX
> Goldwyn> + ACK:CR
> Goldwyn> +
> Goldwyn> + Sender checks that it still needs to send a message. Messages received
> Goldwyn> + or other events that happened while waiting for the TOKEN may have made
> Goldwyn> + this message inappropriate or redundant.
> Goldwyn> +
> Goldwyn> + 3. sender write LVB.
> Goldwyn> + sender down-convert MESSAGE from EX to CR
> Goldwyn> + sender try to get EX of ACK
> Goldwyn> + [ wait until all receiver has *processed* the MESSAGE ]
> Goldwyn> +
> Goldwyn> + [ triggered by bast of ACK ]
> Goldwyn> + receiver get CR of MESSAGE
> Goldwyn> + receiver read LVB
> Goldwyn> + receiver processes the message
> Goldwyn> + [ wait finish ]
> Goldwyn> + receiver release ACK
> Goldwyn> +
> Goldwyn> + sender receiver receiver
> Goldwyn> + TOKEN:EX MESSAGE:CR MESSAGE:CR
> Goldwyn> + MESSAGE:CR
> Goldwyn> + ACK:EX
> Goldwyn> +
> Goldwyn> + 4. triggered by grant of EX on ACK (indicating all receivers have processed
> Goldwyn> + message)
> Goldwyn> + sender down-convert ACK from EX to CR
> Goldwyn> + sender release MESSAGE
> Goldwyn> + sender release TOKEN
> Goldwyn> + receiver upconvert to EX of MESSAGE
> Goldwyn> + receiver get CR of ACK
> Goldwyn> + receiver release MESSAGE
> Goldwyn> +
> Goldwyn> + sender receiver receiver
> Goldwyn> + ACK:CR ACK:CR ACK:CR
> Goldwyn> +
> Goldwyn> +
> Goldwyn> +4. Handling Failures
> Goldwyn> +
> Goldwyn> +4.1 Node Failure
> Goldwyn> + When a node fails, the DLM informs the cluster with the slot. The node
>
> This needs to be re-worded. The cluster is the entire group of
> machines, I think you mean:
>
> The DLM informs the node with the slot.
Correct.
>
> And is a node failure as simple as a reboot? How about if the entire
> cluster crashes, how to do you know which node is the more upto date
> and should be the master?
There is not concept of master here since everything is distributed. We
do not want a central dependency. A node failure is it's inability to
respond. It is usually STONITHd (Shoot the Other node in the Head) by
the cluster resource management.
The concept of bitmap is that data needs to be synced (that what I had
been trying to explain in the point where you mentioned about
filesystem). In case of a cluster failure, The first node to come up
performs the "bitmap recovery" for all the bitmaps.
>
> Goldwyn> + starts a cluster recovery thread. The cluster recovery thread:
> Goldwyn> + - acquires the bitmap<number> lock of the failed node
> Goldwyn> + - opens the bitmap
> Goldwyn> + - reads the bitmap of the failed node
> Goldwyn> + - copies the set bitmap to local node
> Goldwyn> + - cleans the bitmap of the failed node
> Goldwyn> + - releases bitmap<number> lock of the failed node
> Goldwyn> + - initiates resync of the bitmap on the current node
> Goldwyn> +
> Goldwyn> + The resync process, is the regular md resync. However, in a clustered
> Goldwyn> + environment when a resync is performed, it needs to tell other nodes
> Goldwyn> + of the areas which are suspended. Before a resync starts, the node
> Goldwyn> + send out RESYNC_START with the (lo,hi) range of the area which needs
> Goldwyn> + to be suspended. Each node maintains a suspend_list, which contains
> Goldwyn> + the list of ranges which are currently suspended. On receiving
> Goldwyn> + RESYNC_START, the node adds the range to the suspend_list. Similarly,
> Goldwyn> + when the node performing resync finishes, it send RESYNC_FINISHED
> Goldwyn> + to other nodes and other nodes remove the corresponding entry from
> Goldwyn> + the suspend_list.
> Goldwyn> +
> Goldwyn> + A helper function, should_suspend() can be used to check if a particular
> Goldwyn> + I/O range should be suspended or not.
> Goldwyn> +
> Goldwyn> +4.2 Device Failure
> Goldwyn> + Device failures are handled and communicated with the metadata update
> Goldwyn> + routine.
> Goldwyn> +
> Goldwyn> +5. Adding a new Device
> Goldwyn> +For adding a new device, it is necessary that all nodes "see" the new device
> Goldwyn> +to be added. For this, the following algorithm is used:
> Goldwyn> +
> Goldwyn> + 1. Node 1 issues mdadm --manage /dev/mdX --add /dev/sdYY which issues
> Goldwyn> + ioctl(ADD_NEW_DISC with disc.state set to MD_DISK_CLUSTER_ADD)
> Goldwyn> + 2. Node 1 sends NEWDISK with uuid and slot number
> Goldwyn> + 3. Other nodes issue kobject_uevent_env with uuid and slot number
> Goldwyn> + (Steps 4,5 could be a udev rule)
> Goldwyn> + 4. In userspace, the node searches for the disk, perhaps
> Goldwyn> + using blkid -t SUB_UUID=""
> Goldwyn> + 5. Other nodes issue either of the following depending on whether the disk
> Goldwyn> + was found:
> Goldwyn> + ioctl(ADD_NEW_DISK with disc.state set to MD_DISK_CANDIDATE and
> Goldwyn> + disc.number set to slot number)
> Goldwyn> + ioctl(CLUSTERED_DISK_NACK)
> Goldwyn> + 6. Other nodes drop lock on no-new-devs (CR) if device is found
> Goldwyn> + 7. Node 1 attempts EX lock on no-new-devs
> Goldwyn> + 8. If node 1 gets the lock, it sends METADATA_UPDATED after unmarking the disk
> Goldwyn> + as SpareLocal
> Goldwyn> + 9. If not (get no-new-dev lock), it fails the operation and sends METADATA_UPDATED
> Goldwyn> + 10. Other nodes get the information whether a disk is added or not
> Goldwyn> + by the following METADATA_UPDATED.
> Goldwyn> +
> Goldwyn> +
> Goldwyn> --
> Goldwyn> 2.1.2
>
--
Goldwyn
^ permalink raw reply
* 4 disks outage in RAID6
From: Mark Kolama @ 2014-12-20 17:03 UTC (permalink / raw)
To: linux-raid
Dear List,
due to a controller failure, a raid6 with 16 drives
lost 4 drives at once. The failure was noticed a few days later.
a examine output of all 16 drive is listed at
http://pastebin.com/4WH9xp7K
As you can see the event count differs on 4 drives with
about 150 comparing to the other 12 drives.
I have already tried:
mdadm --assemble --scan:
assembled from 12 drives - not enough to start the array.
Then i tried:
mdadm --assemble /dev/md0 /dev/sda1 /dev/sdb1 /dev/sdc1 /dev/sdd1
/dev/sde1 /dev/sdf1 /dev/sdg1 /dev/sdh1 /dev/sdi1 /dev/sdj1 /dev/sdk1
/dev/sdl1 /dev/sdm1 /dev/sdn1 /dev/sdo1 /dev/sdp1 --force
/proc/mdstat after that:
Personalities : [raid6] [raid5] [raid4]
md0 : inactive sda1[0](S) sdp1[15](S) sdo1[14](S) sdn1[13](S)
sdm1[12](S) sdl1[11](S) sdk1[10](S) sdj1[9](S) sdi1[8](S) sdh1[7](S)
sdg1[6](S) sdf1[5](S) sde1[4](S) sdd1[3](S) sdc1[2](S) sdb1[1](S)
62353932288 blocks super 1.2
No success either.
So the next try would be recreating the array ?
Any help is appreciated, thanks in advance.
Mark
^ permalink raw reply
* Re: Expand RAID5 array or switch to RAID10
From: forumscollective @ 2014-12-22 5:57 UTC (permalink / raw)
To: linux-raid@vger.kernel.org
In-Reply-To: <CAG__1a7YqAWMccgDeYksY_t10Zyy8Rc-veZM_P7aaXs-DtFdfQ@mail.gmail.com>
anyone?
Sent from my iPhone
> On 17 Dec 2014, at 12:11, George Duffield <forumscollective@gmail.com> wrote:
>
> Hi
>
> I have a RAID5 array comprising four 3TB drives and I've basically got
> 100GB remaining free so it's time to expand the array. With this in
> mind, and considering my requirement is predominantly to increase
> capacity, I'm wondering whether it'd be best to add another 3TB drive
> and expand the array whilst retaining RAID5 or to switch out to a
> RAID10 array. My thinking is 1) adding a 5th 3TB drive to the
> existing array may result in drive failure during the rebuild; and 2)
> RAID10 is very expensive to implement. Hence, I'm tempted to consider
> other options (recognising that whilst it'd be convenient I don't
> actually need everything stored on a single array). One such option
> would be running two raid5 arrays comprised of 3x 3TB each yielding
> 12TB of storage across the two arrays.
>
>
> Questions:
> + Is my assumption re RAID5 drive failure correct/ likely?
> + Is there a non-destructive way to migrate from RAID5 to RAID10?
> + 2 x RAID5 arrays seems pretty appealing - from a reliability and
> cost effectiveness standpoint - yes/no?
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox