* [PATCH 07/24] Return MD_SB_CLUSTERED if mddev is clustered
From: Goldwyn Rodrigues @ 2014-12-18 16:16 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md.c | 3 +++
include/uapi/linux/raid/md_p.h | 1 +
2 files changed, 4 insertions(+)
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 795d925..5f45951 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -5630,6 +5630,9 @@ static int get_array_info(struct mddev * mddev, void __user * arg)
info.state = (1<<MD_SB_CLEAN);
if (mddev->bitmap && mddev->bitmap_info.offset)
info.state = (1<<MD_SB_BITMAP_PRESENT);
+ if (mddev_is_clustered(mddev))
+ info.state |= (1<<MD_SB_CLUSTERED);
+
info.active_disks = insync;
info.working_disks = working;
info.failed_disks = failed;
diff --git a/include/uapi/linux/raid/md_p.h b/include/uapi/linux/raid/md_p.h
index 49f4210..643489d 100644
--- a/include/uapi/linux/raid/md_p.h
+++ b/include/uapi/linux/raid/md_p.h
@@ -101,6 +101,7 @@ typedef struct mdp_device_descriptor_s {
#define MD_SB_CLEAN 0
#define MD_SB_ERRORS 1
+#define MD_SB_CLUSTERED 5 /* MD is clustered */
#define MD_SB_BITMAP_PRESENT 8 /* bitmap may be present nearby */
/*
--
2.1.2
^ permalink raw reply related
* [PATCH 06/24] Introduce md_cluster_info
From: Goldwyn Rodrigues @ 2014-12-18 16:16 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
md_cluster_info stores the cluster information in the MD device.
The join() is called when mddev detects it is a clustered device.
The main responsibilities are:
1. Setup a DLM lockspace
2. Setup all initial locks such as super block locks and bitmap lock (will come later)
The leave() clears up the lockspace and all the locks held.
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/bitmap.c | 4 +++
drivers/md/md-cluster.c | 67 +++++++++++++++++++++++++++++++++++++++++++++----
drivers/md/md.h | 8 ++++++
3 files changed, 74 insertions(+), 5 deletions(-)
diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c
index 4195a01..33374de 100644
--- a/drivers/md/bitmap.c
+++ b/drivers/md/bitmap.c
@@ -420,6 +420,7 @@ void bitmap_update_sb(struct bitmap *bitmap)
/* This might have been changed by a reshape */
sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
+ sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes);
sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
bitmap_info.space);
kunmap_atomic(sb);
@@ -531,6 +532,7 @@ static int bitmap_read_sb(struct bitmap *bitmap)
bitmap_super_t *sb;
unsigned long chunksize, daemon_sleep, write_behind;
unsigned long long events;
+ int nodes = 0;
unsigned long sectors_reserved = 0;
int err = -EINVAL;
struct page *sb_page;
@@ -570,6 +572,7 @@ static int bitmap_read_sb(struct bitmap *bitmap)
daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
write_behind = le32_to_cpu(sb->write_behind);
sectors_reserved = le32_to_cpu(sb->sectors_reserved);
+ nodes = le32_to_cpu(sb->nodes);
/* verify that the bitmap-specific fields are valid */
if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
@@ -630,6 +633,7 @@ out_no_sb:
bitmap->mddev->bitmap_info.chunksize = chunksize;
bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
bitmap->mddev->bitmap_info.max_write_behind = write_behind;
+ bitmap->mddev->bitmap_info.nodes = nodes;
if (bitmap->mddev->bitmap_info.space == 0 ||
bitmap->mddev->bitmap_info.space > sectors_reserved)
bitmap->mddev->bitmap_info.space = sectors_reserved;
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index d2d4d33..fa28aed 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -25,8 +25,16 @@ struct dlm_lock_resource {
struct dlm_lksb lksb;
char *name; /* lock name. */
uint32_t flags; /* flags to pass to dlm_lock() */
- void (*bast)(void *arg, int mode); /* blocking AST function pointer*/
struct completion completion; /* completion for synchronized locking */
+ struct mddev *mddev; /* pointing back to mddev. */
+ void (*bast)(void *arg, int mode); /* blocking AST function pointer*/
+};
+
+struct md_cluster_info {
+ /* dlm lock space and resources for clustered raid. */
+ dlm_lockspace_t *lockspace;
+ struct dlm_lock_resource *sb_lock;
+ struct mutex sb_mutex;
};
static void sync_ast(void *arg)
@@ -54,16 +62,18 @@ static int dlm_unlock_sync(struct dlm_lock_resource *res)
return dlm_lock_sync(res, DLM_LOCK_NL);
}
-static struct dlm_lock_resource *lockres_init(dlm_lockspace_t *lockspace,
+static struct dlm_lock_resource *lockres_init(struct mddev *mddev,
char *name, void (*bastfn)(void *arg, int mode), int with_lvb)
{
struct dlm_lock_resource *res = NULL;
int ret, namelen;
+ struct md_cluster_info *cinfo = mddev->cluster_info;
res = kzalloc(sizeof(struct dlm_lock_resource), GFP_KERNEL);
if (!res)
return NULL;
- res->ls = lockspace;
+ res->ls = cinfo->lockspace;
+ res->mddev = mddev;
namelen = strlen(name);
res->name = kzalloc(namelen + 1, GFP_KERNEL);
if (!res->name) {
@@ -119,13 +129,60 @@ static void lockres_free(struct dlm_lock_resource *res)
return;
}
-static int join(struct mddev *md, int nodes)
+static char *pretty_uuid(char *dest, char *src)
{
+ int i, len = 0;
+ for (i=0; i<16; i++) {
+ if (i==4 || i==6 || i==8 || i==10)
+ len += sprintf(dest + len, "-");
+ len += sprintf(dest + len, "%02x", (__u8)src[i]);
+ }
+ return dest;
+}
+
+static int join(struct mddev *mddev, int nodes)
+{
+ struct md_cluster_info *cinfo;
+ int ret;
+ char str[64];
+
+ if (!try_module_get(THIS_MODULE))
+ return -ENOENT;
+
+ cinfo = kzalloc(sizeof(struct md_cluster_info), GFP_KERNEL);
+ if (!cinfo)
+ return -ENOMEM;
+
+ memset(str, 0, 64);
+ pretty_uuid(str, mddev->uuid);
+ ret = dlm_new_lockspace(str, NULL, DLM_LSFL_FS, LVB_SIZE,
+ NULL, NULL, NULL, &cinfo->lockspace);
+ if (ret)
+ goto err;
+ cinfo->sb_lock = lockres_init(mddev, "cmd-super",
+ NULL, 0);
+ if (!cinfo->sb_lock) {
+ ret = -ENOMEM;
+ goto err;
+ }
+ mutex_init(&cinfo->sb_mutex);
+ mddev->cluster_info = cinfo;
return 0;
+err:
+ if (cinfo->lockspace)
+ dlm_release_lockspace(cinfo->lockspace, 2);
+ kfree(cinfo);
+ module_put(THIS_MODULE);
+ return ret;
}
-static int leave(struct mddev *md)
+static int leave(struct mddev *mddev)
{
+ struct md_cluster_info *cinfo = mddev->cluster_info;
+ if (!cinfo)
+ return 0;
+ lockres_free(cinfo->sb_lock);
+ dlm_release_lockspace(cinfo->lockspace, 2);
return 0;
}
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 076e1ae..8017e18 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -203,6 +203,8 @@ extern int rdev_clear_badblocks(struct md_rdev *rdev, sector_t s, int sectors,
int is_new);
extern void md_ack_all_badblocks(struct badblocks *bb);
+struct md_cluster_info;
+
struct mddev {
void *private;
struct md_personality *pers;
@@ -420,6 +422,7 @@ struct mddev {
unsigned long daemon_sleep; /* how many jiffies between updates? */
unsigned long max_write_behind; /* write-behind mode */
int external;
+ int nodes;
} bitmap_info;
atomic_t max_corr_read_errors; /* max read retries */
@@ -438,6 +441,7 @@ struct mddev {
struct work_struct flush_work;
struct work_struct event_work; /* used by dm to report failure event */
void (*sync_super)(struct mddev *mddev, struct md_rdev *rdev);
+ struct md_cluster_info *cluster_info;
};
@@ -634,4 +638,8 @@ static inline int mddev_check_plugged(struct mddev *mddev)
sizeof(struct blk_plug_cb));
}
extern struct md_cluster_operations *md_cluster_ops;
+static inline int mddev_is_clustered(struct mddev *mddev)
+{
+ return mddev->cluster_info && mddev->bitmap_info.nodes > 1;
+}
#endif /* _MD_MD_H */
--
2.1.2
^ permalink raw reply related
* [PATCH 05/24] Introduce md_cluster_operations to handle cluster functions
From: Goldwyn Rodrigues @ 2014-12-18 16:16 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
This allows dynamic registering of cluster hooks.
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md-cluster.c | 18 +++++++++++++++++
drivers/md/md-cluster.h | 15 +++++++++++++++
drivers/md/md.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++++
drivers/md/md.h | 7 +++++++
4 files changed, 91 insertions(+)
create mode 100644 drivers/md/md-cluster.h
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index e555b76..d2d4d33 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -16,6 +16,7 @@
#include <linux/dlm.h>
#include <linux/sched.h>
#include "md.h"
+#include "md-cluster.h"
#define LVB_SIZE 64
@@ -118,14 +119,31 @@ static void lockres_free(struct dlm_lock_resource *res)
return;
}
+static int join(struct mddev *md, int nodes)
+{
+ return 0;
+}
+
+static int leave(struct mddev *md)
+{
+ return 0;
+}
+
+static struct md_cluster_operations cluster_ops = {
+ .join = join,
+ .leave = leave,
+};
+
static int __init cluster_init(void)
{
pr_info("Registering Cluster MD functions\n");
+ register_md_cluster_operations(&cluster_ops, THIS_MODULE);
return 0;
}
static void cluster_exit(void)
{
+ unregister_md_cluster_operations();
}
module_init(cluster_init);
diff --git a/drivers/md/md-cluster.h b/drivers/md/md-cluster.h
new file mode 100644
index 0000000..aa9f07b
--- /dev/null
+++ b/drivers/md/md-cluster.h
@@ -0,0 +1,15 @@
+
+
+#ifndef _MD_CLUSTER_H
+#define _MD_CLUSTER_H
+
+#include "md.h"
+
+struct mddev;
+
+struct md_cluster_operations {
+ int (*join)(struct mddev *mddev);
+ int (*leave)(struct mddev *mddev);
+};
+
+#endif /* _MD_CLUSTER_H */
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 40959ee..795d925 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -53,6 +53,7 @@
#include <linux/slab.h>
#include "md.h"
#include "bitmap.h"
+#include "md-cluster.h"
#ifndef MODULE
static void autostart_arrays(int part);
@@ -66,6 +67,10 @@ static void autostart_arrays(int part);
static LIST_HEAD(pers_list);
static DEFINE_SPINLOCK(pers_lock);
+struct md_cluster_operations *md_cluster_ops;
+struct module *md_cluster_mod;
+EXPORT_SYMBOL(md_cluster_mod);
+
static void md_print_devices(void);
static DECLARE_WAIT_QUEUE_HEAD(resync_wait);
@@ -7228,6 +7233,52 @@ int unregister_md_personality(struct md_personality *p)
return 0;
}
+int register_md_cluster_operations(struct md_cluster_operations *ops, struct module *module)
+{
+ if (md_cluster_ops != NULL)
+ return -EALREADY;
+ spin_lock(&pers_lock);
+ md_cluster_ops = ops;
+ md_cluster_mod = module;
+ spin_unlock(&pers_lock);
+ return 0;
+}
+EXPORT_SYMBOL(register_md_cluster_operations);
+
+int unregister_md_cluster_operations(void)
+{
+ spin_lock(&pers_lock);
+ md_cluster_ops = NULL;
+ spin_unlock(&pers_lock);
+ return 0;
+}
+EXPORT_SYMBOL(unregister_md_cluster_operations);
+
+int md_setup_cluster(struct mddev *mddev, int nodes)
+{
+ int err;
+ err = request_module("md-cluster");
+ if (err) {
+ pr_err("md-cluster module not found.\n");
+ return err;
+ }
+
+ spin_lock(&pers_lock);
+ if (!md_cluster_ops || !try_module_get(md_cluster_mod)) {
+ spin_unlock(&pers_lock);
+ return -ENOENT;
+ }
+ spin_unlock(&pers_lock);
+
+ return md_cluster_ops->join(mddev);
+}
+
+void md_cluster_stop(struct mddev *mddev)
+{
+ md_cluster_ops->leave(mddev);
+ module_put(md_cluster_mod);
+}
+
static int is_mddev_idle(struct mddev *mddev, int init)
{
struct md_rdev * rdev;
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 07bba96..076e1ae 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -23,6 +23,7 @@
#include <linux/timer.h>
#include <linux/wait.h>
#include <linux/workqueue.h>
+#include "md-cluster.h"
#define MaxSector (~(sector_t)0)
@@ -575,6 +576,11 @@ static inline void safe_put_page(struct page *p)
extern int register_md_personality(struct md_personality *p);
extern int unregister_md_personality(struct md_personality *p);
+extern int register_md_cluster_operations(struct md_cluster_operations *ops,
+ struct module *module);
+extern int unregister_md_cluster_operations(void);
+extern int md_setup_cluster(struct mddev *mddev, int nodes);
+extern void md_cluster_stop(struct mddev *mddev);
extern struct md_thread *md_register_thread(
void (*run)(struct md_thread *thread),
struct mddev *mddev,
@@ -627,4 +633,5 @@ static inline int mddev_check_plugged(struct mddev *mddev)
return !!blk_check_plugged(md_unplug, mddev,
sizeof(struct blk_plug_cb));
}
+extern struct md_cluster_operations *md_cluster_ops;
#endif /* _MD_MD_H */
--
2.1.2
^ permalink raw reply related
* [PATCH 04/24] DLM lock and unlock functions
From: Goldwyn Rodrigues @ 2014-12-18 16:15 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
A dlm_lock_resource is a structure which contains all information
required for locking using DLM. The init function allocates the
lock and acquires the lock in NL mode. The unlock function
converts the lock resource to NL mode. This is done to preserve
LVB and for faster processing of locks. The lock resource is
DLM unlocked only in the lockres_free function, which is the end
of life of the lock resource.
Signed-off-by: Lidong Zhong <lzhong@suse.com>
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/md-cluster.c | 104 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 104 insertions(+)
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
index 6f21bba..e555b76 100644
--- a/drivers/md/md-cluster.c
+++ b/drivers/md/md-cluster.c
@@ -13,6 +13,110 @@
#include <linux/module.h>
+#include <linux/dlm.h>
+#include <linux/sched.h>
+#include "md.h"
+
+#define LVB_SIZE 64
+
+struct dlm_lock_resource {
+ dlm_lockspace_t *ls;
+ struct dlm_lksb lksb;
+ char *name; /* lock name. */
+ uint32_t flags; /* flags to pass to dlm_lock() */
+ void (*bast)(void *arg, int mode); /* blocking AST function pointer*/
+ struct completion completion; /* completion for synchronized locking */
+};
+
+static void sync_ast(void *arg)
+{
+ struct dlm_lock_resource *res;
+ res = (struct dlm_lock_resource *) arg;
+ complete(&res->completion);
+}
+
+static int dlm_lock_sync(struct dlm_lock_resource *res, int mode)
+{
+ int ret = 0;
+ init_completion(&res->completion);
+ ret = dlm_lock(res->ls, mode, &res->lksb,
+ res->flags, res->name, strlen(res->name),
+ 0, sync_ast, res, res->bast);
+ if (ret)
+ return ret;
+ wait_for_completion(&res->completion);
+ return res->lksb.sb_status;
+}
+
+static int dlm_unlock_sync(struct dlm_lock_resource *res)
+{
+ return dlm_lock_sync(res, DLM_LOCK_NL);
+}
+
+static struct dlm_lock_resource *lockres_init(dlm_lockspace_t *lockspace,
+ char *name, void (*bastfn)(void *arg, int mode), int with_lvb)
+{
+ struct dlm_lock_resource *res = NULL;
+ int ret, namelen;
+
+ res = kzalloc(sizeof(struct dlm_lock_resource), GFP_KERNEL);
+ if (!res)
+ return NULL;
+ res->ls = lockspace;
+ namelen = strlen(name);
+ res->name = kzalloc(namelen + 1, GFP_KERNEL);
+ if (!res->name) {
+ pr_err("md-cluster: Unable to allocate resource name for"
+ " resource %s\n", name);
+ goto out_err;
+ }
+ strlcpy(res->name, name, namelen + 1);
+ if (with_lvb) {
+ res->lksb.sb_lvbptr = kzalloc(LVB_SIZE, GFP_KERNEL);
+ if (!res->lksb.sb_lvbptr) {
+ pr_err("md-cluster: Unable to allocate LVB for"
+ " resource %s\n", name);
+ goto out_err;
+ }
+ res->flags = DLM_LKF_VALBLK;
+ }
+
+ if (bastfn)
+ res->bast = bastfn;
+
+ res->flags |= DLM_LKF_EXPEDITE;
+
+ ret = dlm_lock_sync(res, DLM_LOCK_NL);
+ if (ret) {
+ pr_err("md-cluster: Unable to lock NL on new lock"
+ " resource %s\n", name);
+ goto out_err;
+ }
+ res->flags &= ~DLM_LKF_EXPEDITE;
+ res->flags |= DLM_LKF_CONVERT;
+
+ return res;
+out_err:
+ kfree(res->lksb.sb_lvbptr);
+ kfree(res->name);
+ kfree(res);
+ return NULL;
+}
+
+static void lockres_free(struct dlm_lock_resource *res)
+{
+ if (!res)
+ return;
+
+ init_completion(&res->completion);
+ dlm_unlock(res->ls, res->lksb.sb_lkid, 0, &res->lksb, res);
+ wait_for_completion(&res->completion);
+
+ kfree(res->name);
+ kfree(res->lksb.sb_lvbptr);
+ kfree(res);
+ return;
+}
static int __init cluster_init(void)
{
--
2.1.2
^ permalink raw reply related
* [PATCH 03/24] Create a separate module for clustering support
From: Goldwyn Rodrigues @ 2014-12-18 16:15 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/Kconfig | 15 +++++++++++++++
drivers/md/Makefile | 1 +
drivers/md/md-cluster.c | 31 +++++++++++++++++++++++++++++++
3 files changed, 47 insertions(+)
create mode 100644 drivers/md/md-cluster.c
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index 95ad936..55ea703 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -174,6 +174,21 @@ config MD_FAULTY
In unsure, say N.
+
+config MD_CLUSTER
+ tristate "Cluster Support for MD (EXPERIMENTAL)"
+ depends on BLK_DEV_MD
+ depends on DLM
+ ---help---
+ Clustering support for MD devices. This enables locking and
+ synchronization across multiple systems on the cluster, so all
+ nodes in the cluster can access the MD devices simultaneously.
+
+ This brings the redundancy (and uptime) of RAID levels across the
+ nodes of the cluster.
+
+ If unsure, say N.
+
source "drivers/md/bcache/Kconfig"
config BLK_DEV_DM_BUILTIN
diff --git a/drivers/md/Makefile b/drivers/md/Makefile
index f26d832..a3d7bf6 100644
--- a/drivers/md/Makefile
+++ b/drivers/md/Makefile
@@ -29,6 +29,7 @@ obj-$(CONFIG_MD_RAID10) += raid10.o
obj-$(CONFIG_MD_RAID456) += raid456.o
obj-$(CONFIG_MD_MULTIPATH) += multipath.o
obj-$(CONFIG_MD_FAULTY) += faulty.o
+obj-$(CONFIG_MD_CLUSTER) += md-cluster.o
obj-$(CONFIG_BCACHE) += bcache/
obj-$(CONFIG_BLK_DEV_MD) += md-mod.o
obj-$(CONFIG_BLK_DEV_DM) += dm-mod.o
diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c
new file mode 100644
index 0000000..6f21bba
--- /dev/null
+++ b/drivers/md/md-cluster.c
@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2014, SUSE
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * You should have received a copy of the GNU General Public License
+ * (for example /usr/src/linux/COPYING); if not, write to the Free
+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+
+#include <linux/module.h>
+
+static int __init cluster_init(void)
+{
+ pr_info("Registering Cluster MD functions\n");
+ return 0;
+}
+
+static void cluster_exit(void)
+{
+}
+
+module_init(cluster_init);
+module_exit(cluster_exit);
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Clustering support for MD");
+
--
2.1.2
^ permalink raw reply related
* [PATCH 02/24] Add number of nodes to bitmap structure for clustering
From: Goldwyn Rodrigues @ 2014-12-18 16:15 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
drivers/md/bitmap.h | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/drivers/md/bitmap.h b/drivers/md/bitmap.h
index 30210b9..6872945 100644
--- a/drivers/md/bitmap.h
+++ b/drivers/md/bitmap.h
@@ -128,10 +128,11 @@ typedef struct bitmap_super_s {
__le32 chunksize; /* 52 the bitmap chunk size in bytes */
__le32 daemon_sleep; /* 56 seconds between disk flushes */
__le32 write_behind; /* 60 number of outstanding write-behind writes */
- __le32 sectors_reserved; /* 64 number of 512-byte sectors that are
+ __le32 nodes; /* 64 the maximum number of nodes in cluster. */
+ __le32 sectors_reserved; /* 68 number of 512-byte sectors that are
* reserved for the bitmap. */
- __u8 pad[256 - 68]; /* set to zero */
+ __u8 pad[256 - 72]; /* set to zero */
} bitmap_super_t;
/* notes:
--
2.1.2
^ permalink raw reply related
* [PATCH 01/24] md-cluster: Design Documentation
From: Goldwyn Rodrigues @ 2014-12-18 16:15 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
Signed-off-by: Goldwyn Rodrigues <rgoldwyn@suse.com>
---
Documentation/md-cluster.txt | 178 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 178 insertions(+)
create mode 100644 Documentation/md-cluster.txt
diff --git a/Documentation/md-cluster.txt b/Documentation/md-cluster.txt
new file mode 100644
index 0000000..038d0f0
--- /dev/null
+++ b/Documentation/md-cluster.txt
@@ -0,0 +1,178 @@
+The cluster MD is a shared-device RAID for a cluster.
+
+
+1. On-disk format
+
+Separate write-intent-bitmap are used for each cluster node.
+The bitmaps record all writes that may have been started on that node,
+and may not yet have finished. The on-disk layout is:
+
+0 4k 8k 12k
+-------------------------------------------------------------------
+| idle | md super | bm super [0] + bits |
+| bm bits[0, contd] | bm super[1] + bits | bm bits[1, contd] |
+| bm super[2] + bits | bm bits [2, contd] | bm super[3] + bits |
+| bm bits [3, contd] | | |
+
+During "normal" functioning we assume the filesystem ensures that only one
+node writes to any given block at a time, so a write
+request will
+ - set the appropriate bit (if not already set)
+ - commit the write to all mirrors
+ - schedule the bit to be cleared after a timeout.
+
+Reads are just handled normally. It is up to the filesystem to
+ensure one node doesn't read from a location where another node (or the same
+node) is writing.
+
+
+2. DLM Locks for management
+
+There are two locks for managing the device:
+
+2.1 Bitmap lock resource (bm_lockres)
+
+ The bm_lockres protects individual node bitmaps. They are named in the
+ form bitmap001 for node 1, bitmap002 for node and so on. When a node
+ joins the cluster, it acquires the lock in PW mode and it stays so
+ during the lifetime the node is part of the cluster. The lock resource
+ number is based on the slot number returned by the DLM subsystem. Since
+ DLM starts node count from one and bitmap slots start from zero, one is
+ subtracted from the DLM slot number to arrive at the bitmap slot number.
+
+3. Communication
+
+Each node has to communicate with other nodes when starting or ending
+resync, and metadata superblock updates.
+
+3.1 Message Types
+
+ There are 3 types, of messages which are passed
+
+ 3.1.1 METADATA_UPDATED: informs other nodes that the metadata has been
+ updated, and the node must re-read the md superblock. This is performed
+ synchronously.
+
+ 3.1.2 RESYNC: informs other nodes that a resync is initiated or ended
+ so that each node may suspend or resume the region.
+
+3.2 Communication mechanism
+
+ The DLM LVB is used to communicate within nodes of the cluster. There
+ are three resources used for the purpose:
+
+ 3.2.1 Token: The resource which protects the entire communication
+ system. The node having the token resource is allowed to
+ communicate.
+
+ 3.2.2 Message: The lock resource which carries the data to
+ communicate.
+
+ 3.2.3 Ack: The resource, acquiring which means the message has been
+ acknowledged by all nodes in the cluster. The BAST of the resource
+ is used to inform the receive node that a node wants to communicate.
+
+The algorithm is:
+
+ 1. receive status
+
+ sender receiver receiver
+ ACK:CR ACK:CR ACK:CR
+
+ 2. sender get EX of TOKEN
+ sender get EX of MESSAGE
+ sender receiver receiver
+ TOKEN:EX ACK:CR ACK:CR
+ MESSAGE:EX
+ ACK:CR
+
+ Sender checks that it still needs to send a message. Messages received
+ or other events that happened while waiting for the TOKEN may have made
+ this message inappropriate or redundant.
+
+ 3. sender write LVB.
+ sender down-convert MESSAGE from EX to CR
+ sender try to get EX of ACK
+ [ wait until all receiver has *processed* the MESSAGE ]
+
+ [ triggered by bast of ACK ]
+ receiver get CR of MESSAGE
+ receiver read LVB
+ receiver processes the message
+ [ wait finish ]
+ receiver release ACK
+
+ sender receiver receiver
+ TOKEN:EX MESSAGE:CR MESSAGE:CR
+ MESSAGE:CR
+ ACK:EX
+
+ 4. triggered by grant of EX on ACK (indicating all receivers have processed
+ message)
+ sender down-convert ACK from EX to CR
+ sender release MESSAGE
+ sender release TOKEN
+ receiver upconvert to EX of MESSAGE
+ receiver get CR of ACK
+ receiver release MESSAGE
+
+ sender receiver receiver
+ ACK:CR ACK:CR ACK:CR
+
+
+4. Handling Failures
+
+4.1 Node Failure
+ When a node fails, the DLM informs the cluster with the slot. The node
+ starts a cluster recovery thread. The cluster recovery thread:
+ - acquires the bitmap<number> lock of the failed node
+ - opens the bitmap
+ - reads the bitmap of the failed node
+ - copies the set bitmap to local node
+ - cleans the bitmap of the failed node
+ - releases bitmap<number> lock of the failed node
+ - initiates resync of the bitmap on the current node
+
+ The resync process, is the regular md resync. However, in a clustered
+ environment when a resync is performed, it needs to tell other nodes
+ of the areas which are suspended. Before a resync starts, the node
+ send out RESYNC_START with the (lo,hi) range of the area which needs
+ to be suspended. Each node maintains a suspend_list, which contains
+ the list of ranges which are currently suspended. On receiving
+ RESYNC_START, the node adds the range to the suspend_list. Similarly,
+ when the node performing resync finishes, it send RESYNC_FINISHED
+ to other nodes and other nodes remove the corresponding entry from
+ the suspend_list.
+
+ A helper function, should_suspend() can be used to check if a particular
+ I/O range should be suspended or not.
+
+4.2 Device Failure
+ Device failures are handled and communicated with the metadata update
+ routine.
+
+5. Adding a new Device
+For adding a new device, it is necessary that all nodes "see" the new device
+to be added. For this, the following algorithm is used:
+
+ 1. Node 1 issues mdadm --manage /dev/mdX --add /dev/sdYY which issues
+ ioctl(ADD_NEW_DISC with disc.state set to MD_DISK_CLUSTER_ADD)
+ 2. Node 1 sends NEWDISK with uuid and slot number
+ 3. Other nodes issue kobject_uevent_env with uuid and slot number
+ (Steps 4,5 could be a udev rule)
+ 4. In userspace, the node searches for the disk, perhaps
+ using blkid -t SUB_UUID=""
+ 5. Other nodes issue either of the following depending on whether the disk
+ was found:
+ ioctl(ADD_NEW_DISK with disc.state set to MD_DISK_CANDIDATE and
+ disc.number set to slot number)
+ ioctl(CLUSTERED_DISK_NACK)
+ 6. Other nodes drop lock on no-new-devs (CR) if device is found
+ 7. Node 1 attempts EX lock on no-new-devs
+ 8. If node 1 gets the lock, it sends METADATA_UPDATED after unmarking the disk
+ as SpareLocal
+ 9. If not (get no-new-dev lock), it fails the operation and sends METADATA_UPDATED
+ 10. Other nodes get the information whether a disk is added or not
+ by the following METADATA_UPDATED.
+
+
--
2.1.2
^ permalink raw reply related
* [PATCH 00/24] Clustered MD RAID1
From: Goldwyn Rodrigues @ 2014-12-18 16:14 UTC (permalink / raw)
To: neilb; +Cc: lzhong, linux-raid
Hello,
This is an attempt to make MD-RAID cluster-aware. The advantage of
redundancy can help highly available systems to improve uptime.
Currently, the implementation is limited to RAID1 but with further work
(and some positive feedback), we could extend this to other compatible
RAID scenarios.
The design document (first patch) is pretty descriptive of how
the md has been made cluster-aware and how DLM is used to safeguard data
and communication.
This work requires some patches to the mdadm tool [1]
A quick howto:
1. With your corosync/pacemaker based cluster running execute:
# mdadm --create md0 --bitmap=clustered --raid-devices=2 --level=mirror --assume-clean /dev/sda /dev/sdb
2. On other nodes, issue:
# mdadm --assemble md0 /dev/sda /dev/sdb
References:
[1] mdadm tool changes: https://github.com/goldwynr/mdadm branch:cluster-md
[2] Patches against stable 3.14: https://github.com/goldwynr/linux branch: cluster-md-devel
Regards,
--
Goldwyn
^ permalink raw reply
* Re: Panic doing BLKDISCARD on a raid 5 array on linux 3.17.3
From: Anthony Wright @ 2014-12-18 10:58 UTC (permalink / raw)
To: NeilBrown; +Cc: linux-raid
In-Reply-To: <20141218162858.47310158@notabene.brown>
On 18/12/2014 05:28, NeilBrown wrote:
> I suspect md/raid5 is sending down a discard request in some way that the
> scsi/sata layer or driver doesn't like, but without the full oops, I really
> cannot guess what it might be.
We've tried 4 times to reproduce the panic we got originally, but
unforunately with no luck. Below are the outputs from all four crashes
as captured by netconsole, in case they are any help.
Crash #1
[63207.177400] BUG: unable to handle kernel paging request at
0000001e00008000
Crash #2
[ 531.210340] BUG: unable to handle kernel paging request at
0000000100000000
[ 531.210514] IP:[ 531.210340] BUG: unable to handle kernel
[<ffffffff8128788e>] __blk_segment_map_sg+0x5e/0x1b0
paging request[ 531.210632] PGD 20187f067 PUD 0 at 0000000100000000
[ 531.210514] IP: [<ffffffff8128788e>] __blk_segment_map_sg+0x5e/0x1b0
[ 531.210632] PGD 20187f067 PUD 0
[ 531.210783] Oops: 0000 [#1] SMP
[ 531.210932] Modules linked in: eql netconsole configfs raid456[
531.210783] Oops: 0000 [#1] SMP
[ 531.210932] Modules linked in: eql netconsole configfs raid456
async_raid6_recov async_memcpy async_pq async_xor xor async_tx raid6_pq
xt_multiport async_raid6_recov async_memcpy async_pq async_xor xor
async_tx raid6_pq xt_multiport xt_tcpudp iptable_filter ip_tables
x_tables aesni_intel aes_x86_64 glue_helper lrw xt_tcpudp iptable_filter
ip_tables x_tables aesni_intel aes_x86_64 glue_helper lrw gf128mul
ablk_helper cryptd ppdev gf128mul ablk_helper cryptd ppdev
Crash #3
[ 268.115094] general protection fault: 0000 [#1] SMP
[ 268.115263] Modules linked in:[ 268.115094] general protection
fault: 0000 [#1] SMP
[ 268.115263] Modules linked in:
Crash #4
[ 276.325157] general protection fault: 0000 [#1] SMP
^ permalink raw reply
* Re: Panic doing BLKDISCARD on a raid 5 array on linux 3.17.3
From: Anthony Wright @ 2014-12-18 10:21 UTC (permalink / raw)
To: NeilBrown; +Cc: linux-raid
In-Reply-To: <20141218162858.47310158@notabene.brown>
On 18/12/2014 05:28, NeilBrown wrote:
> On Wed, 17 Dec 2014 12:00:13 +0000 Anthony Wright <anthony@overnetdata.com>
> wrote:
>
>> I've hit a panic bug on stock linux 3.17.3 (which includes the recent
>> commit on BLKDISCARD in md/raid5.c) running in Dom0 under Xen 4.1.0 that
>> I've isolated to a BLKDISCARD system call within mkfs.ext3 and only
>> happens on a raid 5 array (it doesn't happen on a raid 1 array).
>>
>> The system it happens on is remote and I don't have physical access to
>> it, but the system administrator there is fairly helpful. We're in the
>> process of commissioning the system which needs to be done tomorrow
>> (thursday), so I've only got 24 hours in which I can run any tests you
>> may want. If necessary I can arrange remote access, but it's a little
>> complex.
>>
>> We have 3 512GB SSDs on the system, all with a GPT partition table and
>> the same partition layout. All the partitions have optimal alignment
>> according to parted. One of the partitions on each SSD is assembled into
>> a raid 1 array, another partition is assembled into a raid 5 array. Each
>> array is the used as the only physical volume in a LVM volume group. I
>> then create a logical volume on each array and format the logical volume
>> with mkfs.ext3. I ran mkfs.ext3 in verbose mode and also ran strace on
>> it in a separate session (though it was over a network) so it's possible
>> I lost the last few packets of data.
>>
>> /dev/Test/Test - 400MB LV on raid 1
>> /dev/Master/Test - 400MB LV on raid 5
>>
>> A) mkfs.ext3 -E nodiscard -v /dev/Test/Test - succeeds
>> B) mkfs.ext3 -v /dev/Test/Test - succeeds
>> C) mkfs.ext3 -E nodiscard -v /dev/Master/Test - succeeds
>> D) mkfs.ext3 -v /dev/Master/Test - panics
>>
>> mkfs.ext3 output from (B)
>> -------------------------
>> mke2fs 1.42.9 (28-Dec-2013)
>> fs_types for mke2fs.conf resolution: 'ext3', 'small'
>> Discarding device blocks: done Discard
>> succeeded and will return 0s - skipping inode table wipe
>> Filesystem label=
>> OS type: Linux
>> Block size=1024 (log=0)
>> Fragment size=1024 (log=0)
>> Stride=4 blocks, Stripe width=4 blocks
>> 51200 inodes, 204800 blocks
>> 10240 blocks (5.00%) reserved for the super user
>> First data block=1
>> Maximum filesystem blocks=67371008
>> 25 block groups
>> 8192 blocks per group, 8192 fragments per group
>> 2048 inodes per group
>> Superblock backups stored on blocks:
>> 8193, 24577, 40961, 57345, 73729
>>
>> Allocating group tables: done Writing inode
>> tables: done Creating journal (4096 blocks): done
>> Writing superblocks and filesystem accounting information: done
>>
>> strace output from (B) around the BLKDISCARD
>> --------------------------------------------
>> gettimeofday({1418806647, 890754}, NULL) = 0
>> gettimeofday({1418806647, 890814}, NULL) = 0
>> ioctl(3, BLKDISCARD, {0, 3000000010}) = 0
>> write(1, "Discarding device blocks: ", 26) = 26
>> write(1, " 1024/204800", 13) = 13
>> write(1, "\10\10\10\10\10\10\10\10\10\10\10\10\10", 13) = 13
>> ioctl(3, BLKDISCARD, {100000, 3000000010}) = 0
>> write(1, " ", 13) = 13
>> write(1, "\10\10\10\10\10\10\10\10\10\10\10\10\10", 13) = 13
>> write(1, "done "..., 33) = 33
>> write(1, "Discard succeeded and will retur"..., 65) = 65
>>
>> mkfs.ext3 output from (D)
>> -------------------------
>> mke2fs 1.42.9 (28-Dec-2013)
>> fs_types for mke2fs.conf resolution: 'ext3', 'small'
>> <Panic>
>>
>> strace output from (D) around the BLKDISCARD
>> --------------------------------------------
>> gettimeofday({1418809706, 244197}, NULL) = 0
>> gettimeofday({1418809706, 244259}, NULL) = 0
>> ioctl(3, BLKDISCARD, {0, 3000000010}
>> <Panic>
>>
>> I have a photograph of the panic output from a previous session which
>> includes raid5d and blk_finish_plug in the stack trace, unfortunately I
>> don't have the top part of the panic and vger won't accept the
>> attachment. I also have a photograph of the console output from the
>> crash at (D), but in this case it outputs to the console every 180 seconds:
>>
>> INFO: rcu_sched self-detected stall on CPU { 1}
>> sending NMI to all CPUs:
>> xen: vector 0x2 is not implemented
>>
>> thanks,
>>
>> Anthony Wright
> Presumably you have deliberately enabled DISCARD support by setting the
> raid456.devices_handle_discard_safely
>
> modules parameters? Otherwise the DISCARD should be a no-op.
I haven't touched the raid456.devices_handle_discard_safely setting, I
only learnt about it when I discovered your patch while I investigated
the crash. I'm presuming it's the default value, but if there's a way to
confirm that please let me know.
> It is very hard to deduce anything without the full Oops. Do you have access
> to another machine on the same subnet? If so you could enable netconsole and
> capture the full oops from the other machines (all console messages are sent
> via UDP at a very low level).
I've got netconsole working, but it doesn't always panic and it takes a
while to get the system reset. Below is the output I got from the most
recent crash:
[63207.177400] BUG: unable to handle kernel paging request at
0000001e00008000
Anthony.
^ permalink raw reply
* Re: Question about my patch
From: Peter Kieser @ 2014-12-18 6:30 UTC (permalink / raw)
To: NeilBrown, nick; +Cc: linux-raid, linux-kernel
In-Reply-To: <20141217162600.69dc1933@notabene.brown>
[-- Attachment #1: Type: text/plain, Size: 179 bytes --]
On 2014-12-16 9:26 PM, NeilBrown wrote:
> i.e. there is no bug here, and nothing to fix.
>
> Thanks,
>
> NeilBrown
FYI:https://lkml.org/lkml/2014/8/4/206
-Peter
[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4291 bytes --]
^ permalink raw reply
* Re: raid10: 6 out of 8 disks marked as stale on every restart
From: Peter Kieser @ 2014-12-18 6:26 UTC (permalink / raw)
To: NeilBrown; +Cc: linux-raid
In-Reply-To: <20141218163632.6cb57524@notabene.brown>
[-- Attachment #1: Type: text/plain, Size: 851 bytes --]
On 2014-12-17 9:36 PM, NeilBrown wrote:
> Curious.
>
> What does "mdadm --examine" report for each device immediately after boot,
> before you try assembling anything?
>
> Maybe also get the output just before you shut down to compare.
>
> NeilBrown
Sadly, I had to disassemble this array and get something workable that
didn't become unassembled on every restart. I did try sticking all
drives on the same AHCI controller, which caused the array to be
resynced on every restart instead of disassembled.
I suspect this issue is caused by bcache (I'm using the md as a backing
device for bcache.) The bcache maintainer states "get the md people or
someone to explain _what_ they want whatever has their device open to do
on reboot."
I'm going to setup a test environment and see if I can reproduce it again.
-Peter
[-- Attachment #2: S/MIME Cryptographic Signature --]
[-- Type: application/pkcs7-signature, Size: 4291 bytes --]
^ permalink raw reply
* RE: md_raid5 using 100% CPU and hang with status resync=PENDING, if a drive is removed during initialization
From: Manibalan P @ 2014-12-18 6:08 UTC (permalink / raw)
To: linux-raid; +Cc: NeilBrown, Vijayarankan Muthirisavengopal, Dinakaran N
Dear neil,
I also compiled the latest 3.18 kernel on CentOS 6.4 with GIT MD pull patches form 3.19, that also ran in to the same issue after removing a drive during resync.
Dec 17 19:07:32 ITX002590129362 kernel: Linux version 3.18.0 (root@mycentos6) (gcc version 4.4.7 20120313 (Red Hat 4.4.7-11) (GCC) ) #1 SMP Wed Dec 17 15:59:09 EST 2014
Dec 17 19:07:32 ITX002590129362 kernel: Command line: ro root=/dev/md255 rd_NO_LVM rd_NO_DM rhgb quiet md_mod.start_ro=1 nmi_watchdog=1 md_mod.start_dirty_degraded=1
…
Dec 17 19:10:15 ITX002590129362 kernel: md: bind<sda6>
Dec 17 19:10:15 ITX002590129362 kernel: md: bind<sdb6>
Dec 17 19:10:15 ITX002590129362 kernel: md: bind<sdc6>
Dec 17 19:10:15 ITX002590129362 kernel: md: bind<sdh6>
Dec 17 19:10:15 ITX002590129362 kernel: md: bind<sdi6>
Dec 17 19:10:15 ITX002590129362 kernel: md: bind<sdj6>
Dec 17 19:10:15 ITX002590129362 kernel: async_tx: api initialized (async)
Dec 17 19:10:15 ITX002590129362 kernel: xor: measuring software checksum speed
Dec 17 19:10:15 ITX002590129362 kernel: prefetch64-sse: 10048.000 MB/sec
Dec 17 19:10:15 ITX002590129362 kernel: generic_sse: 8824.000 MB/sec
Dec 17 19:10:15 ITX002590129362 kernel: xor: using function: prefetch64-sse (10048.000 MB/sec)
Dec 17 19:10:15 ITX002590129362 kernel: raid6: sse2x1 5921 MB/s
Dec 17 19:10:15 ITX002590129362 kernel: raid6: sse2x2 6933 MB/s
Dec 17 19:10:15 ITX002590129362 kernel: raid6: sse2x4 7476 MB/s
Dec 17 19:10:15 ITX002590129362 kernel: raid6: using algorithm sse2x4 (7476 MB/s)
Dec 17 19:10:15 ITX002590129362 kernel: raid6: using ssse3x2 recovery algorithm
Dec 17 19:10:15 ITX002590129362 kernel: md: raid6 personality registered for level 6
Dec 17 19:10:15 ITX002590129362 kernel: md: raid5 personality registered for level 5
Dec 17 19:10:15 ITX002590129362 kernel: md: raid4 personality registered for level 4
Dec 17 19:10:15 ITX002590129362 kernel: md/raid:md0: not clean -- starting background reconstruction
Dec 17 19:10:15 ITX002590129362 kernel: md/raid:md0: device sdj6 operational as raid disk 5
Dec 17 19:10:15 ITX002590129362 kernel: md/raid:md0: device sdi6 operational as raid disk 4
Dec 17 19:10:15 ITX002590129362 kernel: md/raid:md0: device sdh6 operational as raid disk 3
Dec 17 19:10:15 ITX002590129362 kernel: md/raid:md0: device sdc6 operational as raid disk 2
Dec 17 19:10:15 ITX002590129362 kernel: md/raid:md0: device sdb6 operational as raid disk 1
Dec 17 19:10:15 ITX002590129362 kernel: md/raid:md0: device sda6 operational as raid disk 0
Dec 17 19:10:15 ITX002590129362 kernel: md/raid:md0: allocated 0kB
Dec 17 19:10:15 ITX002590129362 kernel: md/raid:md0: raid level 5 active with 6 out of 6 devices, algorithm 2
Dec 17 19:10:15 ITX002590129362 kernel: md0: detected capacity change from 0 to 2361059573760
Dec 17 19:10:15 ITX002590129362 kernel: md0: unknown partition table
Dec 17 19:10:35 ITX002590129362 kernel: md: md0 switched to read-write mode.
Dec 17 19:10:35 ITX002590129362 kernel: md: resync of RAID array md0
Dec 17 19:10:35 ITX002590129362 kernel: md: minimum _guaranteed_ speed: 10000 KB/sec/disk.
Dec 17 19:10:35 ITX002590129362 kernel: md: using maximum available idle IO bandwidth (but not more than 30000 KB/sec) for resync.
Dec 17 19:10:35 ITX002590129362 kernel: md: using 128k window, over a total of 461144448k.
…
Started IOs using fio tool.
./fio --name=md0 --filename=/dev/md0 --thread --numjobs=10 --direct=1 --group_reporting --unlink=0 --loops=1 --offset=0 --randrepeat=1 --norandommap --scramble_buffers=1 --stonewall --ioengine=libaio --rw=randwrite --bs=8704 --iodepth=4000 --runtime=3000 --blockalign=512
…
Removed a drive form the system..
Dec 17 19:13:23 ITX002590129362 kernel: mpt2sas0: log_info(0x31120101): originator(PL), code(0x12), sub_code(0x0101)
Dec 17 19:13:23 ITX002590129362 kernel: mpt2sas0: log_info(0x31120101): originator(PL), code(0x12), sub_code(0x0101)
Dec 17 19:13:23 ITX002590129362 kernel: mpt2sas0: log_info(0x31120101): originator(PL), code(0x12), sub_code(0x0101)
Dec 17 19:13:23 ITX002590129362 kernel: mpt2sas0: log_info(0x31120101): originator(PL), code(0x12), sub_code(0x0101)
..
Dec 17 19:13:23 ITX002590129362 kernel: sd 0:0:7:0: [sdh]
Dec 17 19:13:23 ITX002590129362 kernel: Result: hostbyte=DID_TRANSPORT_DISRUPTED driverbyte=DRIVER_OK
Dec 17 19:13:23 ITX002590129362 kernel: sd 0:0:7:0: [sdh] CDB:
Dec 17 19:13:23 ITX002590129362 kernel: Read(10): 28 00 02 69 03 70 00 00 10 00
Dec 17 19:13:23 ITX002590129362 kernel: blk_update_request: I/O error, dev sdh, sector 40436592
Dec 17 19:13:23 ITX002590129362 kernel: sd 0:0:7:0: [sdh]
Dec 17 19:13:23 ITX002590129362 kernel: Result: hostbyte=DID_TRANSPORT_DISRUPTED driverbyte=DRIVER_OK
Dec 17 19:13:23 ITX002590129362 kernel: sd 0:0:7:0: [sdh] CDB:
Dec 17 19:13:23 ITX002590129362 kernel: Read(10): 28 00 0c 51 b3 d0 00 00 18 00
Dec 17 19:13:23 ITX002590129362 kernel: blk_update_request: I/O error, dev sdh, sector 206681040
Dec 17 19:13:23 ITX002590129362 kernel: sd 0:0:7:0: [sdh]
Dec 17 19:13:23 ITX002590129362 kernel: Result: hostbyte=DID_TRANSPORT_DISRUPTED driverbyte=DRIVER_OK
Dec 17 19:13:23 ITX002590129362 kernel: sd 0:0:7:0: [sdh] CDB:
Dec 17 19:13:23 ITX002590129362 kernel: Read(10): 28 00 0c 3a f3 40 00 00 18 00
Dec 17 19:13:23 ITX002590129362 kernel: blk_update_request: I/O error, dev sdh, sector 205189952
Dec 17 19:13:23 ITX002590129362 kernel: sd 0:0:7:0: [sdh]
Dec 17 19:13:23 ITX002590129362 kernel: Result: hostbyte=DID_TRANSPORT_DISRUPTED driverbyte=DRIVER_OK
…
Dec 17 19:13:25 ITX002590129362 kernel: sd 0:0:7:0: [sdh] CDB:
Dec 17 19:13:25 ITX002590129362 kernel: Read(10): 28 00 26 8d eb 00 00 00 08 00
Dec 17 19:13:25 ITX002590129362 kernel: sd 0:0:7:0: [sdh]
Dec 17 19:13:25 ITX002590129362 kernel: Result: hostbyte=DID_NO_CONNECT driverbyte=DRIVER_OK
Dec 17 19:13:25 ITX002590129362 kernel: sd 0:0:7:0: [sdh] CDB:
Dec 17 19:13:25 ITX002590129362 kernel: Read(10): 28 00 26 8d eb f0 00 00 10 00
Dec 17 19:13:25 ITX002590129362 aghswap: devpath [0:0:7:0] action [remove] devtype [scsi_disk]
Dec 17 19:13:25 ITX002590129362 aghswap: MHSA: Sent event 0 0 7 0 remove scsi_disk
Dec 17 19:13:25 ITX002590129362 kernel: mpt2sas0: removing handle(0x0011), sas_addr(0x500605ba0101e305)
Dec 17 19:13:25 ITX002590129362 kernel: md/raid:md0: Disk failure on sdh6, disabling device.
Dec 17 19:13:25 ITX002590129362 kernel: md/raid:md0: Operation continuing on 5 devices.
Dec 17 19:13:25 ITX002590129362 kernel: md: md0: resync interrupted.
Dec 17 19:13:25 ITX002590129362 kernel: md: checkpointing resync of md0.
..
Log messages after enabling debufgs on raid5.c, it is getting repeated continuously.
__get_priority_stripe: handle: busy hold: empty full_writes: 0 bypass_count: 0
__get_priority_stripe: handle: busy hold: empty full_writes: 0 bypass_count: 0
__get_priority_stripe: handle: busy hold: empty full_writes: 0 bypass_count: 0
__get_priority_stripe: handle: busy hold: empty full_writes: 0 bypass_count: 0
__get_priority_stripe: handle: busy hold: empty full_writes: 0 bypass_count: 0
handling stripe 273480328, state=0x2041 cnt=1, pd_idx=5, qd_idx=-1
, check:0, reconstruct:0
check 5: state 0x10 read (null) write (null) written (null)
check 4: state 0x11 read (null) write (null) written (null)
check 3: state 0x0 read (null) write (null) written (null)
check 2: state 0x11 read (null) write (null) written (null)
check 1: state 0x11 read (null) write (null) written (null)
check 0: state 0x18 read (null) write ffff8808029b6b00 written (null)
locked=0 uptodate=3 to_read=0 to_write=1 failed=1 failed_num=3,-1
force RCW max_degraded=1, recovery_cp=7036944 sh->sector=273480328
for sector 273480328, rmw=2 rcw=1
handling stripe 65238568, state=0x2041 cnt=1, pd_idx=5, qd_idx=-1
, check:0, reconstruct:0
check 5: state 0x10 read (null) write (null) written (null)
check 4: state 0x11 read (null) write (null) written (null)
check 3: state 0x0 read (null) write (null) written (null)
check 2: state 0x18 read (null) write ffff88081a956b00 written (null)
check 1: state 0x11 read (null) write (null) written (null)
check 0: state 0x11 read (null) write (null) written (null)
locked=0 uptodate=3 to_read=0 to_write=1 failed=1 failed_num=3,-1
force RCW max_degraded=1, recovery_cp=7036944 sh->sector=65238568
for sector 65238568, rmw=2 rcw=1
handling stripe 713868672, state=0x2041 cnt=1, pd_idx=4, qd_idx=-1
, check:0, reconstruct:0
check 5: state 0x11 read (null) write (null) written (null)
check 4: state 0x10 read (null) write (null) written (null)
check 3: state 0x0 read (null) write (null) written (null)
check 2: state 0x18 read (null) write ffff88081f020100 written (null)
check 1: state 0x11 read (null) write (null) written (null)
check 0: state 0x11 read (null) write (null) written (null)
locked=0 uptodate=3 to_read=0 to_write=1 failed=1 failed_num=3,-1
force RCW max_degraded=1, recovery_cp=7036944 sh->sector=713868672
for sector 713868672, rmw=2 rcw=1
handling stripe 729622496, state=0x2041 cnt=1, pd_idx=2, qd_idx=-1
, check:0, reconstruct:0
check 5: state 0x11 read (null) write (null) written (null)
check 4: state 0x11 read (null) write (null) written (null)
check 3: state 0x0 read (null) write (null) written (null)
check 2: state 0x10 read (null) write (null) written (null)
check 1: state 0x18 read (null) write ffff88081b9bae00 written (null)
check 0: state 0x11 read (null) write (null) written (null)
locked=0 uptodate=3 to_read=0 to_write=1 failed=1 failed_num=3,-1
force RCW max_degraded=1, recovery_cp=7036944 sh->sector=729622496
for sector 729622496, rmw=2 rcw=1
handling stripe 729622504, state=0x2041 cnt=1, pd_idx=2, qd_idx=-1
, check:0, reconstruct:0
check 5: state 0x11 read (null) write (null) written (null)
check 4: state 0x11 read (null) write (null) written (null)
check 3: state 0x0 read (null) write (null) written (null)
check 2: state 0x10 read (null) write (null) written (null)
check 1: state 0x18 read (null) write ffff88081b9bae00 written (null)
check 0: state 0x11 read (null) write (null) written (null)
locked=0 uptodate=3 to_read=0 to_write=1 failed=1 failed_num=3,-1
force RCW max_degraded=1, recovery_cp=7036944 sh->sector=729622504
for sector 729622504, rmw=2 rcw=1
handling stripe 245773680, state=0x2041 cnt=1, pd_idx=0, qd_idx=-1
, check:0, reconstruct:0
check 5: state 0x11 read (null) write (null) written (null)
check 4: state 0x11 read (null) write (null) written (null)
check 3: state 0x0 read (null) write (null) written (null)
check 2: state 0x11 read (null) write (null) written (null)
check 1: state 0x18 read (null) write ffff88081cab7a00 written (null)
check 0: state 0x10 read (null) write (null) written (null)
locked=0 uptodate=3 to_read=0 to_write=1 failed=1 failed_num=3,-1
force RCW max_degraded=1, recovery_cp=7036944 sh->sector=245773680
for sector 245773680, rmw=2 rcw=1
handling stripe 867965560, state=0x2041 cnt=1, pd_idx=1, qd_idx=-1
, check:0, reconstruct:0
check 5: state 0x11 read (null) write (null) written (null)
check 4: state 0x11 read (null) write (null) written (null)
check 3: state 0x0 read (null) write (null) written (null)
check 2: state 0x18 read (null) write ffff880802b2bf00 written (null)
check 1: state 0x10 read (null) write (null) written (null)
check 0: state 0x11 read (null) write (null) written (null)
locked=0 uptodate=3 to_read=0 to_write=1 failed=1 failed_num=3,-1
force RCW max_degraded=1, recovery_cp=7036944 sh->sector=867965560
for sector 867965560, rmw=2 rcw=1
handling stripe 550162280, state=0x2041 cnt=1, pd_idx=2, qd_idx=-1
, check:0, reconstruct:0
check 5: state 0x11 read (null) write (null) written (null)
check 4: state 0x18 read (null) write ffff880802b08800 written (null)
check 3: state 0x0 read (null) write (null) written (null)
check 2: state 0x10 read (null) write (null) written (null)
check 1: state 0x11 read (null) write (null) written (null)
check 0: state 0x11 read (null) write (null) written (null)
locked=0 uptodate=3 to_read=0 to_write=1 failed=1 failed_num=3,-1
force RCW max_degraded=1, recovery_cp=7036944 sh->sector=550162280
for sector 550162280, rmw=2 rcw=1
Thanks,
Manibalan
-----Original Message-----
From: Manibalan P
Sent: Wednesday, December 17, 2014 12:11 PM
To: 'linux-raid'
Cc: 'NeilBrown'; Vijayarankan Muthirisavengopal; Dinakaran N
Subject: RE: md_raid5 using 100% CPU and hang with status resync=PENDING, if a drive is removed during initialization
Dear Neil,
The same Issue is reproducible in the latest upstream kernel also.
Tested in "3.17.6" latest stable upstream kernel and find the same issue.
[root@root ~]# modinfo raid456
filename: /lib/modules/3.17.6/kernel/drivers/md/raid456.ko
alias: raid6
alias: raid5
alias: md-level-6
alias: md-raid6
alias: md-personality-8
alias: md-level-4
alias: md-level-5
alias: md-raid4
alias: md-raid5
alias: md-personality-4
description: RAID4/5/6 (striping with parity) personality for MD
license: GPL
srcversion: 0EEF680023FDC7410F7989A
depends: async_raid6_recov,async_pq,async_tx,async_memcpy,async_xor
intree: Y
vermagic: 3.17.6 SMP mod_unload modversions
parm: devices_handle_discard_safely:Set to Y if all devices in each array reliably return zeroes on reads from discarded regions (bool)
Thanks,
Manibalan.
-----Original Message-----
From: Manibalan P
Sent: Wednesday, December 17, 2014 12:01 PM
To: 'linux-raid'
Cc: 'NeilBrown'
Subject: RE: md_raid5 using 100% CPU and hang with status resync=PENDING, if a drive is removed during initialization
Dear Neil,
We are facing IO struck issue with raid5 in the following scenario. (please see the attachment for the complete information) In RAID5 array, if a drive is removed while initialization and the same time if IO is happening to that md. Then IO is getting struck, and md_raid5 thread is using 100 % of CPU. Also the md state showing as resync=PENDING
Kernel : Issue found in the following kernels RHEL 6.5 (2.6.32-431.el6.x86_64) CentOS 7 (kernel-3.10.0-123.13.1.el7.x86_64)
Steps to Reproduce the issue:
1. Created a raid 5 md with 4 drives using the below mdadm command.
mdadm -C /dev/md0 -c 64 -l 5 -f -n 4 -e 1.2 /dev/sdb6 /dev/sdc6 /dev/sdd6 /dev/sde6
2. Make the md writable
mdadm –readwrite /dev/md0
3. Now md will start initialization
4. Run FIO Tool, the the below said configuration /usr/bin/fio --name=md0 --filename=/dev/md0 --thread --numjobs=10 --direct=1 --group_reporting --unlink=0 --loops=1 --offset=0 --randrepeat=1 --norandommap --scramble_buffers=1 --stonewall --ioengine=libaio --rw=randwrite --bs=8704 --iodepth=4000 --runtime=3000 --blockalign=512
4. During MD initialzing, remove a drive(either using MDADM set faulty/remove or remove manually)
5. Now the IO will struck, and cat /proc/mdstat shows states with resync=PENDING
---------------------------------------------------------------------------------------------
top - output show, md_raid5 using 100% cpu
top - 17:55:06 up 1:09, 3 users, load average: 11.98, 8.53, 3.99
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
2690 root 20 0 0 0 0 R 100.0 0.0 6:44.41 md0_raid5
---------------------------------------------------------------------------------------------
dmesg - show the stack trace
INFO: task fio:2715 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 000000000000000a 0 2715 2654 0x00000080
ffff88043b623598 0000000000000082 0000000000000000 ffffffff81058d53
ffff88043b623548 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b40b098 ffff88043b623fd8 000000000000fbc8 ffff88043b40b098 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff8140fa39>] ? md_wakeup_thread+0x39/0x70 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffffa0308f66>] ? make_request+0x306/0xc6c [raid456] [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81122283>] ? mempool_alloc+0x63/0x140 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c767a>] do_direct_IO+0x7ca/0xfa0 [<ffffffff811c8196>] __blockdev_direct_IO_newtrunc+0x346/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2717 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000004 0 2717 2654 0x00000080
ffff880439e97698 0000000000000082 ffff880439e97628 ffffffff81058d53
ffff880439e97648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b0adab8 ffff880439e97fd8 000000000000fbc8 ffff88043b0adab8 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8e50>] __blockdev_direct_IO_newtrunc+0x1000/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2718 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000005 0 2718 2654 0x00000080
ffff88043bc13698 0000000000000082 ffff88043bc13628 ffffffff81058d53
ffff88043bc13648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b0ad058 ffff88043bc13fd8 000000000000fbc8 ffff88043b0ad058 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8e50>] __blockdev_direct_IO_newtrunc+0x1000/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2719 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000001 0 2719 2654 0x00000080
ffff880439ebb698 0000000000000082 ffff880439ebb628 ffffffff81058d53
ffff880439ebb648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b0ac5f8 ffff880439ebbfd8 000000000000fbc8 ffff88043b0ac5f8 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2720 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000008 0 2720 2654 0x00000080
ffff88043b8cf698 0000000000000082 ffff88043b8cf628 ffffffff81058d53
ffff88043b8cf648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff880439e89af8 ffff88043b8cffd8 000000000000fbc8 ffff880439e89af8 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2721 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000000 0 2721 2654 0x00000080
ffff88043b047698 0000000000000082 ffff88043b047628 ffffffff81058d53
ffff88043b047648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff880439e89098 ffff88043b047fd8 000000000000fbc8 ffff880439e89098 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2722 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000000 0 2722 2654 0x00000080
ffff880439ea3698 0000000000000082 ffff880439ea3628 ffffffff81058d53
ffff880439ea3648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff880439e88638 ffff880439ea3fd8 000000000000fbc8 ffff880439e88638 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2723 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000006 0 2723 2654 0x00000080
ffff88043bf5f698 0000000000000082 ffff88043bf5f628 ffffffff81058d53
ffff88043bf5f648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043a183ab8 ffff88043bf5ffd8 000000000000fbc8 ffff88043a183ab8 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2724 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 000000000000000b 0 2724 2654 0x00000080
ffff88043be05698 0000000000000082 ffff88043be05628 ffffffff81058d53
ffff88043be05648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043a183058 ffff88043be05fd8 000000000000fbc8 ffff88043a183058 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2725 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000003 0 2725 2654 0x00000080
ffff88043be07698 0000000000000082 ffff88043be07628 ffffffff81058d53
ffff88043be07648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043a1825f8 ffff88043be07fd8 000000000000fbc8 ffff88043a1825f8 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
[root@root ~]# cat /proc/2690/stack
[<ffffffff810686da>] __cond_resched+0x2a/0x40 [<ffffffffa030361c>] ops_run_io+0x2c/0x920 [raid456] [<ffffffffa03052cc>] handle_stripe+0x9cc/0x2980 [raid456] [<ffffffffa03078a4>] raid5d+0x624/0x850 [raid456] [<ffffffff81416f05>] md_thread+0x115/0x150 [<ffffffff8109aef6>] kthread+0x96/0xa0 [<ffffffff8100c20a>] child_rip+0xa/0x20 [<ffffffffffffffff>] 0xffffffffffffffff
[root@root ~]# cat /proc/2690/stat
2690 (md0_raid5) R 2 0 0 0 -1 2149613632 0 0 0 0 0 68495 0 0 20 0 1 0 350990 0 0 18446744073709551615 0 0 0 0 0 0 0 2147483391 256 0 0 0 17 2 0 0 6855 0 0 [root@root ~]# cat /proc/2690/statm
0 0 0 0 0 0 0
[root@root ~]# cat /proc/2690/stat
stat statm status
[root@root ~]# cat /proc/2690/status
Name: md0_raid5
State: R (running)
Tgid: 2690
Pid: 2690
PPid: 2
TracerPid: 0
Uid: 0 0 0 0
Gid: 0 0 0 0
Utrace: 0
FDSize: 64
Groups:
Threads: 1
SigQ: 2/128402
SigPnd: 0000000000000000
ShdPnd: 0000000000000000
SigBlk: 0000000000000000
SigIgn: fffffffffffffeff
SigCgt: 0000000000000100
CapInh: 0000000000000000
CapPrm: ffffffffffffffff
CapEff: fffffffffffffeff
CapBnd: ffffffffffffffff
Cpus_allowed: ffffff
Cpus_allowed_list: 0-23
Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000003
Mems_allowed_list: 0-1
voluntary_ctxt_switches: 5411612
nonvoluntary_ctxt_switches: 257032
Thanks,
Manibalan.
^ permalink raw reply
* Re: Bug#763917: mdadm: rounding errors in human_size()
From: NeilBrown @ 2014-12-18 6:00 UTC (permalink / raw)
To: Michael Tokarev; +Cc: 763917, linux-raid
In-Reply-To: <548188B0.6000309@msgid.tls.msk.ru>
[-- Attachment #1: Type: text/plain, Size: 3411 bytes --]
On Fri, 05 Dec 2014 13:28:00 +0300 Michael Tokarev <mjt@tls.msk.ru> wrote:
> Neil, will you take the patch in this bugreport for the next version?
>
> http://bugs.debian.org/763917.
>
> Thanks,
>
> /mjt
>
> On Fri, 3 Oct 2014 20:24:54 +0200 Jan Echternach <jan@goneko.de> wrote:
> > Package: mdadm
> > Version: 3.3.2-1
> > Severity: minor
> > Tags: patch
> >
> >
> > While setting up a new system, I noticed an incorrect value in the output
> > of mdadm --examine:
> >
> > Avail Dev Size : 2095080 (1023.16 MiB 1072.68 MB)
> >
> > The 1023.16 MiB were quite irritating because the underlying partition has
> > a size of exactly 1023 MiB.
> >
> > The number of sectors seems plausible: 2095080 sectors * 512 bytes/sector
> > are 1022.988 MiB or 1072.681 MB. I looked into the code and found an
> > inaccuracy in human_size() and human_size_brief(). The formula used for
> > the MiB value is essentially
> >
> > long cMiB = (bytes / ( (1LL<<20) / 200LL ) +1) /2;
> >
> > but (1LL<<20) / 200LL is not an integer. It's rounded down and cMiB becomes
> > too large. The quick fix would have been multiplying by 200 before dividing
> > by 1<<20, but that might cause integer overflows in the GiB case.
Given that we are doing 64bit arithmetic, we would need about 56bits of
bytes for there to be a rounding problem. That's 64 petabytes.
I decide to just do the simple transformation. If we get arrays close to
petabytes I would want to make other changes, like reporting the number of
terabytes for larger arrays.
Thanks,
NeilBrown
> >
> > The following patch uses a more complicated formula that computes the
> > fractional portion separately from the integer portion. It also changes some
> > longs to long longs to eliminate a different cause of integer overflows.
> >
> >
> > --- mdadm-3.3.2/util.c 2014-10-03 19:06:51.000000000 +0200
> > +++ mdadm-3.3.2/util.c 2014-10-03 19:08:06.000000000 +0200
> > @@ -671,15 +671,17 @@
> > if (bytes < 5000*1024)
> > buf[0] = 0;
> > else if (bytes < 2*1024LL*1024LL*1024LL) {
> > - long cMiB = (bytes / ( (1LL<<20) / 200LL ) +1) /2;
> > + long cMiB = bytes / (1LL<<20) * 100
> > + + ((bytes % (1LL<<20)) * 200 / (1LL<<20) +1) /2;
> > long cMB = (bytes / ( 1000000LL / 200LL ) +1) /2;
> > snprintf(buf, sizeof(buf), " (%ld.%02ld MiB %ld.%02ld MB)",
> > cMiB/100 , cMiB % 100,
> > cMB/100, cMB % 100);
> > } else {
> > - long cGiB = (bytes / ( (1LL<<30) / 200LL ) +1) /2;
> > - long cGB = (bytes / (1000000000LL/200LL ) +1) /2;
> > - snprintf(buf, sizeof(buf), " (%ld.%02ld GiB %ld.%02ld GB)",
> > + long long cGiB = bytes / (1LL<<30) * 100
> > + + ((bytes % (1LL<<30)) * 200 / (1LL<<30) +1) /2;
> > + long long cGB = (bytes / (1000000000LL/200LL ) +1) /2;
> > + snprintf(buf, sizeof(buf), " (%lld.%02lld GiB %lld.%02lld GB)",
> > cGiB/100 , cGiB % 100,
> > cGB/100, cGB % 100);
> > }
> > @@ -706,12 +708,14 @@
> > buf[0] = 0;
> > else if (prefix == IEC) {
> > if (bytes < 2*1024LL*1024LL*1024LL) {
> > - long cMiB = (bytes / ( (1LL<<20) / 200LL ) +1) /2;
> > + long cMiB = bytes / (1LL<<20) * 100
>
> _______________________________________________
> pkg-mdadm-devel mailing list
> pkg-mdadm-devel@lists.alioth.debian.org
> http://lists.alioth.debian.org/cgi-bin/mailman/listinfo/pkg-mdadm-devel
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 811 bytes --]
^ permalink raw reply
* Re: raid10: 6 out of 8 disks marked as stale on every restart
From: NeilBrown @ 2014-12-18 5:36 UTC (permalink / raw)
To: Peter Kieser; +Cc: linux-raid
In-Reply-To: <548B2033.5030803@kieser.ca>
[-- Attachment #1: Type: text/plain, Size: 5981 bytes --]
On Fri, 12 Dec 2014 09:04:51 -0800 Peter Kieser <peter@kieser.ca> wrote:
> Hello,
>
> I have a 8 disk RAID10 array, 6 of the disks are on an LSISAS2008
> controller and 2 are on a 82801JI (ICH10 Family) SATA AHCI controller.
> I upgraded the kernel from 3.17.1 to 3.17.6 when the issue I am having
> started to occur, but reverting to an older kernel does not resolve the
> issue.
>
> Restarting the machine causes the array not to start (or be visible in
> /proc/mdstat or any mention in kernel messages.) If I try to assemble
> the drives, mdraid complains that 6 out of the 8 disks (coincidentally
> all on the LSISAS2008 controller) are non-fresh:
>
> root@kvm:~# mdadm --assemble /dev/md3 /dev/sde /dev/sdf /dev/sdg
> /dev/sdh /dev/sdi /dev/sdj /dev/sda /dev/sdb
>
> Dec 11 21:08:25 kvm kernel: [ 528.503736] md: kicking non-fresh sdi
> from array!
> Dec 11 21:08:25 kvm kernel: [ 528.503747] md: unbind<sdi>
> Dec 11 21:08:25 kvm kernel: [ 528.523775] md: export_rdev(sdi)
> Dec 11 21:08:25 kvm kernel: [ 528.523802] md: kicking non-fresh sdg
> from array!
> Dec 11 21:08:25 kvm kernel: [ 528.523809] md: unbind<sdg>
> Dec 11 21:08:25 kvm kernel: [ 528.531753] md: export_rdev(sdg)
> Dec 11 21:08:25 kvm kernel: [ 528.531780] md: kicking non-fresh sdf
> from array!
> Dec 11 21:08:25 kvm kernel: [ 528.531788] md: unbind<sdf>
> Dec 11 21:08:25 kvm kernel: [ 528.539749] md: export_rdev(sdf)
> Dec 11 21:08:25 kvm kernel: [ 528.539776] md: kicking non-fresh sdh
> from array!
> Dec 11 21:08:25 kvm kernel: [ 528.539785] md: unbind<sdh>
> Dec 11 21:08:25 kvm kernel: [ 528.547744] md: export_rdev(sdh)
> Dec 11 21:08:25 kvm kernel: [ 528.547771] md: kicking non-fresh sdj
> from array!
> Dec 11 21:08:25 kvm kernel: [ 528.547779] md: unbind<sdj>
> Dec 11 21:08:25 kvm kernel: [ 528.555755] md: export_rdev(sdj)
> Dec 11 21:08:25 kvm kernel: [ 528.555782] md: kicking non-fresh sde
> from array!
> Dec 11 21:08:25 kvm kernel: [ 528.555790] md: unbind<sde>
> Dec 11 21:08:25 kvm kernel: [ 528.563758] md: export_rdev(sde)
> Dec 11 21:08:25 kvm kernel: [ 528.565831] md/raid10:md3: not enough
> operational mirrors.
> Dec 11 21:08:25 kvm kernel: [ 528.567230] md: pers->run() failed ...
>
> /dev/sda and /dev/sdb are the only drives not on the LSI controller. If
> I force the assembly with 6 out of the 8 drives the RAID array comes up:
>
> root@kvm:~# mdadm --assemble /dev/md3 /dev/sde /dev/sdf /dev/sdg
> /dev/sdh /dev/sdi /dev/sdj --run
>
> Then I add the extra drives:
>
> root@kvm:~# mdadm --manage /dev/md3 --add /dev/sda
> root@kvm:~# mdadm --manage /dev/md3 --add /dev/sdb
>
> root@kvm:~# mdadm --detail /dev/md3
> /dev/md3:
> Version : 1.0
> Creation Time : Thu Sep 12 18:43:56 2013
> Raid Level : raid10
> Array Size : 7814055936 (7452.06 GiB 8001.59 GB)
> Used Dev Size : 1953513984 (1863.02 GiB 2000.40 GB)
> Raid Devices : 8
> Total Devices : 8
> Persistence : Superblock is persistent
>
> Update Time : Fri Dec 12 08:58:19 2014
> State : active, degraded, recovering
> Active Devices : 6
> Working Devices : 8
> Failed Devices : 0
> Spare Devices : 2
>
> Layout : near=2
> Chunk Size : 512K
>
> Rebuild Status : 76% complete
>
> Name : kvm.taylor.kieser.ca:3
> UUID : f0bc8469:9879a709:e4cc94a7:521bd273
> Events : 82901
>
> Number Major Minor RaidDevice State
> 0 8 128 0 active sync /dev/sdi
> 8 8 96 1 active sync /dev/sdg
> 11 8 0 2 spare rebuilding /dev/sda
> 3 8 112 3 active sync /dev/sdh
> 4 0 0 4 removed
> 10 8 80 5 active sync /dev/sdf
> 6 8 64 6 active sync /dev/sde
> 9 8 144 7 active sync /dev/sdj
>
> 12 8 16 - spare /dev/sdb
>
> This occurs every time I restart the machine. Thoughts? I tried
> rebuilding the initramfs but this didn't resolve the issue. I'm also
> running bcache on this machine, but on top of the mdraid.
>
> /etc/mdadm.conf:
>
> # definitions of existing MD arrays
> ARRAY /dev/md/0 metadata=1.0 UUID=3b174514:49f3e22e:550cf9a7:8ed93920
> name=linux:0
> ARRAY /dev/md/1 metadata=1.0 UUID=8e23f81d:73f9b393:addd1f7f:5ee1833a
> name=linux:1
> ARRAY /dev/md/2 metadata=1.0 UUID=cc5a0495:b5262855:fb3cd40a:8b237162
> name=kvm.taylor.kieser.ca:2
> ARRAY /dev/md/3 metadata=1.0 UUID=f0bc8469:9879a709:e4cc94a7:521bd273
> name=kvm.taylor.kieser.ca:3
>
>
> root@kvm:~# uname -a
> Linux kvm 3.17.6 #3 SMP Sun Dec 7 12:16:45 PST 2014 x86_64 x86_64 x86_64
> GNU/Linux
>
> root@kvm:~# mdadm -V
> mdadm - v3.2.5 - 18th May 2012
>
> root@kvm:~# cat /proc/mdstat
> Personalities : [raid1] [linear] [multipath] [raid0] [raid6] [raid5]
> [raid4] [raid10]
> md127 : inactive sdk[2](S)
> 1465138448 blocks super 1.0
>
> md3 : active raid10 sdb[12](S) sda[11] sdi[0] sdj[9] sde[6] sdf[10]
> sdh[3] sdg[8]
> 7814055936 blocks super 1.0 512K chunks 2 near-copies [8/6]
> [UU_U_UUU]
> [===============>.....] recovery = 76.6% (1498279040/1953513984)
> finish=4710.1min speed=1610K/sec
>
> md1 : active raid1 sdd5[3] sdc5[2]
> 25164672 blocks super 1.0 [2/2] [UU]
>
> md0 : active raid1 sdd1[3] sdc1[2]
> 16779136 blocks super 1.0 [2/2] [UU]
>
> md2 : active raid1 sdd6[3] sdc6[2]
> 192472960 blocks super 1.0 [2/2] [UU]
>
> unused devices: <none>
>
> -Peter
>
>
Curious.
What does "mdadm --examine" report for each device immediately after boot,
before you try assembling anything?
Maybe also get the output just before you shut down to compare.
NeilBrown
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 811 bytes --]
^ permalink raw reply
* Re: Panic doing BLKDISCARD on a raid 5 array on linux 3.17.3
From: NeilBrown @ 2014-12-18 5:28 UTC (permalink / raw)
To: Anthony Wright; +Cc: linux-raid
In-Reply-To: <5491704D.1080709@overnetdata.com>
[-- Attachment #1: Type: text/plain, Size: 4923 bytes --]
On Wed, 17 Dec 2014 12:00:13 +0000 Anthony Wright <anthony@overnetdata.com>
wrote:
> I've hit a panic bug on stock linux 3.17.3 (which includes the recent
> commit on BLKDISCARD in md/raid5.c) running in Dom0 under Xen 4.1.0 that
> I've isolated to a BLKDISCARD system call within mkfs.ext3 and only
> happens on a raid 5 array (it doesn't happen on a raid 1 array).
>
> The system it happens on is remote and I don't have physical access to
> it, but the system administrator there is fairly helpful. We're in the
> process of commissioning the system which needs to be done tomorrow
> (thursday), so I've only got 24 hours in which I can run any tests you
> may want. If necessary I can arrange remote access, but it's a little
> complex.
>
> We have 3 512GB SSDs on the system, all with a GPT partition table and
> the same partition layout. All the partitions have optimal alignment
> according to parted. One of the partitions on each SSD is assembled into
> a raid 1 array, another partition is assembled into a raid 5 array. Each
> array is the used as the only physical volume in a LVM volume group. I
> then create a logical volume on each array and format the logical volume
> with mkfs.ext3. I ran mkfs.ext3 in verbose mode and also ran strace on
> it in a separate session (though it was over a network) so it's possible
> I lost the last few packets of data.
>
> /dev/Test/Test - 400MB LV on raid 1
> /dev/Master/Test - 400MB LV on raid 5
>
> A) mkfs.ext3 -E nodiscard -v /dev/Test/Test - succeeds
> B) mkfs.ext3 -v /dev/Test/Test - succeeds
> C) mkfs.ext3 -E nodiscard -v /dev/Master/Test - succeeds
> D) mkfs.ext3 -v /dev/Master/Test - panics
>
> mkfs.ext3 output from (B)
> -------------------------
> mke2fs 1.42.9 (28-Dec-2013)
> fs_types for mke2fs.conf resolution: 'ext3', 'small'
> Discarding device blocks: done Discard
> succeeded and will return 0s - skipping inode table wipe
> Filesystem label=
> OS type: Linux
> Block size=1024 (log=0)
> Fragment size=1024 (log=0)
> Stride=4 blocks, Stripe width=4 blocks
> 51200 inodes, 204800 blocks
> 10240 blocks (5.00%) reserved for the super user
> First data block=1
> Maximum filesystem blocks=67371008
> 25 block groups
> 8192 blocks per group, 8192 fragments per group
> 2048 inodes per group
> Superblock backups stored on blocks:
> 8193, 24577, 40961, 57345, 73729
>
> Allocating group tables: done Writing inode
> tables: done Creating journal (4096 blocks): done
> Writing superblocks and filesystem accounting information: done
>
> strace output from (B) around the BLKDISCARD
> --------------------------------------------
> gettimeofday({1418806647, 890754}, NULL) = 0
> gettimeofday({1418806647, 890814}, NULL) = 0
> ioctl(3, BLKDISCARD, {0, 3000000010}) = 0
> write(1, "Discarding device blocks: ", 26) = 26
> write(1, " 1024/204800", 13) = 13
> write(1, "\10\10\10\10\10\10\10\10\10\10\10\10\10", 13) = 13
> ioctl(3, BLKDISCARD, {100000, 3000000010}) = 0
> write(1, " ", 13) = 13
> write(1, "\10\10\10\10\10\10\10\10\10\10\10\10\10", 13) = 13
> write(1, "done "..., 33) = 33
> write(1, "Discard succeeded and will retur"..., 65) = 65
>
> mkfs.ext3 output from (D)
> -------------------------
> mke2fs 1.42.9 (28-Dec-2013)
> fs_types for mke2fs.conf resolution: 'ext3', 'small'
> <Panic>
>
> strace output from (D) around the BLKDISCARD
> --------------------------------------------
> gettimeofday({1418809706, 244197}, NULL) = 0
> gettimeofday({1418809706, 244259}, NULL) = 0
> ioctl(3, BLKDISCARD, {0, 3000000010}
> <Panic>
>
> I have a photograph of the panic output from a previous session which
> includes raid5d and blk_finish_plug in the stack trace, unfortunately I
> don't have the top part of the panic and vger won't accept the
> attachment. I also have a photograph of the console output from the
> crash at (D), but in this case it outputs to the console every 180 seconds:
>
> INFO: rcu_sched self-detected stall on CPU { 1}
> sending NMI to all CPUs:
> xen: vector 0x2 is not implemented
>
> thanks,
>
> Anthony Wright
Presumably you have deliberately enabled DISCARD support by setting the
raid456.devices_handle_discard_safely
modules parameters? Otherwise the DISCARD should be a no-op.
It is very hard to deduce anything without the full Oops. Do you have access
to another machine on the same subnet? If so you could enable netconsole and
capture the full oops from the other machines (all console messages are sent
via UDP at a very low level).
I suspect md/raid5 is sending down a discard request in some way that the
scsi/sata layer or driver doesn't like, but without the full oops, I really
cannot guess what it might be.
NeilBrown
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 811 bytes --]
^ permalink raw reply
* Re: [PATCH] drivers:md: Remove no longer needed fixme comment in raid5.c
From: Valdis.Kletnieks @ 2014-12-18 4:44 UTC (permalink / raw)
To: NeilBrown; +Cc: Nicholas Krause, linux-raid, linux-kernel
In-Reply-To: <20141218101906.55b7772d@notabene.brown>
[-- Attachment #1: Type: text/plain, Size: 1301 bytes --]
On Thu, 18 Dec 2014 10:19:06 +1100, NeilBrown said:
> On Wed, 17 Dec 2014 10:49:33 -0500 Nicholas Krause <xerofoify@gmail.com> wrote:
>
> > Removes the no longer needed fix me comment related to not needing the agrument faster for the function,
> > sync_request. Due to getting warnings when building without the agrument it seems that this fix me was
> > wrong and we must keep this agrument in order to avoid build warnings when building without it.
Examining raid1.c, which *does* use the go_faster parameter, I suspect
that the *real* meaning of the FIXME is that the code in raid5.c *should*
possibly be doing this that's done in raid1.c's sync_request():
/*
* If there is non-resync activity waiting for a turn,
* and resync is going fast enough,
* then let it though before starting on this new sync request.
*/
if (!go_faster && conf->nr_waiting)
msleep_interruptible(1000);
(Which is one of the few flow differences for the sync_request() code
in raid1.c versus raid5.c. I'll however defer to actual RAID experts
on whether raid5 needs resync throttling, or if it's accomplished via
other means I'm not spotting. Finding a FIXME outside of drivers/staging
is usually a "Here be fearsome and nasty dragons" comment....
[-- Attachment #2: Type: application/pgp-signature, Size: 848 bytes --]
^ permalink raw reply
* Hung RAID5 array with discard
From: Terry Hardie @ 2014-12-18 3:08 UTC (permalink / raw)
To: linux-raid
Hi,
I am testing 3 SSDs (1TB Crucial M550 with DRZAT, and I tested they do
return zeros after discard) with RAID5 and discard. I create the array
with a 64k chunk size, and it starts to sync. During it's initial
reconstruction, I do a mkfs.ext4, which starts to do the "Discarding
device blocks". After a short period (I believe when the mkfs reaches
the point where the reconstruction is at, all IO to the disks freezes,
and mkfs does not advance. iostat shows 2 of the 3 drives at 100%
utilization with no data read or written. After 2 minutes, I get the
hung task dump. Most CPUs are idle, and here are a few which are not,
which look like a deadlock to me:
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154399] INFO:
rcu_sched detected stalls on CPUs/tasks: { 4 5} (detected by 3,
t=285032 jiffies, g=1160, c=1159, q=0)
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154902] NMI
backtrace for cpu 4
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154904] CPU: 4 PID:
2146 Comm: md3_raid5 Tainted: G W IOX 3.13.0-43-generic
#72~precise1
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154905] Hardware
name: Supermicro SYS-2028TP-HC1R/X10DRT-P, BIOS 1.0a 08/28/2014
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154906] task:
ffff88202594c800 ti: ffff8810245a0000 task.ti: ffff8810245a0000
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154907] RIP:
0010:[<ffffffff817644c1>] [<ffffffff817644c1>]
_raw_spin_lock_irqsave+0x41/0x60
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154910] RSP:
0018:ffff8810245a1cc8 EFLAGS: 00000006
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154911] RAX:
0000000000002ec5 RBX: ffff882028a6ec00 RCX: 0000000000007b78
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154912] RDX:
0000000000000202 RSI: 0000000000007b78 RDI: ffff882028a6ec10
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154913] RBP:
ffff8810245a1cc8 R08: 0000000000007b76 R09: ffff882023629170
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154914] R10:
0000000000000000 R11: ffff882028a6ec00 R12: ffff882028a6ee68
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154915] R13:
0000000000000003 R14: 0000000000000002 R15: ffff882028a6ec10
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154916] FS:
0000000000000000(0000) GS:ffff88103fc80000(0000)
knlGS:0000000000000000
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154917] CS: 0010
DS: 0000 ES: 0000 CR0: 0000000080050033
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154918] CR2:
00007f208c2d0000 CR3: 0000000001c0d000 CR4: 00000000001407e0
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154919] Stack:
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154920]
ffff8810245a1d18 ffffffffa0149890 0000000000000002 ffff882028a6ee88
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154923]
ffff882028a6ee68 ffff882028a6ec00 0000000000000008 ffff882028a6ee68
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154926]
0000000000000000 ffff882028a6ee50 ffff8810245a1d98 ffffffffa015212f
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154929] Call Trace:
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154933]
[<ffffffffa0149890>] release_inactive_stripe_list+0x50/0x160 [raid456]
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154937]
[<ffffffffa015212f>] handle_active_stripes.isra.38+0x7f/0x190
[raid456]
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154940]
[<ffffffffa0152758>] raid5d+0x198/0x2f0 [raid456]
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154942]
[<ffffffff815d30a7>] md_thread+0x117/0x150
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154945]
[<ffffffff810affe0>] ? __wake_up_sync+0x20/0x20
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154947]
[<ffffffff815d2f90>] ? md_rdev_init+0x110/0x110
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154949]
[<ffffffff8108fb59>] kthread+0xc9/0xe0
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154952]
[<ffffffff8108fa90>] ? flush_kthread_worker+0xb0/0xb0
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154954]
[<ffffffff8176d5bc>] ret_from_fork+0x7c/0xb0
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154956]
[<ffffffff8108fa90>] ? flush_kthread_worker+0xb0/0xb0
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.154957] Code: 1f 44
00 00 b8 00 00 02 00 f0 0f c1 07 89 c1 c1 e9 10 66 39 c1 75 05 48 89
d0 5d c3 83 e1 fe 0f b7 f1 b8 00 80 00 00 44 0f b7 07 <66> 44 39 c1 74
e6 f3 90 83 e8 01 75 ef 0f 1f 80 00 00 00 00 eb
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155001] NMI
backtrace for cpu 5
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155003] CPU: 5 PID:
2147 Comm: md3_resync Tainted: G W IOX 3.13.0-43-generic
#72~precise1
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155004] Hardware
name: Supermicro SYS-2028TP-HC1R/X10DRT-P, BIOS 1.0a 08/28/2014
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155005] task:
ffff88202594b000 ti: ffff8810274a0000 task.ti: ffff8810274a0000
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155006] RIP:
0010:[<ffffffffa01483b7>] [<ffffffffa01483b7>]
__find_stripe+0x57/0xa0 [raid456]
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155010] RSP:
0018:ffff8810274a1b68 EFLAGS: 00000006
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155011] RAX:
ffff882027092da0 RBX: 0000000000a30c10 RCX: 0000000000000001
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155012] RDX:
0000000000000c10 RSI: 0000000000a30c10 RDI: ffff882028a6ec00
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155013] RBP:
ffff8810274a1b88 R08: 0000000000000000 R09: 0000000000000000
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155014] R10:
0000000000000000 R11: 0000000000000001 R12: 0000000000000000
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155015] R13:
ffff882028a6ec00 R14: 0000000000000000 R15: ffff882028a6eda8
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155016] FS:
0000000000000000(0000) GS:ffff88103fca0000(0000)
knlGS:0000000000000000
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155017] CS: 0010
DS: 0000 ES: 0000 CR0: 0000000080050033
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155018] CR2:
00000000006e1dc8 CR3: 0000000001c0d000 CR4: 00000000001407e0
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155019] Stack:
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155020]
ffff8810274a1ba8 ffff882028a6ec00 000000007b767b00 ffff882028a6ec10
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155023]
ffff8810274a1c28 ffffffffa0150555 ffff882023773b50 ffff882028a6eda8
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155026]
0000000200000001 ffff882028a6ec08 0000000000000000 0000000000a30c10
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155029] Call Trace:
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155033]
[<ffffffffa0150555>] get_active_stripe+0x115/0x3e0 [raid456]
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155036]
[<ffffffffa014aea8>] ? release_stripe+0x68/0x100 [raid456]
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155040]
[<ffffffffa0154f3b>] sync_request+0x11b/0x2a0 [raid456]
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155042]
[<ffffffff815d5ccf>] md_do_sync+0x84f/0xdb0
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155046]
[<ffffffff810affe0>] ? __wake_up_sync+0x20/0x20
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155048]
[<ffffffff815d30a7>] md_thread+0x117/0x150
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155050]
[<ffffffff815d2f90>] ? md_rdev_init+0x110/0x110
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155052]
[<ffffffff8108fb59>] kthread+0xc9/0xe0
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155054]
[<ffffffff8108fa90>] ? flush_kthread_worker+0xb0/0xb0
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155057]
[<ffffffff8176d5bc>] ret_from_fork+0x7c/0xb0
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155059]
[<ffffffff8108fa90>] ? flush_kthread_worker+0xb0/0xb0
Dec 18 00:57:41 unassigned-hostname kernel: [ 1606.155060] Code: e2 f8
0f 00 00 48 8b 04 02 48 85 c0 75 25 f6 05 29 25 01 00 04 75 3e 31 c0
48 83 c4 08 5b 41 5c 41 5d 5d c3 66 44 39 60 30 74 ee <48> 8b 00 48 85
c0 74 db 48 39 58 38 75 f2 eb e9 48 89 f2 48 c7
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670082] INFO: task
mkfs.ext4:2235 blocked for more than 120 seconds.
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670109]
Tainted: G W IOX 3.13.0-43-generic #72~precise1
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670130] "echo 0 >
/proc/sys/kernel/hung_task_timeout_secs" disables this message.
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670155] mkfs.ext4
D ffff881024fe39e0 0 2235 2080 0x00000000
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670158]
ffff882026eafa68 0000000000000082 ffff88103fc73480 ffff882026eaffd8
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670162]
0000000000013480 0000000000013480 ffff8820293e8000 ffff88202208b000
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670165]
ffff882026eafa78 ffff882028a6ec00 ffff882028a6ed98 ffff882028a6ec0c
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670169] Call Trace:
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670175]
[<ffffffff81760ae9>] schedule+0x29/0x70
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670181]
[<ffffffffa01506e3>] get_active_stripe+0x2a3/0x3e0 [raid456]
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670185]
[<ffffffff8134c152>] ? blk_check_plugged+0x72/0xb0
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670189]
[<ffffffff810affe0>] ? __wake_up_sync+0x20/0x20
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670193]
[<ffffffffa0155e44>] make_discard_request+0x108/0x12c4 [raid456]
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670196]
[<ffffffff810affe0>] ? __wake_up_sync+0x20/0x20
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670201]
[<ffffffffa0155c91>] make_request+0x581/0x590 [raid456]
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670204]
[<ffffffff8109cfd6>] ? ttwu_do_activate.constprop.82+0x66/0x70
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670207]
[<ffffffff8109d097>] ? ttwu_queue+0xb7/0xd0
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670210]
[<ffffffff8109f950>] ? try_to_wake_up+0x190/0x210
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670212]
[<ffffffff815d2c53>] md_make_request+0xd3/0x230
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670216]
[<ffffffff8115b085>] ? mempool_alloc_slab+0x15/0x20
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670219]
[<ffffffff8134ceb7>] generic_make_request.part.62+0x77/0xb0
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670222]
[<ffffffff8134d428>] generic_make_request+0x68/0x70
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670225]
[<ffffffff8134d4a8>] submit_bio+0x78/0x160
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670228]
[<ffffffff81202f80>] ? bio_alloc_bioset+0xa0/0x1d0
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670232]
[<ffffffff813578c0>] blkdev_issue_discard+0x1f0/0x2a0
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670235]
[<ffffffff8135c1f4>] blkdev_ioctl+0x354/0x810
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670238]
[<ffffffff8101361d>] ? __switch_to+0x16d/0x4d0
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670241]
[<ffffffff81204370>] block_ioctl+0x40/0x50
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670244]
[<ffffffff811dd5c5>] do_vfs_ioctl+0x75/0x2c0
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670247]
[<ffffffff817606be>] ? __schedule+0x38e/0x700
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670249]
[<ffffffff811dd8a1>] SyS_ioctl+0x91/0xb0
Dec 18 00:58:57 unassigned-hostname kernel: [ 1682.670252]
[<ffffffff8176d66d>] system_call_fastpath+0x1a/0x1f
If I do the mkfs.ext4 after the initial reconstruction is done, is
gets all the way through. I don't want to put this system into
production, since this could mean this condition could show up in the
future if the array needs to reconstruct again at a future point while
in service.
This is a test system in a lab, so I'd be happy to try some tests.
Terry
^ permalink raw reply
* Re: [PATCH] drivers:md: Remove no longer needed fixme comment in raid5.c
From: NeilBrown @ 2014-12-17 23:19 UTC (permalink / raw)
To: Nicholas Krause; +Cc: linux-raid, linux-kernel
In-Reply-To: <1418831373-18415-1-git-send-email-xerofoify@gmail.com>
[-- Attachment #1: Type: text/plain, Size: 1123 bytes --]
On Wed, 17 Dec 2014 10:49:33 -0500 Nicholas Krause <xerofoify@gmail.com>
wrote:
> Removes the no longer needed fix me comment related to not needing the agrument faster for the function,
> sync_request. Due to getting warnings when building without the agrument it seems that this fix me was
> wrong and we must keep this agrument in order to avoid build warnings when building without it.
>
> Signed-off-by: Nicholas Krause <xerofoify@gmail.com>
> ---
> drivers/md/raid5.c | 1 -
> 1 file changed, 1 deletion(-)
>
> diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
> index c1b0d52..c0fde2a 100644
> --- a/drivers/md/raid5.c
> +++ b/drivers/md/raid5.c
> @@ -4986,7 +4986,6 @@ ret:
> return reshape_sectors;
> }
>
> -/* FIXME go_faster isn't used */
> static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped, int go_faster)
> {
> struct r5conf *conf = mddev->private;
Why do you think the comment is no longer needed. Is "go_faster" being used
now? Is there a clear explanation somewhere of why "go_faster" should not be
used?
NeilBrown
[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 811 bytes --]
^ permalink raw reply
* indefinite hang when growing/reshaping
From: Chris Murphy @ 2014-12-17 22:20 UTC (permalink / raw)
To: linux-raid
If this is user error I'd expect a discreet error message, but I don't
get one, just a hang during the reshape following successful --grow
command (going from 3x device raid5 to 4x device raid5).
# cat /proc/mdstat
Personalities : [raid6] [raid5] [raid4]
md127 : active raid5 loop3[4] loop2[3] loop1[1] loop0[0]
2095104 blocks super 1.2 level 5, 512k chunk, algorithm 2 [4/4] [UUUU]
[>....................] reshape = 0.0% (0/1047552)
finish=7.0min speed=2402K/sec
Reproduces with:
kernel-3.17.6-300.fc21.x86_64
mdadm-3.3.2-1.fc21.x86_64
and
3.18.0-2.fc22.i686+debug
mdadm-3.3.2-1.fc21.i686
Details are here:
https://bugzilla.kernel.org/show_bug.cgi?id=89851
--
Chris Murphy
^ permalink raw reply
* Panic doing BLKDISCARD on a raid 5 array on linux 3.17.3
From: Anthony Wright @ 2014-12-17 12:00 UTC (permalink / raw)
To: linux-raid
I've hit a panic bug on stock linux 3.17.3 (which includes the recent
commit on BLKDISCARD in md/raid5.c) running in Dom0 under Xen 4.1.0 that
I've isolated to a BLKDISCARD system call within mkfs.ext3 and only
happens on a raid 5 array (it doesn't happen on a raid 1 array).
The system it happens on is remote and I don't have physical access to
it, but the system administrator there is fairly helpful. We're in the
process of commissioning the system which needs to be done tomorrow
(thursday), so I've only got 24 hours in which I can run any tests you
may want. If necessary I can arrange remote access, but it's a little
complex.
We have 3 512GB SSDs on the system, all with a GPT partition table and
the same partition layout. All the partitions have optimal alignment
according to parted. One of the partitions on each SSD is assembled into
a raid 1 array, another partition is assembled into a raid 5 array. Each
array is the used as the only physical volume in a LVM volume group. I
then create a logical volume on each array and format the logical volume
with mkfs.ext3. I ran mkfs.ext3 in verbose mode and also ran strace on
it in a separate session (though it was over a network) so it's possible
I lost the last few packets of data.
/dev/Test/Test - 400MB LV on raid 1
/dev/Master/Test - 400MB LV on raid 5
A) mkfs.ext3 -E nodiscard -v /dev/Test/Test - succeeds
B) mkfs.ext3 -v /dev/Test/Test - succeeds
C) mkfs.ext3 -E nodiscard -v /dev/Master/Test - succeeds
D) mkfs.ext3 -v /dev/Master/Test - panics
mkfs.ext3 output from (B)
-------------------------
mke2fs 1.42.9 (28-Dec-2013)
fs_types for mke2fs.conf resolution: 'ext3', 'small'
Discarding device blocks: done Discard
succeeded and will return 0s - skipping inode table wipe
Filesystem label=
OS type: Linux
Block size=1024 (log=0)
Fragment size=1024 (log=0)
Stride=4 blocks, Stripe width=4 blocks
51200 inodes, 204800 blocks
10240 blocks (5.00%) reserved for the super user
First data block=1
Maximum filesystem blocks=67371008
25 block groups
8192 blocks per group, 8192 fragments per group
2048 inodes per group
Superblock backups stored on blocks:
8193, 24577, 40961, 57345, 73729
Allocating group tables: done Writing inode
tables: done Creating journal (4096 blocks): done
Writing superblocks and filesystem accounting information: done
strace output from (B) around the BLKDISCARD
--------------------------------------------
gettimeofday({1418806647, 890754}, NULL) = 0
gettimeofday({1418806647, 890814}, NULL) = 0
ioctl(3, BLKDISCARD, {0, 3000000010}) = 0
write(1, "Discarding device blocks: ", 26) = 26
write(1, " 1024/204800", 13) = 13
write(1, "\10\10\10\10\10\10\10\10\10\10\10\10\10", 13) = 13
ioctl(3, BLKDISCARD, {100000, 3000000010}) = 0
write(1, " ", 13) = 13
write(1, "\10\10\10\10\10\10\10\10\10\10\10\10\10", 13) = 13
write(1, "done "..., 33) = 33
write(1, "Discard succeeded and will retur"..., 65) = 65
mkfs.ext3 output from (D)
-------------------------
mke2fs 1.42.9 (28-Dec-2013)
fs_types for mke2fs.conf resolution: 'ext3', 'small'
<Panic>
strace output from (D) around the BLKDISCARD
--------------------------------------------
gettimeofday({1418809706, 244197}, NULL) = 0
gettimeofday({1418809706, 244259}, NULL) = 0
ioctl(3, BLKDISCARD, {0, 3000000010}
<Panic>
I have a photograph of the panic output from a previous session which
includes raid5d and blk_finish_plug in the stack trace, unfortunately I
don't have the top part of the panic and vger won't accept the
attachment. I also have a photograph of the console output from the
crash at (D), but in this case it outputs to the console every 180 seconds:
INFO: rcu_sched self-detected stall on CPU { 1}
sending NMI to all CPUs:
xen: vector 0x2 is not implemented
thanks,
Anthony Wright
^ permalink raw reply
* Expand RAID5 array or switch to RAID10
From: George Duffield @ 2014-12-17 10:11 UTC (permalink / raw)
To: linux-raid@vger.kernel.org
Hi
I have a RAID5 array comprising four 3TB drives and I've basically got
100GB remaining free so it's time to expand the array. With this in
mind, and considering my requirement is predominantly to increase
capacity, I'm wondering whether it'd be best to add another 3TB drive
and expand the array whilst retaining RAID5 or to switch out to a
RAID10 array. My thinking is 1) adding a 5th 3TB drive to the
existing array may result in drive failure during the rebuild; and 2)
RAID10 is very expensive to implement. Hence, I'm tempted to consider
other options (recognising that whilst it'd be convenient I don't
actually need everything stored on a single array). One such option
would be running two raid5 arrays comprised of 3x 3TB each yielding
12TB of storage across the two arrays.
Questions:
+ Is my assumption re RAID5 drive failure correct/ likely?
+ Is there a non-destructive way to migrate from RAID5 to RAID10?
+ 2 x RAID5 arrays seems pretty appealing - from a reliability and
cost effectiveness standpoint - yes/no?
^ permalink raw reply
* Re: Split RAID: Proposal for archival RAID using incremental batch checksum
From: Anshuman Aggarwal @ 2014-12-17 6:40 UTC (permalink / raw)
To: NeilBrown; +Cc: Mdadm
In-Reply-To: <20141217084921.360ca442@notabene.brown>
On 17 December 2014 at 03:19, NeilBrown <neilb@suse.de> wrote:
> On Tue, 16 Dec 2014 21:55:15 +0530 Anshuman Aggarwal
> <anshuman.aggarwal@gmail.com> wrote:
>
>> On 2 December 2014 at 17:26, Anshuman Aggarwal
>> <anshuman.aggarwal@gmail.com> wrote:
>> > It works! (Atleast on a sample 5 MB device with 5 x 1MB partitions :-)
>> > will find more space on my drives and do a larger test but don't see
>> > why it shouldn't work)
>> > Here are the following caveats (and questions):
>> > - Neil, like you pointed out, the power of 2 chunk size will probably
>> > need a code change (in the kernel or only in the userspace tool?)
>
> In the kernel too.
Is this something that you would consider implementing soon? Is there
a performance/other impact to any other consideration to remove this
limitation.. could you elaborate on the reason why it was there in the
first place?
If this is a case of patches are welcome, please guide on where to
start looking/working even if its just
>
>> > - Any performance or other reasons why a terabyte size chunk may
>> > not be feasible?
>
> Not that I can think of.
>
>> > - Implications of safe_mode_delay
>> > - Would the metadata be updated on the block device be written to
>> > and the parity device as well?
>
> Probably. Hard to give a specific answer to vague question.
I should clarify.
For example in a 5 device RAID4, lets say block is being written to
device 1 and parity is on device 5 and devices 2,3,4 are sleeping
(spun down). If we set safe_mode_delay to 0 and md decides to update
the parity without involving the blocks on the other 3 devices and
just updates the parity by doing a read, compute, write to device 5
will the metadata be updated on both device 1 and 5 even though
safe_mode_delay is 0?
>
>> > - If the drive fails which is the same as the drive being written
>> > to, would that lack of metadata updates to the other devices affect
>> > reconstruction?
>
> Again, to give a precise answer, a detailed question is needed. Obviously
> any change would have to made in such a way to ensure that things which
> needed to work, did work.
Continuing from the previous example, lets say device 1 fails after a
write which only updated metadata on 1 and 5 while 2,3,4 were
sleeping. In that case to access the data from 1, md will use 2,3,4,5
but will it then update the metadata from 5 onto 2,3,4? I hope I am
making this clear.
>
>
>> > - Adding new devices (is it possible to move the parity to the disk
>> > being added? How does device addition work for RAID4 ...is it added as
>> > a zero-ed out device with parity disk remaining the same)
>
> RAID5 or RAID6 with ALGORITHM_PARITY_0 puts the parity on the early devices.
> Currently if you add a device to such an array ...... I'm not sure what it
> will do. It should be possible to make it just write zeros out.
>
Once again, is this something that can make its way to your roadmap?
If so, great.. otherwise could you steer me towards where in the md
kernel and mdadm source I should be looking to make these changes.
Thanks again.
>
> NeilBrown
>
>
>> >
>> >
>>
>> Neil, sorry to try to bump this thread. Could you please look over the
>> questions and address the points on the remaining items that can make
>> it a working solution? Thanks
>
^ permalink raw reply
* RE: md_raid5 using 100% CPU and hang with status resync=PENDING, if a drive is removed during initialization
From: Manibalan P @ 2014-12-17 6:40 UTC (permalink / raw)
To: linux-raid; +Cc: NeilBrown, Vijayarankan Muthirisavengopal, Dinakaran N
Dear Neil,
The same Issue is reproducible in the latest upstream kernel also.
Tested in "3.17.6" latest stable upstream kernel and find the same issue.
[root@root ~]# modinfo raid456
filename: /lib/modules/3.17.6/kernel/drivers/md/raid456.ko
alias: raid6
alias: raid5
alias: md-level-6
alias: md-raid6
alias: md-personality-8
alias: md-level-4
alias: md-level-5
alias: md-raid4
alias: md-raid5
alias: md-personality-4
description: RAID4/5/6 (striping with parity) personality for MD
license: GPL
srcversion: 0EEF680023FDC7410F7989A
depends: async_raid6_recov,async_pq,async_tx,async_memcpy,async_xor
intree: Y
vermagic: 3.17.6 SMP mod_unload modversions
parm: devices_handle_discard_safely:Set to Y if all devices in each array reliably return zeroes on reads from discarded regions (bool)
Thanks,
Manibalan.
-----Original Message-----
From: Manibalan P
Sent: Wednesday, December 17, 2014 12:01 PM
To: 'linux-raid'
Cc: 'NeilBrown'
Subject: RE: md_raid5 using 100% CPU and hang with status resync=PENDING, if a drive is removed during initialization
Dear Neil,
We are facing IO struck issue with raid5 in the following scenario. (please see the attachment for the complete information) In RAID5 array, if a drive is removed while initialization and the same time if IO is happening to that md. Then IO is getting struck, and md_raid5 thread is using 100 % of CPU. Also the md state showing as resync=PENDING
Kernel : Issue found in the following kernels RHEL 6.5 (2.6.32-431.el6.x86_64) CentOS 7 (kernel-3.10.0-123.13.1.el7.x86_64)
Steps to Reproduce the issue:
1. Created a raid 5 md with 4 drives using the below mdadm command.
mdadm -C /dev/md0 -c 64 -l 5 -f -n 4 -e 1.2 /dev/sdb6 /dev/sdc6 /dev/sdd6 /dev/sde6
2. Make the md writable
mdadm –readwrite /dev/md0
3. Now md will start initialization
4. Run FIO Tool, the the below said configuration /usr/bin/fio --name=md0 --filename=/dev/md0 --thread --numjobs=10 --direct=1 --group_reporting --unlink=0 --loops=1 --offset=0 --randrepeat=1 --norandommap --scramble_buffers=1 --stonewall --ioengine=libaio --rw=randwrite --bs=8704 --iodepth=4000 --runtime=3000 --blockalign=512
4. During MD initialzing, remove a drive(either using MDADM set faulty/remove or remove manually)
5. Now the IO will struck, and cat /proc/mdstat shows states with resync=PENDING
---------------------------------------------------------------------------------------------
top - output show, md_raid5 using 100% cpu
top - 17:55:06 up 1:09, 3 users, load average: 11.98, 8.53, 3.99
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
2690 root 20 0 0 0 0 R 100.0 0.0 6:44.41 md0_raid5
---------------------------------------------------------------------------------------------
dmesg - show the stack trace
INFO: task fio:2715 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 000000000000000a 0 2715 2654 0x00000080
ffff88043b623598 0000000000000082 0000000000000000 ffffffff81058d53
ffff88043b623548 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b40b098 ffff88043b623fd8 000000000000fbc8 ffff88043b40b098 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff8140fa39>] ? md_wakeup_thread+0x39/0x70 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffffa0308f66>] ? make_request+0x306/0xc6c [raid456] [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81122283>] ? mempool_alloc+0x63/0x140 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c767a>] do_direct_IO+0x7ca/0xfa0 [<ffffffff811c8196>] __blockdev_direct_IO_newtrunc+0x346/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2717 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000004 0 2717 2654 0x00000080
ffff880439e97698 0000000000000082 ffff880439e97628 ffffffff81058d53
ffff880439e97648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b0adab8 ffff880439e97fd8 000000000000fbc8 ffff88043b0adab8 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8e50>] __blockdev_direct_IO_newtrunc+0x1000/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2718 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000005 0 2718 2654 0x00000080
ffff88043bc13698 0000000000000082 ffff88043bc13628 ffffffff81058d53
ffff88043bc13648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b0ad058 ffff88043bc13fd8 000000000000fbc8 ffff88043b0ad058 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8e50>] __blockdev_direct_IO_newtrunc+0x1000/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2719 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000001 0 2719 2654 0x00000080
ffff880439ebb698 0000000000000082 ffff880439ebb628 ffffffff81058d53
ffff880439ebb648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b0ac5f8 ffff880439ebbfd8 000000000000fbc8 ffff88043b0ac5f8 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2720 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000008 0 2720 2654 0x00000080
ffff88043b8cf698 0000000000000082 ffff88043b8cf628 ffffffff81058d53
ffff88043b8cf648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff880439e89af8 ffff88043b8cffd8 000000000000fbc8 ffff880439e89af8 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2721 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000000 0 2721 2654 0x00000080
ffff88043b047698 0000000000000082 ffff88043b047628 ffffffff81058d53
ffff88043b047648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff880439e89098 ffff88043b047fd8 000000000000fbc8 ffff880439e89098 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2722 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000000 0 2722 2654 0x00000080
ffff880439ea3698 0000000000000082 ffff880439ea3628 ffffffff81058d53
ffff880439ea3648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff880439e88638 ffff880439ea3fd8 000000000000fbc8 ffff880439e88638 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2723 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000006 0 2723 2654 0x00000080
ffff88043bf5f698 0000000000000082 ffff88043bf5f628 ffffffff81058d53
ffff88043bf5f648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043a183ab8 ffff88043bf5ffd8 000000000000fbc8 ffff88043a183ab8 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2724 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 000000000000000b 0 2724 2654 0x00000080
ffff88043be05698 0000000000000082 ffff88043be05628 ffffffff81058d53
ffff88043be05648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043a183058 ffff88043be05fd8 000000000000fbc8 ffff88043a183058 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2725 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000003 0 2725 2654 0x00000080
ffff88043be07698 0000000000000082 ffff88043be07628 ffffffff81058d53
ffff88043be07648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043a1825f8 ffff88043be07fd8 000000000000fbc8 ffff88043a1825f8 Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70 [<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456] [<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456] [<ffffffff81065df0>] ? default_wake_function+0x0/0x20 [<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80 [<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456] [<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40 [<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20 [<ffffffff81415b41>] md_make_request+0xe1/0x230 [<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110 [<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230 [<ffffffff81266c50>] generic_make_request+0x240/0x5a0 [<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0 [<ffffffff81267020>] submit_bio+0x70/0x120 [<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60 [<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20 [<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170 [<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0 [<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0 [<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200 [<ffffffff811d6924>] aio_run_iocb+0x64/0x170 [<ffffffff811d7d51>] do_io_submit+0x291/0x920 [<ffffffff811d83f0>] sys_io_submit+0x10/0x20 [<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
[root@root ~]# cat /proc/2690/stack
[<ffffffff810686da>] __cond_resched+0x2a/0x40 [<ffffffffa030361c>] ops_run_io+0x2c/0x920 [raid456] [<ffffffffa03052cc>] handle_stripe+0x9cc/0x2980 [raid456] [<ffffffffa03078a4>] raid5d+0x624/0x850 [raid456] [<ffffffff81416f05>] md_thread+0x115/0x150 [<ffffffff8109aef6>] kthread+0x96/0xa0 [<ffffffff8100c20a>] child_rip+0xa/0x20 [<ffffffffffffffff>] 0xffffffffffffffff
[root@root ~]# cat /proc/2690/stat
2690 (md0_raid5) R 2 0 0 0 -1 2149613632 0 0 0 0 0 68495 0 0 20 0 1 0 350990 0 0 18446744073709551615 0 0 0 0 0 0 0 2147483391 256 0 0 0 17 2 0 0 6855 0 0 [root@root ~]# cat /proc/2690/statm
0 0 0 0 0 0 0
[root@root ~]# cat /proc/2690/stat
stat statm status
[root@root ~]# cat /proc/2690/status
Name: md0_raid5
State: R (running)
Tgid: 2690
Pid: 2690
PPid: 2
TracerPid: 0
Uid: 0 0 0 0
Gid: 0 0 0 0
Utrace: 0
FDSize: 64
Groups:
Threads: 1
SigQ: 2/128402
SigPnd: 0000000000000000
ShdPnd: 0000000000000000
SigBlk: 0000000000000000
SigIgn: fffffffffffffeff
SigCgt: 0000000000000100
CapInh: 0000000000000000
CapPrm: ffffffffffffffff
CapEff: fffffffffffffeff
CapBnd: ffffffffffffffff
Cpus_allowed: ffffff
Cpus_allowed_list: 0-23
Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000003
Mems_allowed_list: 0-1
voluntary_ctxt_switches: 5411612
nonvoluntary_ctxt_switches: 257032
Thanks,
Manibalan.
^ permalink raw reply
* RE: md_raid5 using 100% CPU and hang with status resync=PENDING, if a drive is removed during initialization
From: Manibalan P @ 2014-12-17 6:31 UTC (permalink / raw)
To: linux-raid; +Cc: NeilBrown
[-- Attachment #1: Type: text/plain, Size: 23516 bytes --]
Dear Neil,
We are facing IO struck issue with raid5 in the following scenario. (please see the attachment for the complete information)
In RAID5 array, if a drive is removed while initialization and the same time if IO is happening to that md. Then IO is getting struck, and md_raid5 thread is using 100 % of CPU. Also the md state showing as resync=PENDING
Kernel : Issue found in the following kernels
RHEL 6.5 (2.6.32-431.el6.x86_64)
CentOS 7 (kernel-3.10.0-123.13.1.el7.x86_64)
Steps to Reproduce the issue:
1. Created a raid 5 md with 4 drives using the below mdadm command.
mdadm -C /dev/md0 -c 64 -l 5 -f -n 4 -e 1.2 /dev/sdb6 /dev/sdc6 /dev/sdd6 /dev/sde6
2. Make the md writable
mdadm –readwrite /dev/md0
3. Now md will start initialization
4. Run FIO Tool, the the below said configuration
/usr/bin/fio --name=md0 --filename=/dev/md0 --thread --numjobs=10 --direct=1 --group_reporting --unlink=0 --loops=1 --offset=0 --randrepeat=1 --norandommap --scramble_buffers=1 --stonewall --ioengine=libaio --rw=randwrite --bs=8704 --iodepth=4000 --runtime=3000 --blockalign=512
4. During MD initialzing, remove a drive(either using MDADM set faulty/remove or remove manually)
5. Now the IO will struck, and cat /proc/mdstat shows states with resync=PENDING
---------------------------------------------------------------------------------------------
top - output show, md_raid5 using 100% cpu
top - 17:55:06 up 1:09, 3 users, load average: 11.98, 8.53, 3.99
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
2690 root 20 0 0 0 0 R 100.0 0.0 6:44.41 md0_raid5
---------------------------------------------------------------------------------------------
dmesg - show the stack trace
INFO: task fio:2715 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 000000000000000a 0 2715 2654 0x00000080
ffff88043b623598 0000000000000082 0000000000000000 ffffffff81058d53
ffff88043b623548 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b40b098 ffff88043b623fd8 000000000000fbc8 ffff88043b40b098
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff8140fa39>] ? md_wakeup_thread+0x39/0x70
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffffa0308f66>] ? make_request+0x306/0xc6c [raid456]
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81122283>] ? mempool_alloc+0x63/0x140
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c767a>] do_direct_IO+0x7ca/0xfa0
[<ffffffff811c8196>] __blockdev_direct_IO_newtrunc+0x346/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2717 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000004 0 2717 2654 0x00000080
ffff880439e97698 0000000000000082 ffff880439e97628 ffffffff81058d53
ffff880439e97648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b0adab8 ffff880439e97fd8 000000000000fbc8 ffff88043b0adab8
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8e50>] __blockdev_direct_IO_newtrunc+0x1000/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2718 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000005 0 2718 2654 0x00000080
ffff88043bc13698 0000000000000082 ffff88043bc13628 ffffffff81058d53
ffff88043bc13648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b0ad058 ffff88043bc13fd8 000000000000fbc8 ffff88043b0ad058
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8e50>] __blockdev_direct_IO_newtrunc+0x1000/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2719 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000001 0 2719 2654 0x00000080
ffff880439ebb698 0000000000000082 ffff880439ebb628 ffffffff81058d53
ffff880439ebb648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b0ac5f8 ffff880439ebbfd8 000000000000fbc8 ffff88043b0ac5f8
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2720 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000008 0 2720 2654 0x00000080
ffff88043b8cf698 0000000000000082 ffff88043b8cf628 ffffffff81058d53
ffff88043b8cf648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff880439e89af8 ffff88043b8cffd8 000000000000fbc8 ffff880439e89af8
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2721 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000000 0 2721 2654 0x00000080
ffff88043b047698 0000000000000082 ffff88043b047628 ffffffff81058d53
ffff88043b047648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff880439e89098 ffff88043b047fd8 000000000000fbc8 ffff880439e89098
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2722 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000000 0 2722 2654 0x00000080
ffff880439ea3698 0000000000000082 ffff880439ea3628 ffffffff81058d53
ffff880439ea3648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff880439e88638 ffff880439ea3fd8 000000000000fbc8 ffff880439e88638
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2723 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000006 0 2723 2654 0x00000080
ffff88043bf5f698 0000000000000082 ffff88043bf5f628 ffffffff81058d53
ffff88043bf5f648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043a183ab8 ffff88043bf5ffd8 000000000000fbc8 ffff88043a183ab8
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2724 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 000000000000000b 0 2724 2654 0x00000080
ffff88043be05698 0000000000000082 ffff88043be05628 ffffffff81058d53
ffff88043be05648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043a183058 ffff88043be05fd8 000000000000fbc8 ffff88043a183058
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2725 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000003 0 2725 2654 0x00000080
ffff88043be07698 0000000000000082 ffff88043be07628 ffffffff81058d53
ffff88043be07648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043a1825f8 ffff88043be07fd8 000000000000fbc8 ffff88043a1825f8
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
[root@root ~]# cat /proc/2690/stack
[<ffffffff810686da>] __cond_resched+0x2a/0x40
[<ffffffffa030361c>] ops_run_io+0x2c/0x920 [raid456]
[<ffffffffa03052cc>] handle_stripe+0x9cc/0x2980 [raid456]
[<ffffffffa03078a4>] raid5d+0x624/0x850 [raid456]
[<ffffffff81416f05>] md_thread+0x115/0x150
[<ffffffff8109aef6>] kthread+0x96/0xa0
[<ffffffff8100c20a>] child_rip+0xa/0x20
[<ffffffffffffffff>] 0xffffffffffffffff
[root@root ~]# cat /proc/2690/stat
2690 (md0_raid5) R 2 0 0 0 -1 2149613632 0 0 0 0 0 68495 0 0 20 0 1 0 350990 0 0 18446744073709551615 0 0 0 0 0 0 0 2147483391 256 0 0 0 17 2 0 0 6855 0 0
[root@root ~]# cat /proc/2690/statm
0 0 0 0 0 0 0
[root@root ~]# cat /proc/2690/stat
stat statm status
[root@root ~]# cat /proc/2690/status
Name: md0_raid5
State: R (running)
Tgid: 2690
Pid: 2690
PPid: 2
TracerPid: 0
Uid: 0 0 0 0
Gid: 0 0 0 0
Utrace: 0
FDSize: 64
Groups:
Threads: 1
SigQ: 2/128402
SigPnd: 0000000000000000
ShdPnd: 0000000000000000
SigBlk: 0000000000000000
SigIgn: fffffffffffffeff
SigCgt: 0000000000000100
CapInh: 0000000000000000
CapPrm: ffffffffffffffff
CapEff: fffffffffffffeff
CapBnd: ffffffffffffffff
Cpus_allowed: ffffff
Cpus_allowed_list: 0-23
Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000003
Mems_allowed_list: 0-1
voluntary_ctxt_switches: 5411612
nonvoluntary_ctxt_switches: 257032
Thanks,
Manibalan.
[-- Attachment #2: md_raid5-hang-resync-PENDING.txt --]
[-- Type: text/plain, Size: 29686 bytes --]
Issue:
md_raid5 using 100% CPU and hang with resync=PENDING status, if a drive is removed during initialization
Discription:
In RAID5 array, if a drive is removed during initialization and the same time if IO is happening to that md.
Then IO is getting struck, and md_raid5 thread is using 100 % of CPU. Also the md state showing as resync=PENDING
Kernel : Issue found in the following kernels
>RHEL 6.5 (2.6.32-431.el6.x86_64)
>CentOS 7 (kernel-3.10.0-123.13.1.el7.x86_64)
Steps to Reproduce the issue:
1. Created a raid 5 md with 4 drives using the below mdadm command.
mdadm -C /dev/md0 -c 64 -l 5 -f -n 4 -e 1.2 /dev/sdb6 /dev/sdc6 /dev/sdd6 /dev/sde6
2. Make the md writable
mdadm –readwrite /dev/md0
3. Now md will start initialization
4. Run FIO Tool, the the below said configuration
/usr/bin/fio --name=md0 --filename=/dev/md0 --thread --numjobs=10 --direct=1 --group_reporting --unlink=0 --loops=1 --offset=0 --randrepeat=1 --norandommap --scramble_buffers=1 --stonewall --ioengine=libaio --rw=randwrite --bs=8704 --iodepth=4000 --runtime=3000 --blockalign=512
4. During MD initialzing, remove a drive(either using MDADM set faulty/remove or remove manually)
5. Now the IO will struck, and cat /proc/mdstat shows states with resync=PENDING
Step done one by one to reproduce the issue, and the Observation during each step:
1. System Information:
[root@root ~]# uname -a
Linux root 2.6.32-431.el6.x86_64 #1 SMP Sun Nov 10 22:19:54 EST 2013 x86_64 x86_64 x86_64 GNU/Linux
[root@root ~]# mdadm -V
mdadm - v3.2.6 - 25th October 2012
[root@root ~]# fio --version
fio-2.1.10
[root@root ~]# lsscsi
[0:0:0:0] disk SEAGATE ST31000640SS 0003 /dev/sda
[0:0:1:0] disk SEAGATE ST2000NM0001 0002 /dev/sdb
[0:0:2:0] enclosu LSI CORP SAS2X36 0424 -
[0:0:3:0] disk SEAGATE ST200FM0002 0003 /dev/sdc
[0:0:4:0] disk SEAGATE ST200FM0002 0003 /dev/sdd
[0:0:5:0] disk SEAGATE ST200FM0002 0003 /dev/sde
[0:0:6:0] disk SEAGATE ST200FM0002 0003 /dev/sdf
2. Creating raid5 md
[root@root ~]# mdadm -C /dev/md0 -c 64 -l 5 -f -n 4 -e 1.2 /dev/sd[cdef]6
mdadm: /dev/sdc6 appears to be part of a raid array:
level=raid5 devices=4 ctime=Mon Dec 15 18:23:17 2014
mdadm: /dev/sdd6 appears to be part of a raid array:
level=raid5 devices=4 ctime=Mon Dec 15 18:23:17 2014
mdadm: /dev/sde6 appears to be part of a raid array:
level=raid5 devices=4 ctime=Mon Dec 15 18:23:17 2014
mdadm: /dev/sdf6 appears to be part of a raid array:
level=raid5 devices=4 ctime=Mon Dec 15 18:23:17 2014
Continue creating array? y
mdadm: array /dev/md0 started.
dmesg
md: unbind<sdf6>
md: export_rdev(sdf6)
md: unbind<sde6>
md: export_rdev(sde6)
md: unbind<sdd6>
md: export_rdev(sdd6)
md: unbind<sdc6>
md: export_rdev(sdc6)
md: bind<sdc6>
md: bind<sdd6>
md: bind<sde6>
md: bind<sdf6>
async_tx: api initialized (async)
xor: automatically using best checksumming function: generic_sse
generic_sse: 9976.000 MB/sec
xor: using function: generic_sse (9976.000 MB/sec)
raid6: sse2x1 6386 MB/s
raid6: sse2x2 7464 MB/s
raid6: sse2x4 8199 MB/s
raid6: using algorithm sse2x4 (8199 MB/s)
raid6: using ssse3x2 recovery algorithm
md: raid6 personality registered for level 6
md: raid5 personality registered for level 5
md: raid4 personality registered for level 4
bio: create slab <bio-1> at 1
md/raid:md0: not clean -- starting background reconstruction
md/raid:md0: device sdf6 operational as raid disk 3
md/raid:md0: device sde6 operational as raid disk 2
md/raid:md0: device sdd6 operational as raid disk 1
md/raid:md0: device sdc6 operational as raid disk 0
md/raid:md0: allocated 4314kB
md/raid:md0: raid level 5 active with 4 out of 4 devices, algorithm 2
RAID conf printout:
--- level:5 rd:4 wd:4
disk 0, o:1, dev:sdc6
disk 1, o:1, dev:sdd6
disk 2, o:1, dev:sde6
disk 3, o:1, dev:sdf6
md0: detected capacity change from 0 to 576636125184
md: resync of RAID array md0
md: minimum _guaranteed_ speed: 1000 KB/sec/disk.
md: using maximum available idle IO bandwidth (but not more than 200000 KB/sec) for resync.
md: using 128k window, over a total of 187707072k.
md0: unknown partition table
[root@root ~]# cat /proc/mdstat
Personalities : [raid6] [raid5] [raid4]
md0 : active raid5 sdf6[3] sde6[2] sdd6[1] sdc6[0]
563121216 blocks super 1.2 level 5, 64k chunk, algorithm 2 [4/4] [UUUU]
[=>...................] resync = 8.2% (15404908/187707072) finish=16.6min speed=172462K/sec
unused devices: <none>
[root@root ~]# echo 10000 > /sys/block/md0/md/sync_speed_min
[root@root ~]# echo 30000 > /sys/block/md0/md/sync_speed_max
[root@root ~]# cat /proc/mdstat
Personalities : [raid6] [raid5] [raid4]
md0 : active raid5 sdf6[3] sde6[2] sdd6[1] sdc6[0]
563121216 blocks super 1.2 level 5, 64k chunk, algorithm 2 [4/4] [UUUU]
[===>.................] resync = 16.1% (30226432/187707072) finish=47.3min speed=55459K/sec
unused devices: <none>
3. Start FIO
[root@root ~]# /usr/bin/fio --name=md0 --filename=/dev/md0 --thread --numjobs=10 --direct=1 --group_reporting --unlink=0 --loops=1 --offset=0 --randrepeat=1 --norandommap --scramble_buffers=1 --stonewall --ioengine=libaio --rw=randwrite --bs=8704 --iodepth=4000 --runtime=3000 --blockalign=512
md0: (g=0): rw=randwrite, bs=8704-8704/8704-8704/8704-8704, ioengine=libaio, iodepth=4000
...
fio-2.1.10
Starting 10 threads
Jobs: 10 (f=10): [wwwwwwwwww] [12.8% done] [0KB/0KB/0KB /s] [0/0/0 iops] [eta 43m:37s]
4. Remove a drive from md arry using mdadm command
[root@root ~]# mdadm /dev/md0 --set-faulty /dev/sdc6
dmesg
md/raid:md0: Disk failure on sdc6, disabling device.
md/raid:md0: Operation continuing on 3 devices.
md: md0: resync done.
md: checkpointing resync of md0.
5. System state after the drive is removed
[root@root ~]# cat /proc/mdstat
Personalities : [raid6] [raid5] [raid4]
md0 : active raid5 sdf6[3] sde6[2] sdd6[1] sdc6[0](F)
563121216 blocks super 1.2 level 5, 64k chunk, algorithm 2 [4/3] [_UUU]
resync=PENDING
unused devices: <none>
top
top - 17:55:06 up 1:09, 3 users, load average: 11.98, 8.53, 3.99
Tasks: 313 total, 2 running, 311 sleeping, 0 stopped, 0 zombie
Cpu(s): 0.0%us, 6.3%sy, 0.0%ni, 93.7%id, 0.0%wa, 0.0%hi, 0.0%si, 0.0%st
Mem: 16455916k total, 780184k used, 15675732k free, 29628k buffers
Swap: 6127608k total, 0k used, 6127608k free, 116212k cached
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
2690 root 20 0 0 0 0 R 100.0 0.0 6:44.41 md0_raid5
235 root 39 19 0 0 0 S 0.3 0.0 0:12.95 kipmi0
2650 root 20 0 98.1m 4456 3348 S 0.3 0.0 0:00.21 sshd
1 root 20 0 19364 1536 1232 S 0.0 0.0 0:01.42 init
Dmesg
INFO: task fio:2715 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 000000000000000a 0 2715 2654 0x00000080
ffff88043b623598 0000000000000082 0000000000000000 ffffffff81058d53
ffff88043b623548 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b40b098 ffff88043b623fd8 000000000000fbc8 ffff88043b40b098
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff8140fa39>] ? md_wakeup_thread+0x39/0x70
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffffa0308f66>] ? make_request+0x306/0xc6c [raid456]
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81122283>] ? mempool_alloc+0x63/0x140
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c767a>] do_direct_IO+0x7ca/0xfa0
[<ffffffff811c8196>] __blockdev_direct_IO_newtrunc+0x346/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2717 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000004 0 2717 2654 0x00000080
ffff880439e97698 0000000000000082 ffff880439e97628 ffffffff81058d53
ffff880439e97648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b0adab8 ffff880439e97fd8 000000000000fbc8 ffff88043b0adab8
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8e50>] __blockdev_direct_IO_newtrunc+0x1000/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2718 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000005 0 2718 2654 0x00000080
ffff88043bc13698 0000000000000082 ffff88043bc13628 ffffffff81058d53
ffff88043bc13648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b0ad058 ffff88043bc13fd8 000000000000fbc8 ffff88043b0ad058
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8e50>] __blockdev_direct_IO_newtrunc+0x1000/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2719 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000001 0 2719 2654 0x00000080
ffff880439ebb698 0000000000000082 ffff880439ebb628 ffffffff81058d53
ffff880439ebb648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043b0ac5f8 ffff880439ebbfd8 000000000000fbc8 ffff88043b0ac5f8
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2720 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000008 0 2720 2654 0x00000080
ffff88043b8cf698 0000000000000082 ffff88043b8cf628 ffffffff81058d53
ffff88043b8cf648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff880439e89af8 ffff88043b8cffd8 000000000000fbc8 ffff880439e89af8
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2721 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000000 0 2721 2654 0x00000080
ffff88043b047698 0000000000000082 ffff88043b047628 ffffffff81058d53
ffff88043b047648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff880439e89098 ffff88043b047fd8 000000000000fbc8 ffff880439e89098
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2722 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000000 0 2722 2654 0x00000080
ffff880439ea3698 0000000000000082 ffff880439ea3628 ffffffff81058d53
ffff880439ea3648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff880439e88638 ffff880439ea3fd8 000000000000fbc8 ffff880439e88638
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2723 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000006 0 2723 2654 0x00000080
ffff88043bf5f698 0000000000000082 ffff88043bf5f628 ffffffff81058d53
ffff88043bf5f648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043a183ab8 ffff88043bf5ffd8 000000000000fbc8 ffff88043a183ab8
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2724 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 000000000000000b 0 2724 2654 0x00000080
ffff88043be05698 0000000000000082 ffff88043be05628 ffffffff81058d53
ffff88043be05648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043a183058 ffff88043be05fd8 000000000000fbc8 ffff88043a183058
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
INFO: task fio:2725 blocked for more than 120 seconds.
Not tainted 2.6.32-431.el6.x86_64 #1
"echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
fio D 0000000000000003 0 2725 2654 0x00000080
ffff88043be07698 0000000000000082 ffff88043be07628 ffffffff81058d53
ffff88043be07648 ffff880230e49cc0 ffff8802389aa228 ffff88043b2ad1b8
ffff88043a1825f8 ffff88043be07fd8 000000000000fbc8 ffff88043a1825f8
Call Trace:
[<ffffffff81058d53>] ? __wake_up+0x53/0x70
[<ffffffffa030334b>] ? md_raid5_unplug_device+0x7b/0x100 [raid456]
[<ffffffffa0304146>] get_active_stripe+0x236/0x830 [raid456]
[<ffffffff81065df0>] ? default_wake_function+0x0/0x20
[<ffffffff8109b5ce>] ? prepare_to_wait+0x4e/0x80
[<ffffffffa0308e15>] make_request+0x1b5/0xc6c [raid456]
[<ffffffff8109b2a0>] ? autoremove_wake_function+0x0/0x40
[<ffffffff811220e5>] ? mempool_alloc_slab+0x15/0x20
[<ffffffff81415b41>] md_make_request+0xe1/0x230
[<ffffffff811c3fd2>] ? bvec_alloc_bs+0x62/0x110
[<ffffffff811c32f0>] ? __bio_add_page+0x110/0x230
[<ffffffff81266c50>] generic_make_request+0x240/0x5a0
[<ffffffff811c742c>] ? do_direct_IO+0x57c/0xfa0
[<ffffffff81267020>] submit_bio+0x70/0x120
[<ffffffff811c8acd>] __blockdev_direct_IO_newtrunc+0xc7d/0x1270
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c9137>] __blockdev_direct_IO+0x77/0xe0
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff811c53b7>] blkdev_direct_IO+0x57/0x60
[<ffffffff811c4330>] ? blkdev_get_block+0x0/0x20
[<ffffffff81120552>] generic_file_direct_write+0xc2/0x190
[<ffffffff81121e71>] __generic_file_aio_write+0x3a1/0x490
[<ffffffff811d64c0>] ? aio_read_evt+0xa0/0x170
[<ffffffff811c490c>] blkdev_aio_write+0x3c/0xa0
[<ffffffff811c48d0>] ? blkdev_aio_write+0x0/0xa0
[<ffffffff811d4f64>] aio_rw_vect_retry+0x84/0x200
[<ffffffff811d6924>] aio_run_iocb+0x64/0x170
[<ffffffff811d7d51>] do_io_submit+0x291/0x920
[<ffffffff811d83f0>] sys_io_submit+0x10/0x20
[<ffffffff8100b072>] system_call_fastpath+0x16/0x1b
[root@root ~]# cat /proc/2690/stack
[<ffffffff810686da>] __cond_resched+0x2a/0x40
[<ffffffffa030361c>] ops_run_io+0x2c/0x920 [raid456]
[<ffffffffa03052cc>] handle_stripe+0x9cc/0x2980 [raid456]
[<ffffffffa03078a4>] raid5d+0x624/0x850 [raid456]
[<ffffffff81416f05>] md_thread+0x115/0x150
[<ffffffff8109aef6>] kthread+0x96/0xa0
[<ffffffff8100c20a>] child_rip+0xa/0x20
[<ffffffffffffffff>] 0xffffffffffffffff
[root@root ~]# cat /proc/2690/stat
2690 (md0_raid5) R 2 0 0 0 -1 2149613632 0 0 0 0 0 68495 0 0 20 0 1 0 350990 0 0 18446744073709551615 0 0 0 0 0 0 0 2147483391 256 0 0 0 17 2 0 0 6855 0 0
[root@root ~]# cat /proc/2690/statm
0 0 0 0 0 0 0
[root@root ~]# cat /proc/2690/stat
stat statm status
[root@root ~]# cat /proc/2690/status
Name: md0_raid5
State: R (running)
Tgid: 2690
Pid: 2690
PPid: 2
TracerPid: 0
Uid: 0 0 0 0
Gid: 0 0 0 0
Utrace: 0
FDSize: 64
Groups:
Threads: 1
SigQ: 2/128402
SigPnd: 0000000000000000
ShdPnd: 0000000000000000
SigBlk: 0000000000000000
SigIgn: fffffffffffffeff
SigCgt: 0000000000000100
CapInh: 0000000000000000
CapPrm: ffffffffffffffff
CapEff: fffffffffffffeff
CapBnd: ffffffffffffffff
Cpus_allowed: ffffff
Cpus_allowed_list: 0-23
Mems_allowed: 00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000003
Mems_allowed_list: 0-1
voluntary_ctxt_switches: 5411612
nonvoluntary_ctxt_switches: 257032
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox