Re: 2.6.23.1: mdadm/raid5 hung/d-state

Dan Williams
2007-11-06T00:20:04Z

[-- Attachment #1: Type: text/plain, Size: 963 bytes --]

On 11/5/07, Justin Piszcz <jpiszcz@lucidpixels.com> wrote:
[..]
> > Are you seeing the same "md thread takes 100% of the CPU" that Joël is
> > reporting?
> >
>
> Yes, in another e-mail I posted the top output with md3_raid5 at 100%.
>

This seems too similar to Joël's situation for them not to be
correlated, and it shows that iscsi is not a necessary component of
the failure.

The attached patch allows the debug statements in MD to be enabled via
sysfs. Joël, since it is easier for you to reproduce can you capture
the kernel log output after the raid thread goes into the spin? It
will help if you have CONFIG_PRINTK_TIME=y set in your kernel
configuration.

After the failure run:

echo 1 > /sys/block/md_d0/md/debug_print_enable; sleep 5; echo 0 >
/sys/block/md_d0/md/debug_print_enable

...to enable the print messages for a few seconds. Please send the
output in a private message if it proves too big for the mailing list.

[-- Attachment #2: raid5-debug-print-enable.patch --]
[-- Type: application/octet-stream, Size: 1805 bytes --]

raid5: debug print enable

From: Dan Williams <dan.j.williams@intel.com>

---

drivers/md/raid5.c | 36 ++++++++++++++++++++++++++++++++++++
1 files changed, 36 insertions(+), 0 deletions(-)

diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 3808f52..496b9a3 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -54,6 +54,10 @@
#include <linux/raid/bitmap.h>
#include <linux/async_tx.h>

+static int debug_print_enable;
+#undef pr_debug
+#define pr_debug(x...) ((void)(debug_print_enable && printk(x)))
+
/*
* Stripe cache
*/
@@ -4023,6 +4027,37 @@ raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
raid5_store_stripe_cache_size);

static ssize_t
+raid5_show_debug_print_enable(mddev_t *mddev, char *page)
+{
+ return sprintf(page, "%d\n", debug_print_enable);
+}
+
+static ssize_t
+raid5_store_debug_print_enable(mddev_t *mddev, const char *page, size_t len)
+{
+ raid5_conf_t *conf = mddev_to_conf(mddev);
+ char *end;
+ int new;
+ if (len >= PAGE_SIZE)
+ return -EINVAL;
+
+ new = simple_strtoul(page, &end, 10);
+ if (!*page || (*end && *end != '\n') )
+ return -EINVAL;
+ if (new < 0 || new > 1)
+ return -EINVAL;
+
+ debug_print_enable = new;
+
+ return len;
+}
+
+static struct md_sysfs_entry
+raid5_debug_print = __ATTR(debug_print_enable, S_IRUGO | S_IWUSR,
+ raid5_show_debug_print_enable,
+ raid5_store_debug_print_enable);
+
+static ssize_t
stripe_cache_active_show(mddev_t *mddev, char *page)
{
raid5_conf_t *conf = mddev_to_conf(mddev);
@@ -4038,6 +4073,7 @@ raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
static struct attribute *raid5_attrs[] = {
&raid5_stripecache_size.attr,
&raid5_stripecache_active.attr,
+ &raid5_debug_print.attr,
NULL,
};
static struct attribute_group raid5_attrs_group = {