[PATCH] md: Avoid a deadlock when removing a device from an md array via sysfs.

[PATCH] md: Avoid a deadlock when removing a device from an md array via sysfs.

[NeilBrown]

(This patch should go in 2.6.21 as it fixes a recent regression - NB)

A device can be removed from an md array via e.g.
  echo remove > /sys/block/md3/md/dev-sde/state

This will try to remove the 'dev-sde' subtree which will deadlock
since
  commit e7b0d26a86943370c04d6833c6edba2a72a6e240

With this patch we run the kobject_del via schedule_work so as to
avoid the deadlock.

Cc: Alan Stern <stern@rowland.harvard.edu>
Signed-off-by: Neil Brown <neilb@suse.de>

### Diffstat output
 ./drivers/md/md.c           |   13 ++++++++++++-
 ./include/linux/raid/md_k.h |    1 +
 2 files changed, 13 insertions(+), 1 deletion(-)

diff .prev/drivers/md/md.c ./drivers/md/md.c
--- .prev/drivers/md/md.c	2007-04-02 17:43:03.000000000 +1000
+++ ./drivers/md/md.c	2007-04-02 17:38:46.000000000 +1000
@@ -1389,6 +1389,12 @@ static int bind_rdev_to_array(mdk_rdev_t
 	return err;
 }
 
+static void delayed_delete(struct work_struct *ws)
+{
+	mdk_rdev_t *rdev = container_of(ws, mdk_rdev_t, del_work);
+	kobject_del(&rdev->kobj);
+}
+
 static void unbind_rdev_from_array(mdk_rdev_t * rdev)
 {
 	char b[BDEVNAME_SIZE];
@@ -1401,7 +1407,12 @@ static void unbind_rdev_from_array(mdk_r
 	printk(KERN_INFO "md: unbind<%s>\n", bdevname(rdev->bdev,b));
 	rdev->mddev = NULL;
 	sysfs_remove_link(&rdev->kobj, "block");
-	kobject_del(&rdev->kobj);
+
+	/* We need to delay this, otherwise we can deadlock when
+	 * writing to 'remove' to "dev/state"
+	 */
+	INIT_WORK(&rdev->del_work, delayed_delete);
+	schedule_work(&rdev->del_work);
 }
 
 /*

diff .prev/include/linux/raid/md_k.h ./include/linux/raid/md_k.h
--- .prev/include/linux/raid/md_k.h	2007-04-02 17:43:03.000000000 +1000
+++ ./include/linux/raid/md_k.h	2007-04-02 17:36:32.000000000 +1000
@@ -104,6 +104,7 @@ struct mdk_rdev_s
 					   * for reporting to userspace and storing
 					   * in superblock.
 					   */
+	struct work_struct del_work;	/* used for delayed sysfs removal */
 };
 
 struct mddev_s

[PATCH] md: Avoid a deadlock when removing a device from an md array via sysfs.

[NeilBrown]

(This patch should go in 2.6.21 as it fixes a recent regression - NB)

A device can be removed from an md array via e.g.
  echo remove > /sys/block/md3/md/dev-sde/state

This will try to remove the 'dev-sde' subtree which will deadlock
since
  commit e7b0d26a86943370c04d6833c6edba2a72a6e240

With this patch we run the kobject_del via schedule_work so as to
avoid the deadlock.

Cc: Alan Stern <stern@rowland.harvard.edu>
Signed-off-by: Neil Brown <neilb@suse.de>

### Diffstat output
 ./drivers/md/md.c           |   13 ++++++++++++-
 ./include/linux/raid/md_k.h |    1 +
 2 files changed, 13 insertions(+), 1 deletion(-)

diff .prev/drivers/md/md.c ./drivers/md/md.c
--- .prev/drivers/md/md.c	2007-04-02 17:43:03.000000000 +1000
+++ ./drivers/md/md.c	2007-04-02 17:38:46.000000000 +1000
@@ -1389,6 +1389,12 @@ static int bind_rdev_to_array(mdk_rdev_t
 	return err;
 }
 
+static void delayed_delete(struct work_struct *ws)
+{
+	mdk_rdev_t *rdev = container_of(ws, mdk_rdev_t, del_work);
+	kobject_del(&rdev->kobj);
+}
+
 static void unbind_rdev_from_array(mdk_rdev_t * rdev)
 {
 	char b[BDEVNAME_SIZE];
@@ -1401,7 +1407,12 @@ static void unbind_rdev_from_array(mdk_r
 	printk(KERN_INFO "md: unbind<%s>\n", bdevname(rdev->bdev,b));
 	rdev->mddev = NULL;
 	sysfs_remove_link(&rdev->kobj, "block");
-	kobject_del(&rdev->kobj);
+
+	/* We need to delay this, otherwise we can deadlock when
+	 * writing to 'remove' to "dev/state"
+	 */
+	INIT_WORK(&rdev->del_work, delayed_delete);
+	schedule_work(&rdev->del_work);
 }
 
 /*

diff .prev/include/linux/raid/md_k.h ./include/linux/raid/md_k.h
--- .prev/include/linux/raid/md_k.h	2007-04-02 17:43:03.000000000 +1000
+++ ./include/linux/raid/md_k.h	2007-04-02 17:36:32.000000000 +1000
@@ -104,6 +104,7 @@ struct mdk_rdev_s
 					   * for reporting to userspace and storing
 					   * in superblock.
 					   */
+	struct work_struct del_work;	/* used for delayed sysfs removal */
 };
 
 struct mddev_s

Re: [PATCH] md: Avoid a deadlock when removing a device from an md array via sysfs.

[Andrew Morton]

On Mon, 2 Apr 2007 17:44:17 +1000 NeilBrown <neilb@suse.de> wrote:

> (This patch should go in 2.6.21 as it fixes a recent regression - NB)
> 
> A device can be removed from an md array via e.g.
>   echo remove > /sys/block/md3/md/dev-sde/state
> 
> This will try to remove the 'dev-sde' subtree which will deadlock
> since
>   commit e7b0d26a86943370c04d6833c6edba2a72a6e240
> 
> With this patch we run the kobject_del via schedule_work so as to
> avoid the deadlock.
> 
> Cc: Alan Stern <stern@rowland.harvard.edu>
> Signed-off-by: Neil Brown <neilb@suse.de>
> 
> ### Diffstat output
>  ./drivers/md/md.c           |   13 ++++++++++++-
>  ./include/linux/raid/md_k.h |    1 +
>  2 files changed, 13 insertions(+), 1 deletion(-)
> 
> diff .prev/drivers/md/md.c ./drivers/md/md.c
> --- .prev/drivers/md/md.c	2007-04-02 17:43:03.000000000 +1000
> +++ ./drivers/md/md.c	2007-04-02 17:38:46.000000000 +1000
> @@ -1389,6 +1389,12 @@ static int bind_rdev_to_array(mdk_rdev_t
>  	return err;
>  }
>  
> +static void delayed_delete(struct work_struct *ws)
> +{
> +	mdk_rdev_t *rdev = container_of(ws, mdk_rdev_t, del_work);
> +	kobject_del(&rdev->kobj);
> +}
> +
>  static void unbind_rdev_from_array(mdk_rdev_t * rdev)
>  {
>  	char b[BDEVNAME_SIZE];
> @@ -1401,7 +1407,12 @@ static void unbind_rdev_from_array(mdk_r
>  	printk(KERN_INFO "md: unbind<%s>\n", bdevname(rdev->bdev,b));
>  	rdev->mddev = NULL;
>  	sysfs_remove_link(&rdev->kobj, "block");
> -	kobject_del(&rdev->kobj);
> +
> +	/* We need to delay this, otherwise we can deadlock when
> +	 * writing to 'remove' to "dev/state"
> +	 */
> +	INIT_WORK(&rdev->del_work, delayed_delete);
> +	schedule_work(&rdev->del_work);
>  }
>  
>  /*
> 
> diff .prev/include/linux/raid/md_k.h ./include/linux/raid/md_k.h
> --- .prev/include/linux/raid/md_k.h	2007-04-02 17:43:03.000000000 +1000
> +++ ./include/linux/raid/md_k.h	2007-04-02 17:36:32.000000000 +1000
> @@ -104,6 +104,7 @@ struct mdk_rdev_s
>  					   * for reporting to userspace and storing
>  					   * in superblock.
>  					   */
> +	struct work_struct del_work;	/* used for delayed sysfs removal */
>  };
>  

What guarantees that *rdev is still valid when delayed_delete() runs?

And what guarantees that the md module hasn't been rmmodded when
delayed_delete() tries to run?

Re: [PATCH] md: Avoid a deadlock when removing a device from an md array via sysfs.

[Neil Brown]

On Monday April 2, akpm@linux-foundation.org wrote:
> 
> What guarantees that *rdev is still valid when delayed_delete() runs?

Because that is how kobjects and krefs work.  There is an embedded
refcount etc etc..

> 
> And what guarantees that the md module hasn't been rmmodded when
> delayed_delete() tries to run?

Good point.  Nothing.  Maybe this patch is needed.

Thanks,
NeilBrown

---------------------------
Avoid a deadlock when removing a device from an md array via sysfs. - fix

Make sure any delayed_delete calls finish before module unload.
For simplicity, flush the queue when we stop the array.

Signed-off-by: Neil Brown <neilb@suse.de>

### Diffstat output
 ./drivers/md/md.c |    3 +++
 1 file changed, 3 insertions(+)

diff .prev/drivers/md/md.c ./drivers/md/md.c
--- .prev/drivers/md/md.c	2007-04-02 17:38:46.000000000 +1000
+++ ./drivers/md/md.c	2007-04-02 18:49:24.000000000 +1000
@@ -3410,6 +3410,9 @@ static int do_md_stop(mddev_t * mddev, i
 				sysfs_remove_link(&mddev->kobj, nm);
 			}
 
+		/* make sure all delayed_delete calls have finished */
+		flush_scheduled_work();
+
 		export_array(mddev);
 
 		mddev->array_size = 0;