[PATCH 1/1] Btrfs: move super_kobj and device_dir_kobj from fs_info to btrfs_fs_devices

[PATCH 1/1] Btrfs: move super_kobj and device_dir_kobj from fs_info to btrfs_fs_devices

[Anand Jain]

This patch will provide a framework and help to create attributes
from the structure btrfs_fs_devices which are available even before
fs_info is created. So by moving the parent kobject super_kobj from
fs_info to btrfs_fs_devices, it will help to create attributes
from the btrfs_fs_devices as well.

Patches on top of this patch now will be able to create the
sys/fs/btrfs/fsid kobject and attributes from btrfs_fs_devices
when devices are scanned and registered to the kernel.

Just to note, this does not change any of the existing btrfs sysfs
external kobject names and its attributes and not even the life
cycle of them. Changes are internal only. And to ensure the same,
this path has been tested with various device operations and,
checking and comparing the sysfs kobjects and attributes with
sysfs kobject and attributes with out this patch, and they remain
same.

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 fs/btrfs/ctree.h       |   3 --
 fs/btrfs/dev-replace.c |   4 +-
 fs/btrfs/disk-io.c     |   1 -
 fs/btrfs/super.c       |   1 +
 fs/btrfs/sysfs.c       | 116 ++++++++++++++++++++++++++++++++++---------------
 fs/btrfs/sysfs.h       |   6 ++-
 fs/btrfs/volumes.c     |  13 ++++--
 fs/btrfs/volumes.h     |   7 +++
 8 files changed, 105 insertions(+), 46 deletions(-)

diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index abf0c7b..d9604e5 100644
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
@@ -1580,10 +1580,7 @@ struct btrfs_fs_info {
 	struct task_struct *cleaner_kthread;
 	int thread_pool_size;
 
-	struct kobject super_kobj;
 	struct kobject *space_info_kobj;
-	struct kobject *device_dir_kobj;
-	struct completion kobj_unregister;
 	int do_barriers;
 	int closing;
 	int log_root_recovering;
diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c
index ca6a3a3..124b60f 100644
--- a/fs/btrfs/dev-replace.c
+++ b/fs/btrfs/dev-replace.c
@@ -592,8 +592,8 @@ static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
 	mutex_unlock(&uuid_mutex);
 
 	/* replace the sysfs entry */
-	btrfs_kobj_rm_device(fs_info, src_device);
-	btrfs_kobj_add_device(fs_info, tgt_device);
+	btrfs_kobj_rm_device(fs_info->fs_devices, src_device);
+	btrfs_kobj_add_device(fs_info->fs_devices, tgt_device);
 	btrfs_rm_dev_replace_free_srcdev(fs_info, src_device);
 
 	/* write back the superblocks */
diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 86ff8c2..f15d2d9 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -2236,7 +2236,6 @@ int open_ctree(struct super_block *sb,
 	mutex_init(&fs_info->delalloc_root_mutex);
 	seqlock_init(&fs_info->profiles_lock);
 
-	init_completion(&fs_info->kobj_unregister);
 	INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots);
 	INIT_LIST_HEAD(&fs_info->space_info);
 	INIT_LIST_HEAD(&fs_info->tree_mod_seq_list);
diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
index 60f7cbe..a1d0a12 100644
--- a/fs/btrfs/super.c
+++ b/fs/btrfs/super.c
@@ -1334,6 +1334,7 @@ static struct dentry *btrfs_mount(struct file_system_type *fs_type, int flags,
 	}
 
 	fs_info->fs_devices = fs_devices;
+	fs_devices->fs_info = fs_info;
 
 	fs_info->super_copy = kzalloc(BTRFS_SUPER_INFO_SIZE, GFP_NOFS);
 	fs_info->super_for_commit = kzalloc(BTRFS_SUPER_INFO_SIZE, GFP_NOFS);
diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index 92db3f6..775fdb9 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -33,6 +33,7 @@
 #include "volumes.h"
 
 static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj);
+static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj);
 
 static u64 get_features(struct btrfs_fs_info *fs_info,
 			enum btrfs_feature_set set)
@@ -428,7 +429,7 @@ static ssize_t btrfs_clone_alignment_show(struct kobject *kobj,
 
 BTRFS_ATTR(clone_alignment, btrfs_clone_alignment_show);
 
-static struct attribute *btrfs_attrs[] = {
+static const struct attribute *btrfs_attrs[] = {
 	BTRFS_ATTR_PTR(label),
 	BTRFS_ATTR_PTR(nodesize),
 	BTRFS_ATTR_PTR(sectorsize),
@@ -438,21 +439,27 @@ static struct attribute *btrfs_attrs[] = {
 
 static void btrfs_release_super_kobj(struct kobject *kobj)
 {
-	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
-	complete(&fs_info->kobj_unregister);
+	struct btrfs_fs_devices *fs_devs = to_fs_devs(kobj);
+	complete(&fs_devs->kobj_unregister);
 }
 
 static struct kobj_type btrfs_ktype = {
 	.sysfs_ops	= &kobj_sysfs_ops,
 	.release	= btrfs_release_super_kobj,
-	.default_attrs	= btrfs_attrs,
 };
 
+static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj)
+{
+	if (kobj->ktype != &btrfs_ktype)
+		return NULL;
+	return container_of(kobj, struct btrfs_fs_devices, super_kobj);
+}
+
 static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj)
 {
 	if (kobj->ktype != &btrfs_ktype)
 		return NULL;
-	return container_of(kobj, struct btrfs_fs_info, super_kobj);
+	return to_fs_devs(kobj)->fs_info;
 }
 
 #define NUM_FEATURE_BITS 64
@@ -493,12 +500,12 @@ static int addrm_unknown_feature_attrs(struct btrfs_fs_info *fs_info, bool add)
 			attrs[0] = &fa->kobj_attr.attr;
 			if (add) {
 				int ret;
-				ret = sysfs_merge_group(&fs_info->super_kobj,
+				ret = sysfs_merge_group(&fs_info->fs_devices->super_kobj,
 							&agroup);
 				if (ret)
 					return ret;
 			} else
-				sysfs_unmerge_group(&fs_info->super_kobj,
+				sysfs_unmerge_group(&fs_info->fs_devices->super_kobj,
 						    &agroup);
 		}
 
@@ -506,25 +513,44 @@ static int addrm_unknown_feature_attrs(struct btrfs_fs_info *fs_info, bool add)
 	return 0;
 }
 
-static void __btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
+/*
+ * Can be called when a volume is removed from the system
+ * as of now only during modunload
+*/
+static void btrfs_kobject_remove_fsid(struct btrfs_fs_devices *fs_devs)
 {
-	kobject_del(&fs_info->super_kobj);
-	kobject_put(&fs_info->super_kobj);
-	wait_for_completion(&fs_info->kobj_unregister);
+	struct list_head *fs_uuids = btrfs_get_fs_uuids();
+
+	if (fs_devs) {
+		kobject_del(&fs_devs->super_kobj);
+		kobject_put(&fs_devs->super_kobj);
+		wait_for_completion(&fs_devs->kobj_unregister);
+		return;
+	}
+
+	list_for_each_entry(fs_devs, fs_uuids, list) {
+		kobject_del(&fs_devs->super_kobj);
+		kobject_put(&fs_devs->super_kobj);
+		wait_for_completion(&fs_devs->kobj_unregister);
+	}
 }
 
+/* Called by the unmount thread */
 void btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
 {
+	struct kobject *super_kobj = &fs_info->fs_devices->super_kobj;
+
 	if (fs_info->space_info_kobj) {
 		sysfs_remove_files(fs_info->space_info_kobj, allocation_attrs);
 		kobject_del(fs_info->space_info_kobj);
 		kobject_put(fs_info->space_info_kobj);
 	}
-	kobject_del(fs_info->device_dir_kobj);
-	kobject_put(fs_info->device_dir_kobj);
+
+	btrfs_kobj_rm_device(fs_info->fs_devices, NULL);
 	addrm_unknown_feature_attrs(fs_info, false);
-	sysfs_remove_group(&fs_info->super_kobj, &btrfs_feature_attr_group);
-	__btrfs_sysfs_remove_one(fs_info);
+	sysfs_remove_group(super_kobj, &btrfs_feature_attr_group);
+	sysfs_remove_files(super_kobj, btrfs_attrs);
+	btrfs_kobject_remove_fsid(fs_info->fs_devices);
 }
 
 const char * const btrfs_feature_set_names[3] = {
@@ -602,38 +628,43 @@ static void init_feature_attrs(void)
 	}
 }
 
-int btrfs_kobj_rm_device(struct btrfs_fs_info *fs_info,
+/* Called by the volume manager operations */
+int btrfs_kobj_rm_device(struct btrfs_fs_devices *fs_devs,
 		struct btrfs_device *one_device)
 {
 	struct hd_struct *disk;
 	struct kobject *disk_kobj;
 
-	if (!fs_info->device_dir_kobj)
+	if (!fs_devs->device_dir_kobj)
 		return -EINVAL;
 
 	if (one_device && one_device->bdev) {
 		disk = one_device->bdev->bd_part;
 		disk_kobj = &part_to_dev(disk)->kobj;
 
-		sysfs_remove_link(fs_info->device_dir_kobj,
+		sysfs_remove_link(fs_devs->device_dir_kobj,
 						disk_kobj->name);
+		return 0;
 	}
 
+	kobject_del(fs_devs->device_dir_kobj);
+	kobject_put(fs_devs->device_dir_kobj);
+
 	return 0;
 }
 
-int btrfs_kobj_add_device(struct btrfs_fs_info *fs_info,
+/* Called by the volume manager operations */
+int btrfs_kobj_add_device(struct btrfs_fs_devices *fs_devices,
 		struct btrfs_device *one_device)
 {
 	int error = 0;
-	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
 	struct btrfs_device *dev;
 
-	if (!fs_info->device_dir_kobj)
-		fs_info->device_dir_kobj = kobject_create_and_add("devices",
-						&fs_info->super_kobj);
+	if (!fs_devices->device_dir_kobj)
+		fs_devices->device_dir_kobj = kobject_create_and_add("devices",
+						&fs_devices->super_kobj);
 
-	if (!fs_info->device_dir_kobj)
+	if (!fs_devices->device_dir_kobj)
 		return -ENOMEM;
 
 	list_for_each_entry(dev, &fs_devices->devices, dev_list) {
@@ -649,7 +680,7 @@ int btrfs_kobj_add_device(struct btrfs_fs_info *fs_info,
 		disk = dev->bdev->bd_part;
 		disk_kobj = &part_to_dev(disk)->kobj;
 
-		error = sysfs_create_link(fs_info->device_dir_kobj,
+		error = sysfs_create_link(fs_devices->device_dir_kobj,
 					  disk_kobj, disk_kobj->name);
 		if (error)
 			break;
@@ -667,21 +698,37 @@ static struct dentry *btrfs_debugfs_root_dentry;
 /* Debugging tunables and exported data */
 u64 btrfs_debugfs_test;
 
+/* Can be called by the device discovery thread */
+int btrfs_kobj_add_fsid(struct btrfs_fs_devices *fs_devs,
+					struct kobject *parent)
+{
+	int error;
+
+	init_completion(&fs_devs->kobj_unregister);
+	fs_devs->super_kobj.kset = btrfs_kset;
+	error = kobject_init_and_add(&fs_devs->super_kobj, &btrfs_ktype, NULL,
+				     "%pU", fs_devs->fsid);
+	return error;
+}
+
+/* Called when fs_info is created. That would be the mount thread */
 int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
 {
 	int error;
+	struct kobject *super_kobj = &fs_info->fs_devices->super_kobj;
+
+	/* This to be called from the volume.c */
+	error = btrfs_kobj_add_fsid(fs_info->fs_devices, NULL);
+	if (error)
+		return error;
 
-	init_completion(&fs_info->kobj_unregister);
-	fs_info->super_kobj.kset = btrfs_kset;
-	error = kobject_init_and_add(&fs_info->super_kobj, &btrfs_ktype, NULL,
-				     "%pU", fs_info->fsid);
+	error = sysfs_create_files(super_kobj, btrfs_attrs);
 	if (error)
 		return error;
 
-	error = sysfs_create_group(&fs_info->super_kobj,
-				   &btrfs_feature_attr_group);
+	error = sysfs_create_group(super_kobj, &btrfs_feature_attr_group);
 	if (error) {
-		__btrfs_sysfs_remove_one(fs_info);
+		sysfs_remove_files(super_kobj, btrfs_attrs);
 		return error;
 	}
 
@@ -689,12 +736,13 @@ int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
 	if (error)
 		goto failure;
 
-	error = btrfs_kobj_add_device(fs_info, NULL);
+	/* This to be called from the volume.c */
+	error = btrfs_kobj_add_device(fs_info->fs_devices, NULL);
 	if (error)
 		goto failure;
 
 	fs_info->space_info_kobj = kobject_create_and_add("allocation",
-						  &fs_info->super_kobj);
+						  super_kobj);
 	if (!fs_info->space_info_kobj) {
 		error = -ENOMEM;
 		goto failure;
diff --git a/fs/btrfs/sysfs.h b/fs/btrfs/sysfs.h
index f7dd298..737cfd6 100644
--- a/fs/btrfs/sysfs.h
+++ b/fs/btrfs/sysfs.h
@@ -70,8 +70,10 @@ char *btrfs_printable_features(enum btrfs_feature_set set, u64 flags);
 extern const char * const btrfs_feature_set_names[3];
 extern struct kobj_type space_info_ktype;
 extern struct kobj_type btrfs_raid_ktype;
-int btrfs_kobj_add_device(struct btrfs_fs_info *fs_info,
+int btrfs_kobj_add_device(struct btrfs_fs_devices *fs_devs,
 		struct btrfs_device *one_device);
-int btrfs_kobj_rm_device(struct btrfs_fs_info *fs_info,
+int btrfs_kobj_rm_device(struct btrfs_fs_devices *fs_devs,
                 struct btrfs_device *one_device);
+int btrfs_kobj_add_fsid(struct btrfs_fs_devices *fs_devs,
+					struct kobject *parent);
 #endif /* _BTRFS_SYSFS_H_ */
diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
index 0144790..b87a576 100644
--- a/fs/btrfs/volumes.c
+++ b/fs/btrfs/volumes.c
@@ -52,6 +52,10 @@ static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
 
 DEFINE_MUTEX(uuid_mutex);
 static LIST_HEAD(fs_uuids);
+struct list_head *btrfs_get_fs_uuids(void)
+{
+	return &fs_uuids;
+}
 
 static struct btrfs_fs_devices *__alloc_fs_devices(void)
 {
@@ -1697,7 +1701,7 @@ int btrfs_rm_device(struct btrfs_root *root, char *device_path)
 	if (device->bdev) {
 		device->fs_devices->open_devices--;
 		/* remove sysfs entry */
-		btrfs_kobj_rm_device(root->fs_info, device);
+		btrfs_kobj_rm_device(root->fs_info->fs_devices, device);
 	}
 
 	call_rcu(&device->rcu, free_device);
@@ -2095,6 +2099,7 @@ int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
 	u64 tmp;
 	int seeding_dev = 0;
 	int ret = 0;
+	struct kobject *super_kobj = &root->fs_info->fs_devices->super_kobj;
 
 	if ((sb->s_flags & MS_RDONLY) && !root->fs_info->fs_devices->seeding)
 		return -EROFS;
@@ -2202,7 +2207,7 @@ int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
 				    tmp + 1);
 
 	/* add sysfs device entry */
-	btrfs_kobj_add_device(root->fs_info, device);
+	btrfs_kobj_add_device(root->fs_info->fs_devices, device);
 
 	/*
 	 * we've got more storage, clear any full flags on the space
@@ -2243,7 +2248,7 @@ int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
 		 */
 		snprintf(fsid_buf, BTRFS_UUID_UNPARSED_SIZE, "%pU",
 						root->fs_info->fsid);
-		if (kobject_rename(&root->fs_info->super_kobj, fsid_buf))
+		if (kobject_rename(super_kobj, fsid_buf))
 			goto error_trans;
 	}
 
@@ -2280,7 +2285,7 @@ int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
 error_trans:
 	btrfs_end_transaction(trans, root);
 	rcu_string_free(device->name);
-	btrfs_kobj_rm_device(root->fs_info, device);
+	btrfs_kobj_rm_device(root->fs_info->fs_devices, device);
 	kfree(device);
 error:
 	blkdev_put(bdev, FMODE_EXCL);
diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h
index d6fe73c..06fde15 100644
--- a/fs/btrfs/volumes.h
+++ b/fs/btrfs/volumes.h
@@ -253,6 +253,12 @@ struct btrfs_fs_devices {
 	 * nonrot flag set
 	 */
 	int rotating;
+
+	struct btrfs_fs_info *fs_info;
+
+	struct kobject super_kobj;
+	struct kobject *device_dir_kobj;
+	struct completion kobj_unregister;
 };
 
 #define BTRFS_BIO_INLINE_CSUM_SIZE	64
@@ -534,5 +540,6 @@ static inline void unlock_chunks(struct btrfs_root *root)
 	mutex_unlock(&root->fs_info->chunk_mutex);
 }
 
+struct list_head *btrfs_get_fs_uuids(void);
 
 #endif
-- 
2.1.0

Re: [PATCH 1/1] Btrfs: move super_kobj and device_dir_kobj from fs_info to btrfs_fs_devices

[David Sterba]

On Tue, Jan 20, 2015 at 12:02:27AM +0800, Anand Jain wrote:
> This patch will provide a framework and help to create attributes
> from the structure btrfs_fs_devices which are available even before
> fs_info is created. So by moving the parent kobject super_kobj from
> fs_info to btrfs_fs_devices, it will help to create attributes
> from the btrfs_fs_devices as well.

Ok.

> Just to note, this does not change any of the existing btrfs sysfs
> external kobject names and its attributes and not even the life
> cycle of them. Changes are internal only. And to ensure the same,
> this path has been tested with various device operations and,
> checking and comparing the sysfs kobjects and attributes with
> sysfs kobject and attributes with out this patch, and they remain
> same.

Good.

Reviewed-by: David Sterba <dsterba@suse.cz>

RE: [PATCH 1/1] Btrfs: move super_kobj and device_dir_kobj from fs_info to btrfs_fs_devices

[Anand Jain]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset=utf-8, Size: 15286 bytes --]

Recently i found that, the original code with out this patch has a bug during kobject clean up, which is prominent with this patch, i am digging more. Just an early headsup. Thxs. Anand.

Sent from my Sony Xperiaâ„¢ smartphone

---- Anand Jain wrote ----

>This patch will provide a framework and help to create attributes
>from the structure btrfs_fs_devices which are available even before
>fs_info is created. So by moving the parent kobject super_kobj from
>fs_info to btrfs_fs_devices, it will help to create attributes
>from the btrfs_fs_devices as well.
>
>Patches on top of this patch now will be able to create the
>sys/fs/btrfs/fsid kobject and attributes from btrfs_fs_devices
>when devices are scanned and registered to the kernel.
>
>Just to note, this does not change any of the existing btrfs sysfs
>external kobject names and its attributes and not even the life
>cycle of them. Changes are internal only. And to ensure the same,
>this path has been tested with various device operations and,
>checking and comparing the sysfs kobjects and attributes with
>sysfs kobject and attributes with out this patch, and they remain
>same.
>
>Signed-off-by: Anand Jain <anand.jain@oracle.com>
>---
> fs/btrfs/ctree.h       |   3 --
> fs/btrfs/dev-replace.c |   4 +-
> fs/btrfs/disk-io.c     |   1 -
> fs/btrfs/super.c       |   1 +
> fs/btrfs/sysfs.c       | 116 ++++++++++++++++++++++++++++++++++---------------
> fs/btrfs/sysfs.h       |   6 ++-
> fs/btrfs/volumes.c     |  13 ++++--
> fs/btrfs/volumes.h     |   7 +++
> 8 files changed, 105 insertions(+), 46 deletions(-)
>
>diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
>index abf0c7b..d9604e5 100644
>--- a/fs/btrfs/ctree.h
>+++ b/fs/btrfs/ctree.h
>@@ -1580,10 +1580,7 @@ struct btrfs_fs_info {
> 	struct task_struct *cleaner_kthread;
> 	int thread_pool_size;
> 
>-	struct kobject super_kobj;
> 	struct kobject *space_info_kobj;
>-	struct kobject *device_dir_kobj;
>-	struct completion kobj_unregister;
> 	int do_barriers;
> 	int closing;
> 	int log_root_recovering;
>diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c
>index ca6a3a3..124b60f 100644
>--- a/fs/btrfs/dev-replace.c
>+++ b/fs/btrfs/dev-replace.c
>@@ -592,8 +592,8 @@ static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
> 	mutex_unlock(&uuid_mutex);
> 
> 	/* replace the sysfs entry */
>-	btrfs_kobj_rm_device(fs_info, src_device);
>-	btrfs_kobj_add_device(fs_info, tgt_device);
>+	btrfs_kobj_rm_device(fs_info->fs_devices, src_device);
>+	btrfs_kobj_add_device(fs_info->fs_devices, tgt_device);
> 	btrfs_rm_dev_replace_free_srcdev(fs_info, src_device);
> 
> 	/* write back the superblocks */
>diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
>index 86ff8c2..f15d2d9 100644
>--- a/fs/btrfs/disk-io.c
>+++ b/fs/btrfs/disk-io.c
>@@ -2236,7 +2236,6 @@ int open_ctree(struct super_block *sb,
> 	mutex_init(&fs_info->delalloc_root_mutex);
> 	seqlock_init(&fs_info->profiles_lock);
> 
>-	init_completion(&fs_info->kobj_unregister);
> 	INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots);
> 	INIT_LIST_HEAD(&fs_info->space_info);
> 	INIT_LIST_HEAD(&fs_info->tree_mod_seq_list);
>diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
>index 60f7cbe..a1d0a12 100644
>--- a/fs/btrfs/super.c
>+++ b/fs/btrfs/super.c
>@@ -1334,6 +1334,7 @@ static struct dentry *btrfs_mount(struct file_system_type *fs_type, int flags,
> 	}
> 
> 	fs_info->fs_devices = fs_devices;
>+	fs_devices->fs_info = fs_info;
> 
> 	fs_info->super_copy = kzalloc(BTRFS_SUPER_INFO_SIZE, GFP_NOFS);
> 	fs_info->super_for_commit = kzalloc(BTRFS_SUPER_INFO_SIZE, GFP_NOFS);
>diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
>index 92db3f6..775fdb9 100644
>--- a/fs/btrfs/sysfs.c
>+++ b/fs/btrfs/sysfs.c
>@@ -33,6 +33,7 @@
> #include "volumes.h"
> 
> static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj);
>+static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj);
> 
> static u64 get_features(struct btrfs_fs_info *fs_info,
> 			enum btrfs_feature_set set)
>@@ -428,7 +429,7 @@ static ssize_t btrfs_clone_alignment_show(struct kobject *kobj,
> 
> BTRFS_ATTR(clone_alignment, btrfs_clone_alignment_show);
> 
>-static struct attribute *btrfs_attrs[] = {
>+static const struct attribute *btrfs_attrs[] = {
> 	BTRFS_ATTR_PTR(label),
> 	BTRFS_ATTR_PTR(nodesize),
> 	BTRFS_ATTR_PTR(sectorsize),
>@@ -438,21 +439,27 @@ static struct attribute *btrfs_attrs[] = {
> 
> static void btrfs_release_super_kobj(struct kobject *kobj)
> {
>-	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
>-	complete(&fs_info->kobj_unregister);
>+	struct btrfs_fs_devices *fs_devs = to_fs_devs(kobj);
>+	complete(&fs_devs->kobj_unregister);
> }
> 
> static struct kobj_type btrfs_ktype = {
> 	.sysfs_ops	= &kobj_sysfs_ops,
> 	.release	= btrfs_release_super_kobj,
>-	.default_attrs	= btrfs_attrs,
> };
> 
>+static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj)
>+{
>+	if (kobj->ktype != &btrfs_ktype)
>+		return NULL;
>+	return container_of(kobj, struct btrfs_fs_devices, super_kobj);
>+}
>+
> static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj)
> {
> 	if (kobj->ktype != &btrfs_ktype)
> 		return NULL;
>-	return container_of(kobj, struct btrfs_fs_info, super_kobj);
>+	return to_fs_devs(kobj)->fs_info;
> }
> 
> #define NUM_FEATURE_BITS 64
>@@ -493,12 +500,12 @@ static int addrm_unknown_feature_attrs(struct btrfs_fs_info *fs_info, bool add)
> 			attrs[0] = &fa->kobj_attr.attr;
> 			if (add) {
> 				int ret;
>-				ret = sysfs_merge_group(&fs_info->super_kobj,
>+				ret = sysfs_merge_group(&fs_info->fs_devices->super_kobj,
> 							&agroup);
> 				if (ret)
> 					return ret;
> 			} else
>-				sysfs_unmerge_group(&fs_info->super_kobj,
>+				sysfs_unmerge_group(&fs_info->fs_devices->super_kobj,
> 						    &agroup);
> 		}
> 
>@@ -506,25 +513,44 @@ static int addrm_unknown_feature_attrs(struct btrfs_fs_info *fs_info, bool add)
> 	return 0;
> }
> 
>-static void __btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
>+/*
>+ * Can be called when a volume is removed from the system
>+ * as of now only during modunload
>+*/
>+static void btrfs_kobject_remove_fsid(struct btrfs_fs_devices *fs_devs)
> {
>-	kobject_del(&fs_info->super_kobj);
>-	kobject_put(&fs_info->super_kobj);
>-	wait_for_completion(&fs_info->kobj_unregister);
>+	struct list_head *fs_uuids = btrfs_get_fs_uuids();
>+
>+	if (fs_devs) {
>+		kobject_del(&fs_devs->super_kobj);
>+		kobject_put(&fs_devs->super_kobj);
>+		wait_for_completion(&fs_devs->kobj_unregister);
>+		return;
>+	}
>+
>+	list_for_each_entry(fs_devs, fs_uuids, list) {
>+		kobject_del(&fs_devs->super_kobj);
>+		kobject_put(&fs_devs->super_kobj);
>+		wait_for_completion(&fs_devs->kobj_unregister);
>+	}
> }
> 
>+/* Called by the unmount thread */
> void btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
> {
>+	struct kobject *super_kobj = &fs_info->fs_devices->super_kobj;
>+
> 	if (fs_info->space_info_kobj) {
> 		sysfs_remove_files(fs_info->space_info_kobj, allocation_attrs);
> 		kobject_del(fs_info->space_info_kobj);
> 		kobject_put(fs_info->space_info_kobj);
> 	}
>-	kobject_del(fs_info->device_dir_kobj);
>-	kobject_put(fs_info->device_dir_kobj);
>+
>+	btrfs_kobj_rm_device(fs_info->fs_devices, NULL);
> 	addrm_unknown_feature_attrs(fs_info, false);
>-	sysfs_remove_group(&fs_info->super_kobj, &btrfs_feature_attr_group);
>-	__btrfs_sysfs_remove_one(fs_info);
>+	sysfs_remove_group(super_kobj, &btrfs_feature_attr_group);
>+	sysfs_remove_files(super_kobj, btrfs_attrs);
>+	btrfs_kobject_remove_fsid(fs_info->fs_devices);
> }
> 
> const char * const btrfs_feature_set_names[3] = {
>@@ -602,38 +628,43 @@ static void init_feature_attrs(void)
> 	}
> }
> 
>-int btrfs_kobj_rm_device(struct btrfs_fs_info *fs_info,
>+/* Called by the volume manager operations */
>+int btrfs_kobj_rm_device(struct btrfs_fs_devices *fs_devs,
> 		struct btrfs_device *one_device)
> {
> 	struct hd_struct *disk;
> 	struct kobject *disk_kobj;
> 
>-	if (!fs_info->device_dir_kobj)
>+	if (!fs_devs->device_dir_kobj)
> 		return -EINVAL;
> 
> 	if (one_device && one_device->bdev) {
> 		disk = one_device->bdev->bd_part;
> 		disk_kobj = &part_to_dev(disk)->kobj;
> 
>-		sysfs_remove_link(fs_info->device_dir_kobj,
>+		sysfs_remove_link(fs_devs->device_dir_kobj,
> 						disk_kobj->name);
>+		return 0;
> 	}
> 
>+	kobject_del(fs_devs->device_dir_kobj);
>+	kobject_put(fs_devs->device_dir_kobj);
>+
> 	return 0;
> }
> 
>-int btrfs_kobj_add_device(struct btrfs_fs_info *fs_info,
>+/* Called by the volume manager operations */
>+int btrfs_kobj_add_device(struct btrfs_fs_devices *fs_devices,
> 		struct btrfs_device *one_device)
> {
> 	int error = 0;
>-	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
> 	struct btrfs_device *dev;
> 
>-	if (!fs_info->device_dir_kobj)
>-		fs_info->device_dir_kobj = kobject_create_and_add("devices",
>-						&fs_info->super_kobj);
>+	if (!fs_devices->device_dir_kobj)
>+		fs_devices->device_dir_kobj = kobject_create_and_add("devices",
>+						&fs_devices->super_kobj);
> 
>-	if (!fs_info->device_dir_kobj)
>+	if (!fs_devices->device_dir_kobj)
> 		return -ENOMEM;
> 
> 	list_for_each_entry(dev, &fs_devices->devices, dev_list) {
>@@ -649,7 +680,7 @@ int btrfs_kobj_add_device(struct btrfs_fs_info *fs_info,
> 		disk = dev->bdev->bd_part;
> 		disk_kobj = &part_to_dev(disk)->kobj;
> 
>-		error = sysfs_create_link(fs_info->device_dir_kobj,
>+		error = sysfs_create_link(fs_devices->device_dir_kobj,
> 					  disk_kobj, disk_kobj->name);
> 		if (error)
> 			break;
>@@ -667,21 +698,37 @@ static struct dentry *btrfs_debugfs_root_dentry;
> /* Debugging tunables and exported data */
> u64 btrfs_debugfs_test;
> 
>+/* Can be called by the device discovery thread */
>+int btrfs_kobj_add_fsid(struct btrfs_fs_devices *fs_devs,
>+					struct kobject *parent)
>+{
>+	int error;
>+
>+	init_completion(&fs_devs->kobj_unregister);
>+	fs_devs->super_kobj.kset = btrfs_kset;
>+	error = kobject_init_and_add(&fs_devs->super_kobj, &btrfs_ktype, NULL,
>+				     "%pU", fs_devs->fsid);
>+	return error;
>+}
>+
>+/* Called when fs_info is created. That would be the mount thread */
> int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
> {
> 	int error;
>+	struct kobject *super_kobj = &fs_info->fs_devices->super_kobj;
>+
>+	/* This to be called from the volume.c */
>+	error = btrfs_kobj_add_fsid(fs_info->fs_devices, NULL);
>+	if (error)
>+		return error;
> 
>-	init_completion(&fs_info->kobj_unregister);
>-	fs_info->super_kobj.kset = btrfs_kset;
>-	error = kobject_init_and_add(&fs_info->super_kobj, &btrfs_ktype, NULL,
>-				     "%pU", fs_info->fsid);
>+	error = sysfs_create_files(super_kobj, btrfs_attrs);
> 	if (error)
> 		return error;
> 
>-	error = sysfs_create_group(&fs_info->super_kobj,
>-				   &btrfs_feature_attr_group);
>+	error = sysfs_create_group(super_kobj, &btrfs_feature_attr_group);
> 	if (error) {
>-		__btrfs_sysfs_remove_one(fs_info);
>+		sysfs_remove_files(super_kobj, btrfs_attrs);
> 		return error;
> 	}
> 
>@@ -689,12 +736,13 @@ int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
> 	if (error)
> 		goto failure;
> 
>-	error = btrfs_kobj_add_device(fs_info, NULL);
>+	/* This to be called from the volume.c */
>+	error = btrfs_kobj_add_device(fs_info->fs_devices, NULL);
> 	if (error)
> 		goto failure;
> 
> 	fs_info->space_info_kobj = kobject_create_and_add("allocation",
>-						  &fs_info->super_kobj);
>+						  super_kobj);
> 	if (!fs_info->space_info_kobj) {
> 		error = -ENOMEM;
> 		goto failure;
>diff --git a/fs/btrfs/sysfs.h b/fs/btrfs/sysfs.h
>index f7dd298..737cfd6 100644
>--- a/fs/btrfs/sysfs.h
>+++ b/fs/btrfs/sysfs.h
>@@ -70,8 +70,10 @@ char *btrfs_printable_features(enum btrfs_feature_set set, u64 flags);
> extern const char * const btrfs_feature_set_names[3];
> extern struct kobj_type space_info_ktype;
> extern struct kobj_type btrfs_raid_ktype;
>-int btrfs_kobj_add_device(struct btrfs_fs_info *fs_info,
>+int btrfs_kobj_add_device(struct btrfs_fs_devices *fs_devs,
> 		struct btrfs_device *one_device);
>-int btrfs_kobj_rm_device(struct btrfs_fs_info *fs_info,
>+int btrfs_kobj_rm_device(struct btrfs_fs_devices *fs_devs,
>                 struct btrfs_device *one_device);
>+int btrfs_kobj_add_fsid(struct btrfs_fs_devices *fs_devs,
>+					struct kobject *parent);
> #endif /* _BTRFS_SYSFS_H_ */
>diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
>index 0144790..b87a576 100644
>--- a/fs/btrfs/volumes.c
>+++ b/fs/btrfs/volumes.c
>@@ -52,6 +52,10 @@ static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
> 
> DEFINE_MUTEX(uuid_mutex);
> static LIST_HEAD(fs_uuids);
>+struct list_head *btrfs_get_fs_uuids(void)
>+{
>+	return &fs_uuids;
>+}
> 
> static struct btrfs_fs_devices *__alloc_fs_devices(void)
> {
>@@ -1697,7 +1701,7 @@ int btrfs_rm_device(struct btrfs_root *root, char *device_path)
> 	if (device->bdev) {
> 		device->fs_devices->open_devices--;
> 		/* remove sysfs entry */
>-		btrfs_kobj_rm_device(root->fs_info, device);
>+		btrfs_kobj_rm_device(root->fs_info->fs_devices, device);
> 	}
> 
> 	call_rcu(&device->rcu, free_device);
>@@ -2095,6 +2099,7 @@ int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
> 	u64 tmp;
> 	int seeding_dev = 0;
> 	int ret = 0;
>+	struct kobject *super_kobj = &root->fs_info->fs_devices->super_kobj;
> 
> 	if ((sb->s_flags & MS_RDONLY) && !root->fs_info->fs_devices->seeding)
> 		return -EROFS;
>@@ -2202,7 +2207,7 @@ int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
> 				    tmp + 1);
> 
> 	/* add sysfs device entry */
>-	btrfs_kobj_add_device(root->fs_info, device);
>+	btrfs_kobj_add_device(root->fs_info->fs_devices, device);
> 
> 	/*
> 	 * we've got more storage, clear any full flags on the space
>@@ -2243,7 +2248,7 @@ int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
> 		 */
> 		snprintf(fsid_buf, BTRFS_UUID_UNPARSED_SIZE, "%pU",
> 						root->fs_info->fsid);
>-		if (kobject_rename(&root->fs_info->super_kobj, fsid_buf))
>+		if (kobject_rename(super_kobj, fsid_buf))
> 			goto error_trans;
> 	}
> 
>@@ -2280,7 +2285,7 @@ int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
> error_trans:
> 	btrfs_end_transaction(trans, root);
> 	rcu_string_free(device->name);
>-	btrfs_kobj_rm_device(root->fs_info, device);
>+	btrfs_kobj_rm_device(root->fs_info->fs_devices, device);
> 	kfree(device);
> error:
> 	blkdev_put(bdev, FMODE_EXCL);
>diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h
>index d6fe73c..06fde15 100644
>--- a/fs/btrfs/volumes.h
>+++ b/fs/btrfs/volumes.h
>@@ -253,6 +253,12 @@ struct btrfs_fs_devices {
> 	 * nonrot flag set
> 	 */
> 	int rotating;
>+
>+	struct btrfs_fs_info *fs_info;
>+
>+	struct kobject super_kobj;
>+	struct kobject *device_dir_kobj;
>+	struct completion kobj_unregister;
> };
> 
> #define BTRFS_BIO_INLINE_CSUM_SIZE	64
>@@ -534,5 +540,6 @@ static inline void unlock_chunks(struct btrfs_root *root)
> 	mutex_unlock(&root->fs_info->chunk_mutex);
> }
> 
>+struct list_head *btrfs_get_fs_uuids(void);
> 
> #endif
>-- 
>2.1.0
>
>--
>To unsubscribe from this list: send the line "unsubscribe linux-btrfs" in
>the body of a message to majordomo@vger.kernel.org
>More majordomo info at  http://vger.kernel.org/majordomo-info.html
ÿôèº{.nÇ+‰·Ÿ®‰­†+%ŠËÿ±éݶ\x17¥Šwÿº{.nÇ+‰·¥Š{±ý»k~ÏâžØ^n‡r¡ö¦zË\x1aëh™¨è­Ú&£ûàz¿äz¹Þ—ú+€Ê+zf£¢·hšˆ§~†­†Ûiÿÿïêÿ‘êçz_è®\x0fæj:+v‰¨þ)ߣøm

[PATCH 1/1] Btrfs: btrfs_release_super_kobj() should clean up the kobject data

[Anand Jain]

The following test case which is a fair tweak with in the funcation
open_ctree() fails with an error from kobject module, indicating that
thread tried to init an initialized object.

kernel: [232104.016513] kobject (ffff880006c1c980): tried to init an initialized object, something is seriously wrong.

btrfs_sysfs_remove_one() self test code:

open_tree()
{
 ::
        ret = btrfs_sysfs_add_one(fs_info);
	if (ret) {
              pr_err("BTRFS: failed to init sysfs interface: %d\n", ret);
                goto fail_block_groups;
        }
+       btrfs_sysfs_remove_one(fs_info);
+       ret = btrfs_sysfs_add_one(fs_info);
+       if (ret) {
+               pr_err("BTRFS: failed to init sysfs interface: %d\n", ret);
+               goto fail_block_groups;
+       }

cleaning up the unregistered kobject fixes this.

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 fs/btrfs/sysfs.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index 92db3f6..68dcd17 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -439,6 +439,8 @@ static struct attribute *btrfs_attrs[] = {
 static void btrfs_release_super_kobj(struct kobject *kobj)
 {
 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
+
+	memset(&fs_info->super_kobj, 0, sizeof(struct kobject));
 	complete(&fs_info->kobj_unregister);
 }
 
-- 
2.0.0.153.g79dcccc

Re: [PATCH 1/1] Btrfs: move super_kobj and device_dir_kobj from fs_info to btrfs_fs_devices

[Anand Jain]

This patch needs

   [PATCH 1/1] Btrfs: btrfs_release_super_kobj() should clean up the 
kobject data

further, I am planning to clean up and split below patch to show
the enhancements and fix-ups separately. Will send V2 soon. Thanks.

Anand



On 01/20/2015 12:02 AM, Anand Jain wrote:
> This patch will provide a framework and help to create attributes
> from the structure btrfs_fs_devices which are available even before
> fs_info is created. So by moving the parent kobject super_kobj from
> fs_info to btrfs_fs_devices, it will help to create attributes
> from the btrfs_fs_devices as well.
>
> Patches on top of this patch now will be able to create the
> sys/fs/btrfs/fsid kobject and attributes from btrfs_fs_devices
> when devices are scanned and registered to the kernel.
>
> Just to note, this does not change any of the existing btrfs sysfs
> external kobject names and its attributes and not even the life
> cycle of them. Changes are internal only. And to ensure the same,
> this path has been tested with various device operations and,
> checking and comparing the sysfs kobjects and attributes with
> sysfs kobject and attributes with out this patch, and they remain
> same.
>
> Signed-off-by: Anand Jain <anand.jain@oracle.com>
> ---
>   fs/btrfs/ctree.h       |   3 --
>   fs/btrfs/dev-replace.c |   4 +-
>   fs/btrfs/disk-io.c     |   1 -
>   fs/btrfs/super.c       |   1 +
>   fs/btrfs/sysfs.c       | 116 ++++++++++++++++++++++++++++++++++---------------
>   fs/btrfs/sysfs.h       |   6 ++-
>   fs/btrfs/volumes.c     |  13 ++++--
>   fs/btrfs/volumes.h     |   7 +++
>   8 files changed, 105 insertions(+), 46 deletions(-)
>
> diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
> index abf0c7b..d9604e5 100644
> --- a/fs/btrfs/ctree.h
> +++ b/fs/btrfs/ctree.h
> @@ -1580,10 +1580,7 @@ struct btrfs_fs_info {
>   	struct task_struct *cleaner_kthread;
>   	int thread_pool_size;
>
> -	struct kobject super_kobj;
>   	struct kobject *space_info_kobj;
> -	struct kobject *device_dir_kobj;
> -	struct completion kobj_unregister;
>   	int do_barriers;
>   	int closing;
>   	int log_root_recovering;
> diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c
> index ca6a3a3..124b60f 100644
> --- a/fs/btrfs/dev-replace.c
> +++ b/fs/btrfs/dev-replace.c
> @@ -592,8 +592,8 @@ static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info,
>   	mutex_unlock(&uuid_mutex);
>
>   	/* replace the sysfs entry */
> -	btrfs_kobj_rm_device(fs_info, src_device);
> -	btrfs_kobj_add_device(fs_info, tgt_device);
> +	btrfs_kobj_rm_device(fs_info->fs_devices, src_device);
> +	btrfs_kobj_add_device(fs_info->fs_devices, tgt_device);
>   	btrfs_rm_dev_replace_free_srcdev(fs_info, src_device);
>
>   	/* write back the superblocks */
> diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
> index 86ff8c2..f15d2d9 100644
> --- a/fs/btrfs/disk-io.c
> +++ b/fs/btrfs/disk-io.c
> @@ -2236,7 +2236,6 @@ int open_ctree(struct super_block *sb,
>   	mutex_init(&fs_info->delalloc_root_mutex);
>   	seqlock_init(&fs_info->profiles_lock);
>
> -	init_completion(&fs_info->kobj_unregister);
>   	INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots);
>   	INIT_LIST_HEAD(&fs_info->space_info);
>   	INIT_LIST_HEAD(&fs_info->tree_mod_seq_list);
> diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c
> index 60f7cbe..a1d0a12 100644
> --- a/fs/btrfs/super.c
> +++ b/fs/btrfs/super.c
> @@ -1334,6 +1334,7 @@ static struct dentry *btrfs_mount(struct file_system_type *fs_type, int flags,
>   	}
>
>   	fs_info->fs_devices = fs_devices;
> +	fs_devices->fs_info = fs_info;
>
>   	fs_info->super_copy = kzalloc(BTRFS_SUPER_INFO_SIZE, GFP_NOFS);
>   	fs_info->super_for_commit = kzalloc(BTRFS_SUPER_INFO_SIZE, GFP_NOFS);
> diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
> index 92db3f6..775fdb9 100644
> --- a/fs/btrfs/sysfs.c
> +++ b/fs/btrfs/sysfs.c
> @@ -33,6 +33,7 @@
>   #include "volumes.h"
>
>   static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj);
> +static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj);
>
>   static u64 get_features(struct btrfs_fs_info *fs_info,
>   			enum btrfs_feature_set set)
> @@ -428,7 +429,7 @@ static ssize_t btrfs_clone_alignment_show(struct kobject *kobj,
>
>   BTRFS_ATTR(clone_alignment, btrfs_clone_alignment_show);
>
> -static struct attribute *btrfs_attrs[] = {
> +static const struct attribute *btrfs_attrs[] = {
>   	BTRFS_ATTR_PTR(label),
>   	BTRFS_ATTR_PTR(nodesize),
>   	BTRFS_ATTR_PTR(sectorsize),
> @@ -438,21 +439,27 @@ static struct attribute *btrfs_attrs[] = {
>
>   static void btrfs_release_super_kobj(struct kobject *kobj)
>   {
> -	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
> -	complete(&fs_info->kobj_unregister);
> +	struct btrfs_fs_devices *fs_devs = to_fs_devs(kobj);
> +	complete(&fs_devs->kobj_unregister);
>   }
>
>   static struct kobj_type btrfs_ktype = {
>   	.sysfs_ops	= &kobj_sysfs_ops,
>   	.release	= btrfs_release_super_kobj,
> -	.default_attrs	= btrfs_attrs,
>   };
>
> +static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj)
> +{
> +	if (kobj->ktype != &btrfs_ktype)
> +		return NULL;
> +	return container_of(kobj, struct btrfs_fs_devices, super_kobj);
> +}
> +
>   static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj)
>   {
>   	if (kobj->ktype != &btrfs_ktype)
>   		return NULL;
> -	return container_of(kobj, struct btrfs_fs_info, super_kobj);
> +	return to_fs_devs(kobj)->fs_info;
>   }
>
>   #define NUM_FEATURE_BITS 64
> @@ -493,12 +500,12 @@ static int addrm_unknown_feature_attrs(struct btrfs_fs_info *fs_info, bool add)
>   			attrs[0] = &fa->kobj_attr.attr;
>   			if (add) {
>   				int ret;
> -				ret = sysfs_merge_group(&fs_info->super_kobj,
> +				ret = sysfs_merge_group(&fs_info->fs_devices->super_kobj,
>   							&agroup);
>   				if (ret)
>   					return ret;
>   			} else
> -				sysfs_unmerge_group(&fs_info->super_kobj,
> +				sysfs_unmerge_group(&fs_info->fs_devices->super_kobj,
>   						    &agroup);
>   		}
>
> @@ -506,25 +513,44 @@ static int addrm_unknown_feature_attrs(struct btrfs_fs_info *fs_info, bool add)
>   	return 0;
>   }
>
> -static void __btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
> +/*
> + * Can be called when a volume is removed from the system
> + * as of now only during modunload
> +*/
> +static void btrfs_kobject_remove_fsid(struct btrfs_fs_devices *fs_devs)
>   {
> -	kobject_del(&fs_info->super_kobj);
> -	kobject_put(&fs_info->super_kobj);
> -	wait_for_completion(&fs_info->kobj_unregister);
> +	struct list_head *fs_uuids = btrfs_get_fs_uuids();
> +
> +	if (fs_devs) {
> +		kobject_del(&fs_devs->super_kobj);
> +		kobject_put(&fs_devs->super_kobj);
> +		wait_for_completion(&fs_devs->kobj_unregister);
> +		return;
> +	}
> +
> +	list_for_each_entry(fs_devs, fs_uuids, list) {
> +		kobject_del(&fs_devs->super_kobj);
> +		kobject_put(&fs_devs->super_kobj);
> +		wait_for_completion(&fs_devs->kobj_unregister);
> +	}
>   }
>
> +/* Called by the unmount thread */
>   void btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
>   {
> +	struct kobject *super_kobj = &fs_info->fs_devices->super_kobj;
> +
>   	if (fs_info->space_info_kobj) {
>   		sysfs_remove_files(fs_info->space_info_kobj, allocation_attrs);
>   		kobject_del(fs_info->space_info_kobj);
>   		kobject_put(fs_info->space_info_kobj);
>   	}
> -	kobject_del(fs_info->device_dir_kobj);
> -	kobject_put(fs_info->device_dir_kobj);
> +
> +	btrfs_kobj_rm_device(fs_info->fs_devices, NULL);
>   	addrm_unknown_feature_attrs(fs_info, false);
> -	sysfs_remove_group(&fs_info->super_kobj, &btrfs_feature_attr_group);
> -	__btrfs_sysfs_remove_one(fs_info);
> +	sysfs_remove_group(super_kobj, &btrfs_feature_attr_group);
> +	sysfs_remove_files(super_kobj, btrfs_attrs);
> +	btrfs_kobject_remove_fsid(fs_info->fs_devices);
>   }
>
>   const char * const btrfs_feature_set_names[3] = {
> @@ -602,38 +628,43 @@ static void init_feature_attrs(void)
>   	}
>   }
>
> -int btrfs_kobj_rm_device(struct btrfs_fs_info *fs_info,
> +/* Called by the volume manager operations */
> +int btrfs_kobj_rm_device(struct btrfs_fs_devices *fs_devs,
>   		struct btrfs_device *one_device)
>   {
>   	struct hd_struct *disk;
>   	struct kobject *disk_kobj;
>
> -	if (!fs_info->device_dir_kobj)
> +	if (!fs_devs->device_dir_kobj)
>   		return -EINVAL;
>
>   	if (one_device && one_device->bdev) {
>   		disk = one_device->bdev->bd_part;
>   		disk_kobj = &part_to_dev(disk)->kobj;
>
> -		sysfs_remove_link(fs_info->device_dir_kobj,
> +		sysfs_remove_link(fs_devs->device_dir_kobj,
>   						disk_kobj->name);
> +		return 0;
>   	}
>
> +	kobject_del(fs_devs->device_dir_kobj);
> +	kobject_put(fs_devs->device_dir_kobj);
> +
>   	return 0;
>   }
>
> -int btrfs_kobj_add_device(struct btrfs_fs_info *fs_info,
> +/* Called by the volume manager operations */
> +int btrfs_kobj_add_device(struct btrfs_fs_devices *fs_devices,
>   		struct btrfs_device *one_device)
>   {
>   	int error = 0;
> -	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
>   	struct btrfs_device *dev;
>
> -	if (!fs_info->device_dir_kobj)
> -		fs_info->device_dir_kobj = kobject_create_and_add("devices",
> -						&fs_info->super_kobj);
> +	if (!fs_devices->device_dir_kobj)
> +		fs_devices->device_dir_kobj = kobject_create_and_add("devices",
> +						&fs_devices->super_kobj);
>
> -	if (!fs_info->device_dir_kobj)
> +	if (!fs_devices->device_dir_kobj)
>   		return -ENOMEM;
>
>   	list_for_each_entry(dev, &fs_devices->devices, dev_list) {
> @@ -649,7 +680,7 @@ int btrfs_kobj_add_device(struct btrfs_fs_info *fs_info,
>   		disk = dev->bdev->bd_part;
>   		disk_kobj = &part_to_dev(disk)->kobj;
>
> -		error = sysfs_create_link(fs_info->device_dir_kobj,
> +		error = sysfs_create_link(fs_devices->device_dir_kobj,
>   					  disk_kobj, disk_kobj->name);
>   		if (error)
>   			break;
> @@ -667,21 +698,37 @@ static struct dentry *btrfs_debugfs_root_dentry;
>   /* Debugging tunables and exported data */
>   u64 btrfs_debugfs_test;
>
> +/* Can be called by the device discovery thread */
> +int btrfs_kobj_add_fsid(struct btrfs_fs_devices *fs_devs,
> +					struct kobject *parent)
> +{
> +	int error;
> +
> +	init_completion(&fs_devs->kobj_unregister);
> +	fs_devs->super_kobj.kset = btrfs_kset;
> +	error = kobject_init_and_add(&fs_devs->super_kobj, &btrfs_ktype, NULL,
> +				     "%pU", fs_devs->fsid);
> +	return error;
> +}
> +
> +/* Called when fs_info is created. That would be the mount thread */
>   int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
>   {
>   	int error;
> +	struct kobject *super_kobj = &fs_info->fs_devices->super_kobj;
> +
> +	/* This to be called from the volume.c */
> +	error = btrfs_kobj_add_fsid(fs_info->fs_devices, NULL);
> +	if (error)
> +		return error;
>
> -	init_completion(&fs_info->kobj_unregister);
> -	fs_info->super_kobj.kset = btrfs_kset;
> -	error = kobject_init_and_add(&fs_info->super_kobj, &btrfs_ktype, NULL,
> -				     "%pU", fs_info->fsid);
> +	error = sysfs_create_files(super_kobj, btrfs_attrs);
>   	if (error)
>   		return error;
>
> -	error = sysfs_create_group(&fs_info->super_kobj,
> -				   &btrfs_feature_attr_group);
> +	error = sysfs_create_group(super_kobj, &btrfs_feature_attr_group);
>   	if (error) {
> -		__btrfs_sysfs_remove_one(fs_info);
> +		sysfs_remove_files(super_kobj, btrfs_attrs);
>   		return error;
>   	}
>
> @@ -689,12 +736,13 @@ int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
>   	if (error)
>   		goto failure;
>
> -	error = btrfs_kobj_add_device(fs_info, NULL);
> +	/* This to be called from the volume.c */
> +	error = btrfs_kobj_add_device(fs_info->fs_devices, NULL);
>   	if (error)
>   		goto failure;
>
>   	fs_info->space_info_kobj = kobject_create_and_add("allocation",
> -						  &fs_info->super_kobj);
> +						  super_kobj);
>   	if (!fs_info->space_info_kobj) {
>   		error = -ENOMEM;
>   		goto failure;
> diff --git a/fs/btrfs/sysfs.h b/fs/btrfs/sysfs.h
> index f7dd298..737cfd6 100644
> --- a/fs/btrfs/sysfs.h
> +++ b/fs/btrfs/sysfs.h
> @@ -70,8 +70,10 @@ char *btrfs_printable_features(enum btrfs_feature_set set, u64 flags);
>   extern const char * const btrfs_feature_set_names[3];
>   extern struct kobj_type space_info_ktype;
>   extern struct kobj_type btrfs_raid_ktype;
> -int btrfs_kobj_add_device(struct btrfs_fs_info *fs_info,
> +int btrfs_kobj_add_device(struct btrfs_fs_devices *fs_devs,
>   		struct btrfs_device *one_device);
> -int btrfs_kobj_rm_device(struct btrfs_fs_info *fs_info,
> +int btrfs_kobj_rm_device(struct btrfs_fs_devices *fs_devs,
>                   struct btrfs_device *one_device);
> +int btrfs_kobj_add_fsid(struct btrfs_fs_devices *fs_devs,
> +					struct kobject *parent);
>   #endif /* _BTRFS_SYSFS_H_ */
> diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
> index 0144790..b87a576 100644
> --- a/fs/btrfs/volumes.c
> +++ b/fs/btrfs/volumes.c
> @@ -52,6 +52,10 @@ static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
>
>   DEFINE_MUTEX(uuid_mutex);
>   static LIST_HEAD(fs_uuids);
> +struct list_head *btrfs_get_fs_uuids(void)
> +{
> +	return &fs_uuids;
> +}
>
>   static struct btrfs_fs_devices *__alloc_fs_devices(void)
>   {
> @@ -1697,7 +1701,7 @@ int btrfs_rm_device(struct btrfs_root *root, char *device_path)
>   	if (device->bdev) {
>   		device->fs_devices->open_devices--;
>   		/* remove sysfs entry */
> -		btrfs_kobj_rm_device(root->fs_info, device);
> +		btrfs_kobj_rm_device(root->fs_info->fs_devices, device);
>   	}
>
>   	call_rcu(&device->rcu, free_device);
> @@ -2095,6 +2099,7 @@ int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
>   	u64 tmp;
>   	int seeding_dev = 0;
>   	int ret = 0;
> +	struct kobject *super_kobj = &root->fs_info->fs_devices->super_kobj;
>
>   	if ((sb->s_flags & MS_RDONLY) && !root->fs_info->fs_devices->seeding)
>   		return -EROFS;
> @@ -2202,7 +2207,7 @@ int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
>   				    tmp + 1);
>
>   	/* add sysfs device entry */
> -	btrfs_kobj_add_device(root->fs_info, device);
> +	btrfs_kobj_add_device(root->fs_info->fs_devices, device);
>
>   	/*
>   	 * we've got more storage, clear any full flags on the space
> @@ -2243,7 +2248,7 @@ int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
>   		 */
>   		snprintf(fsid_buf, BTRFS_UUID_UNPARSED_SIZE, "%pU",
>   						root->fs_info->fsid);
> -		if (kobject_rename(&root->fs_info->super_kobj, fsid_buf))
> +		if (kobject_rename(super_kobj, fsid_buf))
>   			goto error_trans;
>   	}
>
> @@ -2280,7 +2285,7 @@ int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
>   error_trans:
>   	btrfs_end_transaction(trans, root);
>   	rcu_string_free(device->name);
> -	btrfs_kobj_rm_device(root->fs_info, device);
> +	btrfs_kobj_rm_device(root->fs_info->fs_devices, device);
>   	kfree(device);
>   error:
>   	blkdev_put(bdev, FMODE_EXCL);
> diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h
> index d6fe73c..06fde15 100644
> --- a/fs/btrfs/volumes.h
> +++ b/fs/btrfs/volumes.h
> @@ -253,6 +253,12 @@ struct btrfs_fs_devices {
>   	 * nonrot flag set
>   	 */
>   	int rotating;
> +
> +	struct btrfs_fs_info *fs_info;
> +
> +	struct kobject super_kobj;
> +	struct kobject *device_dir_kobj;
> +	struct completion kobj_unregister;
>   };
>
>   #define BTRFS_BIO_INLINE_CSUM_SIZE	64
> @@ -534,5 +540,6 @@ static inline void unlock_chunks(struct btrfs_root *root)
>   	mutex_unlock(&root->fs_info->chunk_mutex);
>   }
>
> +struct list_head *btrfs_get_fs_uuids(void);
>
>   #endif
>

[PATCH 00/12] provide frame work so that sysfs attributs from the fs_devices can be added

[Anand Jain]

This patch set will provide a framework and help to create attributes
from the structure btrfs_fs_devices which are available even before
fs_info is created. So by moving the parent kobject super_kobj from
fs_info to btrfs_fs_devices, it will help to create attributes
from the btrfs_fs_devices as well.
    
Just to note, this does not change any of the existing btrfs sysfs
external kobject names and its attributes and not even the life
cycle of them. Changes are internal only. And to ensure the same,
this path has been tested with various device operations and,
checking and comparing the sysfs kobjects and attributes with
sysfs kobject and attributes with out this patch, and they remain
same. These test cases are added to the progs as test-btrfs-devmgt.sh,
its patch is below as well.


Anand Jain (12):
  Btrfs: sysfs: fix, btrfs_release_super_kobj() should to clean up the
    kobject data
  Btrfs: sysfs: fix, fs_info kobject_unregister has init_completion()
    twice
  Btrfs: sysfs: fix, undo sysfs device links
  Btrfs: sysfs: fix, kobject pointer clean up needed after kobject
    release
  Btrfc: sysfs: fix, check if device_dir_kobj is init before destroy
  Btrfs: sysfs: reorder the kobject creations
  Btrfs: sysfs: rename __btrfs_sysfs_remove_one to
    btrfs_sysfs_remove_fsid
  Btrfs: sysfs: introduce function btrfs_sysfs_add_fsid() to create
    sysfs fsid
  Btrfs: sysfs: let default_attrs be separate from the kset
  Btrfs: sysfs: separate device kobject and its attribute creation
  Btrfs: sysfs: move super_kobj and device_dir_kobj from fs_info to
    btrfs_fs_devices
  Btrfs: sysfs: add pointer to access fs_info from fs_devices

 fs/btrfs/ctree.h   |   3 --
 fs/btrfs/disk-io.c |   1 -
 fs/btrfs/sysfs.c   | 133 +++++++++++++++++++++++++++++++++++++++--------------
 fs/btrfs/volumes.c |   3 +-
 fs/btrfs/volumes.h |   6 +++
 5 files changed, 107 insertions(+), 39 deletions(-)


Anand Jain (1):
  Btrfs-progs: add regression tests for sysfs contents during btrfs
    device management

 tests/test-btrfs-devmgt.sh | 863 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 863 insertions(+)
 create mode 100755 tests/test-btrfs-devmgt.sh
-- 
1.9.1

[PATCH 01/12] Btrfs: sysfs: fix, btrfs_release_super_kobj() should to clean up the kobject data

[Anand Jain]

The following test case fails indicating that, thread tried to init an initialized object.

kernel: [232104.016513] kobject (ffff880006c1c980): tried to init an initialized object, something is seriously wrong.

btrfs_sysfs_remove_one() self test code:

open_tree()
{
 ::
        ret = btrfs_sysfs_add_one(fs_info);
	if (ret) {
              pr_err("BTRFS: failed to init sysfs interface: %d\n", ret);
                goto fail_block_groups;
        }
+       btrfs_sysfs_remove_one(fs_info);
+       ret = btrfs_sysfs_add_one(fs_info);
+       if (ret) {
+               pr_err("BTRFS: failed to init sysfs interface: %d\n", ret);
+               goto fail_block_groups;
+       }

cleaning up the unregistered kobject fixes this.

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 fs/btrfs/sysfs.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index 92db3f6..68dcd17 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -439,6 +439,8 @@ static struct attribute *btrfs_attrs[] = {
 static void btrfs_release_super_kobj(struct kobject *kobj)
 {
 	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
+
+	memset(&fs_info->super_kobj, 0, sizeof(struct kobject));
 	complete(&fs_info->kobj_unregister);
 }
 
-- 
1.9.1

[PATCH 02/12] Btrfs: sysfs: fix, fs_info kobject_unregister has init_completion() twice

[Anand Jain]

kobject_unregister is to handle the release of the kobject,
its completion init is being called in btrfs_sysfs_add_one(),
so we don't have to do the same in the open_ctree() again.

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 fs/btrfs/disk-io.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c
index 612d46e..2e076b2 100644
--- a/fs/btrfs/disk-io.c
+++ b/fs/btrfs/disk-io.c
@@ -2247,7 +2247,6 @@ int open_ctree(struct super_block *sb,
 	mutex_init(&fs_info->delalloc_root_mutex);
 	seqlock_init(&fs_info->profiles_lock);
 
-	init_completion(&fs_info->kobj_unregister);
 	INIT_LIST_HEAD(&fs_info->dirty_cowonly_roots);
 	INIT_LIST_HEAD(&fs_info->space_info);
 	INIT_LIST_HEAD(&fs_info->tree_mod_seq_list);
-- 
1.9.1

[PATCH 03/12] Btrfs: sysfs: fix, undo sysfs device links

[Anand Jain]

Theoritically need to remove the device links attributes, but since its entire device
kobject was removed, so there wasn't any issue of about it. Just do it nicely.

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 fs/btrfs/sysfs.c | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index 68dcd17..adfac3e 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -522,6 +522,7 @@ void btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
 		kobject_del(fs_info->space_info_kobj);
 		kobject_put(fs_info->space_info_kobj);
 	}
+	btrfs_kobj_rm_device(fs_info, NULL);
 	kobject_del(fs_info->device_dir_kobj);
 	kobject_put(fs_info->device_dir_kobj);
 	addrm_unknown_feature_attrs(fs_info, false);
@@ -604,6 +605,8 @@ static void init_feature_attrs(void)
 	}
 }
 
+/* when one_device is NULL, it removes all device links */
+
 int btrfs_kobj_rm_device(struct btrfs_fs_info *fs_info,
 		struct btrfs_device *one_device)
 {
@@ -621,6 +624,20 @@ int btrfs_kobj_rm_device(struct btrfs_fs_info *fs_info,
 						disk_kobj->name);
 	}
 
+	if (one_device)
+		return 0;
+
+	list_for_each_entry(one_device,
+			&fs_info->fs_devices->devices, dev_list) {
+		if (!one_device->bdev)
+			continue;
+		disk = one_device->bdev->bd_part;
+		disk_kobj = &part_to_dev(disk)->kobj;
+
+		sysfs_remove_link(fs_info->device_dir_kobj,
+						disk_kobj->name);
+	}
+
 	return 0;
 }
 
-- 
1.9.1

[PATCH 04/12] Btrfs: sysfs: fix, kobject pointer clean up needed after kobject release

[Anand Jain]

The sysfs clean up self test like in the below code fails, since
fs_info->device_dir_kobject still points to its stale kobject.
Reseting this pointer will help to fix this.

open_ctree()
{

ret = btrfs_sysfs_add_one(fs_info);
::
+       btrfs_sysfs_remove_one(fs_info);
+       ret = btrfs_sysfs_add_one(fs_info);
+       if (ret) {
+               pr_err("BTRFS: failed to init sysfs interface: %d\n", ret);
+               goto fail_block_groups;
+       }

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 fs/btrfs/sysfs.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index adfac3e..15fead2 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -525,6 +525,7 @@ void btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
 	btrfs_kobj_rm_device(fs_info, NULL);
 	kobject_del(fs_info->device_dir_kobj);
 	kobject_put(fs_info->device_dir_kobj);
+	fs_info->device_dir_kobj = NULL;
 	addrm_unknown_feature_attrs(fs_info, false);
 	sysfs_remove_group(&fs_info->super_kobj, &btrfs_feature_attr_group);
 	__btrfs_sysfs_remove_one(fs_info);
-- 
1.9.1

[PATCH 05/12] Btrfc: sysfs: fix, check if device_dir_kobj is init before destroy

[Anand Jain]

Since the failure code in the btrfs_sysfs_add_one() can
call btrfs_sysfs_remove_one() even before device_dir_kobj
has been created we need to check if its null.

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 fs/btrfs/sysfs.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index 15fead2..506f7e4 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -522,10 +522,12 @@ void btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
 		kobject_del(fs_info->space_info_kobj);
 		kobject_put(fs_info->space_info_kobj);
 	}
-	btrfs_kobj_rm_device(fs_info, NULL);
-	kobject_del(fs_info->device_dir_kobj);
-	kobject_put(fs_info->device_dir_kobj);
-	fs_info->device_dir_kobj = NULL;
+	if (fs_info->device_dir_kobj) {
+		btrfs_kobj_rm_device(fs_info, NULL);
+		kobject_del(fs_info->device_dir_kobj);
+		kobject_put(fs_info->device_dir_kobj);
+		fs_info->device_dir_kobj = NULL;
+	}
 	addrm_unknown_feature_attrs(fs_info, false);
 	sysfs_remove_group(&fs_info->super_kobj, &btrfs_feature_attr_group);
 	__btrfs_sysfs_remove_one(fs_info);
-- 
1.9.1

[PATCH 06/12] Btrfs: sysfs: reorder the kobject creations

[Anand Jain]

As of now the order in which the kobjects are created
at btrfs_sysfs_add_one() is..
 fsid
 features
 unknown features (dynamic features)
 devices.

Since we would move fsid and device kobject to fs_devices
from fs_info structure, this patch will reorder in which
the kobjects are created as below.
 fsid
 devices
 features
 unknown features (dynamic features)

And hence the btrfs_sysfs_remove_one() will follow the same
in reverse order. and the device kobject destroy now can
be moved into the function __btrfs_sysfs_remove_one()

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 fs/btrfs/sysfs.c | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index 506f7e4..c3e7f06 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -510,6 +510,13 @@ static int addrm_unknown_feature_attrs(struct btrfs_fs_info *fs_info, bool add)
 
 static void __btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
 {
+	if (fs_info->device_dir_kobj) {
+		btrfs_kobj_rm_device(fs_info, NULL);
+		kobject_del(fs_info->device_dir_kobj);
+		kobject_put(fs_info->device_dir_kobj);
+		fs_info->device_dir_kobj = NULL;
+	}
+
 	kobject_del(&fs_info->super_kobj);
 	kobject_put(&fs_info->super_kobj);
 	wait_for_completion(&fs_info->kobj_unregister);
@@ -522,12 +529,6 @@ void btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
 		kobject_del(fs_info->space_info_kobj);
 		kobject_put(fs_info->space_info_kobj);
 	}
-	if (fs_info->device_dir_kobj) {
-		btrfs_kobj_rm_device(fs_info, NULL);
-		kobject_del(fs_info->device_dir_kobj);
-		kobject_put(fs_info->device_dir_kobj);
-		fs_info->device_dir_kobj = NULL;
-	}
 	addrm_unknown_feature_attrs(fs_info, false);
 	sysfs_remove_group(&fs_info->super_kobj, &btrfs_feature_attr_group);
 	__btrfs_sysfs_remove_one(fs_info);
@@ -700,6 +701,12 @@ int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
 	if (error)
 		return error;
 
+	error = btrfs_kobj_add_device(fs_info, NULL);
+	if (error) {
+		__btrfs_sysfs_remove_one(fs_info);
+		return error;
+	}
+
 	error = sysfs_create_group(&fs_info->super_kobj,
 				   &btrfs_feature_attr_group);
 	if (error) {
@@ -711,10 +718,6 @@ int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
 	if (error)
 		goto failure;
 
-	error = btrfs_kobj_add_device(fs_info, NULL);
-	if (error)
-		goto failure;
-
 	fs_info->space_info_kobj = kobject_create_and_add("allocation",
 						  &fs_info->super_kobj);
 	if (!fs_info->space_info_kobj) {
-- 
1.9.1

[PATCH 07/12] Btrfs: sysfs: rename __btrfs_sysfs_remove_one to btrfs_sysfs_remove_fsid

[Anand Jain]

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 fs/btrfs/sysfs.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index c3e7f06..c923e8b 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -508,7 +508,7 @@ static int addrm_unknown_feature_attrs(struct btrfs_fs_info *fs_info, bool add)
 	return 0;
 }
 
-static void __btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
+static void btrfs_sysfs_remove_fsid(struct btrfs_fs_info *fs_info)
 {
 	if (fs_info->device_dir_kobj) {
 		btrfs_kobj_rm_device(fs_info, NULL);
@@ -531,7 +531,7 @@ void btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
 	}
 	addrm_unknown_feature_attrs(fs_info, false);
 	sysfs_remove_group(&fs_info->super_kobj, &btrfs_feature_attr_group);
-	__btrfs_sysfs_remove_one(fs_info);
+	btrfs_sysfs_remove_fsid(fs_info);
 }
 
 const char * const btrfs_feature_set_names[3] = {
@@ -703,14 +703,14 @@ int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
 
 	error = btrfs_kobj_add_device(fs_info, NULL);
 	if (error) {
-		__btrfs_sysfs_remove_one(fs_info);
+		btrfs_sysfs_remove_fsid(fs_info);
 		return error;
 	}
 
 	error = sysfs_create_group(&fs_info->super_kobj,
 				   &btrfs_feature_attr_group);
 	if (error) {
-		__btrfs_sysfs_remove_one(fs_info);
+		btrfs_sysfs_remove_fsid(fs_info);
 		return error;
 	}
 
-- 
1.9.1

[PATCH 08/12] Btrfs: sysfs: introduce function btrfs_sysfs_add_fsid() to create sysfs fsid

[Anand Jain]

We need it in a seperate function so that it can be called from the
device discovery thread as well.

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 fs/btrfs/sysfs.c | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index c923e8b..f42d8fd 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -690,7 +690,12 @@ static struct dentry *btrfs_debugfs_root_dentry;
 /* Debugging tunables and exported data */
 u64 btrfs_debugfs_test;
 
-int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
+/*
+ * Can be called by the device discovery thread.
+ * And parent can be specified for seed device
+ */
+int btrfs_sysfs_add_fsid(struct btrfs_fs_info *fs_info,
+				struct kobject *parent)
 {
 	int error;
 
@@ -698,6 +703,14 @@ int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
 	fs_info->super_kobj.kset = btrfs_kset;
 	error = kobject_init_and_add(&fs_info->super_kobj, &btrfs_ktype, NULL,
 				     "%pU", fs_info->fsid);
+	return error;
+}
+
+int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
+{
+	int error;
+
+	error = btrfs_sysfs_add_fsid(fs_info, NULL);
 	if (error)
 		return error;
 
-- 
1.9.1

[PATCH 09/12] Btrfs: sysfs: let default_attrs be separate from the kset

[Anand Jain]

As of now btrfs_attrs are provided using the default_attrs through
the kset. Separate them and create the default_attrs using the
sysfs_create_files instead. By doing this we will have the
flexibility that device discovery thread could create fsid
kobject.

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 fs/btrfs/sysfs.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index f42d8fd..5208a49 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -428,7 +428,7 @@ static ssize_t btrfs_clone_alignment_show(struct kobject *kobj,
 
 BTRFS_ATTR(clone_alignment, btrfs_clone_alignment_show);
 
-static struct attribute *btrfs_attrs[] = {
+static const struct attribute *btrfs_attrs[] = {
 	BTRFS_ATTR_PTR(label),
 	BTRFS_ATTR_PTR(nodesize),
 	BTRFS_ATTR_PTR(sectorsize),
@@ -447,7 +447,6 @@ static void btrfs_release_super_kobj(struct kobject *kobj)
 static struct kobj_type btrfs_ktype = {
 	.sysfs_ops	= &kobj_sysfs_ops,
 	.release	= btrfs_release_super_kobj,
-	.default_attrs	= btrfs_attrs,
 };
 
 static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj)
@@ -531,6 +530,7 @@ void btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
 	}
 	addrm_unknown_feature_attrs(fs_info, false);
 	sysfs_remove_group(&fs_info->super_kobj, &btrfs_feature_attr_group);
+	sysfs_remove_files(&fs_info->super_kobj, btrfs_attrs);
 	btrfs_sysfs_remove_fsid(fs_info);
 }
 
@@ -720,13 +720,17 @@ int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
 		return error;
 	}
 
-	error = sysfs_create_group(&fs_info->super_kobj,
-				   &btrfs_feature_attr_group);
+	error = sysfs_create_files(&fs_info->super_kobj, btrfs_attrs);
 	if (error) {
 		btrfs_sysfs_remove_fsid(fs_info);
 		return error;
 	}
 
+	error = sysfs_create_group(&fs_info->super_kobj,
+				   &btrfs_feature_attr_group);
+	if (error)
+		goto failure;
+
 	error = addrm_unknown_feature_attrs(fs_info, true);
 	if (error)
 		goto failure;
-- 
1.9.1

[PATCH 10/12] Btrfs: sysfs: separate device kobject and its attribute creation

[Anand Jain]

Separate device kobject and its attribute creation so that device
kobject can be created from the device discovery thread.

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 fs/btrfs/sysfs.c | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index 5208a49..2cb4c69 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -645,13 +645,8 @@ int btrfs_kobj_rm_device(struct btrfs_fs_info *fs_info,
 	return 0;
 }
 
-int btrfs_kobj_add_device(struct btrfs_fs_info *fs_info,
-		struct btrfs_device *one_device)
+int btrfs_sysfs_add_device(struct btrfs_fs_info *fs_info)
 {
-	int error = 0;
-	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
-	struct btrfs_device *dev;
-
 	if (!fs_info->device_dir_kobj)
 		fs_info->device_dir_kobj = kobject_create_and_add("devices",
 						&fs_info->super_kobj);
@@ -659,6 +654,20 @@ int btrfs_kobj_add_device(struct btrfs_fs_info *fs_info,
 	if (!fs_info->device_dir_kobj)
 		return -ENOMEM;
 
+	return 0;
+}
+
+int btrfs_kobj_add_device(struct btrfs_fs_info *fs_info,
+		struct btrfs_device *one_device)
+{
+	int error = 0;
+	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
+	struct btrfs_device *dev;
+
+	error = btrfs_sysfs_add_device(fs_info);
+	if (error)
+		return error;
+
 	list_for_each_entry(dev, &fs_devices->devices, dev_list) {
 		struct hd_struct *disk;
 		struct kobject *disk_kobj;
-- 
1.9.1

[PATCH 11/12] Btrfs: sysfs: move super_kobj and device_dir_kobj from fs_info to btrfs_fs_devices

[Anand Jain]

This patch will provide a framework and help to create attributes
from the structure btrfs_fs_devices which are available even before
fs_info is created. So by moving the parent kobject super_kobj from
fs_info to btrfs_fs_devices, it will help to create attributes
from the btrfs_fs_devices as well.

Patches on top of this patch now will be able to create the
sys/fs/btrfs/fsid kobject and attributes from btrfs_fs_devices
when devices are scanned and registered to the kernel.

Just to note, this does not change any of the existing btrfs sysfs
external kobject names and its attributes and not even the life
cycle of them. Changes are internal only. And to ensure the same,
this path has been tested with various device operations and,
checking and comparing the sysfs kobjects and attributes with
sysfs kobject and attributes with out this patch, and they remain
same.

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 fs/btrfs/ctree.h   |  3 --
 fs/btrfs/sysfs.c   | 88 ++++++++++++++++++++++++++++++------------------------
 fs/btrfs/volumes.c |  3 +-
 fs/btrfs/volumes.h |  5 ++++
 4 files changed, 56 insertions(+), 43 deletions(-)

diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h
index 85c697d..ef66d13 100644
--- a/fs/btrfs/ctree.h
+++ b/fs/btrfs/ctree.h
@@ -1584,10 +1584,7 @@ struct btrfs_fs_info {
 	struct task_struct *cleaner_kthread;
 	int thread_pool_size;
 
-	struct kobject super_kobj;
 	struct kobject *space_info_kobj;
-	struct kobject *device_dir_kobj;
-	struct completion kobj_unregister;
 	int do_barriers;
 	int closing;
 	int log_root_recovering;
diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index 2cb4c69..ac15fbb 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -33,6 +33,7 @@
 #include "volumes.h"
 
 static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj);
+static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj);
 
 static u64 get_features(struct btrfs_fs_info *fs_info,
 			enum btrfs_feature_set set)
@@ -438,10 +439,10 @@ static const struct attribute *btrfs_attrs[] = {
 
 static void btrfs_release_super_kobj(struct kobject *kobj)
 {
-	struct btrfs_fs_info *fs_info = to_fs_info(kobj);
+	struct btrfs_fs_devices *fs_devs = to_fs_devs(kobj);
 
-	memset(&fs_info->super_kobj, 0, sizeof(struct kobject));
-	complete(&fs_info->kobj_unregister);
+	memset(&fs_devs->super_kobj, 0, sizeof(struct kobject));
+	complete(&fs_devs->kobj_unregister);
 }
 
 static struct kobj_type btrfs_ktype = {
@@ -449,11 +450,18 @@ static struct kobj_type btrfs_ktype = {
 	.release	= btrfs_release_super_kobj,
 };
 
+static inline struct btrfs_fs_devices *to_fs_devs(struct kobject *kobj)
+{
+	if (kobj->ktype != &btrfs_ktype)
+		return NULL;
+	return container_of(kobj, struct btrfs_fs_devices, super_kobj);
+}
+
 static inline struct btrfs_fs_info *to_fs_info(struct kobject *kobj)
 {
 	if (kobj->ktype != &btrfs_ktype)
 		return NULL;
-	return container_of(kobj, struct btrfs_fs_info, super_kobj);
+	return to_fs_devs(kobj)->fs_info;
 }
 
 #define NUM_FEATURE_BITS 64
@@ -494,12 +502,12 @@ static int addrm_unknown_feature_attrs(struct btrfs_fs_info *fs_info, bool add)
 			attrs[0] = &fa->kobj_attr.attr;
 			if (add) {
 				int ret;
-				ret = sysfs_merge_group(&fs_info->super_kobj,
+				ret = sysfs_merge_group(&fs_info->fs_devices->super_kobj,
 							&agroup);
 				if (ret)
 					return ret;
 			} else
-				sysfs_unmerge_group(&fs_info->super_kobj,
+				sysfs_unmerge_group(&fs_info->fs_devices->super_kobj,
 						    &agroup);
 		}
 
@@ -507,18 +515,17 @@ static int addrm_unknown_feature_attrs(struct btrfs_fs_info *fs_info, bool add)
 	return 0;
 }
 
-static void btrfs_sysfs_remove_fsid(struct btrfs_fs_info *fs_info)
+static void btrfs_sysfs_remove_fsid(struct btrfs_fs_devices *fs_devs)
 {
-	if (fs_info->device_dir_kobj) {
-		btrfs_kobj_rm_device(fs_info, NULL);
-		kobject_del(fs_info->device_dir_kobj);
-		kobject_put(fs_info->device_dir_kobj);
-		fs_info->device_dir_kobj = NULL;
+	if (fs_devs->device_dir_kobj) {
+		kobject_del(fs_devs->device_dir_kobj);
+		kobject_put(fs_devs->device_dir_kobj);
+		fs_devs->device_dir_kobj = NULL;
 	}
 
-	kobject_del(&fs_info->super_kobj);
-	kobject_put(&fs_info->super_kobj);
-	wait_for_completion(&fs_info->kobj_unregister);
+	kobject_del(&fs_devs->super_kobj);
+	kobject_put(&fs_devs->super_kobj);
+	wait_for_completion(&fs_devs->kobj_unregister);
 }
 
 void btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
@@ -529,9 +536,10 @@ void btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
 		kobject_put(fs_info->space_info_kobj);
 	}
 	addrm_unknown_feature_attrs(fs_info, false);
-	sysfs_remove_group(&fs_info->super_kobj, &btrfs_feature_attr_group);
-	sysfs_remove_files(&fs_info->super_kobj, btrfs_attrs);
-	btrfs_sysfs_remove_fsid(fs_info);
+	sysfs_remove_group(&fs_info->fs_devices->super_kobj, &btrfs_feature_attr_group);
+	sysfs_remove_files(&fs_info->fs_devices->super_kobj, btrfs_attrs);
+	btrfs_kobj_rm_device(fs_info, NULL);
+	btrfs_sysfs_remove_fsid(fs_info->fs_devices);
 }
 
 const char * const btrfs_feature_set_names[3] = {
@@ -617,14 +625,14 @@ int btrfs_kobj_rm_device(struct btrfs_fs_info *fs_info,
 	struct hd_struct *disk;
 	struct kobject *disk_kobj;
 
-	if (!fs_info->device_dir_kobj)
+	if (!fs_info->fs_devices->device_dir_kobj)
 		return -EINVAL;
 
 	if (one_device && one_device->bdev) {
 		disk = one_device->bdev->bd_part;
 		disk_kobj = &part_to_dev(disk)->kobj;
 
-		sysfs_remove_link(fs_info->device_dir_kobj,
+		sysfs_remove_link(fs_info->fs_devices->device_dir_kobj,
 						disk_kobj->name);
 	}
 
@@ -638,20 +646,20 @@ int btrfs_kobj_rm_device(struct btrfs_fs_info *fs_info,
 		disk = one_device->bdev->bd_part;
 		disk_kobj = &part_to_dev(disk)->kobj;
 
-		sysfs_remove_link(fs_info->device_dir_kobj,
+		sysfs_remove_link(fs_info->fs_devices->device_dir_kobj,
 						disk_kobj->name);
 	}
 
 	return 0;
 }
 
-int btrfs_sysfs_add_device(struct btrfs_fs_info *fs_info)
+int btrfs_sysfs_add_device(struct btrfs_fs_devices *fs_devs)
 {
-	if (!fs_info->device_dir_kobj)
-		fs_info->device_dir_kobj = kobject_create_and_add("devices",
-						&fs_info->super_kobj);
+	if (!fs_devs->device_dir_kobj)
+		fs_devs->device_dir_kobj = kobject_create_and_add("devices",
+						&fs_devs->super_kobj);
 
-	if (!fs_info->device_dir_kobj)
+	if (!fs_devs->device_dir_kobj)
 		return -ENOMEM;
 
 	return 0;
@@ -664,7 +672,7 @@ int btrfs_kobj_add_device(struct btrfs_fs_info *fs_info,
 	struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
 	struct btrfs_device *dev;
 
-	error = btrfs_sysfs_add_device(fs_info);
+	error = btrfs_sysfs_add_device(fs_devices);
 	if (error)
 		return error;
 
@@ -681,7 +689,7 @@ int btrfs_kobj_add_device(struct btrfs_fs_info *fs_info,
 		disk = dev->bdev->bd_part;
 		disk_kobj = &part_to_dev(disk)->kobj;
 
-		error = sysfs_create_link(fs_info->device_dir_kobj,
+		error = sysfs_create_link(fs_devices->device_dir_kobj,
 					  disk_kobj, disk_kobj->name);
 		if (error)
 			break;
@@ -703,39 +711,41 @@ u64 btrfs_debugfs_test;
  * Can be called by the device discovery thread.
  * And parent can be specified for seed device
  */
-int btrfs_sysfs_add_fsid(struct btrfs_fs_info *fs_info,
+static int btrfs_sysfs_add_fsid(struct btrfs_fs_devices *fs_devs,
 				struct kobject *parent)
 {
 	int error;
 
-	init_completion(&fs_info->kobj_unregister);
-	fs_info->super_kobj.kset = btrfs_kset;
-	error = kobject_init_and_add(&fs_info->super_kobj, &btrfs_ktype, NULL,
-				     "%pU", fs_info->fsid);
+	init_completion(&fs_devs->kobj_unregister);
+	fs_devs->super_kobj.kset = btrfs_kset;
+	error = kobject_init_and_add(&fs_devs->super_kobj, &btrfs_ktype, NULL,
+				     "%pU", fs_devs->fsid);
 	return error;
 }
 
 int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
 {
 	int error;
+	struct btrfs_fs_devices *fs_devs = fs_info->fs_devices;
+	struct kobject *super_kobj = &fs_devs->super_kobj;
 
-	error = btrfs_sysfs_add_fsid(fs_info, NULL);
+	error = btrfs_sysfs_add_fsid(fs_devs, NULL);
 	if (error)
 		return error;
 
 	error = btrfs_kobj_add_device(fs_info, NULL);
 	if (error) {
-		btrfs_sysfs_remove_fsid(fs_info);
+		btrfs_sysfs_remove_fsid(fs_devs);
 		return error;
 	}
 
-	error = sysfs_create_files(&fs_info->super_kobj, btrfs_attrs);
+	error = sysfs_create_files(super_kobj, btrfs_attrs);
 	if (error) {
-		btrfs_sysfs_remove_fsid(fs_info);
+		btrfs_sysfs_remove_fsid(fs_devs);
 		return error;
 	}
 
-	error = sysfs_create_group(&fs_info->super_kobj,
+	error = sysfs_create_group(super_kobj,
 				   &btrfs_feature_attr_group);
 	if (error)
 		goto failure;
@@ -745,7 +755,7 @@ int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
 		goto failure;
 
 	fs_info->space_info_kobj = kobject_create_and_add("allocation",
-						  &fs_info->super_kobj);
+						  super_kobj);
 	if (!fs_info->space_info_kobj) {
 		error = -ENOMEM;
 		goto failure;
diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c
index da7e0e1..6bbee64 100644
--- a/fs/btrfs/volumes.c
+++ b/fs/btrfs/volumes.c
@@ -2243,7 +2243,8 @@ int btrfs_init_new_device(struct btrfs_root *root, char *device_path)
 		 */
 		snprintf(fsid_buf, BTRFS_UUID_UNPARSED_SIZE, "%pU",
 						root->fs_info->fsid);
-		if (kobject_rename(&root->fs_info->super_kobj, fsid_buf))
+		if (kobject_rename(&root->fs_info->fs_devices->super_kobj,
+								fsid_buf))
 			goto error_trans;
 	}
 
diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h
index 83069de..a1df344 100644
--- a/fs/btrfs/volumes.h
+++ b/fs/btrfs/volumes.h
@@ -253,6 +253,11 @@ struct btrfs_fs_devices {
 	 * nonrot flag set
 	 */
 	int rotating;
+
+	/* sysfs kobjects */
+	struct kobject super_kobj;
+	struct kobject *device_dir_kobj;
+	struct completion kobj_unregister;
 };
 
 #define BTRFS_BIO_INLINE_CSUM_SIZE	64
-- 
1.9.1

[PATCH 12/12] Btrfs: sysfs: add pointer to access fs_info from fs_devices

[Anand Jain]

adds fs_info pointer with struct btrfs_fs_devices.

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 fs/btrfs/sysfs.c   | 4 ++++
 fs/btrfs/volumes.h | 1 +
 2 files changed, 5 insertions(+)

diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c
index ac15fbb..4b5bac6 100644
--- a/fs/btrfs/sysfs.c
+++ b/fs/btrfs/sysfs.c
@@ -530,6 +530,8 @@ static void btrfs_sysfs_remove_fsid(struct btrfs_fs_devices *fs_devs)
 
 void btrfs_sysfs_remove_one(struct btrfs_fs_info *fs_info)
 {
+	fs_info->fs_devices->fs_info = NULL;
+
 	if (fs_info->space_info_kobj) {
 		sysfs_remove_files(fs_info->space_info_kobj, allocation_attrs);
 		kobject_del(fs_info->space_info_kobj);
@@ -729,6 +731,8 @@ int btrfs_sysfs_add_one(struct btrfs_fs_info *fs_info)
 	struct btrfs_fs_devices *fs_devs = fs_info->fs_devices;
 	struct kobject *super_kobj = &fs_devs->super_kobj;
 
+	fs_devs->fs_info = fs_info;
+
 	error = btrfs_sysfs_add_fsid(fs_devs, NULL);
 	if (error)
 		return error;
diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h
index a1df344..40c38ab 100644
--- a/fs/btrfs/volumes.h
+++ b/fs/btrfs/volumes.h
@@ -254,6 +254,7 @@ struct btrfs_fs_devices {
 	 */
 	int rotating;
 
+	struct btrfs_fs_info *fs_info;
 	/* sysfs kobjects */
 	struct kobject super_kobj;
 	struct kobject *device_dir_kobj;
-- 
1.9.1

[PATCH] Btrfs-progs: add regression tests for sysfs contents during btrfs device management

[Anand Jain]

This contains a series of btrfs device operations and at each operation
the btrfs sysfs contents are logged.  This helps to check if the patch
is affecting any of the btrfs sysfs contents.

OR This script can be used to test the only the device operations.

as of now there are 32 test cases

Signed-off-by: Anand Jain <anand.jain@oracle.com>
---
 tests/test-btrfs-devmgt.sh | 863 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 863 insertions(+)
 create mode 100755 tests/test-btrfs-devmgt.sh

diff --git a/tests/test-btrfs-devmgt.sh b/tests/test-btrfs-devmgt.sh
new file mode 100755
index 0000000..2f75200
--- /dev/null
+++ b/tests/test-btrfs-devmgt.sh
@@ -0,0 +1,863 @@
+# * GNU General Public License v2. Copyright Oracle 2015
+
+
+# Series of btrfs device operation test cases.
+#
+
+# sysfs: test
+# sysfs contents are taken at reasonable places (but you may disable it).
+# So to compare with the next iteration with your kernel patch. so
+# you can check for the sysfs changes by running diff of TMP_FILE(s)
+
+# Changelog:
+# v1.0 init asj
+
+
+		#When you change something related to device
+		#remember to test on btrfs boot separately
+ #test0: btrfs boot test
+
+		#Replace:
+ #test1: raid1, replace normal
+ #test2: raid1, replace missing
+ #test3: NOP
+ #test4: raid1, replace missing, replace normal
+ #test5: raid1, replace dev2, replace dev1
+
+		#Add sprout, replace:
+ #test6: add sprout, replace seed
+ #test7: raid1 seed, add sprout, replace seed, replace sprout
+ #test8: 3 level nested seeds, add sprout, replace mid level seed
+ #test9: raid1, degraded seed mount, add sprout, replace missing, replace non missing seed
+ #test10: add sprout, replace sprout
+ #test11: raid1 seed, add sprout, replace sprout, replace sprout again
+ #test12: 3 level nested seeds, add sprout, replace sprout
+ #test13: degraded raid1 seed, add sprout, replace sprout
+
+		#Mount sprout, replace:
+ #test14: NOP
+ #test15: mount sprout, replace sprout
+ #test16: mount sprout, replace seed
+ #test17: Raid1, mount sprout, replace sprout
+ #test18: Raid1, mount sprout, replace seed
+ #test19: Raid1 degraded, mount sprout, replace sprout
+ #test20: Raid1 degraded, mount sprout, replace missing
+ #test21: 3 level nested seeds, mount sprout, replace mid level seed
+
+		#seed sprout test:
+ #test22: mount sprout, mount seed
+ #test23: clean, mount -o device sprout
+ #test24: raid1, mount sprout
+ #test25: clean, scan, mount sprout
+ #test26: raid1, clean, mount -o device sprout
+ #test26: raid1, clean, scan, mount sprout
+
+		#dev add del test:
+ #test27: dev add
+ #test28: dev del
+
+		#dev scan test:
+ #test29: scan mount
+ #test30: use -o mount
+
+		#subvol mount test:
+ #test31: mount, mount subvol
+
+		#remount test:
+ #test32: mount, remount
+
+
+
+
+# Devices are hard coded. sorry
+
+DEV0=/dev/sdb
+x=0
+if [ $x -eq 1 ]; then
+DEV1=/dev/sdc
+DEV2=/dev/sdd
+DEV3=/dev/sde
+DEV4=/dev/sdf
+DEV5=/dev/sdg
+else
+DEV1=/dev/sdd
+DEV2=/dev/sde
+DEV3=/dev/sdf
+DEV4=/dev/sdg
+DEV5=/dev/sdc
+fi
+
+TEST_FSID=1c52f894-0ead-43d6-847a-d42359f78370
+
+#Enable or disable sysfs data collection by set/unset the below
+#TMP_FILE=''
+TMP_FILE=`mktemp`
+
+ent_cont()
+{
+	echo -n "Enter to continue: "
+	#read
+	echo "wait for input is disabled, uncomment above to wait."
+}
+
+erase()
+{
+	for i in $DEV0 $DEV1 $DEV2 $DEV3 $DEV4 $DEV5; do wipefs -a $i > /dev/null; done
+}
+
+clean()
+{
+	modprobe -r btrfs && modprobe btrfs
+}
+
+collect_sysfs()
+{
+	# see above to disable sysfs data collection
+	[[ -z $TMP_FILE ]] && return
+
+	echo ---------------- $1 ------------- >> $TMP_FILE
+	find /sys/fs/btrfs -type f -exec cat {} \; -print >> $TMP_FILE
+}
+
+_mkfs.btrfs()
+{
+	mkfs.btrfs $* > /dev/null
+}
+
+
+test1()
+{
+	TEST="test1"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID -d raid1 -m raid1 $DEV1 $DEV2 -f
+	collect_sysfs "$TEST"
+mount $DEV2 /btrfs
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV2 $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test2()
+{
+	TEST="test2"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID -d raid1 -m raid1 $DEV1 $DEV2 -f
+	clean
+mount -o degraded $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs rep start -B 2 $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test4()
+{
+	TEST="test4"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID -d raid1 -m raid1 $DEV1 $DEV2 -f
+	clean
+mount -o degraded $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs rep start -B 2 $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV1 $DEV4 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test5()
+{
+	TEST="test5"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID -d raid1 -m raid1 $DEV1 $DEV2 -f
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV2 $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV1 $DEV4 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+
+##### Seed device test cases replace the seed device ###########
+test6()
+{
+	TEST="test6"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV2 /btrfs -f
+	collect_sysfs "$TEST"
+btrfs repl start -B $DEV1 $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test7()
+{
+	TEST="test7"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID -draid1 -mraid1 $DEV1 $DEV2 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV1 $DEV4 /btrfs -f
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV2 $DEV5 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test8()
+{
+	TEST="test8"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV2 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+btrfstune -S 1 $DEV2
+mount $DEV2 /btrfs1
+	collect_sysfs "$TEST"
+btrfs dev add $DEV3 /btrfs1 -f
+	collect_sysfs "$TEST"
+umount /btrfs1
+	collect_sysfs "$TEST"
+btrfstune -S 1 $DEV3
+mount $DEV3 /btrfs2
+	collect_sysfs "$TEST"
+btrfs dev add $DEV4 /btrfs2 -f
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV2 $DEV5 /btrfs2 -f
+	collect_sysfs "$TEST"
+umount /btrfs2
+	collect_sysfs "$TEST"
+mount $DEV5 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test9()
+{
+	TEST="test9"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID -d raid1 -m raid1 $DEV1 $DEV2 -f
+	collect_sysfs "$TEST"
+btrfstune -S 1 $DEV1
+	clean
+mount -o degraded $DEV1 /btrfs
+	collect_sysfs "$TEST"
+	echo -e add
+btrfs dev add $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+	echo -e replace1
+btrfs rep start -B 2 $DEV4 /btrfs -f
+	collect_sysfs "$TEST"
+	echo -e replace2
+btrfs rep start -B $DEV1 $DEV5 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+##### Seed device test cases replace the sprout device ###########
+test10()
+{
+	TEST="test10"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV2 /btrfs -f
+	collect_sysfs "$TEST"
+btrfs repl start -B $DEV2 $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test11()
+{
+	TEST="test11"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID -draid1 -mraid1 $DEV1 $DEV2 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV3 $DEV4 /btrfs -f
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV4 $DEV5 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test12()
+{
+	TEST="test12"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV2 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+btrfstune -S 1 $DEV2
+mount $DEV2 /btrfs1
+	collect_sysfs "$TEST"
+btrfs dev add $DEV3 /btrfs1 -f
+	collect_sysfs "$TEST"
+umount /btrfs1
+	collect_sysfs "$TEST"
+btrfstune -S 1 $DEV3
+mount $DEV3 /btrfs2
+	collect_sysfs "$TEST"
+btrfs dev add $DEV4 /btrfs2 -f
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV4 $DEV5 /btrfs2 -f
+	collect_sysfs "$TEST"
+umount /btrfs2
+	collect_sysfs "$TEST"
+mount $DEV5 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test13()
+{
+	TEST="test13"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID -d raid1 -m raid1 $DEV1 $DEV2 -f
+	collect_sysfs "$TEST"
+btrfstune -S 1 $DEV1
+	clean
+mount -o degraded $DEV1 /btrfs
+	collect_sysfs "$TEST"
+	echo -e add
+btrfs dev add $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+	echo -e replace1
+btrfs rep start -B $DEV3 $DEV4 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+
+test15()
+{
+	TEST="test15"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV2 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+mount $DEV2 /btrfs
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV1 $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test16()
+{
+	TEST="test16"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV2 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+mount $DEV2 /btrfs
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV2 $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test17()
+{
+	TEST="test17"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID -draid1 -mraid1 $DEV1 $DEV2 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+mount $DEV3 /btrfs
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV3 $DEV4 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test18()
+{
+	TEST="test18"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID -draid1 -mraid1 $DEV1 $DEV2 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+mount $DEV3 /btrfs
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV1 $DEV4 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test19()
+{
+	TEST="test19"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID -draid1 -mraid1 $DEV1 $DEV2 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+mount -o degraded -o device=$DEV2 $DEV3 /btrfs
+	collect_sysfs "$TEST"
+btrfs rep start -B $DEV3 $DEV4 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test20()
+{
+	TEST="test20"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID -draid1 -mraid1 $DEV1 $DEV2 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+mount -o degraded -o device=$DEV2 $DEV3 /btrfs
+	collect_sysfs "$TEST"
+btrfs rep start -B 1 $DEV4 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test21()
+{
+	TEST="test21"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV2 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+btrfstune -S 1 $DEV2
+mount $DEV2 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV3 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+btrfstune -S 1 $DEV3
+mount $DEV3 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV4 /btrfs2 -f
+	collect_sysfs "$TEST"
+umount /btrfs
+mount $DEV4 /btrfs
+btrfs rep start -B $DEV2 $DEV5 /btrfs -f
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+mount $DEV5 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test22()
+{
+	TEST="test22"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV2 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+mount $DEV2 /btrfs
+	collect_sysfs "$TEST"
+mount $DEV1 /btrfs1
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs1
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+
+test23()
+{
+	TEST="test23"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV2 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+
+	clean
+
+mount -o device=$DEV1 $DEV2 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test24()
+{
+	TEST="test24"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 $DEV2 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV3 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+mount $DEV3 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test25()
+{
+	TEST="test25"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV2 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+
+	clean
+
+btrfs dev scan
+mount $DEV2 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test26()
+{
+	TEST="test26"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 $DEV2 -f
+btrfstune -S 1 $DEV1
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV3 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+
+	clean
+
+mount -o device=$DEV1 -o device=$DEV2 $DEV3 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	clean
+	ent_cont
+}
+
+test27()
+{
+	TEST="test27"
+	erase
+	echo -e "\n$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 $DEV2 -f
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev add $DEV3 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev del $DEV1 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+	ent_cont
+}
+
+test28()
+{
+	TEST="test28" && erase && echo -e "\n$TEST"
+
+_mkfs.btrfs -dsingle -msingle -L $TEST -U $TEST_FSID $DEV1 $DEV2 -f
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs dev del $DEV1 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+
+	ent_cont
+}
+
+test29()
+{
+	TEST="test29" && erase && echo -e "\n$TEST"
+clean
+	collect_sysfs "$TEST"
+_mkfs.btrfs -dsingle -msingle -L $TEST -U $TEST_FSID $DEV1 $DEV2 -f
+	collect_sysfs "$TEST"
+clean
+	collect_sysfs "$TEST"
+btrfs dev scan
+	collect_sysfs "$TEST"
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+clean
+	collect_sysfs "$TEST"
+
+	ent_cont
+
+}
+
+test30()
+{
+	TEST="test30" && erase && echo -e "\n$TEST"
+clean
+	collect_sysfs "$TEST"
+_mkfs.btrfs -dsingle -msingle -L $TEST -U $TEST_FSID $DEV1 $DEV2 $DEV3 -f
+	collect_sysfs "$TEST"
+clean
+	collect_sysfs "$TEST"
+mount -o device=$DEV1,device=$DEV2 $DEV3 /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+clean
+	collect_sysfs "$TEST"
+
+	ent_cont
+}
+
+test31()
+{
+	TEST="test31" && erase && echo -e "\n$TEST"
+clean
+	collect_sysfs "$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 -f
+	collect_sysfs "$TEST"
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+btrfs su create /btrfs/sv1
+mount -o subvol=sv1 $DEV1 /btrfs1
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs1
+	collect_sysfs "$TEST"
+clean
+	collect_sysfs "$TEST"
+
+	ent_cont
+}
+
+test32()
+{
+	TEST="test32" && erase && echo -e "\n$TEST"
+clean
+	collect_sysfs "$TEST"
+_mkfs.btrfs -L $TEST -U $TEST_FSID $DEV1 -f
+	collect_sysfs "$TEST"
+mount $DEV1 /btrfs
+	collect_sysfs "$TEST"
+mount -o remount /btrfs
+	collect_sysfs "$TEST"
+umount /btrfs
+	collect_sysfs "$TEST"
+clean
+	collect_sysfs "$TEST"
+
+	ent_cont
+}
+
+test0()
+{
+  echo "Have you tested with btrfs boot, you can't do that here\n"
+}
+
+clean
+sleep 2; test1
+sleep 2; test2
+sleep 2; test3
+sleep 2; test4
+sleep 2; test5
+
+# Due to a bug in the ref below, don't enable 6-21 yet.
+#   ref: email: Sub: "replace seed/sprout hangs (regression ?)"
+#sleep 2; test6
+#sleep 2; test7
+#sleep 2; test8
+#sleep 2; test9
+#sleep 2; test9
+#sleep 2; test10
+#sleep 2; test11
+#sleep 2; test12
+#sleep 2; test13
+
+#sleep 2; test14
+#sleep 2; test15
+#sleep 2; test16
+#sleep 2; test17
+#sleep 2; test18
+#sleep 2; test19
+#sleep 2; test20
+#sleep 2; test21
+
+sleep 2; test22
+sleep 2; test23
+sleep 2; test24
+sleep 2; test25
+sleep 2; test26
+
+sleep 2; test27
+sleep 2; test28
+sleep 2; test29
+sleep 2; test30
+sleep 2; test31
+sleep 2; test32
+
+[[ -z $TMP_FILE ]] || echo -e "\nTMP_FILE= $TMP_FILE"
-- 
1.9.1