[PATCH 1/3] bcache: make writeback inflight configurable in sysfs

[PATCH 1/3] bcache: make writeback inflight configurable in sysfs

[mingzhe.zou]

From: Dongsheng Yang <dongsheng.yang@easystack.cn>

This commit introduce a new sysfs file:
/sys/block/bcache0/bcache/writeback_inflight (read only)
/sys/block/bcache0/bcache/writeback_inflight_max (read write)

(1) read the writeback_inflight will output the current inflight writeback op.
(2）read the writeback_inflight_max will output the max number of writeback inflight.
(3) write the writeback_inflight_max can set the max number of writeback inflight,
valid range is [1, INT_MAX).

E.g:
 $ ll /sys/block/bcache0/bcache/writeback_inflight*
-r--r--r-- 1 root root 4096 Oct 27 08:45 /sys/block/bcache0/bcache/writeback_inflight
-rw-r--r-- 1 root root 4096 Oct 27 08:45 /sys/block/bcache0/bcache/writeback_inflight_max
 $ cat /sys/block/bcache0/bcache/writeback_inflight
0
 $ cat /sys/block/bcache0/bcache/writeback_inflight_max
64
 $ echo 1024 > /sys/block/bcache0/bcache/writeback_inflight_max
 $ cat /sys/block/bcache0/bcache/writeback_inflight_max
1024

Signed-off-by: Dongsheng Yang <dongsheng.yang@easystack.cn>
Signed-off-by: mingzhe <mingzhe.zou@easystack.cn>
---
 drivers/md/bcache/bcache.h    |  6 ++++-
 drivers/md/bcache/sysfs.c     | 21 +++++++++++++++++
 drivers/md/bcache/writeback.c | 43 ++++++++++++++++++++++++++++++++---
 3 files changed, 66 insertions(+), 4 deletions(-)

diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h
index aebb7ef10e63..74434a7730bb 100644
--- a/drivers/md/bcache/bcache.h
+++ b/drivers/md/bcache/bcache.h
@@ -337,7 +337,11 @@ struct cached_dev {
 	struct delayed_work	writeback_rate_update;
 
 	/* Limit number of writeback bios in flight */
-	struct semaphore	in_flight;
+	atomic_t		wb_inflight;
+	unsigned long		wb_inflight_max;
+	spinlock_t		wb_inflight_lock;
+	wait_queue_head_t	wb_inflight_wait;
+
 	struct task_struct	*writeback_thread;
 	struct workqueue_struct	*writeback_write_wq;
 
diff --git a/drivers/md/bcache/sysfs.c b/drivers/md/bcache/sysfs.c
index c6f677059214..0382b70c29d5 100644
--- a/drivers/md/bcache/sysfs.c
+++ b/drivers/md/bcache/sysfs.c
@@ -119,6 +119,9 @@ rw_attribute(writeback_delay);
 rw_attribute(writeback_rate);
 rw_attribute(writeback_consider_fragment);
 
+read_attribute(writeback_inflight);
+rw_attribute(writeback_inflight_max);
+
 rw_attribute(writeback_rate_update_seconds);
 rw_attribute(writeback_rate_i_term_inverse);
 rw_attribute(writeback_rate_p_term_inverse);
@@ -201,6 +204,8 @@ SHOW(__bch_cached_dev)
 	var_printf(writeback_consider_fragment,	"%i");
 	var_print(writeback_delay);
 	var_print(writeback_percent);
+	sysfs_printf(writeback_inflight, "%i", atomic_read(&dc->wb_inflight));
+	sysfs_printf(writeback_inflight_max, "%li", dc->wb_inflight_max);
 	sysfs_hprint(writeback_rate,
 		     wb ? atomic_long_read(&dc->writeback_rate.rate) << 9 : 0);
 	sysfs_printf(io_errors,		"%i", atomic_read(&dc->io_errors));
@@ -448,6 +453,20 @@ STORE(__cached_dev)
 	if (attr == &sysfs_detach && dc->disk.c)
 		bch_cached_dev_detach(dc);
 
+	if (attr == &sysfs_writeback_inflight_max) {
+		ssize_t ret;
+		unsigned long v;
+
+		ret = strtoul_safe_clamp(buf, v, 1, INT_MAX);
+		if (ret)
+			return ret;
+
+		spin_lock(&dc->wb_inflight_lock);
+		dc->wb_inflight_max = v;
+		spin_unlock(&dc->wb_inflight_lock);
+		wake_up(&dc->wb_inflight_wait);
+	}
+
 	if (attr == &sysfs_stop)
 		bcache_device_stop(&dc->disk);
 
@@ -514,6 +533,8 @@ static struct attribute *bch_cached_dev_attrs[] = {
 	&sysfs_writeback_running,
 	&sysfs_writeback_delay,
 	&sysfs_writeback_percent,
+	&sysfs_writeback_inflight,
+	&sysfs_writeback_inflight_max,
 	&sysfs_writeback_rate,
 	&sysfs_writeback_consider_fragment,
 	&sysfs_writeback_rate_update_seconds,
diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c
index d4a5fc0650bb..0c5f25816e2e 100644
--- a/drivers/md/bcache/writeback.c
+++ b/drivers/md/bcache/writeback.c
@@ -348,6 +348,7 @@ static void dirty_io_destructor(struct closure *cl)
 	kfree(io);
 }
 
+static void end_wb_inflight(struct cached_dev *dc);
 static void write_dirty_finish(struct closure *cl)
 {
 	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
@@ -382,7 +383,7 @@ static void write_dirty_finish(struct closure *cl)
 	}
 
 	bch_keybuf_del(&dc->writeback_keys, w);
-	up(&dc->in_flight);
+	end_wb_inflight(dc);
 
 	closure_return_with_destructor(cl, dirty_io_destructor);
 }
@@ -471,6 +472,38 @@ static void read_dirty_submit(struct closure *cl)
 	continue_at(cl, write_dirty, io->dc->writeback_write_wq);
 }
 
+static void start_wb_inflight(struct cached_dev *dc)
+{
+	DEFINE_WAIT(w);
+
+	spin_lock(&dc->wb_inflight_lock);
+	if (atomic_read(&dc->wb_inflight) < dc->wb_inflight_max)
+		goto out;
+
+	do {
+		prepare_to_wait(&dc->wb_inflight_wait, &w,
+				TASK_UNINTERRUPTIBLE);
+
+		spin_unlock(&dc->wb_inflight_lock);
+		schedule();
+		spin_lock(&dc->wb_inflight_lock);
+	} while (atomic_read(&dc->wb_inflight) >= dc->wb_inflight_max);
+
+	finish_wait(&dc->wb_inflight_wait, &w);
+
+out:
+	BUG_ON(atomic_inc_return(&dc->wb_inflight) > dc->wb_inflight_max);
+	spin_unlock(&dc->wb_inflight_lock);
+}
+
+static void end_wb_inflight(struct cached_dev *dc)
+{
+	spin_lock(&dc->wb_inflight_lock);
+	BUG_ON(atomic_dec_return(&dc->wb_inflight) < 0);
+	spin_unlock(&dc->wb_inflight_lock);
+	wake_up(&dc->wb_inflight_wait);
+}
+
 static void read_dirty(struct cached_dev *dc)
 {
 	unsigned int delay = 0;
@@ -557,7 +590,7 @@ static void read_dirty(struct cached_dev *dc)
 
 			trace_bcache_writeback(&w->key);
 
-			down(&dc->in_flight);
+			start_wb_inflight(dc);
 
 			/*
 			 * We've acquired a semaphore for the maximum
@@ -1025,7 +1058,11 @@ void bch_sectors_dirty_init(struct bcache_device *d)
 
 void bch_cached_dev_writeback_init(struct cached_dev *dc)
 {
-	sema_init(&dc->in_flight, 64);
+	atomic_set(&dc->wb_inflight, 0);
+	dc->wb_inflight_max = 64;
+	spin_lock_init(&dc->wb_inflight_lock);
+	init_waitqueue_head(&dc->wb_inflight_wait);
+
 	init_rwsem(&dc->writeback_lock);
 	bch_keybuf_init(&dc->writeback_keys);
 
-- 
2.17.1

[PATCH 2/3] bcache: submit writeback inflight dirty writes in batch

[mingzhe.zou]

From: Dongsheng Yang <dongsheng.yang@easystack.cn>

If we have a backing device of log-structured block device (such as bcache flash dev),
there is a possibility to merge the writes in writeback, as the all writes into bcache flash_dev
are stored in bucket as log-structured.

That means, if we have a cached_dev as below:
        ----------------------------
        | bcache2 (cached_dev)     |
        | ------------------------ |
        | |   sdb (cache_dev)    | |
        | ------------------------ |
        | ------------------------ |
        | |   bcache1 (flash_dev)| |
        | ------------------------ |
        ----------------------------

we can merge the dirty writes in writeback, if we can submit the dirty writes in batch and around start_plug/finish_plug.

So this commit change the dirty_write to add the a dirty_io into a rb_tree, and queue a worker to submit all dirty_io,
This provide a timing to merge these writes, which can improve the writeback bandwidth.

Signed-off-by: Dongsheng Yang <dongsheng.yang@easystack.cn>
Signed-off-by: mingzhe <mingzhe.zou@easystack.cn>
---
 drivers/md/bcache/bcache.h    |   4 ++
 drivers/md/bcache/writeback.c | 102 ++++++++++++++++++++++------------
 2 files changed, 72 insertions(+), 34 deletions(-)

diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h
index 74434a7730bb..a82974aefc90 100644
--- a/drivers/md/bcache/bcache.h
+++ b/drivers/md/bcache/bcache.h
@@ -356,6 +356,10 @@ struct cached_dev {
 	struct closure_waitlist writeback_ordering_wait;
 	atomic_t		writeback_sequence_next;
 
+	struct rb_root		writeback_ios;
+	spinlock_t		writeback_ios_lock;
+	struct work_struct	write_dirty_work;
+
 	/* For tracking sequential IO */
 #define RECENT_IO_BITS	7
 #define RECENT_IO	(1 << RECENT_IO_BITS)
diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c
index 0c5f25816e2e..315fb91a8066 100644
--- a/drivers/md/bcache/writeback.c
+++ b/drivers/md/bcache/writeback.c
@@ -323,6 +323,7 @@ struct dirty_io {
 	struct closure		cl;
 	struct cached_dev	*dc;
 	uint16_t		sequence;
+	struct rb_node		node;
 	struct bio		bio;
 };
 
@@ -401,53 +402,81 @@ static void dirty_endio(struct bio *bio)
 	closure_put(&io->cl);
 }
 
-static void write_dirty(struct closure *cl)
+static inline int dirty_io_cmp(struct dirty_io *l, struct dirty_io *r)
+{
+	return (l->sequence < r->sequence) ? -1 : (l->sequence > r->sequence);
+}
+
+static void queue_dirty_write(struct closure *cl)
 {
 	struct dirty_io *io = container_of(cl, struct dirty_io, cl);
-	struct keybuf_key *w = io->bio.bi_private;
 	struct cached_dev *dc = io->dc;
 
-	uint16_t next_sequence;
+	spin_lock(&dc->writeback_ios_lock);
+	BUG_ON(RB_INSERT(&dc->writeback_ios, io, node, dirty_io_cmp));
+	spin_unlock(&dc->writeback_ios_lock);
 
-	if (atomic_read(&dc->writeback_sequence_next) != io->sequence) {
-		/* Not our turn to write; wait for a write to complete */
-		closure_wait(&dc->writeback_ordering_wait, cl);
+	queue_work(dc->writeback_write_wq, &dc->write_dirty_work);
+}
 
-		if (atomic_read(&dc->writeback_sequence_next) == io->sequence) {
-			/*
-			 * Edge case-- it happened in indeterminate order
-			 * relative to when we were added to wait list..
-			 */
-			closure_wake_up(&dc->writeback_ordering_wait);
-		}
+static void write_dirty(struct work_struct *work)
+{
+	struct cached_dev *dc = container_of(work, struct cached_dev,
+						write_dirty_work);
+	struct dirty_io *io;
+	struct keybuf_key *w;
+	uint16_t next_sequence;
+	struct blk_plug plug;
 
-		continue_at(cl, write_dirty, io->dc->writeback_write_wq);
+	spin_lock(&dc->writeback_ios_lock);
+	if (RB_EMPTY_ROOT(&dc->writeback_ios)) {
+		spin_unlock(&dc->writeback_ios_lock);
 		return;
 	}
 
-	next_sequence = io->sequence + 1;
+	io = RB_FIRST(&dc->writeback_ios, struct dirty_io, node);
+	if (io->sequence != atomic_read(&dc->writeback_sequence_next)) {
+		spin_unlock(&dc->writeback_ios_lock);
+		return;
+	}
 
-	/*
-	 * IO errors are signalled using the dirty bit on the key.
-	 * If we failed to read, we should not attempt to write to the
-	 * backing device.  Instead, immediately go to write_dirty_finish
-	 * to clean up.
-	 */
-	if (KEY_DIRTY(&w->key)) {
-		dirty_init(w);
-		io->bio.bi_opf = REQ_OP_WRITE;
-		io->bio.bi_iter.bi_sector = KEY_START(&w->key);
-		bio_set_dev(&io->bio, io->dc->bdev);
-		io->bio.bi_end_io	= dirty_endio;
-
-		/* I/O request sent to backing device */
-		closure_bio_submit(io->dc->disk.c, &io->bio, cl);
+	blk_start_plug(&plug);
+	next_sequence = io->sequence;
+
+	while(io) {
+		if (io->sequence != next_sequence)
+			break;
+
+		rb_erase(&io->node, &dc->writeback_ios);
+		spin_unlock(&dc->writeback_ios_lock);
+		w = io->bio.bi_private;
+		/*
+		 * IO errors are signalled using the dirty bit on the key.
+		 * If we failed to read, we should not attempt to write to the
+		 * backing device.  Instead, immediately go to write_dirty_finish
+		 * to clean up.
+		 */
+		if (KEY_DIRTY(&w->key)) {
+			dirty_init(w);
+			io->bio.bi_opf = REQ_OP_WRITE;
+			io->bio.bi_iter.bi_sector = KEY_START(&w->key);
+			bio_set_dev(&io->bio, io->dc->bdev);
+			io->bio.bi_end_io	= dirty_endio;
+
+			/* I/O request sent to backing device */
+			closure_bio_submit(io->dc->disk.c, &io->bio, &io->cl);
+		}
+
+		continue_at(&io->cl, write_dirty_finish, io->dc->writeback_write_wq);
+
+		spin_lock(&dc->writeback_ios_lock);
+		io = RB_FIRST(&dc->writeback_ios, struct dirty_io, node);
+		next_sequence++;
 	}
 
 	atomic_set(&dc->writeback_sequence_next, next_sequence);
-	closure_wake_up(&dc->writeback_ordering_wait);
-
-	continue_at(cl, write_dirty_finish, io->dc->writeback_write_wq);
+	spin_unlock(&dc->writeback_ios_lock);
+	blk_finish_plug(&plug);
 }
 
 static void read_dirty_endio(struct bio *bio)
@@ -469,7 +498,7 @@ static void read_dirty_submit(struct closure *cl)
 
 	closure_bio_submit(io->dc->disk.c, &io->bio, cl);
 
-	continue_at(cl, write_dirty, io->dc->writeback_write_wq);
+	continue_at(cl, queue_dirty_write, io->dc->writeback_write_wq);
 }
 
 static void start_wb_inflight(struct cached_dev *dc)
@@ -578,6 +607,7 @@ static void read_dirty(struct cached_dev *dc)
 			w->private	= io;
 			io->dc		= dc;
 			io->sequence    = sequence++;
+			RB_CLEAR_NODE(&io->node);
 
 			dirty_init(w);
 			io->bio.bi_opf = REQ_OP_READ;
@@ -1066,6 +1096,10 @@ void bch_cached_dev_writeback_init(struct cached_dev *dc)
 	init_rwsem(&dc->writeback_lock);
 	bch_keybuf_init(&dc->writeback_keys);
 
+	spin_lock_init(&dc->writeback_ios_lock);
+	dc->writeback_ios		= RB_ROOT;
+	INIT_WORK(&dc->write_dirty_work, write_dirty);
+
 	dc->writeback_metadata		= true;
 	dc->writeback_running		= false;
 	dc->writeback_consider_fragment = true;
-- 
2.17.1

[PATCH 3/3] bcache: support overlay bcache

[mingzhe.zou]

From: Dongsheng Yang <dongsheng.yang@easystack.cn>

If we want to build a bcache device with backing device of a bcache flash device,
we will fail with creating a duplicated sysfs filename.

E.g:
(1) we create bcache0 with vdc, then there is "/sys/block/bcache0/bcache" as a link to "/sys/block/vdc/bcache"
 $ readlink /sys/block/bcache0/bcache
../../../pci0000:00/0000:00:0b.0/virtio4/block/vdc/bcache

(2) if we continue to create bcache1 with bcache0:
 $ make-bcache -B /dev/bcache0

We will fail to register bdev with "sysfs: cannot create duplicate filename '/devices/virtual/block/bcache0/bcache'"

How this commit solving this problem?
E.g:
   we have vdf as cache disk, vdc as backing disk.

 $ make-bcache -C /dev/vdf -B /dev/vdc --wipe-bcache
 $ echo 100G > /sys/block/vdf/bcache_cache/set/flash_vol_create
 $ lsblk
NAME                       MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
vdf                        252:80   0   50G  0 disk
├─bcache0                  251:0    0  100G  0 disk
└─bcache1                  251:128  0  100G  0 disk
vdc                        252:32   0  100G  0 disk
└─bcache0                  251:0    0  100G  0 disk

(a) rename sysfs file to more meaningful name:
(a.2) bcahce_cache -> sysfs filename under cache disk (/sys/block/vdf/bcache_cache)
(a.1) bcache_fdev -> flash bcache device (/sys/block/bcache1/bcache_fdev)
(a.4) bcache_bdev -> sysfs filename for backing disk (/sys/block/vdc/bcache_bdev)
(a.3) bcache_cdev -> link to /sys/block/vdc/bcache_bdev (/sys/block/bcache0/bcache_cdev)

(b) create ->bcache lagacy link file for backward compatability
$ ll /sys/block/vdc/bcache
lrwxrwxrwx 1 root root 0 Oct 26 11:21 /sys/block/vdc/bcache -> bcache_bdev
$ ll /sys/block/bcache0/bcache
lrwxrwxrwx 1 root root 0 Oct 26 11:21 /sys/block/bcache0/bcache -> ../../../pci0000:00/0000:00:0b.0/virtio4/block/vdc/bcache_bdev
$ ll /sys/block/bcache1/bcache
lrwxrwxrwx 1 root root 0 Oct 26 11:19 /sys/block/bcache1/bcache -> bcache_fdev
$ ll /sys/block/vdf/bcache
lrwxrwxrwx 1 root root 0 Oct 26 11:17 /sys/block/vdf/bcache -> bcache_cache

These link are created with sysfs_create_link_nowarn(), that means, we dont
care about the failure when creating if these links are already created.
Because these lagacy sysfile are only for backwards compatability in no-overlay usecase
of bcache, in the no-overlay use, bcache will never create duplicated link.

In overlay usecase after this commit, please dont use bcache link any more, instead
use bcache_cdev, bcache_fdev, bcache_bdev or bcache_cache.

Then we can create a cached_dev with bcache1 (flash dev) as backing dev.
$ make-bcache -B /dev/bcache1
$ lsblk
NAME                       MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
vdf                        252:80   0   50G  0 disk
├─bcache0                  251:0    0  100G  0 disk
└─bcache1                  251:128  0  100G  0 disk
  └─bcache2                251:256  0  100G  0 disk

As a result there is a cached device bcache2 with backing device of a flash device bcache1.
        ----------------------------
        | bcache2 (cached_dev)     |
        | ------------------------ |
        | |   sdb (cache_dev)    | |
        | ------------------------ |
        | ------------------------ |
        | |   bcache1 (flash_dev)| |
        | ------------------------ |
        ----------------------------

Signed-off-by: Dongsheng Yang <dongsheng.yang@easystack.cn>
Signed-off-by: mingzhe <mingzhe.zou@easystack.cn>
---
 drivers/md/bcache/super.c | 40 +++++++++++++++++++++++++++++++++++----
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
index ba3909bb6bea..0ca8c05831c9 100644
--- a/drivers/md/bcache/super.c
+++ b/drivers/md/bcache/super.c
@@ -1087,12 +1087,19 @@ int bch_cached_dev_run(struct cached_dev *dc)
 
 	if (sysfs_create_link(&d->kobj, &disk_to_dev(d->disk)->kobj, "dev") ||
 	    sysfs_create_link(&disk_to_dev(d->disk)->kobj,
-			      &d->kobj, "bcache")) {
+			      &d->kobj, "bcache_cdev")) {
 		pr_err("Couldn't create bcache dev <-> disk sysfs symlinks\n");
 		ret = -ENOMEM;
 		goto out;
 	}
 
+	ret = sysfs_create_link_nowarn(&disk_to_dev(d->disk)->kobj,
+				       &d->kobj, "bcache");
+	if (ret && ret != -EEXIST) {
+		pr_err("Couldn't create lagacy disk sysfs ->bcache symlinks\n");
+		goto out;
+	}
+
 	dc->status_update_thread = kthread_run(cached_dev_status_update,
 					       dc, "bcache_status_update");
 	if (IS_ERR(dc->status_update_thread)) {
@@ -1461,8 +1468,17 @@ static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
 		goto err;
 
 	err = "error creating kobject";
-	if (kobject_add(&dc->disk.kobj, bdev_kobj(bdev), "bcache"))
+	if (kobject_add(&dc->disk.kobj, bdev_kobj(bdev), "bcache_bdev"))
 		goto err;
+
+	err = "error creating lagacy sysfs link";
+	ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
+				       &dc->disk.kobj, "bcache");
+	if (ret && ret != -EEXIST) {
+		pr_err("Couldn't create lagacy disk sysfs ->bcache");
+		goto err;
+	}
+
 	if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj))
 		goto err;
 
@@ -1524,6 +1540,7 @@ static void flash_dev_flush(struct closure *cl)
 
 static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
 {
+	int ret;
 	int err = -ENOMEM;
 	struct bcache_device *d = kzalloc(sizeof(struct bcache_device),
 					  GFP_KERNEL);
@@ -1546,10 +1563,17 @@ static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
 	if (err)
 		goto err;
 
-	err = kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache");
+	err = kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache_fdev");
 	if (err)
 		goto err;
 
+	ret = sysfs_create_link_nowarn(&disk_to_dev(d->disk)->kobj,
+				       &d->kobj, "bcache");
+	if (ret && ret != -EEXIST) {
+		pr_err("Couldn't create lagacy flash dev ->bcache");
+		goto err;
+	}
+
 	bcache_device_link(d, c, "volume");
 
 	if (bch_has_feature_obso_large_bucket(&c->cache->sb)) {
@@ -2370,12 +2394,20 @@ static int register_cache(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
 		goto err;
 	}
 
-	if (kobject_add(&ca->kobj, bdev_kobj(bdev), "bcache")) {
+	if (kobject_add(&ca->kobj, bdev_kobj(bdev), "bcache_cache")) {
 		err = "error calling kobject_add";
 		ret = -ENOMEM;
 		goto out;
 	}
 
+	ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
+				       &ca->kobj, "bcache");
+	if (ret && ret != -EEXIST) {
+		pr_err("Couldn't create lagacy disk sysfs ->cache symlinks\n");
+		goto out;
+	} else
+		ret = 0;
+
 	mutex_lock(&bch_register_lock);
 	err = register_cache_set(ca);
 	mutex_unlock(&bch_register_lock);
-- 
2.17.1

Re: [PATCH 3/3] bcache: support overlay bcache

[kernel test robot]

Hi,

I love your patch! Yet something to improve:

[auto build test ERROR on linus/master]
[also build test ERROR on v6.2-rc6 next-20230201]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/mingzhe-zou-easystack-cn/bcache-submit-writeback-inflight-dirty-writes-in-batch/20230201-145421
patch link:    https://lore.kernel.org/r/20230201065202.17610-3-mingzhe.zou%40easystack.cn
patch subject: [PATCH 3/3] bcache: support overlay bcache
config: alpha-randconfig-r026-20230129 (https://download.01.org/0day-ci/archive/20230202/202302020120.FNJ9XwCm-lkp@intel.com/config)
compiler: alpha-linux-gcc (GCC) 12.1.0
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/intel-lab-lkp/linux/commit/785b6ea709e3008e2df009d5555c80db709e6d5f
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review mingzhe-zou-easystack-cn/bcache-submit-writeback-inflight-dirty-writes-in-batch/20230201-145421
        git checkout 785b6ea709e3008e2df009d5555c80db709e6d5f
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=alpha olddefconfig
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=alpha SHELL=/bin/bash drivers/md/bcache/

If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

   drivers/md/bcache/super.c: In function 'register_bdev':
>> drivers/md/bcache/super.c:1475:41: error: implicit declaration of function 'part_to_dev'; did you mean 'part_devt'? [-Werror=implicit-function-declaration]
    1475 |         ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
         |                                         ^~~~~~~~~~~
         |                                         part_devt
>> drivers/md/bcache/super.c:1475:59: error: 'struct block_device' has no member named 'bd_part'; did you mean 'bd_partno'?
    1475 |         ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
         |                                                           ^~~~~~~
         |                                                           bd_partno
   drivers/md/bcache/super.c: In function 'register_cache':
   drivers/md/bcache/super.c:2403:59: error: 'struct block_device' has no member named 'bd_part'; did you mean 'bd_partno'?
    2403 |         ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
         |                                                           ^~~~~~~
         |                                                           bd_partno
   cc1: some warnings being treated as errors


vim +1475 drivers/md/bcache/super.c

  1453	
  1454	static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
  1455					 struct block_device *bdev,
  1456					 struct cached_dev *dc)
  1457	{
  1458		const char *err = "cannot allocate memory";
  1459		struct cache_set *c;
  1460		int ret = -ENOMEM;
  1461	
  1462		memcpy(&dc->sb, sb, sizeof(struct cache_sb));
  1463		dc->bdev = bdev;
  1464		dc->bdev->bd_holder = dc;
  1465		dc->sb_disk = sb_disk;
  1466	
  1467		if (cached_dev_init(dc, sb->block_size << 9))
  1468			goto err;
  1469	
  1470		err = "error creating kobject";
  1471		if (kobject_add(&dc->disk.kobj, bdev_kobj(bdev), "bcache_bdev"))
  1472			goto err;
  1473	
  1474		err = "error creating lagacy sysfs link";
> 1475		ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
  1476					       &dc->disk.kobj, "bcache");
  1477		if (ret && ret != -EEXIST) {
  1478			pr_err("Couldn't create lagacy disk sysfs ->bcache");
  1479			goto err;
  1480		}
  1481	
  1482		if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj))
  1483			goto err;
  1484	
  1485		pr_info("registered backing device %pg\n", dc->bdev);
  1486	
  1487		list_add(&dc->list, &uncached_devices);
  1488		/* attach to a matched cache set if it exists */
  1489		list_for_each_entry(c, &bch_cache_sets, list)
  1490			bch_cached_dev_attach(dc, c, NULL);
  1491	
  1492		if (BDEV_STATE(&dc->sb) == BDEV_STATE_NONE ||
  1493		    BDEV_STATE(&dc->sb) == BDEV_STATE_STALE) {
  1494			err = "failed to run cached device";
  1495			ret = bch_cached_dev_run(dc);
  1496			if (ret)
  1497				goto err;
  1498		}
  1499	
  1500		return 0;
  1501	err:
  1502		pr_notice("error %pg: %s\n", dc->bdev, err);
  1503		bcache_device_stop(&dc->disk);
  1504		return ret;
  1505	}
  1506	

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests

Re: [PATCH 3/3] bcache: support overlay bcache

[kernel test robot]

Hi,

I love your patch! Yet something to improve:

[auto build test ERROR on linus/master]
[also build test ERROR on v6.2-rc6 next-20230201]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/mingzhe-zou-easystack-cn/bcache-submit-writeback-inflight-dirty-writes-in-batch/20230201-145421
patch link:    https://lore.kernel.org/r/20230201065202.17610-3-mingzhe.zou%40easystack.cn
patch subject: [PATCH 3/3] bcache: support overlay bcache
config: x86_64-randconfig-a014-20230130 (https://download.01.org/0day-ci/archive/20230202/202302020147.4HT8cQF3-lkp@intel.com/config)
compiler: clang version 14.0.6 (https://github.com/llvm/llvm-project f28c006a5895fc0e329fe15fead81e37457cb1d1)
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # https://github.com/intel-lab-lkp/linux/commit/785b6ea709e3008e2df009d5555c80db709e6d5f
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review mingzhe-zou-easystack-cn/bcache-submit-writeback-inflight-dirty-writes-in-batch/20230201-145421
        git checkout 785b6ea709e3008e2df009d5555c80db709e6d5f
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=x86_64 olddefconfig
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=x86_64 SHELL=/bin/bash

If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

>> drivers/md/bcache/super.c:1475:34: error: implicit declaration of function 'part_to_dev' is invalid in C99 [-Werror,-Wimplicit-function-declaration]
           ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
                                           ^
>> drivers/md/bcache/super.c:1475:62: error: member reference type 'int' is not a pointer
           ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
                                           ~~~~~~~~~~~~~~~~~~~~~~~~~~  ^
>> drivers/md/bcache/super.c:1475:52: error: no member named 'bd_part' in 'struct block_device'
           ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
                                                       ~~~~  ^
   drivers/md/bcache/super.c:2403:34: error: implicit declaration of function 'part_to_dev' is invalid in C99 [-Werror,-Wimplicit-function-declaration]
           ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
                                           ^
   drivers/md/bcache/super.c:2403:62: error: member reference type 'int' is not a pointer
           ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
                                           ~~~~~~~~~~~~~~~~~~~~~~~~~~  ^
   drivers/md/bcache/super.c:2403:52: error: no member named 'bd_part' in 'struct block_device'
           ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
                                                       ~~~~  ^
   6 errors generated.


vim +/part_to_dev +1475 drivers/md/bcache/super.c

  1453	
  1454	static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
  1455					 struct block_device *bdev,
  1456					 struct cached_dev *dc)
  1457	{
  1458		const char *err = "cannot allocate memory";
  1459		struct cache_set *c;
  1460		int ret = -ENOMEM;
  1461	
  1462		memcpy(&dc->sb, sb, sizeof(struct cache_sb));
  1463		dc->bdev = bdev;
  1464		dc->bdev->bd_holder = dc;
  1465		dc->sb_disk = sb_disk;
  1466	
  1467		if (cached_dev_init(dc, sb->block_size << 9))
  1468			goto err;
  1469	
  1470		err = "error creating kobject";
  1471		if (kobject_add(&dc->disk.kobj, bdev_kobj(bdev), "bcache_bdev"))
  1472			goto err;
  1473	
  1474		err = "error creating lagacy sysfs link";
> 1475		ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
  1476					       &dc->disk.kobj, "bcache");
  1477		if (ret && ret != -EEXIST) {
  1478			pr_err("Couldn't create lagacy disk sysfs ->bcache");
  1479			goto err;
  1480		}
  1481	
  1482		if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj))
  1483			goto err;
  1484	
  1485		pr_info("registered backing device %pg\n", dc->bdev);
  1486	
  1487		list_add(&dc->list, &uncached_devices);
  1488		/* attach to a matched cache set if it exists */
  1489		list_for_each_entry(c, &bch_cache_sets, list)
  1490			bch_cached_dev_attach(dc, c, NULL);
  1491	
  1492		if (BDEV_STATE(&dc->sb) == BDEV_STATE_NONE ||
  1493		    BDEV_STATE(&dc->sb) == BDEV_STATE_STALE) {
  1494			err = "failed to run cached device";
  1495			ret = bch_cached_dev_run(dc);
  1496			if (ret)
  1497				goto err;
  1498		}
  1499	
  1500		return 0;
  1501	err:
  1502		pr_notice("error %pg: %s\n", dc->bdev, err);
  1503		bcache_device_stop(&dc->disk);
  1504		return ret;
  1505	}
  1506	

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests

Re: [PATCH 3/3] bcache: support overlay bcache

[Eric Wheeler]

[-- Attachment #1: Type: text/plain, Size: 7151 bytes --]

On Wed, 1 Feb 2023, mingzhe.zou@easystack.cn wrote:
> From: Dongsheng Yang <dongsheng.yang@easystack.cn>
> 
> If we want to build a bcache device with backing device of a bcache flash device,
> we will fail with creating a duplicated sysfs filename.
> 
> E.g:
> (1) we create bcache0 with vdc, then there is "/sys/block/bcache0/bcache" as a link to "/sys/block/vdc/bcache"
>  $ readlink /sys/block/bcache0/bcache
> ../../../pci0000:00/0000:00:0b.0/virtio4/block/vdc/bcache
> 
> (2) if we continue to create bcache1 with bcache0:
>  $ make-bcache -B /dev/bcache0
> 
> We will fail to register bdev with "sysfs: cannot create duplicate 
> filename '/devices/virtual/block/bcache0/bcache'"
> 
> How this commit solving this problem?
> E.g:
>    we have vdf as cache disk, vdc as backing disk.
>
>  $ make-bcache -C /dev/vdf -B /dev/vdc --wipe-bcache
>  $ echo 100G > /sys/block/vdf/bcache_cache/set/flash_vol_create
>  $ lsblk
> NAME                       MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
> vdf                        252:80   0   50G  0 disk
> ├─bcache0                  251:0    0  100G  0 disk
> └─bcache1                  251:128  0  100G  0 disk
> vdc                        252:32   0  100G  0 disk
> └─bcache0                  251:0    0  100G  0 disk
> 
> (a) rename sysfs file to more meaningful name:
> (a.2) bcahce_cache -> sysfs filename under cache disk (/sys/block/vdf/bcache_cache)
> (a.1) bcache_fdev -> flash bcache device (/sys/block/bcache1/bcache_fdev)
> (a.4) bcache_bdev -> sysfs filename for backing disk (/sys/block/vdc/bcache_bdev)
> (a.3) bcache_cdev -> link to /sys/block/vdc/bcache_bdev (/sys/block/bcache0/bcache_cdev)

Good idea.

> (b) create ->bcache lagacy link file for backward compatability
> $ ll /sys/block/vdc/bcache
> lrwxrwxrwx 1 root root 0 Oct 26 11:21 /sys/block/vdc/bcache -> bcache_bdev
> $ ll /sys/block/bcache0/bcache
> lrwxrwxrwx 1 root root 0 Oct 26 11:21 /sys/block/bcache0/bcache -> ../../../pci0000:00/0000:00:0b.0/virtio4/block/vdc/bcache_bdev
> $ ll /sys/block/bcache1/bcache
> lrwxrwxrwx 1 root root 0 Oct 26 11:19 /sys/block/bcache1/bcache -> bcache_fdev
> $ ll /sys/block/vdf/bcache
> lrwxrwxrwx 1 root root 0 Oct 26 11:17 /sys/block/vdf/bcache -> bcache_cache
> 
> These link are created with sysfs_create_link_nowarn(), that means, we dont
> care about the failure when creating if these links are already created.
> Because these lagacy sysfile are only for backwards compatability in no-overlay usecase
> of bcache, in the no-overlay use, bcache will never create duplicated link.

awesome.
 
> In overlay usecase after this commit, please dont use bcache link any more, instead
> use bcache_cdev, bcache_fdev, bcache_bdev or bcache_cache.
> 
> Then we can create a cached_dev with bcache1 (flash dev) as backing dev.
> $ make-bcache -B /dev/bcache1
> $ lsblk
> NAME                       MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
> vdf                        252:80   0   50G  0 disk
> ├─bcache0                  251:0    0  100G  0 disk
> └─bcache1                  251:128  0  100G  0 disk
>   └─bcache2                251:256  0  100G  0 disk
> 
> As a result there is a cached device bcache2 with backing device of a flash device bcache1.
>         ----------------------------
>         | bcache2 (cached_dev)     |
>         | ------------------------ |
>         | |   sdb (cache_dev)    | |
>         | ------------------------ |
>         | ------------------------ |
>         | |   bcache1 (flash_dev)| |
>         | ------------------------ |
>         ----------------------------

Does this allow an arbitrary depth of bcache stacking?

-Eric

> 
> Signed-off-by: Dongsheng Yang <dongsheng.yang@easystack.cn>
> Signed-off-by: mingzhe <mingzhe.zou@easystack.cn>
> ---
>  drivers/md/bcache/super.c | 40 +++++++++++++++++++++++++++++++++++----
>  1 file changed, 36 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
> index ba3909bb6bea..0ca8c05831c9 100644
> --- a/drivers/md/bcache/super.c
> +++ b/drivers/md/bcache/super.c
> @@ -1087,12 +1087,19 @@ int bch_cached_dev_run(struct cached_dev *dc)
>  
>  	if (sysfs_create_link(&d->kobj, &disk_to_dev(d->disk)->kobj, "dev") ||
>  	    sysfs_create_link(&disk_to_dev(d->disk)->kobj,
> -			      &d->kobj, "bcache")) {
> +			      &d->kobj, "bcache_cdev")) {
>  		pr_err("Couldn't create bcache dev <-> disk sysfs symlinks\n");
>  		ret = -ENOMEM;
>  		goto out;
>  	}
>  
> +	ret = sysfs_create_link_nowarn(&disk_to_dev(d->disk)->kobj,
> +				       &d->kobj, "bcache");
> +	if (ret && ret != -EEXIST) {
> +		pr_err("Couldn't create lagacy disk sysfs ->bcache symlinks\n");
> +		goto out;
> +	}
> +
>  	dc->status_update_thread = kthread_run(cached_dev_status_update,
>  					       dc, "bcache_status_update");
>  	if (IS_ERR(dc->status_update_thread)) {
> @@ -1461,8 +1468,17 @@ static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
>  		goto err;
>  
>  	err = "error creating kobject";
> -	if (kobject_add(&dc->disk.kobj, bdev_kobj(bdev), "bcache"))
> +	if (kobject_add(&dc->disk.kobj, bdev_kobj(bdev), "bcache_bdev"))
>  		goto err;
> +
> +	err = "error creating lagacy sysfs link";
> +	ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
> +				       &dc->disk.kobj, "bcache");
> +	if (ret && ret != -EEXIST) {
> +		pr_err("Couldn't create lagacy disk sysfs ->bcache");
> +		goto err;
> +	}
> +
>  	if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj))
>  		goto err;
>  
> @@ -1524,6 +1540,7 @@ static void flash_dev_flush(struct closure *cl)
>  
>  static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
>  {
> +	int ret;
>  	int err = -ENOMEM;
>  	struct bcache_device *d = kzalloc(sizeof(struct bcache_device),
>  					  GFP_KERNEL);
> @@ -1546,10 +1563,17 @@ static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
>  	if (err)
>  		goto err;
>  
> -	err = kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache");
> +	err = kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache_fdev");
>  	if (err)
>  		goto err;
>  
> +	ret = sysfs_create_link_nowarn(&disk_to_dev(d->disk)->kobj,
> +				       &d->kobj, "bcache");
> +	if (ret && ret != -EEXIST) {
> +		pr_err("Couldn't create lagacy flash dev ->bcache");
> +		goto err;
> +	}
> +
>  	bcache_device_link(d, c, "volume");
>  
>  	if (bch_has_feature_obso_large_bucket(&c->cache->sb)) {
> @@ -2370,12 +2394,20 @@ static int register_cache(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
>  		goto err;
>  	}
>  
> -	if (kobject_add(&ca->kobj, bdev_kobj(bdev), "bcache")) {
> +	if (kobject_add(&ca->kobj, bdev_kobj(bdev), "bcache_cache")) {
>  		err = "error calling kobject_add";
>  		ret = -ENOMEM;
>  		goto out;
>  	}
>  
> +	ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
> +				       &ca->kobj, "bcache");
> +	if (ret && ret != -EEXIST) {
> +		pr_err("Couldn't create lagacy disk sysfs ->cache symlinks\n");
> +		goto out;
> +	} else
> +		ret = 0;
> +
>  	mutex_lock(&bch_register_lock);
>  	err = register_cache_set(ca);
>  	mutex_unlock(&bch_register_lock);
> -- 
> 2.17.1
> 
> 

Re: [PATCH 3/3] bcache: support overlay bcache

[mingzhe]

在 2023/2/2 02:13, Eric Wheeler 写道:
> On Wed, 1 Feb 2023, mingzhe.zou@easystack.cn wrote:
>> From: Dongsheng Yang <dongsheng.yang@easystack.cn>
>>
>> If we want to build a bcache device with backing device of a bcache flash device,
>> we will fail with creating a duplicated sysfs filename.
>>
>> E.g:
>> (1) we create bcache0 with vdc, then there is "/sys/block/bcache0/bcache" as a link to "/sys/block/vdc/bcache"
>>   $ readlink /sys/block/bcache0/bcache
>> ../../../pci0000:00/0000:00:0b.0/virtio4/block/vdc/bcache
>>
>> (2) if we continue to create bcache1 with bcache0:
>>   $ make-bcache -B /dev/bcache0
>>
>> We will fail to register bdev with "sysfs: cannot create duplicate
>> filename '/devices/virtual/block/bcache0/bcache'"
>>
>> How this commit solving this problem?
>> E.g:
>>     we have vdf as cache disk, vdc as backing disk.
>>
>>   $ make-bcache -C /dev/vdf -B /dev/vdc --wipe-bcache
>>   $ echo 100G > /sys/block/vdf/bcache_cache/set/flash_vol_create
>>   $ lsblk
>> NAME                       MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
>> vdf                        252:80   0   50G  0 disk
>> ├─bcache0                  251:0    0  100G  0 disk
>> └─bcache1                  251:128  0  100G  0 disk
>> vdc                        252:32   0  100G  0 disk
>> └─bcache0                  251:0    0  100G  0 disk
>>
>> (a) rename sysfs file to more meaningful name:
>> (a.2) bcahce_cache -> sysfs filename under cache disk (/sys/block/vdf/bcache_cache)
>> (a.1) bcache_fdev -> flash bcache device (/sys/block/bcache1/bcache_fdev)
>> (a.4) bcache_bdev -> sysfs filename for backing disk (/sys/block/vdc/bcache_bdev)
>> (a.3) bcache_cdev -> link to /sys/block/vdc/bcache_bdev (/sys/block/bcache0/bcache_cdev)
> 
> Good idea.
> 
>> (b) create ->bcache lagacy link file for backward compatability
>> $ ll /sys/block/vdc/bcache
>> lrwxrwxrwx 1 root root 0 Oct 26 11:21 /sys/block/vdc/bcache -> bcache_bdev
>> $ ll /sys/block/bcache0/bcache
>> lrwxrwxrwx 1 root root 0 Oct 26 11:21 /sys/block/bcache0/bcache -> ../../../pci0000:00/0000:00:0b.0/virtio4/block/vdc/bcache_bdev
>> $ ll /sys/block/bcache1/bcache
>> lrwxrwxrwx 1 root root 0 Oct 26 11:19 /sys/block/bcache1/bcache -> bcache_fdev
>> $ ll /sys/block/vdf/bcache
>> lrwxrwxrwx 1 root root 0 Oct 26 11:17 /sys/block/vdf/bcache -> bcache_cache
>>
>> These link are created with sysfs_create_link_nowarn(), that means, we dont
>> care about the failure when creating if these links are already created.
>> Because these lagacy sysfile are only for backwards compatability in no-overlay usecase
>> of bcache, in the no-overlay use, bcache will never create duplicated link.
> 
> awesome.
>   
>> In overlay usecase after this commit, please dont use bcache link any more, instead
>> use bcache_cdev, bcache_fdev, bcache_bdev or bcache_cache.
>>
>> Then we can create a cached_dev with bcache1 (flash dev) as backing dev.
>> $ make-bcache -B /dev/bcache1
>> $ lsblk
>> NAME                       MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
>> vdf                        252:80   0   50G  0 disk
>> ├─bcache0                  251:0    0  100G  0 disk
>> └─bcache1                  251:128  0  100G  0 disk
>>    └─bcache2                251:256  0  100G  0 disk
>>
>> As a result there is a cached device bcache2 with backing device of a flash device bcache1.
>>          ----------------------------
>>          | bcache2 (cached_dev)     |
>>          | ------------------------ |
>>          | |   sdb (cache_dev)    | |
>>          | ------------------------ |
>>          | ------------------------ |
>>          | |   bcache1 (flash_dev)| |
>>          | ------------------------ |
>>          ----------------------------
> 
> Does this allow an arbitrary depth of bcache stacking?
> 
> -Eric
> 
More than 2 layers we did not test, but should be allowed.

mingzhe
>>
>> Signed-off-by: Dongsheng Yang <dongsheng.yang@easystack.cn>
>> Signed-off-by: mingzhe <mingzhe.zou@easystack.cn>
>> ---
>>   drivers/md/bcache/super.c | 40 +++++++++++++++++++++++++++++++++++----
>>   1 file changed, 36 insertions(+), 4 deletions(-)
>>
>> diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
>> index ba3909bb6bea..0ca8c05831c9 100644
>> --- a/drivers/md/bcache/super.c
>> +++ b/drivers/md/bcache/super.c
>> @@ -1087,12 +1087,19 @@ int bch_cached_dev_run(struct cached_dev *dc)
>>   
>>   	if (sysfs_create_link(&d->kobj, &disk_to_dev(d->disk)->kobj, "dev") ||
>>   	    sysfs_create_link(&disk_to_dev(d->disk)->kobj,
>> -			      &d->kobj, "bcache")) {
>> +			      &d->kobj, "bcache_cdev")) {
>>   		pr_err("Couldn't create bcache dev <-> disk sysfs symlinks\n");
>>   		ret = -ENOMEM;
>>   		goto out;
>>   	}
>>   
>> +	ret = sysfs_create_link_nowarn(&disk_to_dev(d->disk)->kobj,
>> +				       &d->kobj, "bcache");
>> +	if (ret && ret != -EEXIST) {
>> +		pr_err("Couldn't create lagacy disk sysfs ->bcache symlinks\n");
>> +		goto out;
>> +	}
>> +
>>   	dc->status_update_thread = kthread_run(cached_dev_status_update,
>>   					       dc, "bcache_status_update");
>>   	if (IS_ERR(dc->status_update_thread)) {
>> @@ -1461,8 +1468,17 @@ static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
>>   		goto err;
>>   
>>   	err = "error creating kobject";
>> -	if (kobject_add(&dc->disk.kobj, bdev_kobj(bdev), "bcache"))
>> +	if (kobject_add(&dc->disk.kobj, bdev_kobj(bdev), "bcache_bdev"))
>>   		goto err;
>> +
>> +	err = "error creating lagacy sysfs link";
>> +	ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
>> +				       &dc->disk.kobj, "bcache");
>> +	if (ret && ret != -EEXIST) {
>> +		pr_err("Couldn't create lagacy disk sysfs ->bcache");
>> +		goto err;
>> +	}
>> +
>>   	if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj))
>>   		goto err;
>>   
>> @@ -1524,6 +1540,7 @@ static void flash_dev_flush(struct closure *cl)
>>   
>>   static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
>>   {
>> +	int ret;
>>   	int err = -ENOMEM;
>>   	struct bcache_device *d = kzalloc(sizeof(struct bcache_device),
>>   					  GFP_KERNEL);
>> @@ -1546,10 +1563,17 @@ static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
>>   	if (err)
>>   		goto err;
>>   
>> -	err = kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache");
>> +	err = kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache_fdev");
>>   	if (err)
>>   		goto err;
>>   
>> +	ret = sysfs_create_link_nowarn(&disk_to_dev(d->disk)->kobj,
>> +				       &d->kobj, "bcache");
>> +	if (ret && ret != -EEXIST) {
>> +		pr_err("Couldn't create lagacy flash dev ->bcache");
>> +		goto err;
>> +	}
>> +
>>   	bcache_device_link(d, c, "volume");
>>   
>>   	if (bch_has_feature_obso_large_bucket(&c->cache->sb)) {
>> @@ -2370,12 +2394,20 @@ static int register_cache(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
>>   		goto err;
>>   	}
>>   
>> -	if (kobject_add(&ca->kobj, bdev_kobj(bdev), "bcache")) {
>> +	if (kobject_add(&ca->kobj, bdev_kobj(bdev), "bcache_cache")) {
>>   		err = "error calling kobject_add";
>>   		ret = -ENOMEM;
>>   		goto out;
>>   	}
>>   
>> +	ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
>> +				       &ca->kobj, "bcache");
>> +	if (ret && ret != -EEXIST) {
>> +		pr_err("Couldn't create lagacy disk sysfs ->cache symlinks\n");
>> +		goto out;
>> +	} else
>> +		ret = 0;
>> +
>>   	mutex_lock(&bch_register_lock);
>>   	err = register_cache_set(ca);
>>   	mutex_unlock(&bch_register_lock);
>> -- 
>> 2.17.1
>>

Re: [PATCH 3/3] bcache: support overlay bcache

[Cedric de Wijs]

<snip>
>>>
>>> Then we can create a cached_dev with bcache1 (flash dev) as backing dev.
>>> $ make-bcache -B /dev/bcache1
>>> $ lsblk
>>> NAME                       MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
>>> vdf                        252:80   0   50G  0 disk
>>> ├─bcache0                  251:0    0  100G  0 disk
>>> └─bcache1                  251:128  0  100G  0 disk
>>>    └─bcache2                251:256  0  100G  0 disk
>>>
>>> As a result there is a cached device bcache2 with backing device of a 
>>> flash device bcache1.
>>>          ----------------------------
>>>          | bcache2 (cached_dev)     |
>>>          | ------------------------ |
>>>          | |   sdb (cache_dev)    | |
>>>          | ------------------------ |
>>>          | ------------------------ |
>>>          | |   bcache1 (flash_dev)| |
>>>          | ------------------------ |
>>>          ----------------------------
>>
>> Does this allow an arbitrary depth of bcache stacking?
>>
>> -Eric
>>
> More than 2 layers we did not test, but should be allowed.
> 
> mingzhe
>>>
>>> Signed-off-by: Dongsheng Yang <dongsheng.yang@easystack.cn>
>>> Signed-off-by: mingzhe <mingzhe.zou@easystack.cn>
>>> ---
>>>   drivers/md/bcache/super.c | 40 +++++++++++++++++++++++++++++++++++----
>>>   1 file changed, 36 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
>>> index ba3909bb6bea..0ca8c05831c9 100644
>>> --- a/drivers/md/bcache/super.c
>>> +++ b/drivers/md/bcache/super.c
<snip>

Hi All,

I've not seen this commit appear in the mainline kernel yet. In 2023, 
only this commit changed super.c in 2023:
2023-04-25	block/drivers: remove dead clear of random flag
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/log/drivers/md/bcache/super.c?h=v6.4-rc4

What's preventing this patch from going into the mainline kernel?

Cheers,
Cedric

_________________________________________________________________
________________________________________________________
Your E-Mail. Your Cloud. Your Office. eclipso Mail & Cloud. https://www.eclipso.de

Re: [PATCH 3/3] bcache: support overlay bcache

[Coly Li]

> 2023年5月30日 17:12，Cedric de Wijs <cedric.dewijs@eclipso.eu> 写道：
> 
> <snip>
>>>> 
>>>> Then we can create a cached_dev with bcache1 (flash dev) as backing dev.
>>>> $ make-bcache -B /dev/bcache1
>>>> $ lsblk
>>>> NAME                       MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
>>>> vdf                        252:80   0   50G  0 disk
>>>> ├─bcache0                  251:0    0  100G  0 disk
>>>> └─bcache1                  251:128  0  100G  0 disk
>>>>    └─bcache2                251:256  0  100G  0 disk
>>>> 
>>>> As a result there is a cached device bcache2 with backing device of a flash device bcache1.
>>>>          ----------------------------
>>>>          | bcache2 (cached_dev)     |
>>>>          | ------------------------ |
>>>>          | |   sdb (cache_dev)    | |
>>>>          | ------------------------ |
>>>>          | ------------------------ |
>>>>          | |   bcache1 (flash_dev)| |
>>>>          | ------------------------ |
>>>>          ----------------------------
>>> 
>>> Does this allow an arbitrary depth of bcache stacking?
>>> 
>>> -Eric
>>> 
>> More than 2 layers we did not test, but should be allowed.
>> mingzhe
>>>> 
>>>> Signed-off-by: Dongsheng Yang <dongsheng.yang@easystack.cn>
>>>> Signed-off-by: mingzhe <mingzhe.zou@easystack.cn>
>>>> ---
>>>>   drivers/md/bcache/super.c | 40 +++++++++++++++++++++++++++++++++++----
>>>>   1 file changed, 36 insertions(+), 4 deletions(-)
>>>> 
>>>> diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c
>>>> index ba3909bb6bea..0ca8c05831c9 100644
>>>> --- a/drivers/md/bcache/super.c
>>>> +++ b/drivers/md/bcache/super.c
> <snip>
> 
> Hi All,
> 
> I've not seen this commit appear in the mainline kernel yet. In 2023, only this commit changed super.c in 2023:
> 2023-04-25 block/drivers: remove dead clear of random flag
> https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/log/drivers/md/bcache/super.c?h=v6.4-rc4
> 
> What's preventing this patch from going into the mainline kernel?

Code reviewer is the bottleneck. This series is in my todo list, but not on it yet. If Junhui Tang, or Guoju Fang may help to review the code, it can be a bit faster.
Of course if Kent reviews the code and supportive, I will take it immediately.

Thanks.

Coly Li

Re: [PATCH 3/3] bcache: support overlay bcache

[kernel test robot]

Hi,

I love your patch! Yet something to improve:

[auto build test ERROR on linus/master]
[also build test ERROR on v6.2-rc6 next-20230201]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/mingzhe-zou-easystack-cn/bcache-submit-writeback-inflight-dirty-writes-in-batch/20230201-145421
patch link:    https://lore.kernel.org/r/20230201065202.17610-3-mingzhe.zou%40easystack.cn
patch subject: [PATCH 3/3] bcache: support overlay bcache
config: riscv-randconfig-r042-20230130 (https://download.01.org/0day-ci/archive/20230202/202302020507.y0NPeWHf-lkp@intel.com/config)
compiler: clang version 16.0.0 (https://github.com/llvm/llvm-project 4196ca3278f78c6e19246e54ab0ecb364e37d66a)
reproduce (this is a W=1 build):
        wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # install riscv cross compiling tool for clang build
        # apt-get install binutils-riscv64-linux-gnu
        # https://github.com/intel-lab-lkp/linux/commit/785b6ea709e3008e2df009d5555c80db709e6d5f
        git remote add linux-review https://github.com/intel-lab-lkp/linux
        git fetch --no-tags linux-review mingzhe-zou-easystack-cn/bcache-submit-writeback-inflight-dirty-writes-in-batch/20230201-145421
        git checkout 785b6ea709e3008e2df009d5555c80db709e6d5f
        # save the config file
        mkdir build_dir && cp config build_dir/.config
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=riscv olddefconfig
        COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=riscv SHELL=/bin/bash

If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <lkp@intel.com>

All errors (new ones prefixed by >>):

>> drivers/md/bcache/super.c:1475:34: error: call to undeclared function 'part_to_dev'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
           ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
                                           ^
   drivers/md/bcache/super.c:1475:62: error: member reference type 'int' is not a pointer
           ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
                                           ~~~~~~~~~~~~~~~~~~~~~~~~~~  ^
   drivers/md/bcache/super.c:1475:52: error: no member named 'bd_part' in 'struct block_device'
           ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
                                                       ~~~~  ^
   drivers/md/bcache/super.c:2403:34: error: call to undeclared function 'part_to_dev'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
           ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
                                           ^
   drivers/md/bcache/super.c:2403:62: error: member reference type 'int' is not a pointer
           ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
                                           ~~~~~~~~~~~~~~~~~~~~~~~~~~  ^
   drivers/md/bcache/super.c:2403:52: error: no member named 'bd_part' in 'struct block_device'
           ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
                                                       ~~~~  ^
   6 errors generated.


vim +/part_to_dev +1475 drivers/md/bcache/super.c

  1453	
  1454	static int register_bdev(struct cache_sb *sb, struct cache_sb_disk *sb_disk,
  1455					 struct block_device *bdev,
  1456					 struct cached_dev *dc)
  1457	{
  1458		const char *err = "cannot allocate memory";
  1459		struct cache_set *c;
  1460		int ret = -ENOMEM;
  1461	
  1462		memcpy(&dc->sb, sb, sizeof(struct cache_sb));
  1463		dc->bdev = bdev;
  1464		dc->bdev->bd_holder = dc;
  1465		dc->sb_disk = sb_disk;
  1466	
  1467		if (cached_dev_init(dc, sb->block_size << 9))
  1468			goto err;
  1469	
  1470		err = "error creating kobject";
  1471		if (kobject_add(&dc->disk.kobj, bdev_kobj(bdev), "bcache_bdev"))
  1472			goto err;
  1473	
  1474		err = "error creating lagacy sysfs link";
> 1475		ret = sysfs_create_link_nowarn(&part_to_dev(bdev->bd_part)->kobj,
  1476					       &dc->disk.kobj, "bcache");
  1477		if (ret && ret != -EEXIST) {
  1478			pr_err("Couldn't create lagacy disk sysfs ->bcache");
  1479			goto err;
  1480		}
  1481	
  1482		if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj))
  1483			goto err;
  1484	
  1485		pr_info("registered backing device %pg\n", dc->bdev);
  1486	
  1487		list_add(&dc->list, &uncached_devices);
  1488		/* attach to a matched cache set if it exists */
  1489		list_for_each_entry(c, &bch_cache_sets, list)
  1490			bch_cached_dev_attach(dc, c, NULL);
  1491	
  1492		if (BDEV_STATE(&dc->sb) == BDEV_STATE_NONE ||
  1493		    BDEV_STATE(&dc->sb) == BDEV_STATE_STALE) {
  1494			err = "failed to run cached device";
  1495			ret = bch_cached_dev_run(dc);
  1496			if (ret)
  1497				goto err;
  1498		}
  1499	
  1500		return 0;
  1501	err:
  1502		pr_notice("error %pg: %s\n", dc->bdev, err);
  1503		bcache_device_stop(&dc->disk);
  1504		return ret;
  1505	}
  1506	

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests