Re: lock, Workqueue: bcache bch_data_insert_keys [bcache]

[Konstantin Shalygin <k0ste@k0ste.ru>]

[-- Attachment #1: Type: text/plain, Size: 111 bytes --]

After month of usage I can confirm that this patch resolve this issue.

-- 
Best regards,
Konstantin Shalygin


[-- Attachment #2: linux-4.11-bcache_fix_for_gc_and_write-back_race.diff --]
[-- Type: text/x-patch, Size: 3576 bytes --]

From: Tang Junhui <tang.junhui@xxxxxxxxxx>

gc and write-back get raced (see the email "bcache get stucked" I sended
before):
gc thread						write-back thread
|							|bch_writeback_thread()
|bch_gc_thread()					|
|							|==>read_dirty()
|==>bch_btree_gc()					|
|==>btree_root() //get btree root			|
|			node write locker		|
|==>bch_btree_gc_root()					|
|							|==>read_dirty_submit()
|							|==>write_dirty()
|							|==>continue_at(cl, write_dirty_finish, system_wq);
|							|==>write_dirty_finish()//excute in system_wq
|							|==>bch_btree_insert()
|							|==>bch_btree_map_leaf_nodes()
|							|==>__bch_btree_map_nodes()
|							|==>btree_root //try to get btree root node read lock
|							|-----stuck here
|==>bch_btree_set_root()				|
|==>bch_journal_meta()					|
|==>bch_journal()					|
|==>journal_try_write()					|
|==>journal_write_unlocked() //journal_full(&c->journal) condition satisfied
|==>continue_at(cl, journal_write, system_wq); //try to excute journal_write in system_wq
|					//but work queue is excuting write_dirty_finish()
|==>closure_sync(); //wait journal_write execute over and wake up gc,
|			--stuck here
|==>release root node write locker

This patch alloc a separate work-queue for write-back thread to avoid such
race.

Signed-off-by: Tang Junhui <tang.junhui@xxxxxxxxxx>
---
diff -Naupr linux-4.11_orig/drivers/md/bcache/bcache.h linux-4.11/drivers/md/bcache/bcache.h
--- linux-4.11_orig/drivers/md/bcache/bcache.h	2017-05-01 09:47:48.000000000 +0700
+++ linux-4.11/drivers/md/bcache/bcache.h	2017-07-08 11:49:25.044671097 +0700
@@ -333,6 +333,7 @@ struct cached_dev {
 	/* Limit number of writeback bios in flight */
 	struct semaphore	in_flight;
 	struct task_struct	*writeback_thread;
+	struct workqueue_struct	*writeback_write_wq;
 
 	struct keybuf		writeback_keys;
 
diff -Naupr linux-4.11_orig/drivers/md/bcache/super.c linux-4.11/drivers/md/bcache/super.c
--- linux-4.11_orig/drivers/md/bcache/super.c	2017-05-01 09:47:48.000000000 +0700
+++ linux-4.11/drivers/md/bcache/super.c	2017-07-08 11:50:16.574952148 +0700
@@ -1061,6 +1061,8 @@ static void cached_dev_free(struct closu
 	cancel_delayed_work_sync(&dc->writeback_rate_update);
 	if (!IS_ERR_OR_NULL(dc->writeback_thread))
 		kthread_stop(dc->writeback_thread);
+		if (dc->writeback_write_wq)
+			destroy_workqueue(dc->writeback_write_wq);
 
 	mutex_lock(&bch_register_lock);
 
diff -Naupr linux-4.11_orig/drivers/md/bcache/writeback.c linux-4.11/drivers/md/bcache/writeback.c
--- linux-4.11_orig/drivers/md/bcache/writeback.c	2017-05-01 09:47:48.000000000 +0700
+++ linux-4.11/drivers/md/bcache/writeback.c	2017-07-08 11:52:21.799010810 +0700
@@ -186,7 +186,7 @@ static void write_dirty(struct closure *
 
 	closure_bio_submit(&io->bio, cl);
 
-	continue_at(cl, write_dirty_finish, system_wq);
+	continue_at(cl, write_dirty_finish, io->dc->writeback_write_wq);
 }
 
 static void read_dirty_endio(struct bio *bio)
@@ -206,7 +206,7 @@ static void read_dirty_submit(struct clo
 
 	closure_bio_submit(&io->bio, cl);
 
-	continue_at(cl, write_dirty, system_wq);
+	continue_at(cl, write_dirty, io->dc->writeback_write_wq);
 }
 
 static void read_dirty(struct cached_dev *dc)
@@ -516,6 +516,10 @@ void bch_cached_dev_writeback_init(struc
 
 int bch_cached_dev_writeback_start(struct cached_dev *dc)
 {
+	dc->writeback_write_wq = alloc_workqueue("bcache_writeback_wq", WQ_MEM_RECLAIM, 0);
+	if (!dc->writeback_write_wq)
+		return -ENOMEM;
+
 	dc->writeback_thread = kthread_create(bch_writeback_thread, dc,
 					      "bcache_writeback");
 	if (IS_ERR(dc->writeback_thread))