[PATCH 1/2] nbd: replace wait_for_reconnect with non-blocking retry

[Yun Zhou <yun.zhou@windriver.com>]

wait_for_reconnect() sleeps in the block dispatch path holding the
SRCU read lock.  This blocks blk_mq_quiesce_queue() (which needs
synchronize_srcu) for the entire dead_conn_timeout duration, triggering
hung task warnings.

Replace it with a non-blocking nbd_reconnect_possible() check that
returns BLK_STS_DEV_RESOURCE to keep the request on the dispatch list,
with a 1-second delayed queue run to re-evaluate.  Once the timeout
expires or the device is disconnected, fail the I/O immediately.

Fixes: 560bc4b39952 ("nbd: handle dead connections")
Reported-by: syzbot+30c16035531e3248dcbc@syzkaller.appspotmail.com
Closes: https://syzkaller.appspot.com/bug?extid=30c16035531e3248dcbc
Signed-off-by: Yun Zhou <yun.zhou@windriver.com>
---
 drivers/block/nbd.c | 61 +++++++++++++++++++++++++++------------------
 1 file changed, 37 insertions(+), 24 deletions(-)

diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index 70a04d541ea4..43aa4121f0c5 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -94,11 +94,11 @@ struct nbd_config {
 	u32 flags;
 	unsigned long runtime_flags;
 	u64 dead_conn_timeout;
+	unsigned long dead_conn_start; /* jiffies, 0 = not waiting */
 
 	struct nbd_sock **socks;
 	int num_connections;
 	atomic_t live_connections;
-	wait_queue_head_t conn_wait;
 
 	atomic_t recv_threads;
 	wait_queue_head_t recv_wq;
@@ -1124,20 +1124,25 @@ static int find_fallback(struct nbd_device *nbd, int index)
 	return new_index;
 }
 
-static int wait_for_reconnect(struct nbd_device *nbd)
+static bool nbd_reconnect_possible(struct nbd_device *nbd)
 {
 	struct nbd_config *config = nbd->config;
+
 	if (!config->dead_conn_timeout)
-		return 0;
+		return false;
+	if (test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags))
+		return false;
 
-	if (!wait_event_timeout(config->conn_wait,
-				test_bit(NBD_RT_DISCONNECTED,
-					 &config->runtime_flags) ||
-				atomic_read(&config->live_connections) > 0,
-				config->dead_conn_timeout))
-		return 0;
+	/* Record when all connections first went dead */
+	if (!READ_ONCE(config->dead_conn_start))
+		WRITE_ONCE(config->dead_conn_start, jiffies ? : 1);
+
+	/* Check if we've exceeded the reconnect timeout */
+	if (time_after(jiffies, READ_ONCE(config->dead_conn_start) +
+		       (unsigned long)config->dead_conn_timeout))
+		return false;
 
-	return !test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags);
+	return true;
 }
 
 static blk_status_t nbd_handle_cmd(struct nbd_cmd *cmd, int index)
@@ -1168,23 +1173,24 @@ static blk_status_t nbd_handle_cmd(struct nbd_cmd *cmd, int index)
 	nsock = config->socks[index];
 	mutex_lock(&nsock->tx_lock);
 	if (nsock->dead) {
-		int old_index = index;
 		index = find_fallback(nbd, index);
 		mutex_unlock(&nsock->tx_lock);
 		if (index < 0) {
-			if (wait_for_reconnect(nbd)) {
-				index = old_index;
-				goto again;
+			if (!nbd_reconnect_possible(nbd)) {
+				sock_shutdown(nbd);
+				nbd_config_put(nbd);
+				return BLK_STS_IOERR;
 			}
-			/* All the sockets should already be down at this point,
-			 * we just want to make sure that DISCONNECTED is set so
-			 * any requests that come in that were queue'ed waiting
-			 * for the reconnect timer don't trigger the timer again
-			 * and instead just error out.
+			/*
+			 * All connections are dead but reconnect timeout
+			 * has not expired.  Return BLK_STS_DEV_RESOURCE
+			 * so the request stays on the dispatch list, and
+			 * schedule a delayed queue run after 1 second to
+			 * re-evaluate.
 			 */
-			sock_shutdown(nbd);
+			blk_mq_delay_run_hw_queues(nbd->disk->queue, 1000);
 			nbd_config_put(nbd);
-			return BLK_STS_IOERR;
+			return BLK_STS_DEV_RESOURCE;
 		}
 		goto again;
 	}
@@ -1437,7 +1443,9 @@ static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)
 		queue_work(nbd->recv_workq, &args->work);
 
 		atomic_inc(&config->live_connections);
-		wake_up(&config->conn_wait);
+		/* Reconnected -- stop waiting for dead_conn_timeout */
+		WRITE_ONCE(config->dead_conn_start, 0);
+		blk_mq_run_hw_queues(nbd->disk->queue, true);
 		return 0;
 	}
 	sk_clear_memalloc(sock->sk);
@@ -1499,6 +1507,13 @@ static int nbd_disconnect(struct nbd_device *nbd)
 	dev_info(disk_to_dev(nbd->disk), "NBD_DISCONNECT\n");
 	set_bit(NBD_RT_DISCONNECT_REQUESTED, &config->runtime_flags);
 	set_bit(NBD_DISCONNECT_REQUESTED, &nbd->flags);
+	/*
+	 * If all connections are already dead, no sock_shutdown callback
+	 * will fire to set NBD_RT_DISCONNECTED.  Set it here so
+	 * nbd_reconnect_possible() stops waiting immediately.
+	 */
+	if (atomic_read(&config->live_connections) == 0)
+		set_bit(NBD_RT_DISCONNECTED, &config->runtime_flags);
 	send_disconnects(nbd);
 	return 0;
 }
@@ -1766,7 +1781,6 @@ static int nbd_alloc_and_init_config(struct nbd_device *nbd)
 
 	atomic_set(&config->recv_threads, 0);
 	init_waitqueue_head(&config->recv_wq);
-	init_waitqueue_head(&config->conn_wait);
 	config->blksize_bits = NBD_DEF_BLKSIZE_BITS;
 	atomic_set(&config->live_connections, 0);
 
@@ -2337,7 +2351,6 @@ static void nbd_disconnect_and_put(struct nbd_device *nbd)
 	mutex_lock(&nbd->config_lock);
 	nbd_disconnect(nbd);
 	sock_shutdown(nbd);
-	wake_up(&nbd->config->conn_wait);
 	/*
 	 * Clear NBD_RT_BOUND before releasing config_lock so that
 	 * nbd_genl_reconfigure() won't queue new recv_work between
-- 
2.43.0