[PATCH 1/2] nbd: replace wait_for_reconnect with non-blocking retry

[PATCH 1/2] nbd: replace wait_for_reconnect with non-blocking retry

[Yun Zhou]

wait_for_reconnect() sleeps in the block dispatch path holding the
SRCU read lock.  This blocks blk_mq_quiesce_queue() (which needs
synchronize_srcu) for the entire dead_conn_timeout duration, triggering
hung task warnings.

Replace it with a non-blocking nbd_reconnect_possible() check that
returns BLK_STS_DEV_RESOURCE to keep the request on the dispatch list,
with a 1-second delayed queue run to re-evaluate.  Once the timeout
expires or the device is disconnected, fail the I/O immediately.

Fixes: 560bc4b39952 ("nbd: handle dead connections")
Reported-by: syzbot+30c16035531e3248dcbc@syzkaller.appspotmail.com
Closes: https://syzkaller.appspot.com/bug?extid=30c16035531e3248dcbc
Signed-off-by: Yun Zhou <yun.zhou@windriver.com>
---
 drivers/block/nbd.c | 61 +++++++++++++++++++++++++++------------------
 1 file changed, 37 insertions(+), 24 deletions(-)

diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index 70a04d541ea4..43aa4121f0c5 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -94,11 +94,11 @@ struct nbd_config {
 	u32 flags;
 	unsigned long runtime_flags;
 	u64 dead_conn_timeout;
+	unsigned long dead_conn_start; /* jiffies, 0 = not waiting */
 
 	struct nbd_sock **socks;
 	int num_connections;
 	atomic_t live_connections;
-	wait_queue_head_t conn_wait;
 
 	atomic_t recv_threads;
 	wait_queue_head_t recv_wq;
@@ -1124,20 +1124,25 @@ static int find_fallback(struct nbd_device *nbd, int index)
 	return new_index;
 }
 
-static int wait_for_reconnect(struct nbd_device *nbd)
+static bool nbd_reconnect_possible(struct nbd_device *nbd)
 {
 	struct nbd_config *config = nbd->config;
+
 	if (!config->dead_conn_timeout)
-		return 0;
+		return false;
+	if (test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags))
+		return false;
 
-	if (!wait_event_timeout(config->conn_wait,
-				test_bit(NBD_RT_DISCONNECTED,
-					 &config->runtime_flags) ||
-				atomic_read(&config->live_connections) > 0,
-				config->dead_conn_timeout))
-		return 0;
+	/* Record when all connections first went dead */
+	if (!READ_ONCE(config->dead_conn_start))
+		WRITE_ONCE(config->dead_conn_start, jiffies ? : 1);
+
+	/* Check if we've exceeded the reconnect timeout */
+	if (time_after(jiffies, READ_ONCE(config->dead_conn_start) +
+		       (unsigned long)config->dead_conn_timeout))
+		return false;
 
-	return !test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags);
+	return true;
 }
 
 static blk_status_t nbd_handle_cmd(struct nbd_cmd *cmd, int index)
@@ -1168,23 +1173,24 @@ static blk_status_t nbd_handle_cmd(struct nbd_cmd *cmd, int index)
 	nsock = config->socks[index];
 	mutex_lock(&nsock->tx_lock);
 	if (nsock->dead) {
-		int old_index = index;
 		index = find_fallback(nbd, index);
 		mutex_unlock(&nsock->tx_lock);
 		if (index < 0) {
-			if (wait_for_reconnect(nbd)) {
-				index = old_index;
-				goto again;
+			if (!nbd_reconnect_possible(nbd)) {
+				sock_shutdown(nbd);
+				nbd_config_put(nbd);
+				return BLK_STS_IOERR;
 			}
-			/* All the sockets should already be down at this point,
-			 * we just want to make sure that DISCONNECTED is set so
-			 * any requests that come in that were queue'ed waiting
-			 * for the reconnect timer don't trigger the timer again
-			 * and instead just error out.
+			/*
+			 * All connections are dead but reconnect timeout
+			 * has not expired.  Return BLK_STS_DEV_RESOURCE
+			 * so the request stays on the dispatch list, and
+			 * schedule a delayed queue run after 1 second to
+			 * re-evaluate.
 			 */
-			sock_shutdown(nbd);
+			blk_mq_delay_run_hw_queues(nbd->disk->queue, 1000);
 			nbd_config_put(nbd);
-			return BLK_STS_IOERR;
+			return BLK_STS_DEV_RESOURCE;
 		}
 		goto again;
 	}
@@ -1437,7 +1443,9 @@ static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)
 		queue_work(nbd->recv_workq, &args->work);
 
 		atomic_inc(&config->live_connections);
-		wake_up(&config->conn_wait);
+		/* Reconnected -- stop waiting for dead_conn_timeout */
+		WRITE_ONCE(config->dead_conn_start, 0);
+		blk_mq_run_hw_queues(nbd->disk->queue, true);
 		return 0;
 	}
 	sk_clear_memalloc(sock->sk);
@@ -1499,6 +1507,13 @@ static int nbd_disconnect(struct nbd_device *nbd)
 	dev_info(disk_to_dev(nbd->disk), "NBD_DISCONNECT\n");
 	set_bit(NBD_RT_DISCONNECT_REQUESTED, &config->runtime_flags);
 	set_bit(NBD_DISCONNECT_REQUESTED, &nbd->flags);
+	/*
+	 * If all connections are already dead, no sock_shutdown callback
+	 * will fire to set NBD_RT_DISCONNECTED.  Set it here so
+	 * nbd_reconnect_possible() stops waiting immediately.
+	 */
+	if (atomic_read(&config->live_connections) == 0)
+		set_bit(NBD_RT_DISCONNECTED, &config->runtime_flags);
 	send_disconnects(nbd);
 	return 0;
 }
@@ -1766,7 +1781,6 @@ static int nbd_alloc_and_init_config(struct nbd_device *nbd)
 
 	atomic_set(&config->recv_threads, 0);
 	init_waitqueue_head(&config->recv_wq);
-	init_waitqueue_head(&config->conn_wait);
 	config->blksize_bits = NBD_DEF_BLKSIZE_BITS;
 	atomic_set(&config->live_connections, 0);
 
@@ -2337,7 +2351,6 @@ static void nbd_disconnect_and_put(struct nbd_device *nbd)
 	mutex_lock(&nbd->config_lock);
 	nbd_disconnect(nbd);
 	sock_shutdown(nbd);
-	wake_up(&nbd->config->conn_wait);
 	/*
 	 * Clear NBD_RT_BOUND before releasing config_lock so that
 	 * nbd_genl_reconfigure() won't queue new recv_work between
-- 
2.43.0

[PATCH 2/2] nbd: detect management process exit via netlink notifier

[Yun Zhou]

Register a netlink notifier to detect when the NBD management process
closes its netlink socket (on exit or crash).

Without this, in-flight I/O waits for socket timeout while udevd holds
disk->open_mutex for partition scanning, blocking poweroff's
sync_bdevs() and causing a hung task during shutdown.

Fix this by tracking the management process's netlink portid and, on
NETLINK_URELEASE, calling sock_shutdown() on all devices it owns.
This fails in-flight I/O immediately so shutdown can proceed.

Signed-off-by: Yun Zhou <yun.zhou@windriver.com>
---
 drivers/block/nbd.c | 59 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index 43aa4121f0c5..3bf88f7b3535 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -45,6 +45,7 @@
 #include <linux/nbd.h>
 #include <linux/nbd-netlink.h>
 #include <net/genetlink.h>
+#include <linux/netlink.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/nbd.h>
@@ -131,6 +132,7 @@ struct nbd_device {
 
 	unsigned long flags;
 	pid_t pid; /* pid of nbd-client, if attached */
+	u32 owner_portid; /* netlink portid of management process */
 
 	char *backend;
 };
@@ -2331,12 +2333,19 @@ static int nbd_genl_connect(struct sk_buff *skb, struct genl_info *info)
 	}
 	set_bit(NBD_RT_HAS_BACKEND_FILE, &config->runtime_flags);
 
+	/*
+	 * Set owner before starting device so that the netlink notifier
+	 * can clean up if the management process exits during setup.
+	 */
+	nbd->owner_portid = info->snd_portid;
 	ret = nbd_start_device(nbd);
 out:
 	if (!ret) {
 		set_bit(NBD_RT_HAS_CONFIG_REF, &config->runtime_flags);
 		refcount_inc(&nbd->config_refs);
 		nbd_connect_reply(info, nbd->index);
+	} else {
+		nbd->owner_portid = 0;
 	}
 	mutex_unlock(&nbd->config_lock);
 
@@ -2541,6 +2550,12 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
 		}
 	}
 out:
+	/*
+	 * Update owner to the reconfiguring process so the notifier
+	 * tracks the correct management process.
+	 */
+	if (!ret)
+		nbd->owner_portid = info->snd_portid;
 	mutex_unlock(&nbd->config_lock);
 	nbd_config_put(nbd);
 	nbd_put(nbd);
@@ -2743,6 +2758,48 @@ static void nbd_dead_link_work(struct work_struct *work)
 	kfree(args);
 }
 
+/*
+ * Detect when the management process (the one that issued NBD_CMD_CONNECT)
+ * closes its netlink socket (e.g., exit or crash).  When this happens,
+ * no one can reconnect, so mark all its devices as disconnected to stop
+ * waiting for reconnect and fail pending I/O promptly.
+ */
+static int nbd_netlink_event(struct notifier_block *this,
+			     unsigned long event, void *ptr)
+{
+	struct netlink_notify *n = ptr;
+	struct nbd_device *nbd;
+	int id;
+
+	if (event != NETLINK_URELEASE || n->protocol != NETLINK_GENERIC)
+		return NOTIFY_DONE;
+
+	mutex_lock(&nbd_index_mutex);
+	idr_for_each_entry(&nbd_index_idr, nbd, id) {
+		if (nbd->owner_portid != n->portid)
+			continue;
+
+		nbd->owner_portid = 0;
+		if (refcount_inc_not_zero(&nbd->config_refs)) {
+			/*
+			 * Shut down all sockets.  sock_shutdown() holds
+			 * tx_lock for each socket, preventing races with
+			 * reconnect, and sets NBD_RT_DISCONNECTED so that
+			 * nbd_reconnect_possible() fails immediately.
+			 */
+			sock_shutdown(nbd);
+			nbd_config_put(nbd);
+		}
+	}
+	mutex_unlock(&nbd_index_mutex);
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block nbd_netlink_notifier = {
+	.notifier_call = nbd_netlink_event,
+};
+
 static int __init nbd_init(void)
 {
 	int i;
@@ -2789,6 +2846,7 @@ static int __init nbd_init(void)
 		unregister_blkdev(NBD_MAJOR, "nbd");
 		return -EINVAL;
 	}
+	netlink_register_notifier(&nbd_netlink_notifier);
 	nbd_dbg_init();
 
 	for (i = 0; i < nbds_max; i++)
@@ -2818,6 +2876,7 @@ static void __exit nbd_cleanup(void)
 	 * for the completion of netlink commands.
 	 */
 	genl_unregister_family(&nbd_genl_family);
+	netlink_unregister_notifier(&nbd_netlink_notifier);
 
 	nbd_dbg_close();
 
-- 
2.43.0