[PATCH 0/2] nbd: local daemon restart support

[PATCH 0/2] nbd: local daemon restart support

[Mike Christie]

The following patches made over Linus's tree allow setups that are
using AF_UNIX sockets with a local daemon to recover from crashes
or to upgrade the daemon while IO is running without having to
disrupt the application (no need to reopen the device or handle IO
errors). They basically just use the existing failover
infrastructure, but to failover to a new socket from a non-dead
socket.

[PATCH 1/2] nbd: move send_disconnects

[Mike Christie]

Just move send_disconnects so we can call it earlier in the file in the
next patch.

Signed-off-by: Mike Christie <mchristi@redhat.com>
---
 drivers/block/nbd.c | 48 ++++++++++++++++++++++-----------------------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index a94ee45440b3..06b63c11ae50 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -980,6 +980,30 @@ static blk_status_t nbd_queue_rq(struct blk_mq_hw_ctx *hctx,
 	return ret;
 }
 
+static void send_disconnects(struct nbd_device *nbd)
+{
+	struct nbd_config *config = nbd->config;
+	struct nbd_request request = {
+		.magic = htonl(NBD_REQUEST_MAGIC),
+		.type = htonl(NBD_CMD_DISC),
+	};
+	struct kvec iov = {.iov_base = &request, .iov_len = sizeof(request)};
+	struct iov_iter from;
+	int i, ret;
+
+	for (i = 0; i < config->num_connections; i++) {
+		struct nbd_sock *nsock = config->socks[i];
+
+		iov_iter_kvec(&from, WRITE, &iov, 1, sizeof(request));
+		mutex_lock(&nsock->tx_lock);
+		ret = sock_xmit(nbd, i, 1, &from, 0, NULL);
+		if (ret <= 0)
+			dev_err(disk_to_dev(nbd->disk),
+				"Send disconnect failed %d\n", ret);
+		mutex_unlock(&nsock->tx_lock);
+	}
+}
+
 static struct socket *nbd_get_socket(struct nbd_device *nbd, unsigned long fd,
 				     int *err)
 {
@@ -1139,30 +1163,6 @@ static void nbd_parse_flags(struct nbd_device *nbd)
 		blk_queue_write_cache(nbd->disk->queue, false, false);
 }
 
-static void send_disconnects(struct nbd_device *nbd)
-{
-	struct nbd_config *config = nbd->config;
-	struct nbd_request request = {
-		.magic = htonl(NBD_REQUEST_MAGIC),
-		.type = htonl(NBD_CMD_DISC),
-	};
-	struct kvec iov = {.iov_base = &request, .iov_len = sizeof(request)};
-	struct iov_iter from;
-	int i, ret;
-
-	for (i = 0; i < config->num_connections; i++) {
-		struct nbd_sock *nsock = config->socks[i];
-
-		iov_iter_kvec(&from, WRITE, &iov, 1, sizeof(request));
-		mutex_lock(&nsock->tx_lock);
-		ret = sock_xmit(nbd, i, 1, &from, 0, NULL);
-		if (ret <= 0)
-			dev_err(disk_to_dev(nbd->disk),
-				"Send disconnect failed %d\n", ret);
-		mutex_unlock(&nsock->tx_lock);
-	}
-}
-
 static int nbd_disconnect(struct nbd_device *nbd)
 {
 	struct nbd_config *config = nbd->config;
-- 
2.21.0

[PATCH 2/2] nbd: add support to replace running daemon

[Mike Christie]

This patch allows userspace to switch daemons while IO is running or
to recovery from a crashed daemon. With the patch when using AF_UNIX
sockets and a local running daemon, the new daemon can perform a
NBD_CMD_RECONFIGURE nl command with the NBD_ATTR_SWAP_SOCKETS attr
set to 0. The kernel will then switch over to using the new socket
similar to how we handle failing over from a dead socket.

Signed-off-by: Mike Christie <mchristi@redhat.com>
---
 drivers/block/nbd.c              | 44 +++++++++++++++++++++++++++++---
 include/uapi/linux/nbd-netlink.h |  1 +
 2 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index 06b63c11ae50..6aa396bea401 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -1076,7 +1076,14 @@ static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg,
 	return 0;
 }
 
-static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)
+static bool nbd_requeue_req(struct request *req, void *data, bool reserved)
+{
+	nbd_requeue_cmd(blk_mq_rq_to_pdu(req));
+	return true;
+}
+
+static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg,
+				u8 swap_socks)
 {
 	struct nbd_config *config = nbd->config;
 	struct socket *sock, *old;
@@ -1094,6 +1101,16 @@ static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)
 		return -ENOMEM;
 	}
 
+	if (swap_socks) {
+		dev_dbg(disk_to_dev(nbd->disk), "stopping queue\n");
+
+		blk_mq_quiesce_queue(nbd->disk->queue);
+		send_disconnects(nbd);
+		sock_shutdown(nbd);
+		flush_workqueue(nbd->recv_workq);
+		blk_mq_tagset_busy_iter(&nbd->tag_set, nbd_requeue_req, NULL);
+	}
+
 	for (i = 0; i < config->num_connections; i++) {
 		struct nbd_sock *nsock = config->socks[i];
 
@@ -1130,11 +1147,20 @@ static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)
 
 		atomic_inc(&config->live_connections);
 		wake_up(&config->conn_wait);
-		return 0;
+		err = 0;
+		goto start_queue;
 	}
 	sockfd_put(sock);
 	kfree(args);
-	return -ENOSPC;
+	err = -ENOSPC;
+
+start_queue:
+	if (swap_socks) {
+		nbd->task_recv = current;
+		blk_mq_unquiesce_queue(nbd->disk->queue);
+		dev_dbg(disk_to_dev(nbd->disk), "queue restarted\n");
+	}
+	return err;
 }
 
 static void nbd_bdev_reset(struct block_device *bdev)
@@ -2027,6 +2053,7 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
 {
 	struct nbd_device *nbd = NULL;
 	struct nbd_config *config;
+	u8 swap_socks = 0;
 	int index;
 	int ret = 0;
 	bool put_dev = false;
@@ -2107,6 +2134,15 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
 		}
 	}
 
+	if (info->attrs[NBD_ATTR_SWAP_SOCKETS]) {
+		swap_socks = nla_get_u8(info->attrs[NBD_ATTR_SWAP_SOCKETS]);
+		if (swap_socks != 1 && swap_socks != 0) {
+			printk(KERN_ERR "nbd: NBD_SOCK_SWAP must be 0 or 1\n");
+			ret = -EINVAL;
+			goto out;
+		}
+	}
+
 	if (info->attrs[NBD_ATTR_SOCKETS]) {
 		struct nlattr *attr;
 		int rem, fd;
@@ -2132,7 +2168,7 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
 			if (!socks[NBD_SOCK_FD])
 				continue;
 			fd = (int)nla_get_u32(socks[NBD_SOCK_FD]);
-			ret = nbd_reconnect_socket(nbd, fd);
+			ret = nbd_reconnect_socket(nbd, fd, swap_socks);
 			if (ret) {
 				if (ret == -ENOSPC)
 					ret = 0;
diff --git a/include/uapi/linux/nbd-netlink.h b/include/uapi/linux/nbd-netlink.h
index c5d0ef7aa7d5..0e406445f06c 100644
--- a/include/uapi/linux/nbd-netlink.h
+++ b/include/uapi/linux/nbd-netlink.h
@@ -35,6 +35,7 @@ enum {
 	NBD_ATTR_SOCKETS,
 	NBD_ATTR_DEAD_CONN_TIMEOUT,
 	NBD_ATTR_DEVICE_LIST,
+	NBD_ATTR_SWAP_SOCKETS,
 	__NBD_ATTR_MAX,
 };
 #define NBD_ATTR_MAX (__NBD_ATTR_MAX - 1)
-- 
2.21.0

Re: [PATCH 0/2] nbd: local daemon restart support

[Mike Christie]

On 11/15/2019 11:50 PM, Mike Christie wrote:
> The following patches made over Linus's tree allow setups that are
> using AF_UNIX sockets with a local daemon to recover from crashes
> or to upgrade the daemon while IO is running without having to
> disrupt the application (no need to reopen the device or handle IO
> errors). They basically just use the existing failover
> infrastructure, but to failover to a new socket from a non-dead
> socket.

Josef and Jens,

I am dropping this patchset. Do not bother reviewing it. I will post
something else later.