[PATCH blktests 0/2] blktests: Add ublk testcases

[PATCH blktests 0/2] blktests: Add ublk testcases

[Ziyang Zhang]

Hi,

ublk can passthrough I/O requests to userspce daemons. It is very important
to test ublk crash handling since the userspace part is not reliable.
Especially we should test removing device, killing ublk daemons and user
recovery feature.

The first patch add user recovery support in miniublk.

The second patch add five new tests for ublk to cover above cases.

Ziyang Zhang (2):
  src/miniublk: add user recovery
  tests: Add ublk tests

 src/miniublk.c     | 207 +++++++++++++++++++++++++++++++++++++++++++--
 tests/ublk/001     |  39 +++++++++
 tests/ublk/001.out |   2 +
 tests/ublk/002     |  53 ++++++++++++
 tests/ublk/002.out |   2 +
 tests/ublk/003     |  43 ++++++++++
 tests/ublk/003.out |   2 +
 tests/ublk/004     |  63 ++++++++++++++
 tests/ublk/004.out |   2 +
 tests/ublk/005     |  66 +++++++++++++++
 tests/ublk/005.out |   2 +
 11 files changed, 472 insertions(+), 9 deletions(-)
 create mode 100644 tests/ublk/001
 create mode 100644 tests/ublk/001.out
 create mode 100644 tests/ublk/002
 create mode 100644 tests/ublk/002.out
 create mode 100644 tests/ublk/003
 create mode 100644 tests/ublk/003.out
 create mode 100644 tests/ublk/004
 create mode 100644 tests/ublk/004.out
 create mode 100644 tests/ublk/005
 create mode 100644 tests/ublk/005.out

-- 
2.31.1

[PATCH blktests 1/2] src/miniublk: add user recovery

[Ziyang Zhang]

We are going to test ublk's user recovery feature so add support in
miniublk.

Signed-off-by: Ziyang Zhang <ZiyangZhang@linux.alibaba.com>
---
 src/miniublk.c | 207 ++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 198 insertions(+), 9 deletions(-)

diff --git a/src/miniublk.c b/src/miniublk.c
index fe10291..558bb7b 100644
--- a/src/miniublk.c
+++ b/src/miniublk.c
@@ -74,6 +74,7 @@ struct ublk_tgt_ops {
 	int (*queue_io)(struct ublk_queue *, int tag);
 	void (*tgt_io_done)(struct ublk_queue *,
 			int tag, const struct io_uring_cqe *);
+	int (*recover_tgt)(struct ublk_dev *);
 };
 
 struct ublk_tgt {
@@ -372,6 +373,29 @@ static int ublk_ctrl_get_params(struct ublk_dev *dev,
 	return __ublk_ctrl_cmd(dev, &data);
 }
 
+static int ublk_ctrl_start_user_recover(struct ublk_dev *dev)
+{
+	struct ublk_ctrl_cmd_data data = {
+		.cmd_op	= UBLK_CMD_START_USER_RECOVERY,
+		.flags	= 0,
+	};
+
+	return __ublk_ctrl_cmd(dev, &data);
+}
+
+static int ublk_ctrl_end_user_recover(struct ublk_dev *dev,
+		int daemon_pid)
+{
+	struct ublk_ctrl_cmd_data data = {
+		.cmd_op	= UBLK_CMD_END_USER_RECOVERY,
+		.flags	= CTRL_CMD_HAS_DATA,
+	};
+
+	dev->dev_info.ublksrv_pid = data.data[0] = daemon_pid;
+
+	return __ublk_ctrl_cmd(dev, &data);
+}
+
 static const char *ublk_dev_state_desc(struct ublk_dev *dev)
 {
 	switch (dev->dev_info.state) {
@@ -379,6 +403,8 @@ static const char *ublk_dev_state_desc(struct ublk_dev *dev)
 		return "DEAD";
 	case UBLK_S_DEV_LIVE:
 		return "LIVE";
+	case UBLK_S_DEV_QUIESCED:
+		return "QUIESCED";
 	default:
 		return "UNKNOWN";
 	};
@@ -550,9 +576,12 @@ static int ublk_dev_prep(struct ublk_dev *dev)
 		goto fail;
 	}
 
-	if (dev->tgt.ops->init_tgt)
+	if (dev->dev_info.state != UBLK_S_DEV_QUIESCED && dev->tgt.ops->init_tgt)
 		ret = dev->tgt.ops->init_tgt(dev);
 
+	else if (dev->dev_info.state == UBLK_S_DEV_QUIESCED && dev->tgt.ops->recover_tgt)
+		ret = dev->tgt.ops->recover_tgt(dev);
+
 	return ret;
 fail:
 	close(dev->fds[0]);
@@ -831,7 +860,7 @@ static void ublk_set_parameters(struct ublk_dev *dev)
 				dev->dev_info.dev_id, ret);
 }
 
-static int ublk_start_daemon(struct ublk_dev *dev)
+static int ublk_start_daemon(struct ublk_dev *dev, bool recovery)
 {
 	int ret, i;
 	void *thread_ret;
@@ -853,12 +882,22 @@ static int ublk_start_daemon(struct ublk_dev *dev)
 				&dev->q[i]);
 	}
 
-	ublk_set_parameters(dev);
 
 	/* everything is fine now, start us */
-	ret = ublk_ctrl_start_dev(dev, getpid());
-	if (ret < 0)
-		goto fail;
+	if (recovery) {
+		ret = ublk_ctrl_end_user_recover(dev, getpid());
+		if (ret < 0) {
+			ublk_err("%s: ublk_ctrl_end_user_recover failed: %d\n", __func__, ret);
+			goto fail;
+		}
+	} else {
+		ublk_set_parameters(dev);
+		ret = ublk_ctrl_start_dev(dev, getpid());
+		if (ret < 0) {
+			ublk_err("%s: ublk_ctrl_start_dev failed: %d\n", __func__, ret);
+			goto fail;
+		}
+	}
 
 	ublk_ctrl_get_info(dev);
 	ublk_ctrl_dump(dev, true);
@@ -880,6 +919,7 @@ static int cmd_dev_add(int argc, char *argv[])
 		{ "number",		1,	NULL, 'n' },
 		{ "queues",		1,	NULL, 'q' },
 		{ "depth",		1,	NULL, 'd' },
+		{ "recovery",		1,	NULL, 'r' },
 		{ "debug_mask",	1,	NULL, 0},
 		{ "quiet",	0,	NULL, 0},
 		{ NULL }
@@ -891,8 +931,9 @@ static int cmd_dev_add(int argc, char *argv[])
 	const char *tgt_type = NULL;
 	int dev_id = -1;
 	unsigned nr_queues = 2, depth = UBLK_QUEUE_DEPTH;
+	int user_recovery = 0;
 
-	while ((opt = getopt_long(argc, argv, "-:t:n:d:q:",
+	while ((opt = getopt_long(argc, argv, "-:t:n:d:q:r:",
 				  longopts, &option_idx)) != -1) {
 		switch (opt) {
 		case 'n':
@@ -907,6 +948,9 @@ static int cmd_dev_add(int argc, char *argv[])
 		case 'd':
 			depth = strtol(optarg, NULL, 10);
 			break;
+		case 'r':
+			user_recovery = strtol(optarg, NULL, 10);
+			break;
 		case 0:
 			if (!strcmp(longopts[option_idx].name, "debug_mask"))
 				ublk_dbg_mask = strtol(optarg, NULL, 16);
@@ -942,6 +986,8 @@ static int cmd_dev_add(int argc, char *argv[])
 	info->dev_id = dev_id;
         info->nr_hw_queues = nr_queues;
         info->queue_depth = depth;
+	if (user_recovery)
+		info->flags |= UBLK_F_USER_RECOVERY;
 	dev->tgt.ops = ops;
 	dev->tgt.argc = argc;
 	dev->tgt.argv = argv;
@@ -953,7 +999,95 @@ static int cmd_dev_add(int argc, char *argv[])
 		goto fail;
 	}
 
-	ret = ublk_start_daemon(dev);
+	ret = ublk_start_daemon(dev, false);
+	if (ret < 0) {
+		ublk_err("%s: can't start daemon id %d, type %s\n",
+				__func__, dev_id, tgt_type);
+		goto fail_del;
+	}
+
+fail_del:
+	ublk_ctrl_del_dev(dev);
+fail:
+	ublk_ctrl_deinit(dev);
+	return ret;
+}
+
+static int cmd_dev_recover(int argc, char *argv[])
+{
+	static const struct option longopts[] = {
+		{ "type",		1,	NULL, 't' },
+		{ "number",		1,	NULL, 'n' },
+		{ "debug_mask",	1,	NULL, 0},
+		{ "quiet",	0,	NULL, 0},
+		{ NULL }
+	};
+	const struct ublk_tgt_ops *ops;
+	struct ublksrv_ctrl_dev_info *info;
+	struct ublk_dev *dev;
+	int ret, option_idx, opt;
+	const char *tgt_type = NULL;
+	int dev_id = -1;
+
+	while ((opt = getopt_long(argc, argv, "-:t:n:d:q:",
+				  longopts, &option_idx)) != -1) {
+		switch (opt) {
+		case 'n':
+			dev_id = strtol(optarg, NULL, 10);
+			break;
+		case 't':
+			tgt_type = optarg;
+			break;
+		case 0:
+			if (!strcmp(longopts[option_idx].name, "debug_mask"))
+				ublk_dbg_mask = strtol(optarg, NULL, 16);
+			if (!strcmp(longopts[option_idx].name, "quiet"))
+				ublk_dbg_mask = 0;
+			break;
+		}
+	}
+
+	optind = 0;
+
+	ops = ublk_find_tgt(tgt_type);
+	if (!ops) {
+		ublk_err("%s: no such tgt type, type %s\n",
+				__func__, tgt_type);
+		return -ENODEV;
+	}
+
+	dev = ublk_ctrl_init();
+	if (!dev) {
+		ublk_err("%s: can't alloc dev id %d, type %s\n",
+				__func__, dev_id, tgt_type);
+		return -ENOMEM;
+	}
+
+	info = &dev->dev_info;
+	info->dev_id = dev_id;
+	ret = ublk_ctrl_get_info(dev);
+	if (ret < 0) {
+		ublk_err("%s: can't get dev info from %d\n", __func__, dev_id);
+		goto fail;
+	}
+
+	ret = ublk_ctrl_get_params(dev, &dev->tgt.params);
+	if (ret) {
+		ublk_err("dev %d set basic parameter failed %d\n",
+				dev->dev_info.dev_id, ret);
+		goto fail;
+	}
+
+	dev->tgt.ops = ops;
+	dev->tgt.argc = argc;
+	dev->tgt.argv = argv;
+	ret = ublk_ctrl_start_user_recover(dev);
+	if (ret < 0) {
+		ublk_err("%s: can't start recovery for %d\n", __func__, dev_id);
+		goto fail;
+	}
+
+	ret = ublk_start_daemon(dev, true);
 	if (ret < 0) {
 		ublk_err("%s: can't start daemon id %d, type %s\n",
 				__func__, dev_id, tgt_type);
@@ -1125,7 +1259,10 @@ static int cmd_dev_help(int argc, char *argv[])
 	printf("\t -a delete all devices -n delete specified device\n");
 	printf("%s list [-n dev_id] -a \n", argv[0]);
 	printf("\t -a list all devices, -n list specified device, default -a \n");
-
+	printf("%s recover -t {null|loop} [-n dev_id] \n",
+			argv[0]);
+	printf("\t -t loop -f backing_file \n");
+	printf("\t -t null\n");
 	return 0;
 }
 
@@ -1150,6 +1287,12 @@ static int ublk_null_tgt_init(struct ublk_dev *dev)
 	return 0;
 }
 
+static int ublk_null_tgt_recover(struct ublk_dev *dev)
+{
+	dev->tgt.dev_size = dev->tgt.params.basic.dev_sectors << 9;
+	return 0;
+}
+
 static int ublk_null_queue_io(struct ublk_queue *q, int tag)
 {
 	const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);
@@ -1313,11 +1456,54 @@ static int ublk_loop_tgt_init(struct ublk_dev *dev)
 	return 0;
 }
 
+static int ublk_loop_tgt_recover(struct ublk_dev *dev)
+{
+	static const struct option lo_longopts[] = {
+		{ "file",		1,	NULL, 'f' },
+		{ NULL }
+	};
+	char **argv = dev->tgt.argv;
+	int argc = dev->tgt.argc;
+	char *file = NULL;
+	int fd, opt;
+
+	while ((opt = getopt_long(argc, argv, "-:f:",
+				  lo_longopts, NULL)) != -1) {
+		switch (opt) {
+		case 'f':
+			file = strdup(optarg);
+			break;
+		}
+	}
+
+	ublk_dbg(UBLK_DBG_DEV, "%s: file %s\n", __func__, file);
+
+	if (!file)
+		return -EINVAL;
+
+	fd = open(file, O_RDWR);
+	if (fd < 0) {
+		ublk_err( "%s: backing file %s can't be opened\n",
+				__func__, file);
+		return -EBADF;
+	}
+
+	if (fcntl(fd, F_SETFL, O_DIRECT))
+		ublk_log("%s: ublk-loop fallback to buffered IO\n", __func__);
+
+	dev->tgt.dev_size = dev->tgt.params.basic.dev_sectors << 9;
+	dev->fds[1] = fd;
+	dev->nr_fds += 1;
+
+	return 0;
+}
+
 const struct ublk_tgt_ops tgt_ops_list[] = {
 	{
 		.name = "null",
 		.init_tgt = ublk_null_tgt_init,
 		.queue_io = ublk_null_queue_io,
+		.recover_tgt = ublk_null_tgt_recover,
 	},
 
 	{
@@ -1326,6 +1512,7 @@ const struct ublk_tgt_ops tgt_ops_list[] = {
 		.deinit_tgt = ublk_loop_tgt_deinit,
 		.queue_io = ublk_loop_queue_io,
 		.tgt_io_done = ublk_loop_io_done,
+		.recover_tgt = ublk_loop_tgt_recover,
 	},
 };
 
@@ -1359,6 +1546,8 @@ int main(int argc, char *argv[])
 		ret = cmd_dev_list(argc, argv);
 	else if (!strcmp(cmd, "help"))
 		ret = cmd_dev_help(argc, argv);
+	else if (!strcmp(cmd, "recover"))
+		ret = cmd_dev_recover(argc, argv);
 out:
 	if (ret)
 		cmd_dev_help(argc, argv);
-- 
2.31.1

[PATCH blktests 2/2] tests: Add ublk tests

[Ziyang Zhang]

It is very important to test ublk crash handling since the userspace
part is not reliable. Especially we should test removing device, killing
ublk daemons and user recovery feature.

Add five new test for ublk to cover these cases.

Signed-off-by: Ziyang Zhang <ZiyangZhang@linux.alibaba.com>
---
 tests/ublk/001     | 39 +++++++++++++++++++++++++++
 tests/ublk/001.out |  2 ++
 tests/ublk/002     | 53 +++++++++++++++++++++++++++++++++++++
 tests/ublk/002.out |  2 ++
 tests/ublk/003     | 43 ++++++++++++++++++++++++++++++
 tests/ublk/003.out |  2 ++
 tests/ublk/004     | 63 +++++++++++++++++++++++++++++++++++++++++++
 tests/ublk/004.out |  2 ++
 tests/ublk/005     | 66 ++++++++++++++++++++++++++++++++++++++++++++++
 tests/ublk/005.out |  2 ++
 10 files changed, 274 insertions(+)
 create mode 100644 tests/ublk/001
 create mode 100644 tests/ublk/001.out
 create mode 100644 tests/ublk/002
 create mode 100644 tests/ublk/002.out
 create mode 100644 tests/ublk/003
 create mode 100644 tests/ublk/003.out
 create mode 100644 tests/ublk/004
 create mode 100644 tests/ublk/004.out
 create mode 100644 tests/ublk/005
 create mode 100644 tests/ublk/005.out

diff --git a/tests/ublk/001 b/tests/ublk/001
new file mode 100644
index 0000000..fe158ba
--- /dev/null
+++ b/tests/ublk/001
@@ -0,0 +1,39 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2023 Ziyang Zhang
+#
+# Test ublk by deleting ublk device while running fio
+
+. tests/block/rc
+. common/ublk
+
+DESCRIPTION="test ublk deletion"
+QUICK=1
+
+requires() {
+	_have_ublk
+}
+
+test() {
+	local ublk_prog="src/miniublk"
+
+	echo "Running ${TEST_NAME}"
+
+	if ! _init_ublk; then
+		return 1
+	fi
+
+	${ublk_prog} add -t null -n 0 > "$FULL"
+	udevadm settle
+	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
+		echo "fail to list dev"
+	fi
+
+	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
+
+        ${ublk_prog} del -n 0 >> "$FULL"
+
+	_exit_ublk
+
+	echo "Test complete"
+}
diff --git a/tests/ublk/001.out b/tests/ublk/001.out
new file mode 100644
index 0000000..0d070b3
--- /dev/null
+++ b/tests/ublk/001.out
@@ -0,0 +1,2 @@
+Running ublk/001
+Test complete
diff --git a/tests/ublk/002 b/tests/ublk/002
new file mode 100644
index 0000000..25cad13
--- /dev/null
+++ b/tests/ublk/002
@@ -0,0 +1,53 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2023 Ziyang Zhang
+#
+# Test ublk by killing ublk daemon while running fio
+# Delete the device after it is dead
+
+. tests/block/rc
+. common/ublk
+
+DESCRIPTION="test ublk crash(1)"
+QUICK=1
+
+requires() {
+	_have_ublk
+}
+
+test() {
+	local ublk_prog="src/miniublk"
+
+	echo "Running ${TEST_NAME}"
+
+	if ! _init_ublk; then
+		return 1
+	fi
+
+	${ublk_prog} add -t null -n 0 > "$FULL"
+	udevadm settle
+	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
+		echo "fail to list dev"
+	fi
+
+	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
+
+        pid=`${ublk_prog} list -n 0 | grep "pid" | awk '{print $7}'`
+        kill -9 $pid
+
+        sleep 2
+        secs=0
+        while [ $secs -lt 10 ]; do
+                state=`${ublk_prog} list -n 0 | grep "state" | awk '{print $11}'`
+                [ "$state" == "DEAD" ] && break
+		sleep 1
+		let secs++
+        done
+        [ "$state" != "DEAD" ] && echo "device isn't dead after killing queue daemon"
+
+        ${ublk_prog} del -n 0 >> "$FULL"
+
+	_exit_ublk
+
+	echo "Test complete"
+}
diff --git a/tests/ublk/002.out b/tests/ublk/002.out
new file mode 100644
index 0000000..93039b7
--- /dev/null
+++ b/tests/ublk/002.out
@@ -0,0 +1,2 @@
+Running ublk/002
+Test complete
diff --git a/tests/ublk/003 b/tests/ublk/003
new file mode 100644
index 0000000..34bce74
--- /dev/null
+++ b/tests/ublk/003
@@ -0,0 +1,43 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2023 Ziyang Zhang
+#
+# Test ublk by killing ublk daemon while running fio
+# Delete the device immediately
+
+. tests/block/rc
+. common/ublk
+
+DESCRIPTION="test ublk crash(2)"
+QUICK=1
+
+requires() {
+	_have_ublk
+}
+
+test() {
+	local ublk_prog="src/miniublk"
+
+	echo "Running ${TEST_NAME}"
+
+	if ! _init_ublk; then
+		return 1
+	fi
+
+	${ublk_prog} add -t null -n 0 > "$FULL"
+	udevadm settle
+	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
+		echo "fail to list dev"
+	fi
+
+	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
+
+        pid=`${ublk_prog} list -n 0 | grep "pid" | awk '{print $7}'`
+        kill -9 $pid
+
+        ${ublk_prog} del -n 0 >> "$FULL"
+
+	_exit_ublk
+
+	echo "Test complete"
+}
diff --git a/tests/ublk/003.out b/tests/ublk/003.out
new file mode 100644
index 0000000..90a3bfa
--- /dev/null
+++ b/tests/ublk/003.out
@@ -0,0 +1,2 @@
+Running ublk/003
+Test complete
diff --git a/tests/ublk/004 b/tests/ublk/004
new file mode 100644
index 0000000..c5d0694
--- /dev/null
+++ b/tests/ublk/004
@@ -0,0 +1,63 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2023 Ziyang Zhang
+#
+# Test ublk user recovery by run fio with dev recovery:
+# (1)kill all ubq_deamon, (2)recover with new ubq_daemon,
+# (3)delete dev
+
+. tests/block/rc
+. common/ublk
+
+DESCRIPTION="test ublk recovery(1)"
+QUICK=1
+
+requires() {
+	_have_ublk
+}
+
+test() {
+	local ublk_prog="src/miniublk"
+
+	echo "Running ${TEST_NAME}"
+
+	if ! _init_ublk; then
+		return 1
+	fi
+
+	${ublk_prog} add -t null -n 0 -r 1 > "$FULL"
+	udevadm settle
+	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
+		echo "fail to list dev"
+	fi
+
+	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
+        pid=`${ublk_prog} list -n 0 | grep "pid" | awk '{print $7}'`
+        kill -9 $pid
+
+        sleep 2
+        secs=0
+        while [ $secs -lt 10 ]; do
+                state=`${ublk_prog} list -n 0 | grep "state" | awk '{print $11}'`
+                [ "$state" == "QUIESCED" ] && break
+		sleep 1
+		let secs++
+        done
+        [ "$state" != "QUIESCED" ] && echo "device isn't quiesced after killing queue daemon"
+ 
+        secs=0
+        while [ $secs -lt 10 ]; do
+                ${ublk_prog} recover -t null -n 0 >> "$FULL"
+                [ $? -eq 0 ] && break 
+                sleep 1
+                let secs++
+        done
+        state=`${ublk_prog} list -n 0 | grep "state" | awk '{print $11}'`
+        [ "$state" == "QUIESCED" ] && echo "failed to recover dev"
+
+        ${ublk_prog} del -n 0 >> "$FULL"
+
+	_exit_ublk
+
+	echo "Test complete"
+}
diff --git a/tests/ublk/004.out b/tests/ublk/004.out
new file mode 100644
index 0000000..a92cd50
--- /dev/null
+++ b/tests/ublk/004.out
@@ -0,0 +1,2 @@
+Running ublk/004
+Test complete
diff --git a/tests/ublk/005 b/tests/ublk/005
new file mode 100644
index 0000000..23c0555
--- /dev/null
+++ b/tests/ublk/005
@@ -0,0 +1,66 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2023 Ziyang Zhang
+#
+# Test ublk user recovery by run fio with dev recovery:
+# (1)kill all ubq_deamon, (2)recover with new ubq_daemon,
+# (3)kill all ubq_deamon, (4)delete dev
+
+. tests/block/rc
+. common/ublk
+
+DESCRIPTION="test ublk recovery(2)"
+QUICK=1
+
+requires() {
+	_have_ublk
+}
+
+test() {
+	local ublk_prog="src/miniublk"
+
+	echo "Running ${TEST_NAME}"
+
+	if ! _init_ublk; then
+		return 1
+	fi
+
+	${ublk_prog} add -t null -n 0 -r 1 > "$FULL"
+	udevadm settle
+	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
+		echo "fail to list dev"
+	fi
+
+	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
+        pid=`${ublk_prog} list -n 0 | grep "pid" | awk '{print $7}'`
+        kill -9 $pid
+
+        sleep 2
+        secs=0
+        while [ $secs -lt 10 ]; do
+                state=`${ublk_prog} list -n 0 | grep "state" | awk '{print $11}'`
+                [ "$state" == "QUIESCED" ] && break
+		sleep 1
+		let secs++
+        done
+        [ "$state" != "QUIESCED" ] && echo "device isn't quiesced after killing queue daemon"
+
+        secs=0
+        while [ $secs -lt 10 ]; do
+                ${ublk_prog} recover -t null -n 0 >> "$FULL"
+                [ $? -eq 0 ] && break 
+                sleep 1
+                let secs++
+        done
+        state=`${ublk_prog} list -n 0 | grep "state" | awk '{print $11}'`
+        [ "$state" == "QUIESCED" ] && echo "failed to recover dev"
+
+        pid=`${ublk_prog} list -n 0 | grep "pid" | awk '{print $7}'`
+        kill -9 $pid
+
+        ${ublk_prog} del -n 0 >> "$FULL"
+
+	_exit_ublk
+
+	echo "Test complete"
+}
diff --git a/tests/ublk/005.out b/tests/ublk/005.out
new file mode 100644
index 0000000..20d7b38
--- /dev/null
+++ b/tests/ublk/005.out
@@ -0,0 +1,2 @@
+Running ublk/005
+Test complete
-- 
2.31.1

Re: [PATCH blktests 1/2] src/miniublk: add user recovery

[Chaitanya Kulkarni]

On 4/27/23 03:32, Ziyang Zhang wrote:
> We are going to test ublk's user recovery feature so add support in
> miniublk.
>
> Signed-off-by: Ziyang Zhang <ZiyangZhang@linux.alibaba.com>
> ---
>   src/miniublk.c | 207 ++++++++++++++++++++++++++++++++++++++++++++++---
>   1 file changed, 198 insertions(+), 9 deletions(-)
>
> diff --git a/src/miniublk.c b/src/miniublk.c
> index fe10291..558bb7b 100644
> --- a/src/miniublk.c
> +++ b/src/miniublk.c
> @@ -74,6 +74,7 @@ struct ublk_tgt_ops {
>   	int (*queue_io)(struct ublk_queue *, int tag);
>   	void (*tgt_io_done)(struct ublk_queue *,
>   			int tag, const struct io_uring_cqe *);
> +	int (*recover_tgt)(struct ublk_dev *);
>   };
>   
>   struct ublk_tgt {
> @@ -372,6 +373,29 @@ static int ublk_ctrl_get_params(struct ublk_dev *dev,
>   	return __ublk_ctrl_cmd(dev, &data);
>   }
>   
> +static int ublk_ctrl_start_user_recover(struct ublk_dev *dev)
> +{
> +	struct ublk_ctrl_cmd_data data = {
> +		.cmd_op	= UBLK_CMD_START_USER_RECOVERY,
> +		.flags	= 0,
> +	};
> +
> +	return __ublk_ctrl_cmd(dev, &data);
> +}
> +
> +static int ublk_ctrl_end_user_recover(struct ublk_dev *dev,
> +		int daemon_pid)
> +{
> +	struct ublk_ctrl_cmd_data data = {
> +		.cmd_op	= UBLK_CMD_END_USER_RECOVERY,
> +		.flags	= CTRL_CMD_HAS_DATA,
> +	};
> +
> +	dev->dev_info.ublksrv_pid = data.data[0] = daemon_pid;
> +
> +	return __ublk_ctrl_cmd(dev, &data);
> +}
> +
>   static const char *ublk_dev_state_desc(struct ublk_dev *dev)
>   {
>   	switch (dev->dev_info.state) {
> @@ -379,6 +403,8 @@ static const char *ublk_dev_state_desc(struct ublk_dev *dev)
>   		return "DEAD";
>   	case UBLK_S_DEV_LIVE:
>   		return "LIVE";
> +	case UBLK_S_DEV_QUIESCED:
> +		return "QUIESCED";
>   	default:
>   		return "UNKNOWN";
>   	};
> @@ -550,9 +576,12 @@ static int ublk_dev_prep(struct ublk_dev *dev)
>   		goto fail;
>   	}
>   
> -	if (dev->tgt.ops->init_tgt)
> +	if (dev->dev_info.state != UBLK_S_DEV_QUIESCED && dev->tgt.ops->init_tgt)
>   		ret = dev->tgt.ops->init_tgt(dev);
>   
> +	else if (dev->dev_info.state == UBLK_S_DEV_QUIESCED && dev->tgt.ops->recover_tgt)
> +		ret = dev->tgt.ops->recover_tgt(dev);
> +
>   	return ret;
>   fail:
>   	close(dev->fds[0]);
> @@ -831,7 +860,7 @@ static void ublk_set_parameters(struct ublk_dev *dev)
>   				dev->dev_info.dev_id, ret);
>   }
>   
> -static int ublk_start_daemon(struct ublk_dev *dev)
> +static int ublk_start_daemon(struct ublk_dev *dev, bool recovery)
>   {
>   	int ret, i;
>   	void *thread_ret;
> @@ -853,12 +882,22 @@ static int ublk_start_daemon(struct ublk_dev *dev)
>   				&dev->q[i]);
>   	}
>   
> -	ublk_set_parameters(dev);
>   
>   	/* everything is fine now, start us */
> -	ret = ublk_ctrl_start_dev(dev, getpid());
> -	if (ret < 0)
> -		goto fail;
> +	if (recovery) {
> +		ret = ublk_ctrl_end_user_recover(dev, getpid());
> +		if (ret < 0) {
> +			ublk_err("%s: ublk_ctrl_end_user_recover failed: %d\n", __func__, ret);
> +			goto fail;
> +		}
> +	} else {
> +		ublk_set_parameters(dev);
> +		ret = ublk_ctrl_start_dev(dev, getpid());
> +		if (ret < 0) {
> +			ublk_err("%s: ublk_ctrl_start_dev failed: %d\n", __func__, ret);
> +			goto fail;
> +		}
> +	}
>   
>   	ublk_ctrl_get_info(dev);
>   	ublk_ctrl_dump(dev, true);
> @@ -880,6 +919,7 @@ static int cmd_dev_add(int argc, char *argv[])
>   		{ "number",		1,	NULL, 'n' },
>   		{ "queues",		1,	NULL, 'q' },
>   		{ "depth",		1,	NULL, 'd' },
> +		{ "recovery",		1,	NULL, 'r' },
>   		{ "debug_mask",	1,	NULL, 0},
>   		{ "quiet",	0,	NULL, 0},
>   		{ NULL }
> @@ -891,8 +931,9 @@ static int cmd_dev_add(int argc, char *argv[])
>   	const char *tgt_type = NULL;
>   	int dev_id = -1;
>   	unsigned nr_queues = 2, depth = UBLK_QUEUE_DEPTH;
> +	int user_recovery = 0;
>   
> -	while ((opt = getopt_long(argc, argv, "-:t:n:d:q:",
> +	while ((opt = getopt_long(argc, argv, "-:t:n:d:q:r:",
>   				  longopts, &option_idx)) != -1) {
>   		switch (opt) {
>   		case 'n':
> @@ -907,6 +948,9 @@ static int cmd_dev_add(int argc, char *argv[])
>   		case 'd':
>   			depth = strtol(optarg, NULL, 10);
>   			break;
> +		case 'r':
> +			user_recovery = strtol(optarg, NULL, 10);

user_recovery is int strtol returns long ?

> +			break;
>   		case 0:
>   			if (!strcmp(longopts[option_idx].name, "debug_mask"))
>   				ublk_dbg_mask = strtol(optarg, NULL, 16);
> @@ -942,6 +986,8 @@ static int cmd_dev_add(int argc, char *argv[])
>   	info->dev_id = dev_id;
>           info->nr_hw_queues = nr_queues;
>           info->queue_depth = depth;
> +	if (user_recovery)
> +		info->flags |= UBLK_F_USER_RECOVERY;
>   	dev->tgt.ops = ops;
>   	dev->tgt.argc = argc;
>   	dev->tgt.argv = argv;
> @@ -953,7 +999,95 @@ static int cmd_dev_add(int argc, char *argv[])
>   		goto fail;
>   	}
>   
> -	ret = ublk_start_daemon(dev);
> +	ret = ublk_start_daemon(dev, false);
> +	if (ret < 0) {
> +		ublk_err("%s: can't start daemon id %d, type %s\n",
> +				__func__, dev_id, tgt_type);
> +		goto fail_del;
> +	}
> +
> +fail_del:
> +	ublk_ctrl_del_dev(dev);
> +fail:
> +	ublk_ctrl_deinit(dev);
> +	return ret;
> +}
> +
> +static int cmd_dev_recover(int argc, char *argv[])
> +{
> +	static const struct option longopts[] = {
> +		{ "type",		1,	NULL, 't' },
> +		{ "number",		1,	NULL, 'n' },
> +		{ "debug_mask",	1,	NULL, 0},
> +		{ "quiet",	0,	NULL, 0},
> +		{ NULL }
> +	};
> +	const struct ublk_tgt_ops *ops;
> +	struct ublksrv_ctrl_dev_info *info;
> +	struct ublk_dev *dev;
> +	int ret, option_idx, opt;
> +	const char *tgt_type = NULL;
> +	int dev_id = -1;
> +
> +	while ((opt = getopt_long(argc, argv, "-:t:n:d:q:",
> +				  longopts, &option_idx)) != -1) {
> +		switch (opt) {
> +		case 'n':
> +			dev_id = strtol(optarg, NULL, 10);
> +			break;
> +		case 't':
> +			tgt_type = optarg;
> +			break;
> +		case 0:
> +			if (!strcmp(longopts[option_idx].name, "debug_mask"))
> +				ublk_dbg_mask = strtol(optarg, NULL, 16);
> +			if (!strcmp(longopts[option_idx].name, "quiet"))
> +				ublk_dbg_mask = 0;
> +			break;
> +		}
> +	}
> +
> +	optind = 0;
> +
> +	ops = ublk_find_tgt(tgt_type);
> +	if (!ops) {
> +		ublk_err("%s: no such tgt type, type %s\n",
> +				__func__, tgt_type);
> +		return -ENODEV;
> +	}
> +
> +	dev = ublk_ctrl_init();
> +	if (!dev) {
> +		ublk_err("%s: can't alloc dev id %d, type %s\n",
> +				__func__, dev_id, tgt_type);
> +		return -ENOMEM;
> +	}
> +
> +	info = &dev->dev_info;
> +	info->dev_id = dev_id;
> +	ret = ublk_ctrl_get_info(dev);
> +	if (ret < 0) {
> +		ublk_err("%s: can't get dev info from %d\n", __func__, dev_id);
> +		goto fail;
> +	}
> +
> +	ret = ublk_ctrl_get_params(dev, &dev->tgt.params);
> +	if (ret) {
> +		ublk_err("dev %d set basic parameter failed %d\n",
> +				dev->dev_info.dev_id, ret);
> +		goto fail;
> +	}
> +
> +	dev->tgt.ops = ops;
> +	dev->tgt.argc = argc;
> +	dev->tgt.argv = argv;
> +	ret = ublk_ctrl_start_user_recover(dev);
> +	if (ret < 0) {
> +		ublk_err("%s: can't start recovery for %d\n", __func__, dev_id);
> +		goto fail;
> +	}
> +
> +	ret = ublk_start_daemon(dev, true);
>   	if (ret < 0) {
>   		ublk_err("%s: can't start daemon id %d, type %s\n",
>   				__func__, dev_id, tgt_type);
> @@ -1125,7 +1259,10 @@ static int cmd_dev_help(int argc, char *argv[])
>   	printf("\t -a delete all devices -n delete specified device\n");
>   	printf("%s list [-n dev_id] -a \n", argv[0]);
>   	printf("\t -a list all devices, -n list specified device, default -a \n");
> -
> +	printf("%s recover -t {null|loop} [-n dev_id] \n",
> +			argv[0]);
> +	printf("\t -t loop -f backing_file \n");
> +	printf("\t -t null\n");
>   	return 0;
>   }
>   
> @@ -1150,6 +1287,12 @@ static int ublk_null_tgt_init(struct ublk_dev *dev)
>   	return 0;
>   }
>   
> +static int ublk_null_tgt_recover(struct ublk_dev *dev)
> +{
> +	dev->tgt.dev_size = dev->tgt.params.basic.dev_sectors << 9;
> +	return 0;
> +}
> +
>   static int ublk_null_queue_io(struct ublk_queue *q, int tag)
>   {
>   	const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);
> @@ -1313,11 +1456,54 @@ static int ublk_loop_tgt_init(struct ublk_dev *dev)
>   	return 0;
>   }
>   
> +static int ublk_loop_tgt_recover(struct ublk_dev *dev)
> +{
> +	static const struct option lo_longopts[] = {
> +		{ "file",		1,	NULL, 'f' },
> +		{ NULL }
> +	};
> +	char **argv = dev->tgt.argv;
> +	int argc = dev->tgt.argc;
> +	char *file = NULL;
> +	int fd, opt;
> +
> +	while ((opt = getopt_long(argc, argv, "-:f:",
> +				  lo_longopts, NULL)) != -1) {
> +		switch (opt) {
> +		case 'f':
> +			file = strdup(optarg);
> +			break;
> +		}
> +	}
> +
> +	ublk_dbg(UBLK_DBG_DEV, "%s: file %s\n", __func__, file);

why print file above before checking for NULL below?

> +
> +	if (!file)
> +		return -EINVAL;
> +
> +	fd = open(file, O_RDWR);
> +	if (fd < 0) {
> +		ublk_err( "%s: backing file %s can't be opened\n",
> +				__func__, file);

for system call related errors printing error-no string is always useful.

> +		return -EBADF;
> +	}
> +
> +	if (fcntl(fd, F_SETFL, O_DIRECT))
> +		ublk_log("%s: ublk-loop fallback to buffered IO\n", __func__);

same here ...

> +
> +	dev->tgt.dev_size = dev->tgt.params.basic.dev_sectors << 9;
> +	dev->fds[1] = fd;
> +	dev->nr_fds += 1;
> +
> +	return 0;
> +}
> +
>   const struct ublk_tgt_ops tgt_ops_list[] = {
>   	{
>   		.name = "null",
>   		.init_tgt = ublk_null_tgt_init,
>   		.queue_io = ublk_null_queue_io,
> +		.recover_tgt = ublk_null_tgt_recover,
>   	},
>   
>   	{
> @@ -1326,6 +1512,7 @@ const struct ublk_tgt_ops tgt_ops_list[] = {
>   		.deinit_tgt = ublk_loop_tgt_deinit,
>   		.queue_io = ublk_loop_queue_io,
>   		.tgt_io_done = ublk_loop_io_done,
> +		.recover_tgt = ublk_loop_tgt_recover,
>   	},
>   };
>   
> @@ -1359,6 +1546,8 @@ int main(int argc, char *argv[])
>   		ret = cmd_dev_list(argc, argv);
>   	else if (!strcmp(cmd, "help"))
>   		ret = cmd_dev_help(argc, argv);
> +	else if (!strcmp(cmd, "recover"))
> +		ret = cmd_dev_recover(argc, argv);
>   out:
>   	if (ret)
>   		cmd_dev_help(argc, argv);

Re: [PATCH blktests 1/2] src/miniublk: add user recovery

[Ming Lei]

On Thu, Apr 27, 2023 at 06:32:41PM +0800, Ziyang Zhang wrote:
> We are going to test ublk's user recovery feature so add support in
> miniublk.
> 
> Signed-off-by: Ziyang Zhang <ZiyangZhang@linux.alibaba.com>
> ---
>  src/miniublk.c | 207 ++++++++++++++++++++++++++++++++++++++++++++++---
>  1 file changed, 198 insertions(+), 9 deletions(-)
> 
> diff --git a/src/miniublk.c b/src/miniublk.c
> index fe10291..558bb7b 100644
> --- a/src/miniublk.c
> +++ b/src/miniublk.c
> @@ -74,6 +74,7 @@ struct ublk_tgt_ops {
>  	int (*queue_io)(struct ublk_queue *, int tag);
>  	void (*tgt_io_done)(struct ublk_queue *,
>  			int tag, const struct io_uring_cqe *);
> +	int (*recover_tgt)(struct ublk_dev *);
>  };
>  
>  struct ublk_tgt {
> @@ -372,6 +373,29 @@ static int ublk_ctrl_get_params(struct ublk_dev *dev,
>  	return __ublk_ctrl_cmd(dev, &data);
>  }
>  
> +static int ublk_ctrl_start_user_recover(struct ublk_dev *dev)
> +{
> +	struct ublk_ctrl_cmd_data data = {
> +		.cmd_op	= UBLK_CMD_START_USER_RECOVERY,
> +		.flags	= 0,
> +	};
> +
> +	return __ublk_ctrl_cmd(dev, &data);
> +}
> +
> +static int ublk_ctrl_end_user_recover(struct ublk_dev *dev,
> +		int daemon_pid)
> +{
> +	struct ublk_ctrl_cmd_data data = {
> +		.cmd_op	= UBLK_CMD_END_USER_RECOVERY,
> +		.flags	= CTRL_CMD_HAS_DATA,
> +	};
> +
> +	dev->dev_info.ublksrv_pid = data.data[0] = daemon_pid;
> +
> +	return __ublk_ctrl_cmd(dev, &data);
> +}
> +
>  static const char *ublk_dev_state_desc(struct ublk_dev *dev)
>  {
>  	switch (dev->dev_info.state) {
> @@ -379,6 +403,8 @@ static const char *ublk_dev_state_desc(struct ublk_dev *dev)
>  		return "DEAD";
>  	case UBLK_S_DEV_LIVE:
>  		return "LIVE";
> +	case UBLK_S_DEV_QUIESCED:
> +		return "QUIESCED";
>  	default:
>  		return "UNKNOWN";
>  	};
> @@ -550,9 +576,12 @@ static int ublk_dev_prep(struct ublk_dev *dev)
>  		goto fail;
>  	}
>  
> -	if (dev->tgt.ops->init_tgt)
> +	if (dev->dev_info.state != UBLK_S_DEV_QUIESCED && dev->tgt.ops->init_tgt)
>  		ret = dev->tgt.ops->init_tgt(dev);
>  
> +	else if (dev->dev_info.state == UBLK_S_DEV_QUIESCED && dev->tgt.ops->recover_tgt)
> +		ret = dev->tgt.ops->recover_tgt(dev);
> +
>  	return ret;
>  fail:
>  	close(dev->fds[0]);
> @@ -831,7 +860,7 @@ static void ublk_set_parameters(struct ublk_dev *dev)
>  				dev->dev_info.dev_id, ret);
>  }
>  
> -static int ublk_start_daemon(struct ublk_dev *dev)
> +static int ublk_start_daemon(struct ublk_dev *dev, bool recovery)
>  {
>  	int ret, i;
>  	void *thread_ret;
> @@ -853,12 +882,22 @@ static int ublk_start_daemon(struct ublk_dev *dev)
>  				&dev->q[i]);
>  	}
>  
> -	ublk_set_parameters(dev);
>  
>  	/* everything is fine now, start us */
> -	ret = ublk_ctrl_start_dev(dev, getpid());
> -	if (ret < 0)
> -		goto fail;
> +	if (recovery) {
> +		ret = ublk_ctrl_end_user_recover(dev, getpid());
> +		if (ret < 0) {
> +			ublk_err("%s: ublk_ctrl_end_user_recover failed: %d\n", __func__, ret);
> +			goto fail;
> +		}
> +	} else {
> +		ublk_set_parameters(dev);
> +		ret = ublk_ctrl_start_dev(dev, getpid());
> +		if (ret < 0) {
> +			ublk_err("%s: ublk_ctrl_start_dev failed: %d\n", __func__, ret);
> +			goto fail;
> +		}
> +	}
>  
>  	ublk_ctrl_get_info(dev);
>  	ublk_ctrl_dump(dev, true);
> @@ -880,6 +919,7 @@ static int cmd_dev_add(int argc, char *argv[])
>  		{ "number",		1,	NULL, 'n' },
>  		{ "queues",		1,	NULL, 'q' },
>  		{ "depth",		1,	NULL, 'd' },
> +		{ "recovery",		1,	NULL, 'r' },
>  		{ "debug_mask",	1,	NULL, 0},
>  		{ "quiet",	0,	NULL, 0},
>  		{ NULL }
> @@ -891,8 +931,9 @@ static int cmd_dev_add(int argc, char *argv[])
>  	const char *tgt_type = NULL;
>  	int dev_id = -1;
>  	unsigned nr_queues = 2, depth = UBLK_QUEUE_DEPTH;
> +	int user_recovery = 0;
>  
> -	while ((opt = getopt_long(argc, argv, "-:t:n:d:q:",
> +	while ((opt = getopt_long(argc, argv, "-:t:n:d:q:r:",
>  				  longopts, &option_idx)) != -1) {
>  		switch (opt) {
>  		case 'n':
> @@ -907,6 +948,9 @@ static int cmd_dev_add(int argc, char *argv[])
>  		case 'd':
>  			depth = strtol(optarg, NULL, 10);
>  			break;
> +		case 'r':
> +			user_recovery = strtol(optarg, NULL, 10);
> +			break;
>  		case 0:
>  			if (!strcmp(longopts[option_idx].name, "debug_mask"))
>  				ublk_dbg_mask = strtol(optarg, NULL, 16);
> @@ -942,6 +986,8 @@ static int cmd_dev_add(int argc, char *argv[])
>  	info->dev_id = dev_id;
>          info->nr_hw_queues = nr_queues;
>          info->queue_depth = depth;
> +	if (user_recovery)
> +		info->flags |= UBLK_F_USER_RECOVERY;
>  	dev->tgt.ops = ops;
>  	dev->tgt.argc = argc;
>  	dev->tgt.argv = argv;
> @@ -953,7 +999,95 @@ static int cmd_dev_add(int argc, char *argv[])
>  		goto fail;
>  	}
>  
> -	ret = ublk_start_daemon(dev);
> +	ret = ublk_start_daemon(dev, false);
> +	if (ret < 0) {
> +		ublk_err("%s: can't start daemon id %d, type %s\n",
> +				__func__, dev_id, tgt_type);
> +		goto fail_del;
> +	}
> +
> +fail_del:
> +	ublk_ctrl_del_dev(dev);
> +fail:
> +	ublk_ctrl_deinit(dev);
> +	return ret;
> +}
> +
> +static int cmd_dev_recover(int argc, char *argv[])
> +{
> +	static const struct option longopts[] = {
> +		{ "type",		1,	NULL, 't' },
> +		{ "number",		1,	NULL, 'n' },
> +		{ "debug_mask",	1,	NULL, 0},
> +		{ "quiet",	0,	NULL, 0},
> +		{ NULL }
> +	};
> +	const struct ublk_tgt_ops *ops;
> +	struct ublksrv_ctrl_dev_info *info;
> +	struct ublk_dev *dev;
> +	int ret, option_idx, opt;
> +	const char *tgt_type = NULL;
> +	int dev_id = -1;
> +
> +	while ((opt = getopt_long(argc, argv, "-:t:n:d:q:",
> +				  longopts, &option_idx)) != -1) {
> +		switch (opt) {
> +		case 'n':
> +			dev_id = strtol(optarg, NULL, 10);
> +			break;
> +		case 't':
> +			tgt_type = optarg;
> +			break;
> +		case 0:
> +			if (!strcmp(longopts[option_idx].name, "debug_mask"))
> +				ublk_dbg_mask = strtol(optarg, NULL, 16);
> +			if (!strcmp(longopts[option_idx].name, "quiet"))
> +				ublk_dbg_mask = 0;
> +			break;
> +		}
> +	}
> +
> +	optind = 0;
> +
> +	ops = ublk_find_tgt(tgt_type);
> +	if (!ops) {
> +		ublk_err("%s: no such tgt type, type %s\n",
> +				__func__, tgt_type);
> +		return -ENODEV;
> +	}
> +
> +	dev = ublk_ctrl_init();
> +	if (!dev) {
> +		ublk_err("%s: can't alloc dev id %d, type %s\n",
> +				__func__, dev_id, tgt_type);
> +		return -ENOMEM;
> +	}
> +
> +	info = &dev->dev_info;
> +	info->dev_id = dev_id;
> +	ret = ublk_ctrl_get_info(dev);
> +	if (ret < 0) {
> +		ublk_err("%s: can't get dev info from %d\n", __func__, dev_id);
> +		goto fail;
> +	}
> +
> +	ret = ublk_ctrl_get_params(dev, &dev->tgt.params);
> +	if (ret) {
> +		ublk_err("dev %d set basic parameter failed %d\n",
> +				dev->dev_info.dev_id, ret);
> +		goto fail;
> +	}
> +
> +	dev->tgt.ops = ops;
> +	dev->tgt.argc = argc;
> +	dev->tgt.argv = argv;
> +	ret = ublk_ctrl_start_user_recover(dev);
> +	if (ret < 0) {
> +		ublk_err("%s: can't start recovery for %d\n", __func__, dev_id);
> +		goto fail;
> +	}
> +
> +	ret = ublk_start_daemon(dev, true);
>  	if (ret < 0) {
>  		ublk_err("%s: can't start daemon id %d, type %s\n",
>  				__func__, dev_id, tgt_type);
> @@ -1125,7 +1259,10 @@ static int cmd_dev_help(int argc, char *argv[])
>  	printf("\t -a delete all devices -n delete specified device\n");
>  	printf("%s list [-n dev_id] -a \n", argv[0]);
>  	printf("\t -a list all devices, -n list specified device, default -a \n");
> -
> +	printf("%s recover -t {null|loop} [-n dev_id] \n",
> +			argv[0]);
> +	printf("\t -t loop -f backing_file \n");
> +	printf("\t -t null\n");
>  	return 0;
>  }
>  
> @@ -1150,6 +1287,12 @@ static int ublk_null_tgt_init(struct ublk_dev *dev)
>  	return 0;
>  }
>  
> +static int ublk_null_tgt_recover(struct ublk_dev *dev)
> +{
> +	dev->tgt.dev_size = dev->tgt.params.basic.dev_sectors << 9;
> +	return 0;
> +}
> +
>  static int ublk_null_queue_io(struct ublk_queue *q, int tag)
>  {
>  	const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);
> @@ -1313,11 +1456,54 @@ static int ublk_loop_tgt_init(struct ublk_dev *dev)
>  	return 0;
>  }
>  
> +static int ublk_loop_tgt_recover(struct ublk_dev *dev)
> +{
> +	static const struct option lo_longopts[] = {
> +		{ "file",		1,	NULL, 'f' },
> +		{ NULL }
> +	};
> +	char **argv = dev->tgt.argv;
> +	int argc = dev->tgt.argc;
> +	char *file = NULL;
> +	int fd, opt;
> +
> +	while ((opt = getopt_long(argc, argv, "-:f:",
> +				  lo_longopts, NULL)) != -1) {
> +		switch (opt) {
> +		case 'f':
> +			file = strdup(optarg);
> +			break;
> +		}
> +	}
> +
> +	ublk_dbg(UBLK_DBG_DEV, "%s: file %s\n", __func__, file);
> +
> +	if (!file)
> +		return -EINVAL;
> +
> +	fd = open(file, O_RDWR);
> +	if (fd < 0) {
> +		ublk_err( "%s: backing file %s can't be opened\n",
> +				__func__, file);
> +		return -EBADF;
> +	}
> +
> +	if (fcntl(fd, F_SETFL, O_DIRECT))
> +		ublk_log("%s: ublk-loop fallback to buffered IO\n", __func__);
> +
> +	dev->tgt.dev_size = dev->tgt.params.basic.dev_sectors << 9;

miniublk doesn't support parameter serialization, so you have to pass file
to 'recover' command.

This way could cause trouble, what if size of the new file is less than
dev->tgt.params.basic.dev_sectors? what if different block size used?

From test purpose, it might be fine to allow different backing file used
for recovery, but the related parameters have to keep same, such as
device size and block size, given ublk driver doesn't support to
change these parameters after starting up.

Thanks,
Ming

Re: [PATCH blktests 2/2] tests: Add ublk tests

[Ming Lei]

On Thu, Apr 27, 2023 at 06:32:42PM +0800, Ziyang Zhang wrote:
> It is very important to test ublk crash handling since the userspace
> part is not reliable. Especially we should test removing device, killing
> ublk daemons and user recovery feature.

Indeed, good job!

> 
> Add five new test for ublk to cover these cases.
> 
> Signed-off-by: Ziyang Zhang <ZiyangZhang@linux.alibaba.com>
> ---
>  tests/ublk/001     | 39 +++++++++++++++++++++++++++
>  tests/ublk/001.out |  2 ++
>  tests/ublk/002     | 53 +++++++++++++++++++++++++++++++++++++
>  tests/ublk/002.out |  2 ++
>  tests/ublk/003     | 43 ++++++++++++++++++++++++++++++
>  tests/ublk/003.out |  2 ++
>  tests/ublk/004     | 63 +++++++++++++++++++++++++++++++++++++++++++
>  tests/ublk/004.out |  2 ++
>  tests/ublk/005     | 66 ++++++++++++++++++++++++++++++++++++++++++++++
>  tests/ublk/005.out |  2 ++
>  10 files changed, 274 insertions(+)
>  create mode 100644 tests/ublk/001
>  create mode 100644 tests/ublk/001.out
>  create mode 100644 tests/ublk/002
>  create mode 100644 tests/ublk/002.out
>  create mode 100644 tests/ublk/003
>  create mode 100644 tests/ublk/003.out
>  create mode 100644 tests/ublk/004
>  create mode 100644 tests/ublk/004.out
>  create mode 100644 tests/ublk/005
>  create mode 100644 tests/ublk/005.out

Please run 'make check' and fix the following warning:

tests/ublk/002:35:13: note: Use $(...) notation instead of legacy backticks `...`. [SC2006]
tests/ublk/002:36:17: note: Double quote to prevent globbing and word splitting. [SC2086]
tests/ublk/002:41:23: note: Use $(...) notation instead of legacy backticks `...`. [SC2006]
tests/ublk/002:44:3: note: Instead of 'let expr', prefer (( expr )) . [SC2219]
tests/ublk/003:35:13: note: Use $(...) notation instead of legacy backticks `...`. [SC2006]
tests/ublk/003:36:17: note: Double quote to prevent globbing and word splitting. [SC2086]
tests/ublk/004:35:13: note: Use $(...) notation instead of legacy backticks `...`. [SC2006]
tests/ublk/004:36:17: note: Double quote to prevent globbing and word splitting. [SC2086]
tests/ublk/004:41:23: note: Use $(...) notation instead of legacy backticks `...`. [SC2006]
tests/ublk/004:44:3: note: Instead of 'let expr', prefer (( expr )) . [SC2219]
tests/ublk/004:51:19: note: Check exit code directly with e.g. 'if mycmd;', not indirectly with $?. [SC2181]
tests/ublk/004:53:17: note: Instead of 'let expr', prefer (( expr )) . [SC2219]
tests/ublk/004:55:15: note: Use $(...) notation instead of legacy backticks `...`. [SC2006]
tests/ublk/005:35:13: note: Use $(...) notation instead of legacy backticks `...`. [SC2006]
tests/ublk/005:36:17: note: Double quote to prevent globbing and word splitting. [SC2086]
tests/ublk/005:41:23: note: Use $(...) notation instead of legacy backticks `...`. [SC2006]
tests/ublk/005:44:3: note: Instead of 'let expr', prefer (( expr )) . [SC2219]
tests/ublk/005:51:19: note: Check exit code directly with e.g. 'if mycmd;', not indirectly with $?. [SC2181]
tests/ublk/005:53:17: note: Instead of 'let expr', prefer (( expr )) . [SC2219]
tests/ublk/005:55:15: note: Use $(...) notation instead of legacy backticks `...`. [SC2006]
tests/ublk/005:58:13: note: Use $(...) notation instead of legacy backticks `...`. [SC2006]
tests/ublk/005:59:17: note: Double quote to prevent globbing and word splitting. [SC2086]

> 
> diff --git a/tests/ublk/001 b/tests/ublk/001
> new file mode 100644
> index 0000000..fe158ba
> --- /dev/null
> +++ b/tests/ublk/001
> @@ -0,0 +1,39 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-3.0+
> +# Copyright (C) 2023 Ziyang Zhang
> +#
> +# Test ublk by deleting ublk device while running fio
> +
> +. tests/block/rc
> +. common/ublk
> +
> +DESCRIPTION="test ublk deletion"
> +QUICK=1
> +
> +requires() {
> +	_have_ublk
> +}
> +
> +test() {
> +	local ublk_prog="src/miniublk"
> +
> +	echo "Running ${TEST_NAME}"
> +
> +	if ! _init_ublk; then
> +		return 1
> +	fi
> +
> +	${ublk_prog} add -t null -n 0 > "$FULL"
> +	udevadm settle
> +	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
> +		echo "fail to list dev"
> +	fi
> +
> +	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
> +
> +        ${ublk_prog} del -n 0 >> "$FULL"

Indent.

Also probably, add one small delay before deleting device, then we can
make sure there are enough in-flight IOs before starting to delete.

> +
> +	_exit_ublk
> +
> +	echo "Test complete"
> +}
> diff --git a/tests/ublk/001.out b/tests/ublk/001.out
> new file mode 100644
> index 0000000..0d070b3
> --- /dev/null
> +++ b/tests/ublk/001.out
> @@ -0,0 +1,2 @@
> +Running ublk/001
> +Test complete
> diff --git a/tests/ublk/002 b/tests/ublk/002
> new file mode 100644
> index 0000000..25cad13
> --- /dev/null
> +++ b/tests/ublk/002
> @@ -0,0 +1,53 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-3.0+
> +# Copyright (C) 2023 Ziyang Zhang
> +#
> +# Test ublk by killing ublk daemon while running fio
> +# Delete the device after it is dead
> +
> +. tests/block/rc
> +. common/ublk
> +
> +DESCRIPTION="test ublk crash(1)"
> +QUICK=1
> +
> +requires() {
> +	_have_ublk
> +}
> +
> +test() {
> +	local ublk_prog="src/miniublk"
> +
> +	echo "Running ${TEST_NAME}"
> +
> +	if ! _init_ublk; then
> +		return 1
> +	fi
> +
> +	${ublk_prog} add -t null -n 0 > "$FULL"
> +	udevadm settle
> +	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
> +		echo "fail to list dev"
> +	fi
> +
> +	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
> +
> +        pid=`${ublk_prog} list -n 0 | grep "pid" | awk '{print $7}'`
> +        kill -9 $pid
> +
> +        sleep 2
> +        secs=0
> +        while [ $secs -lt 10 ]; do
> +                state=`${ublk_prog} list -n 0 | grep "state" | awk '{print $11}'`

It could be cleaner to add one function for retrieving ublk state, given
the func is needed in many tests.

> +                [ "$state" == "DEAD" ] && break
> +		sleep 1
> +		let secs++
> +        done

Indent.

> +        [ "$state" != "DEAD" ] && echo "device isn't dead after killing queue daemon"
> +
> +        ${ublk_prog} del -n 0 >> "$FULL"
> +
> +	_exit_ublk
> +
> +	echo "Test complete"
> +}
> diff --git a/tests/ublk/002.out b/tests/ublk/002.out
> new file mode 100644
> index 0000000..93039b7
> --- /dev/null
> +++ b/tests/ublk/002.out
> @@ -0,0 +1,2 @@
> +Running ublk/002
> +Test complete
> diff --git a/tests/ublk/003 b/tests/ublk/003
> new file mode 100644
> index 0000000..34bce74
> --- /dev/null
> +++ b/tests/ublk/003
> @@ -0,0 +1,43 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-3.0+
> +# Copyright (C) 2023 Ziyang Zhang
> +#
> +# Test ublk by killing ublk daemon while running fio
> +# Delete the device immediately
> +
> +. tests/block/rc
> +. common/ublk
> +
> +DESCRIPTION="test ublk crash(2)"
> +QUICK=1
> +
> +requires() {
> +	_have_ublk
> +}
> +
> +test() {
> +	local ublk_prog="src/miniublk"
> +
> +	echo "Running ${TEST_NAME}"
> +
> +	if ! _init_ublk; then
> +		return 1
> +	fi
> +
> +	${ublk_prog} add -t null -n 0 > "$FULL"
> +	udevadm settle
> +	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
> +		echo "fail to list dev"
> +	fi
> +
> +	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
> +
> +        pid=`${ublk_prog} list -n 0 | grep "pid" | awk '{print $7}'`
> +        kill -9 $pid
> +
> +        ${ublk_prog} del -n 0 >> "$FULL"

Indent.

> +
> +	_exit_ublk
> +
> +	echo "Test complete"
> +}
> diff --git a/tests/ublk/003.out b/tests/ublk/003.out
> new file mode 100644
> index 0000000..90a3bfa
> --- /dev/null
> +++ b/tests/ublk/003.out
> @@ -0,0 +1,2 @@
> +Running ublk/003
> +Test complete
> diff --git a/tests/ublk/004 b/tests/ublk/004
> new file mode 100644
> index 0000000..c5d0694
> --- /dev/null
> +++ b/tests/ublk/004
> @@ -0,0 +1,63 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-3.0+
> +# Copyright (C) 2023 Ziyang Zhang
> +#
> +# Test ublk user recovery by run fio with dev recovery:
> +# (1)kill all ubq_deamon, (2)recover with new ubq_daemon,
> +# (3)delete dev
> +
> +. tests/block/rc
> +. common/ublk
> +
> +DESCRIPTION="test ublk recovery(1)"
> +QUICK=1
> +
> +requires() {
> +	_have_ublk
> +}
> +
> +test() {
> +	local ublk_prog="src/miniublk"
> +
> +	echo "Running ${TEST_NAME}"
> +
> +	if ! _init_ublk; then
> +		return 1
> +	fi
> +
> +	${ublk_prog} add -t null -n 0 -r 1 > "$FULL"
> +	udevadm settle
> +	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
> +		echo "fail to list dev"
> +	fi
> +
> +	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
> +        pid=`${ublk_prog} list -n 0 | grep "pid" | awk '{print $7}'`
> +        kill -9 $pid
> +
> +        sleep 2
> +        secs=0
> +        while [ $secs -lt 10 ]; do
> +                state=`${ublk_prog} list -n 0 | grep "state" | awk '{print $11}'`
> +                [ "$state" == "QUIESCED" ] && break
> +		sleep 1
> +		let secs++
> +        done

Indent.

> +        [ "$state" != "QUIESCED" ] && echo "device isn't quiesced after killing queue daemon"
> + 
> +        secs=0
> +        while [ $secs -lt 10 ]; do
> +                ${ublk_prog} recover -t null -n 0 >> "$FULL"
> +                [ $? -eq 0 ] && break 
> +                sleep 1
> +                let secs++
> +        done

Do we need to send multiple recover commands?

> +        state=`${ublk_prog} list -n 0 | grep "state" | awk '{print $11}'`
> +        [ "$state" == "QUIESCED" ] && echo "failed to recover dev"
> +
> +        ${ublk_prog} del -n 0 >> "$FULL"
> +
> +	_exit_ublk
> +
> +	echo "Test complete"
> +}
> diff --git a/tests/ublk/004.out b/tests/ublk/004.out
> new file mode 100644
> index 0000000..a92cd50
> --- /dev/null
> +++ b/tests/ublk/004.out
> @@ -0,0 +1,2 @@
> +Running ublk/004
> +Test complete
> diff --git a/tests/ublk/005 b/tests/ublk/005
> new file mode 100644
> index 0000000..23c0555
> --- /dev/null
> +++ b/tests/ublk/005
> @@ -0,0 +1,66 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-3.0+
> +# Copyright (C) 2023 Ziyang Zhang
> +#
> +# Test ublk user recovery by run fio with dev recovery:
> +# (1)kill all ubq_deamon, (2)recover with new ubq_daemon,
> +# (3)kill all ubq_deamon, (4)delete dev
> +
> +. tests/block/rc
> +. common/ublk
> +
> +DESCRIPTION="test ublk recovery(2)"
> +QUICK=1
> +
> +requires() {
> +	_have_ublk
> +}
> +
> +test() {
> +	local ublk_prog="src/miniublk"
> +
> +	echo "Running ${TEST_NAME}"
> +
> +	if ! _init_ublk; then
> +		return 1
> +	fi
> +
> +	${ublk_prog} add -t null -n 0 -r 1 > "$FULL"
> +	udevadm settle
> +	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
> +		echo "fail to list dev"
> +	fi
> +
> +	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
> +        pid=`${ublk_prog} list -n 0 | grep "pid" | awk '{print $7}'`
> +        kill -9 $pid
> +
> +        sleep 2
> +        secs=0
> +        while [ $secs -lt 10 ]; do
> +                state=`${ublk_prog} list -n 0 | grep "state" | awk '{print $11}'`
> +                [ "$state" == "QUIESCED" ] && break
> +		sleep 1
> +		let secs++
> +        done
> +        [ "$state" != "QUIESCED" ] && echo "device isn't quiesced after killing queue daemon"

Indent.

> +
> +        secs=0
> +        while [ $secs -lt 10 ]; do
> +                ${ublk_prog} recover -t null -n 0 >> "$FULL"
> +                [ $? -eq 0 ] && break 
> +                sleep 1
> +                let secs++
> +        done

Same question with above.


Thanks, 
Ming

Re: [PATCH blktests 2/2] tests: Add ublk tests

[Shinichiro Kawasaki]

On Apr 27, 2023 / 18:32, Ziyang Zhang wrote:
> It is very important to test ublk crash handling since the userspace
> part is not reliable. Especially we should test removing device, killing
> ublk daemons and user recovery feature.

Hello Ziyang, thanks for the patches. I think it is valuable to add the ublk
test group and the five test cases. Please find my comments in line. Also, I
saw some trailing spaces to remove.

I tried the two patches on the kernel v6.3, and observed ublk/004 and ublk/005
fail. Do I need some fixes on kernel side to make them pass?

> 
> Add five new test for ublk to cover these cases.

nit: s/test/tests/

> 
> Signed-off-by: Ziyang Zhang <ZiyangZhang@linux.alibaba.com>
> ---
>  tests/ublk/001     | 39 +++++++++++++++++++++++++++
>  tests/ublk/001.out |  2 ++
>  tests/ublk/002     | 53 +++++++++++++++++++++++++++++++++++++
>  tests/ublk/002.out |  2 ++
>  tests/ublk/003     | 43 ++++++++++++++++++++++++++++++
>  tests/ublk/003.out |  2 ++
>  tests/ublk/004     | 63 +++++++++++++++++++++++++++++++++++++++++++
>  tests/ublk/004.out |  2 ++
>  tests/ublk/005     | 66 ++++++++++++++++++++++++++++++++++++++++++++++
>  tests/ublk/005.out |  2 ++
>  10 files changed, 274 insertions(+)
>  create mode 100644 tests/ublk/001

For historical reason, we add executable mode to the test script files. Please
add the file mode 755 to tests/ublk/001. Same for 002-005.

>  create mode 100644 tests/ublk/001.out
>  create mode 100644 tests/ublk/002
>  create mode 100644 tests/ublk/002.out
>  create mode 100644 tests/ublk/003
>  create mode 100644 tests/ublk/003.out
>  create mode 100644 tests/ublk/004
>  create mode 100644 tests/ublk/004.out
>  create mode 100644 tests/ublk/005
>  create mode 100644 tests/ublk/005.out
> 
> diff --git a/tests/ublk/001 b/tests/ublk/001
> new file mode 100644
> index 0000000..fe158ba
> --- /dev/null
> +++ b/tests/ublk/001
> @@ -0,0 +1,39 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-3.0+
> +# Copyright (C) 2023 Ziyang Zhang
> +#
> +# Test ublk by deleting ublk device while running fio
> +
> +. tests/block/rc
> +. common/ublk
> +
> +DESCRIPTION="test ublk deletion"
> +QUICK=1
> +
> +requires() {
> +	_have_ublk
> +}

I suggest to add group_requires() in tests/ublk/rc file, and call _have_ublk
in it. With this, the five test cases do not repeat the same requires().

> +
> +test() {
> +	local ublk_prog="src/miniublk"
> +
> +	echo "Running ${TEST_NAME}"
> +
> +	if ! _init_ublk; then
> +		return 1
> +	fi
> +
> +	${ublk_prog} add -t null -n 0 > "$FULL"
> +	udevadm settle
> +	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
> +		echo "fail to list dev"
> +	fi
> +
> +	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
> +
> +        ${ublk_prog} del -n 0 >> "$FULL"
> +
> +	_exit_ublk
> +
> +	echo "Test complete"
> +}
> diff --git a/tests/ublk/001.out b/tests/ublk/001.out
> new file mode 100644
> index 0000000..0d070b3
> --- /dev/null
> +++ b/tests/ublk/001.out
> @@ -0,0 +1,2 @@
> +Running ublk/001
> +Test complete
> diff --git a/tests/ublk/002 b/tests/ublk/002
> new file mode 100644
> index 0000000..25cad13
> --- /dev/null
> +++ b/tests/ublk/002
> @@ -0,0 +1,53 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-3.0+
> +# Copyright (C) 2023 Ziyang Zhang
> +#
> +# Test ublk by killing ublk daemon while running fio
> +# Delete the device after it is dead
> +
> +. tests/block/rc
> +. common/ublk
> +
> +DESCRIPTION="test ublk crash(1)"

This description and that of ublk/003 are not so informative. Could you try to
enrich them? Maybe "test ublk crash with delete after dead confirmation" and
"test ublk crash with delete just after daemon kill".

> +QUICK=1
> +
> +requires() {
> +	_have_ublk
> +}
> +
> +test() {
> +	local ublk_prog="src/miniublk"
> +
> +	echo "Running ${TEST_NAME}"
> +
> +	if ! _init_ublk; then
> +		return 1
> +	fi
> +
> +	${ublk_prog} add -t null -n 0 > "$FULL"
> +	udevadm settle
> +	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
> +		echo "fail to list dev"
> +	fi
> +
> +	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
> +
> +        pid=`${ublk_prog} list -n 0 | grep "pid" | awk '{print $7}'`
> +        kill -9 $pid
> +
> +        sleep 2
> +        secs=0
> +        while [ $secs -lt 10 ]; do
> +                state=`${ublk_prog} list -n 0 | grep "state" | awk '{print $11}'`
> +                [ "$state" == "DEAD" ] && break
> +		sleep 1
> +		let secs++
> +        done
> +        [ "$state" != "DEAD" ] && echo "device isn't dead after killing queue daemon"
> +
> +        ${ublk_prog} del -n 0 >> "$FULL"
> +
> +	_exit_ublk
> +
> +	echo "Test complete"
> +}
> diff --git a/tests/ublk/002.out b/tests/ublk/002.out
> new file mode 100644
> index 0000000..93039b7
> --- /dev/null
> +++ b/tests/ublk/002.out
> @@ -0,0 +1,2 @@
> +Running ublk/002
> +Test complete
> diff --git a/tests/ublk/003 b/tests/ublk/003
> new file mode 100644
> index 0000000..34bce74
> --- /dev/null
> +++ b/tests/ublk/003
> @@ -0,0 +1,43 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-3.0+
> +# Copyright (C) 2023 Ziyang Zhang
> +#
> +# Test ublk by killing ublk daemon while running fio
> +# Delete the device immediately
> +
> +. tests/block/rc
> +. common/ublk
> +
> +DESCRIPTION="test ublk crash(2)"
> +QUICK=1
> +
> +requires() {
> +	_have_ublk
> +}
> +
> +test() {
> +	local ublk_prog="src/miniublk"
> +
> +	echo "Running ${TEST_NAME}"
> +
> +	if ! _init_ublk; then
> +		return 1
> +	fi
> +
> +	${ublk_prog} add -t null -n 0 > "$FULL"
> +	udevadm settle
> +	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
> +		echo "fail to list dev"
> +	fi
> +
> +	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
> +
> +        pid=`${ublk_prog} list -n 0 | grep "pid" | awk '{print $7}'`
> +        kill -9 $pid
> +
> +        ${ublk_prog} del -n 0 >> "$FULL"
> +
> +	_exit_ublk
> +
> +	echo "Test complete"
> +}
> diff --git a/tests/ublk/003.out b/tests/ublk/003.out
> new file mode 100644
> index 0000000..90a3bfa
> --- /dev/null
> +++ b/tests/ublk/003.out
> @@ -0,0 +1,2 @@
> +Running ublk/003
> +Test complete
> diff --git a/tests/ublk/004 b/tests/ublk/004
> new file mode 100644
> index 0000000..c5d0694
> --- /dev/null
> +++ b/tests/ublk/004
> @@ -0,0 +1,63 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-3.0+
> +# Copyright (C) 2023 Ziyang Zhang
> +#
> +# Test ublk user recovery by run fio with dev recovery:
> +# (1)kill all ubq_deamon, (2)recover with new ubq_daemon,
> +# (3)delete dev
> +
> +. tests/block/rc
> +. common/ublk
> +
> +DESCRIPTION="test ublk recovery(1)"

Same comment as DESCRIPTIONS for crash(1) and crash(2). The descriptions can be
"test ublk recovery with one time daemon kill" and "test ublk recovery with
two times daemon kill", or something like that.

> +QUICK=1
> +
> +requires() {
> +	_have_ublk
> +}
> +
> +test() {
> +	local ublk_prog="src/miniublk"
> +
> +	echo "Running ${TEST_NAME}"
> +
> +	if ! _init_ublk; then
> +		return 1
> +	fi
> +
> +	${ublk_prog} add -t null -n 0 -r 1 > "$FULL"
> +	udevadm settle
> +	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
> +		echo "fail to list dev"
> +	fi
> +
> +	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
> +        pid=`${ublk_prog} list -n 0 | grep "pid" | awk '{print $7}'`
> +        kill -9 $pid
> +
> +        sleep 2
> +        secs=0
> +        while [ $secs -lt 10 ]; do
> +                state=`${ublk_prog} list -n 0 | grep "state" | awk '{print $11}'`
> +                [ "$state" == "QUIESCED" ] && break
> +		sleep 1
> +		let secs++
> +        done
> +        [ "$state" != "QUIESCED" ] && echo "device isn't quiesced after killing queue daemon"
> + 
> +        secs=0
> +        while [ $secs -lt 10 ]; do
> +                ${ublk_prog} recover -t null -n 0 >> "$FULL"
> +                [ $? -eq 0 ] && break 
> +                sleep 1
> +                let secs++
> +        done
> +        state=`${ublk_prog} list -n 0 | grep "state" | awk '{print $11}'`
> +        [ "$state" == "QUIESCED" ] && echo "failed to recover dev"
> +
> +        ${ublk_prog} del -n 0 >> "$FULL"
> +
> +	_exit_ublk
> +
> +	echo "Test complete"
> +}
> diff --git a/tests/ublk/004.out b/tests/ublk/004.out
> new file mode 100644
> index 0000000..a92cd50
> --- /dev/null
> +++ b/tests/ublk/004.out
> @@ -0,0 +1,2 @@
> +Running ublk/004
> +Test complete
> diff --git a/tests/ublk/005 b/tests/ublk/005
> new file mode 100644
> index 0000000..23c0555
> --- /dev/null
> +++ b/tests/ublk/005
> @@ -0,0 +1,66 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-3.0+
> +# Copyright (C) 2023 Ziyang Zhang
> +#
> +# Test ublk user recovery by run fio with dev recovery:
> +# (1)kill all ubq_deamon, (2)recover with new ubq_daemon,
> +# (3)kill all ubq_deamon, (4)delete dev
> +
> +. tests/block/rc
> +. common/ublk
> +
> +DESCRIPTION="test ublk recovery(2)"
> +QUICK=1
> +
> +requires() {
> +	_have_ublk
> +}
> +
> +test() {
> +	local ublk_prog="src/miniublk"
> +
> +	echo "Running ${TEST_NAME}"
> +
> +	if ! _init_ublk; then
> +		return 1
> +	fi
> +
> +	${ublk_prog} add -t null -n 0 -r 1 > "$FULL"
> +	udevadm settle
> +	if ! ${ublk_prog} list -n 0 >> "$FULL"; then
> +		echo "fail to list dev"
> +	fi
> +
> +	_run_fio_rand_io --filename=/dev/ublkb0 --time_based --runtime=30 > /dev/null 2>&1 &
> +        pid=`${ublk_prog} list -n 0 | grep "pid" | awk '{print $7}'`
> +        kill -9 $pid
> +
> +        sleep 2
> +        secs=0
> +        while [ $secs -lt 10 ]; do
> +                state=`${ublk_prog} list -n 0 | grep "state" | awk '{print $11}'`
> +                [ "$state" == "QUIESCED" ] && break
> +		sleep 1
> +		let secs++
> +        done
> +        [ "$state" != "QUIESCED" ] && echo "device isn't quiesced after killing queue daemon"
> +
> +        secs=0
> +        while [ $secs -lt 10 ]; do
> +                ${ublk_prog} recover -t null -n 0 >> "$FULL"
> +                [ $? -eq 0 ] && break 
> +                sleep 1
> +                let secs++
> +        done
> +        state=`${ublk_prog} list -n 0 | grep "state" | awk '{print $11}'`
> +        [ "$state" == "QUIESCED" ] && echo "failed to recover dev"
> +
> +        pid=`${ublk_prog} list -n 0 | grep "pid" | awk '{print $7}'`
> +        kill -9 $pid
> +
> +        ${ublk_prog} del -n 0 >> "$FULL"
> +
> +	_exit_ublk
> +
> +	echo "Test complete"
> +}
> diff --git a/tests/ublk/005.out b/tests/ublk/005.out
> new file mode 100644
> index 0000000..20d7b38
> --- /dev/null
> +++ b/tests/ublk/005.out
> @@ -0,0 +1,2 @@
> +Running ublk/005
> +Test complete
> -- 
> 2.31.1
>