[PATCH blktests v2 0/2] md: add regression test for "md/md-bitmap: fix writing non bitmap pages"

[PATCH blktests v2 0/2] md: add regression test for "md/md-bitmap: fix writing non bitmap pages"

[Ofir Gal]

This patchset adds a regression test for "md/md-bitmap: fix writing non
bitmap pages".

The regression test requires a network layer as the underlying layer of
md, it use nvme-tcp as the network layer.

Changelog:
v2 - applied Shinichiro's comments, use common/nvme instead of
   tests/nvme/rc, disconnecting nvme controller on cleanup_nvme_over_tcp

Ofir Gal (2):
  nvme: move helper functions to common/nvme
  md: add regression test for "md/md-bitmap: fix writing non bitmap
    pages"

 common/brd       |  28 +++
 common/nvme      | 595 +++++++++++++++++++++++++++++++++++++++++++++++
 tests/md/001     |  88 +++++++
 tests/md/001.out |   2 +
 tests/md/rc      |  12 +
 tests/nvme/rc    | 591 +---------------------------------------------
 6 files changed, 726 insertions(+), 590 deletions(-)
 create mode 100644 common/brd
 create mode 100644 common/nvme
 create mode 100755 tests/md/001
 create mode 100644 tests/md/001.out
 create mode 100644 tests/md/rc

-- 
2.45.1

[PATCH blktests v2 1/2] nvme: move helper functions to common/nvme

[Ofir Gal]

Move functions from tests/nvme/rc to common/nvme to be able to reuse
them in other tests groups.

Signed-off-by: Ofir Gal <ofir.gal@volumez.com>
---
I have moved the function that are nessecary for the regression test I
add. Let me know if we want to move more or all the functions to
common/nvme.

shellcheck detects 2 new warnings:
common/nvme:35:10: warning: nvme_trtype is referenced but not assigned. [SC2154]
common/nvme:234:11: warning: nvme_adrfam is referenced but not assigned. [SC2154]

I have tried to figure out why but I couldn't figure out what is
different than the upstream version. How can I fix them?


 common/nvme   | 595 ++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/rc | 591 +------------------------------------------------
 2 files changed, 596 insertions(+), 590 deletions(-)
 create mode 100644 common/nvme

diff --git a/common/nvme b/common/nvme
new file mode 100644
index 0000000..1800263
--- /dev/null
+++ b/common/nvme
@@ -0,0 +1,595 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+#
+# nvme helper functions.
+
+. common/shellcheck
+
+def_traddr="127.0.0.1"
+def_adrfam="ipv4"
+def_trsvcid="4420"
+def_remote_wwnn="0x10001100aa000001"
+def_remote_wwpn="0x20001100aa000001"
+def_local_wwnn="0x10001100aa000002"
+def_local_wwpn="0x20001100aa000002"
+def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349"
+def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}"
+export def_subsysnqn="blktests-subsystem-1"
+export def_subsys_uuid="91fdba0d-f87b-4c25-b80f-db7be1418b9e"
+_check_conflict_and_set_default NVMET_TRTYPES nvme_trtype "loop"
+_check_conflict_and_set_default NVME_IMG_SIZE nvme_img_size 1G
+_check_conflict_and_set_default NVME_NUM_ITER nvme_num_iter 1000
+nvmet_blkdev_type=${nvmet_blkdev_type:-"device"}
+NVMET_BLKDEV_TYPES=${NVMET_BLKDEV_TYPES:-"device file"}
+NVMET_CFS="/sys/kernel/config/nvmet/"
+
+# TMPDIR can not be referred out of test() or test_device() context. Instead of
+# global variable def_flie_path, use this getter function.
+_nvme_def_file_path() {
+	echo "${TMPDIR}/img"
+}
+
+_require_nvme_trtype() {
+	local trtype
+	for trtype in "$@"; do
+		if [[ "${nvme_trtype}" == "$trtype" ]]; then
+			return 0
+		fi
+	done
+	SKIP_REASONS+=("nvme_trtype=${nvme_trtype} is not supported in this test")
+	return 1
+}
+
+_require_nvme_trtype_is_loop() {
+	if ! _require_nvme_trtype loop; then
+		return 1
+	fi
+	return 0
+}
+
+_require_nvme_trtype_is_fabrics() {
+	if ! _require_nvme_trtype loop fc rdma tcp; then
+		return 1
+	fi
+	return 0
+}
+
+_nvme_fcloop_add_rport() {
+	local local_wwnn="$1"
+	local local_wwpn="$2"
+	local remote_wwnn="$3"
+	local remote_wwpn="$4"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port
+}
+
+_nvme_fcloop_add_lport() {
+	local wwnn="$1"
+	local wwpn="$2"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port
+}
+
+_nvme_fcloop_add_tport() {
+	local wwnn="$1"
+	local wwpn="$2"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port
+}
+
+_setup_fcloop() {
+	local local_wwnn="${1:-$def_local_wwnn}"
+	local local_wwpn="${2:-$def_local_wwpn}"
+	local remote_wwnn="${3:-$def_remote_wwnn}"
+	local remote_wwpn="${4:-$def_remote_wwpn}"
+
+	_nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}"
+	_nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}"
+	_nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \
+		               "${remote_wwnn}" "${remote_wwpn}"
+}
+
+_nvme_fcloop_del_rport() {
+	local local_wwnn="$1"
+	local local_wwpn="$2"
+	local remote_wwnn="$3"
+	local remote_wwpn="$4"
+	local loopctl=/sys/class/fcloop/ctl
+
+	if [[ ! -f "${loopctl}/del_remote_port" ]]; then
+		return
+	fi
+	echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > "${loopctl}/del_remote_port"
+}
+
+_nvme_fcloop_del_lport() {
+	local wwnn="$1"
+	local wwpn="$2"
+	local loopctl=/sys/class/fcloop/ctl
+
+	if [[ ! -f "${loopctl}/del_local_port" ]]; then
+		return
+	fi
+	echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_local_port"
+}
+
+_nvme_fcloop_del_tport() {
+	local wwnn="$1"
+	local wwpn="$2"
+	local loopctl=/sys/class/fcloop/ctl
+
+	if [[ ! -f "${loopctl}/del_target_port" ]]; then
+		return
+	fi
+	echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port"
+}
+
+_cleanup_fcloop() {
+	local local_wwnn="${1:-$def_local_wwnn}"
+	local local_wwpn="${2:-$def_local_wwpn}"
+	local remote_wwnn="${3:-$def_remote_wwnn}"
+	local remote_wwpn="${4:-$def_remote_wwpn}"
+
+	_nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}"
+	_nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}"
+	_nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \
+			       "${remote_wwnn}" "${remote_wwpn}"
+}
+
+_cleanup_blkdev() {
+	local blkdev
+	local dev
+
+	blkdev="$(losetup -l | awk '$6 == "'"$(_nvme_def_file_path)"'" { print $1 }')"
+	for dev in ${blkdev}; do
+		losetup -d "${dev}"
+	done
+	rm -f "$(_nvme_def_file_path)"
+}
+
+_cleanup_nvmet() {
+	local dev
+	local port
+	local subsys
+	local transport
+	local name
+
+	if [[ ! -d "${NVMET_CFS}" ]]; then
+		return 0
+	fi
+
+	# Don't let successive Ctrl-Cs interrupt the cleanup processes
+	trap '' SIGINT
+
+	shopt -s nullglob
+
+	for dev in /sys/class/nvme/nvme*; do
+		dev="$(basename "$dev")"
+		transport="$(cat "/sys/class/nvme/${dev}/transport" 2>/dev/null)"
+		if [[ "$transport" == "${nvme_trtype}" ]]; then
+			# if udev auto connect is enabled for FC we get false positives
+			if [[ "$transport" != "fc" ]]; then
+				echo "WARNING: Test did not clean up ${nvme_trtype} device: ${dev}"
+			fi
+			_nvme_disconnect_ctrl "${dev}" 2>/dev/null
+		fi
+	done
+
+	for port in "${NVMET_CFS}"/ports/*; do
+		name=$(basename "${port}")
+		echo "WARNING: Test did not clean up port: ${name}"
+		rm -f "${port}"/subsystems/*
+		rmdir "${port}"
+	done
+
+	for subsys in "${NVMET_CFS}"/subsystems/*; do
+		name=$(basename "${subsys}")
+		echo "WARNING: Test did not clean up subsystem: ${name}"
+		for ns in "${subsys}"/namespaces/*; do
+			rmdir "${ns}"
+		done
+		rmdir "${subsys}"
+	done
+
+	for host in "${NVMET_CFS}"/hosts/*; do
+		name=$(basename "${host}")
+		echo "WARNING: Test did not clean up host: ${name}"
+		rmdir "${host}"
+	done
+
+	shopt -u nullglob
+	trap SIGINT
+
+	if [[ "${nvme_trtype}" == "fc" ]]; then
+		_cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
+				"${def_remote_wwnn}" "${def_remote_wwpn}"
+		modprobe -rq nvme-fcloop 2>/dev/null
+	fi
+	modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null
+	if [[ "${nvme_trtype}" != "loop" ]]; then
+		modprobe -rq nvmet-"${nvme_trtype}" 2>/dev/null
+	fi
+	modprobe -rq nvmet 2>/dev/null
+	if [[ "${nvme_trtype}" == "rdma" ]]; then
+		stop_soft_rdma
+	fi
+
+	_cleanup_blkdev
+}
+
+_setup_nvmet() {
+	_register_test_cleanup _cleanup_nvmet
+	modprobe -q nvmet
+	if [[ "${nvme_trtype}" != "loop" ]]; then
+		modprobe -q nvmet-"${nvme_trtype}"
+	fi
+	modprobe -q nvme-"${nvme_trtype}"
+	if [[ "${nvme_trtype}" == "rdma" ]]; then
+		start_soft_rdma
+		for i in $(rdma_network_interfaces)
+		do
+			if [[ "${nvme_adrfam}" == "ipv6" ]]; then
+				ipv6_addr=$(get_ipv6_ll_addr "$i")
+				if [[ -n "${ipv6_addr}" ]]; then
+					def_traddr=${ipv6_addr}
+				fi
+			else
+				ipv4_addr=$(get_ipv4_addr "$i")
+				if [[ -n "${ipv4_addr}" ]]; then
+					def_traddr=${ipv4_addr}
+				fi
+			fi
+		done
+	fi
+	if [[ "${nvme_trtype}" = "fc" ]]; then
+		modprobe -q nvme-fcloop
+		_setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
+			      "${def_remote_wwnn}" "${def_remote_wwpn}"
+
+		def_traddr=$(printf "nn-%s:pn-%s" \
+				    "${def_remote_wwnn}" \
+				    "${def_remote_wwpn}")
+		def_host_traddr=$(printf "nn-%s:pn-%s" \
+					 "${def_local_wwnn}" \
+					 "${def_local_wwpn}")
+	fi
+}
+
+_nvme_disconnect_ctrl() {
+	local ctrl="$1"
+
+	nvme disconnect --device "${ctrl}"
+}
+
+_nvme_connect_subsys() {
+	local subsysnqn="$def_subsysnqn"
+	local hostnqn="$def_hostnqn"
+	local hostid="$def_hostid"
+	local hostkey=""
+	local ctrlkey=""
+	local nr_io_queues=""
+	local nr_write_queues=""
+	local nr_poll_queues=""
+	local keep_alive_tmo=""
+	local reconnect_delay=""
+	local ctrl_loss_tmo=""
+	local no_wait=false
+	local i
+
+	while [[ $# -gt 0 ]]; do
+		case $1 in
+			--subsysnqn)
+				subsysnqn="$2"
+				shift 2
+				;;
+			--hostnqn)
+				hostnqn="$2"
+				shift 2
+				;;
+			--hostid)
+				hostid="$2"
+				shift 2
+				;;
+			--dhchap-secret)
+				hostkey="$2"
+				shift 2
+				;;
+			--dhchap-ctrl-secret)
+				ctrlkey="$2"
+				shift 2
+				;;
+			--nr-io-queues)
+				nr_io_queues="$2"
+				shift 2
+				;;
+			--nr-write-queues)
+				nr_write_queues="$2"
+				shift 2
+				;;
+			--nr-poll-queues)
+				nr_poll_queues="$2"
+				shift 2
+				;;
+			--keep-alive-tmo)
+				keep_alive_tmo="$2"
+				shift 2
+				;;
+			--reconnect-delay)
+				reconnect_delay="$2"
+				shift 2
+				;;
+			--ctrl-loss-tmo)
+				ctrl_loss_tmo="$2"
+				shift 2
+				;;
+			--no-wait)
+				no_wait=true
+				shift 1
+				;;
+			*)
+				echo "WARNING: unknown argument: $1"
+				shift
+				;;
+		esac
+	done
+
+	ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}")
+	if [[ "${nvme_trtype}" == "fc" ]] ; then
+		ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}")
+	elif [[ "${nvme_trtype}" != "loop" ]]; then
+		ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}")
+	fi
+	ARGS+=(--hostnqn="${hostnqn}")
+	ARGS+=(--hostid="${hostid}")
+	if [[ -n "${hostkey}" ]]; then
+		ARGS+=(--dhchap-secret="${hostkey}")
+	fi
+	if [[ -n "${ctrlkey}" ]]; then
+		ARGS+=(--dhchap-ctrl-secret="${ctrlkey}")
+	fi
+	if [[ -n "${nr_io_queues}" ]]; then
+		ARGS+=(--nr-io-queues="${nr_io_queues}")
+	fi
+	if [[ -n "${nr_write_queues}" ]]; then
+		ARGS+=(--nr-write-queues="${nr_write_queues}")
+	fi
+	if [[ -n "${nr_poll_queues}" ]]; then
+		ARGS+=(--nr-poll-queues="${nr_poll_queues}")
+	fi
+	if [[ -n "${keep_alive_tmo}" ]]; then
+		ARGS+=(--keep-alive-tmo="${keep_alive_tmo}")
+	fi
+	if [[ -n "${reconnect_delay}" ]]; then
+		ARGS+=(--reconnect-delay="${reconnect_delay}")
+	fi
+	if [[ -n "${ctrl_loss_tmo}" ]]; then
+		ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}")
+	fi
+
+	nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:"
+
+	# Wait until device file and uuid/wwid sysfs attributes get ready for
+	# all namespaces.
+	if [[ ${no_wait} = false ]]; then
+		udevadm settle
+		for ((i = 0; i < 10; i++)); do
+			_nvme_ns_ready "${subsysnqn}" && return
+			sleep .1
+		done
+	fi
+}
+
+_nvme_ns_ready() {
+	local subsysnqn="${1}"
+	local ns_path ns_id dev
+	local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn"
+
+	dev=$(_find_nvme_dev "$subsysnqn")
+	for ns_path in "${cfs_path}/namespaces/"*; do
+		ns_id=${ns_path##*/}
+		if [[ ! -b /dev/${dev}n${ns_id} ||
+			   ! -e /sys/block/${dev}n${ns_id}/uuid ||
+			   ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then
+			return 1
+		fi
+	done
+	return 0
+}
+
+_create_nvmet_port() {
+	local trtype="$1"
+	local traddr="${2:-$def_traddr}"
+	local adrfam="${3:-$def_adrfam}"
+	local trsvcid="${4:-$def_trsvcid}"
+
+	local port
+	for ((port = 0; ; port++)); do
+		if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then
+			break
+		fi
+	done
+
+	mkdir "${NVMET_CFS}/ports/${port}"
+	echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
+	echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
+	echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam"
+	if [[ "${adrfam}" != "fc" ]]; then
+		echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid"
+	fi
+
+	echo "${port}"
+}
+
+_remove_nvmet_port() {
+	local port="$1"
+	rmdir "${NVMET_CFS}/ports/${port}"
+}
+
+_create_nvmet_ns() {
+	local nvmet_subsystem="$1"
+	local nsid="$2"
+	local blkdev="$3"
+	local uuid="00000000-0000-0000-0000-000000000000"
+	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
+	local ns_path="${subsys_path}/namespaces/${nsid}"
+
+	if [[ $# -eq 4 ]]; then
+		uuid="$4"
+	fi
+
+	mkdir "${ns_path}"
+	printf "%s" "${blkdev}" > "${ns_path}/device_path"
+	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
+	printf 1 > "${ns_path}/enable"
+}
+
+_create_nvmet_subsystem() {
+	local nvmet_subsystem="$1"
+	local blkdev="$2"
+	local uuid=$3
+	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
+
+	mkdir -p "${cfs_path}"
+	echo 0 > "${cfs_path}/attr_allow_any_host"
+	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
+}
+
+_add_nvmet_allow_hosts() {
+	local nvmet_subsystem="$1"
+	local nvmet_hostnqn="$2"
+	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
+	local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
+
+	ln -s "${host_path}" "${cfs_path}/allowed_hosts/${nvmet_hostnqn}"
+}
+
+_create_nvmet_host() {
+	local nvmet_subsystem="$1"
+	local nvmet_hostnqn="$2"
+	local nvmet_hostkey="$3"
+	local nvmet_ctrlkey="$4"
+	local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
+
+	if [[ -d "${host_path}" ]]; then
+		echo "FAIL target setup failed. stale host configuration found"
+		return 1;
+	fi
+
+	mkdir "${host_path}"
+	_add_nvmet_allow_hosts "${nvmet_subsystem}" "${nvmet_hostnqn}"
+	if [[ "${nvmet_hostkey}" ]] ; then
+		echo "${nvmet_hostkey}" > "${host_path}/dhchap_key"
+	fi
+	if [[ "${nvmet_ctrlkey}" ]] ; then
+		echo "${nvmet_ctrlkey}" > "${host_path}/dhchap_ctrl_key"
+	fi
+}
+
+_remove_nvmet_ns() {
+	local nvmet_subsystem="$1"
+	local nsid=$2
+	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
+	local nvmet_ns_path="${subsys_path}/namespaces/${nsid}"
+
+	echo 0 > "${nvmet_ns_path}/enable"
+	rmdir "${nvmet_ns_path}"
+}
+
+_remove_nvmet_subsystem() {
+	local nvmet_subsystem="$1"
+	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
+
+	_remove_nvmet_ns "${nvmet_subsystem}" "1"
+	rm -f "${subsys_path}"/allowed_hosts/*
+	rmdir "${subsys_path}"
+}
+
+_remove_nvmet_host() {
+	local nvmet_host="$1"
+	local host_path="${NVMET_CFS}/hosts/${nvmet_host}"
+
+	rmdir "${host_path}"
+}
+
+_add_nvmet_subsys_to_port() {
+	local port="$1"
+	local nvmet_subsystem="$2"
+
+	ln -s "${NVMET_CFS}/subsystems/${nvmet_subsystem}" \
+		"${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
+}
+
+_remove_nvmet_subsystem_from_port() {
+	local port="$1"
+	local nvmet_subsystem="$2"
+
+	rm "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
+}
+
+_get_nvmet_ports() {
+	local nvmet_subsystem="$1"
+	local -n nvmet_ports="$2"
+	local cfs_path="${NVMET_CFS}/ports"
+	local sarg
+
+	sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p"
+
+	for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do
+		nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")")
+	done
+}
+
+_find_nvme_dev() {
+	local subsys=$1
+	local subsysnqn
+	local dev
+	for dev in /sys/class/nvme/nvme*; do
+		[ -e "$dev" ] || continue
+		dev="$(basename "$dev")"
+		subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)"
+		if [[ "$subsysnqn" == "$subsys" ]]; then
+			echo "$dev"
+		fi
+	done
+}
+
+_nvmet_target_cleanup() {
+	local ports
+	local port
+	local blkdev
+	local subsysnqn="${def_subsysnqn}"
+	local blkdev_type=""
+
+	while [[ $# -gt 0 ]]; do
+		case $1 in
+			--blkdev)
+				blkdev_type="$2"
+				shift 2
+				;;
+			--subsysnqn)
+				subsysnqn="$2"
+				shift 2
+				;;
+			*)
+				echo "WARNING: unknown argument: $1"
+				shift
+				;;
+		esac
+	done
+
+	_get_nvmet_ports "${subsysnqn}" ports
+
+	for port in "${ports[@]}"; do
+		_remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}"
+		_remove_nvmet_port "${port}"
+	done
+	_remove_nvmet_subsystem "${subsysnqn}"
+	_remove_nvmet_host "${def_hostnqn}"
+
+	if [[ "${blkdev_type}" == "device" ]]; then
+		_cleanup_blkdev
+	fi
+}
diff --git a/tests/nvme/rc b/tests/nvme/rc
index c1ddf41..3462f2e 100644
--- a/tests/nvme/rc
+++ b/tests/nvme/rc
@@ -5,25 +5,9 @@
 # Test specific to NVMe devices
 
 . common/rc
+. common/nvme
 . common/multipath-over-rdma
 
-def_traddr="127.0.0.1"
-def_adrfam="ipv4"
-def_trsvcid="4420"
-def_remote_wwnn="0x10001100aa000001"
-def_remote_wwpn="0x20001100aa000001"
-def_local_wwnn="0x10001100aa000002"
-def_local_wwpn="0x20001100aa000002"
-def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349"
-def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}"
-export def_subsysnqn="blktests-subsystem-1"
-export def_subsys_uuid="91fdba0d-f87b-4c25-b80f-db7be1418b9e"
-_check_conflict_and_set_default NVMET_TRTYPES nvme_trtype "loop"
-_check_conflict_and_set_default NVME_IMG_SIZE nvme_img_size 1G
-_check_conflict_and_set_default NVME_NUM_ITER nvme_num_iter 1000
-nvmet_blkdev_type=${nvmet_blkdev_type:-"device"}
-NVMET_BLKDEV_TYPES=${NVMET_BLKDEV_TYPES:-"device file"}
-
 _NVMET_TRTYPES_is_valid() {
 	local type
 
@@ -70,12 +54,6 @@ _set_nvmet_blkdev_type() {
 	COND_DESC="bd=${nvmet_blkdev_type}"
 }
 
-# TMPDIR can not be referred out of test() or test_device() context. Instead of
-# global variable def_flie_path, use this getter function.
-_nvme_def_file_path() {
-	echo "${TMPDIR}/img"
-}
-
 _nvme_requires() {
 	_have_program nvme
 	_require_nvme_test_img_size 4m
@@ -144,8 +122,6 @@ group_device_requires() {
 	_require_test_dev_is_nvme
 }
 
-NVMET_CFS="/sys/kernel/config/nvmet/"
-
 _require_test_dev_is_nvme() {
 	if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then
 		SKIP_REASONS+=("$TEST_DEV is not a NVMe device")
@@ -168,31 +144,6 @@ _require_nvme_test_img_size() {
 	return 0
 }
 
-_require_nvme_trtype() {
-	local trtype
-	for trtype in "$@"; do
-		if [[ "${nvme_trtype}" == "$trtype" ]]; then
-			return 0
-		fi
-	done
-	SKIP_REASONS+=("nvme_trtype=${nvme_trtype} is not supported in this test")
-	return 1
-}
-
-_require_nvme_trtype_is_loop() {
-	if ! _require_nvme_trtype loop; then
-		return 1
-	fi
-	return 0
-}
-
-_require_nvme_trtype_is_fabrics() {
-	if ! _require_nvme_trtype loop fc rdma tcp; then
-		return 1
-	fi
-	return 0
-}
-
 _require_nvme_cli_auth() {
 	if ! nvme gen-dhchap-key --nqn nvmf-test-subsys > /dev/null 2>&1 ; then
 		SKIP_REASONS+=("nvme gen-dhchap-key command missing")
@@ -235,216 +186,6 @@ _nvme_calc_rand_io_size() {
 	echo "${io_size_kb}k"
 }
 
-_nvme_fcloop_add_rport() {
-	local local_wwnn="$1"
-	local local_wwpn="$2"
-	local remote_wwnn="$3"
-	local remote_wwpn="$4"
-	local loopctl=/sys/class/fcloop/ctl
-
-	echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port
-}
-
-_nvme_fcloop_add_lport() {
-	local wwnn="$1"
-	local wwpn="$2"
-	local loopctl=/sys/class/fcloop/ctl
-
-	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port
-}
-
-_nvme_fcloop_add_tport() {
-	local wwnn="$1"
-	local wwpn="$2"
-	local loopctl=/sys/class/fcloop/ctl
-
-	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port
-}
-
-_setup_fcloop() {
-	local local_wwnn="${1:-$def_local_wwnn}"
-	local local_wwpn="${2:-$def_local_wwpn}"
-	local remote_wwnn="${3:-$def_remote_wwnn}"
-	local remote_wwpn="${4:-$def_remote_wwpn}"
-
-	_nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}"
-	_nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}"
-	_nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \
-		               "${remote_wwnn}" "${remote_wwpn}"
-}
-
-_nvme_fcloop_del_rport() {
-	local local_wwnn="$1"
-	local local_wwpn="$2"
-	local remote_wwnn="$3"
-	local remote_wwpn="$4"
-	local loopctl=/sys/class/fcloop/ctl
-
-	if [[ ! -f "${loopctl}/del_remote_port" ]]; then
-		return
-	fi
-	echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > "${loopctl}/del_remote_port"
-}
-
-_nvme_fcloop_del_lport() {
-	local wwnn="$1"
-	local wwpn="$2"
-	local loopctl=/sys/class/fcloop/ctl
-
-	if [[ ! -f "${loopctl}/del_local_port" ]]; then
-		return
-	fi
-	echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_local_port"
-}
-
-_nvme_fcloop_del_tport() {
-	local wwnn="$1"
-	local wwpn="$2"
-	local loopctl=/sys/class/fcloop/ctl
-
-	if [[ ! -f "${loopctl}/del_target_port" ]]; then
-		return
-	fi
-	echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port"
-}
-
-_cleanup_fcloop() {
-	local local_wwnn="${1:-$def_local_wwnn}"
-	local local_wwpn="${2:-$def_local_wwpn}"
-	local remote_wwnn="${3:-$def_remote_wwnn}"
-	local remote_wwpn="${4:-$def_remote_wwpn}"
-
-	_nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}"
-	_nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}"
-	_nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \
-			       "${remote_wwnn}" "${remote_wwpn}"
-}
-
-_cleanup_blkdev() {
-	local blkdev
-	local dev
-
-	blkdev="$(losetup -l | awk '$6 == "'"$(_nvme_def_file_path)"'" { print $1 }')"
-	for dev in ${blkdev}; do
-		losetup -d "${dev}"
-	done
-	rm -f "$(_nvme_def_file_path)"
-}
-
-_cleanup_nvmet() {
-	local dev
-	local port
-	local subsys
-	local transport
-	local name
-
-	if [[ ! -d "${NVMET_CFS}" ]]; then
-		return 0
-	fi
-
-	# Don't let successive Ctrl-Cs interrupt the cleanup processes
-	trap '' SIGINT
-
-	shopt -s nullglob
-
-	for dev in /sys/class/nvme/nvme*; do
-		dev="$(basename "$dev")"
-		transport="$(cat "/sys/class/nvme/${dev}/transport" 2>/dev/null)"
-		if [[ "$transport" == "${nvme_trtype}" ]]; then
-			# if udev auto connect is enabled for FC we get false positives
-			if [[ "$transport" != "fc" ]]; then
-				echo "WARNING: Test did not clean up ${nvme_trtype} device: ${dev}"
-			fi
-			_nvme_disconnect_ctrl "${dev}" 2>/dev/null
-		fi
-	done
-
-	for port in "${NVMET_CFS}"/ports/*; do
-		name=$(basename "${port}")
-		echo "WARNING: Test did not clean up port: ${name}"
-		rm -f "${port}"/subsystems/*
-		rmdir "${port}"
-	done
-
-	for subsys in "${NVMET_CFS}"/subsystems/*; do
-		name=$(basename "${subsys}")
-		echo "WARNING: Test did not clean up subsystem: ${name}"
-		for ns in "${subsys}"/namespaces/*; do
-			rmdir "${ns}"
-		done
-		rmdir "${subsys}"
-	done
-
-	for host in "${NVMET_CFS}"/hosts/*; do
-		name=$(basename "${host}")
-		echo "WARNING: Test did not clean up host: ${name}"
-		rmdir "${host}"
-	done
-
-	shopt -u nullglob
-	trap SIGINT
-
-	if [[ "${nvme_trtype}" == "fc" ]]; then
-		_cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
-				"${def_remote_wwnn}" "${def_remote_wwpn}"
-		modprobe -rq nvme-fcloop 2>/dev/null
-	fi
-	modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null
-	if [[ "${nvme_trtype}" != "loop" ]]; then
-		modprobe -rq nvmet-"${nvme_trtype}" 2>/dev/null
-	fi
-	modprobe -rq nvmet 2>/dev/null
-	if [[ "${nvme_trtype}" == "rdma" ]]; then
-		stop_soft_rdma
-	fi
-
-	_cleanup_blkdev
-}
-
-_setup_nvmet() {
-	_register_test_cleanup _cleanup_nvmet
-	modprobe -q nvmet
-	if [[ "${nvme_trtype}" != "loop" ]]; then
-		modprobe -q nvmet-"${nvme_trtype}"
-	fi
-	modprobe -q nvme-"${nvme_trtype}"
-	if [[ "${nvme_trtype}" == "rdma" ]]; then
-		start_soft_rdma
-		for i in $(rdma_network_interfaces)
-		do
-			if [[ "${nvme_adrfam}" == "ipv6" ]]; then
-				ipv6_addr=$(get_ipv6_ll_addr "$i")
-				if [[ -n "${ipv6_addr}" ]]; then
-					def_traddr=${ipv6_addr}
-				fi
-			else
-				ipv4_addr=$(get_ipv4_addr "$i")
-				if [[ -n "${ipv4_addr}" ]]; then
-					def_traddr=${ipv4_addr}
-				fi
-			fi
-		done
-	fi
-	if [[ "${nvme_trtype}" = "fc" ]]; then
-		modprobe -q nvme-fcloop
-		_setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
-			      "${def_remote_wwnn}" "${def_remote_wwpn}"
-
-		def_traddr=$(printf "nn-%s:pn-%s" \
-				    "${def_remote_wwnn}" \
-				    "${def_remote_wwpn}")
-		def_host_traddr=$(printf "nn-%s:pn-%s" \
-					 "${def_local_wwnn}" \
-					 "${def_local_wwpn}")
-	fi
-}
-
-_nvme_disconnect_ctrl() {
-	local ctrl="$1"
-
-	nvme disconnect --device "${ctrl}"
-}
-
 _nvme_disconnect_subsys() {
 	local subsysnqn="$def_subsysnqn"
 
@@ -465,141 +206,6 @@ _nvme_disconnect_subsys() {
 		grep -o "disconnected.*"
 }
 
-_nvme_connect_subsys() {
-	local subsysnqn="$def_subsysnqn"
-	local hostnqn="$def_hostnqn"
-	local hostid="$def_hostid"
-	local hostkey=""
-	local ctrlkey=""
-	local nr_io_queues=""
-	local nr_write_queues=""
-	local nr_poll_queues=""
-	local keep_alive_tmo=""
-	local reconnect_delay=""
-	local ctrl_loss_tmo=""
-	local no_wait=false
-	local i
-
-	while [[ $# -gt 0 ]]; do
-		case $1 in
-			--subsysnqn)
-				subsysnqn="$2"
-				shift 2
-				;;
-			--hostnqn)
-				hostnqn="$2"
-				shift 2
-				;;
-			--hostid)
-				hostid="$2"
-				shift 2
-				;;
-			--dhchap-secret)
-				hostkey="$2"
-				shift 2
-				;;
-			--dhchap-ctrl-secret)
-				ctrlkey="$2"
-				shift 2
-				;;
-			--nr-io-queues)
-				nr_io_queues="$2"
-				shift 2
-				;;
-			--nr-write-queues)
-				nr_write_queues="$2"
-				shift 2
-				;;
-			--nr-poll-queues)
-				nr_poll_queues="$2"
-				shift 2
-				;;
-			--keep-alive-tmo)
-				keep_alive_tmo="$2"
-				shift 2
-				;;
-			--reconnect-delay)
-				reconnect_delay="$2"
-				shift 2
-				;;
-			--ctrl-loss-tmo)
-				ctrl_loss_tmo="$2"
-				shift 2
-				;;
-			--no-wait)
-				no_wait=true
-				shift 1
-				;;
-			*)
-				echo "WARNING: unknown argument: $1"
-				shift
-				;;
-		esac
-	done
-
-	ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}")
-	if [[ "${nvme_trtype}" == "fc" ]] ; then
-		ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}")
-	elif [[ "${nvme_trtype}" != "loop" ]]; then
-		ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}")
-	fi
-	ARGS+=(--hostnqn="${hostnqn}")
-	ARGS+=(--hostid="${hostid}")
-	if [[ -n "${hostkey}" ]]; then
-		ARGS+=(--dhchap-secret="${hostkey}")
-	fi
-	if [[ -n "${ctrlkey}" ]]; then
-		ARGS+=(--dhchap-ctrl-secret="${ctrlkey}")
-	fi
-	if [[ -n "${nr_io_queues}" ]]; then
-		ARGS+=(--nr-io-queues="${nr_io_queues}")
-	fi
-	if [[ -n "${nr_write_queues}" ]]; then
-		ARGS+=(--nr-write-queues="${nr_write_queues}")
-	fi
-	if [[ -n "${nr_poll_queues}" ]]; then
-		ARGS+=(--nr-poll-queues="${nr_poll_queues}")
-	fi
-	if [[ -n "${keep_alive_tmo}" ]]; then
-		ARGS+=(--keep-alive-tmo="${keep_alive_tmo}")
-	fi
-	if [[ -n "${reconnect_delay}" ]]; then
-		ARGS+=(--reconnect-delay="${reconnect_delay}")
-	fi
-	if [[ -n "${ctrl_loss_tmo}" ]]; then
-		ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}")
-	fi
-
-	nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:"
-
-	# Wait until device file and uuid/wwid sysfs attributes get ready for
-	# all namespaces.
-	if [[ ${no_wait} = false ]]; then
-		udevadm settle
-		for ((i = 0; i < 10; i++)); do
-			_nvme_ns_ready "${subsysnqn}" && return
-			sleep .1
-		done
-	fi
-}
-
-_nvme_ns_ready() {
-	local subsysnqn="${1}"
-	local ns_path ns_id dev
-	local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn"
-
-	dev=$(_find_nvme_dev "$subsysnqn")
-	for ns_path in "${cfs_path}/namespaces/"*; do
-		ns_id=${ns_path##*/}
-		if [[ ! -b /dev/${dev}n${ns_id} ||
-			   ! -e /sys/block/${dev}n${ns_id}/uuid ||
-			   ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then
-			return 1
-		fi
-	done
-	return 0
-}
-
 _nvme_discover() {
 	local trtype="$1"
 	local traddr="${2:-$def_traddr}"
@@ -617,73 +223,6 @@ _nvme_discover() {
 	nvme discover "${ARGS[@]}"
 }
 
-_create_nvmet_port() {
-	local trtype="$1"
-	local traddr="${2:-$def_traddr}"
-	local adrfam="${3:-$def_adrfam}"
-	local trsvcid="${4:-$def_trsvcid}"
-
-	local port
-	for ((port = 0; ; port++)); do
-		if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then
-			break
-		fi
-	done
-
-	mkdir "${NVMET_CFS}/ports/${port}"
-	echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
-	echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
-	echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam"
-	if [[ "${adrfam}" != "fc" ]]; then
-		echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid"
-	fi
-
-	echo "${port}"
-}
-
-_remove_nvmet_port() {
-	local port="$1"
-	rmdir "${NVMET_CFS}/ports/${port}"
-}
-
-_create_nvmet_ns() {
-	local nvmet_subsystem="$1"
-	local nsid="$2"
-	local blkdev="$3"
-	local uuid="00000000-0000-0000-0000-000000000000"
-	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
-	local ns_path="${subsys_path}/namespaces/${nsid}"
-
-	if [[ $# -eq 4 ]]; then
-		uuid="$4"
-	fi
-
-	mkdir "${ns_path}"
-	printf "%s" "${blkdev}" > "${ns_path}/device_path"
-	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
-	printf 1 > "${ns_path}/enable"
-}
-
-_create_nvmet_subsystem() {
-	local nvmet_subsystem="$1"
-	local blkdev="$2"
-	local uuid=$3
-	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
-
-	mkdir -p "${cfs_path}"
-	echo 0 > "${cfs_path}/attr_allow_any_host"
-	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
-}
-
-_add_nvmet_allow_hosts() {
-	local nvmet_subsystem="$1"
-	local nvmet_hostnqn="$2"
-	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
-	local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
-
-	ln -s "${host_path}" "${cfs_path}/allowed_hosts/${nvmet_hostnqn}"
-}
-
 _remove_nvmet_allow_hosts() {
 	local nvmet_subsystem="$1"
 	local nvmet_hostnqn="$2"
@@ -692,54 +231,6 @@ _remove_nvmet_allow_hosts() {
 	rm "${cfs_path}/allowed_hosts/${nvmet_hostnqn}"
 }
 
-_create_nvmet_host() {
-	local nvmet_subsystem="$1"
-	local nvmet_hostnqn="$2"
-	local nvmet_hostkey="$3"
-	local nvmet_ctrlkey="$4"
-	local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
-
-	if [[ -d "${host_path}" ]]; then
-		echo "FAIL target setup failed. stale host configuration found"
-		return 1;
-	fi
-
-	mkdir "${host_path}"
-	_add_nvmet_allow_hosts "${nvmet_subsystem}" "${nvmet_hostnqn}"
-	if [[ "${nvmet_hostkey}" ]] ; then
-		echo "${nvmet_hostkey}" > "${host_path}/dhchap_key"
-	fi
-	if [[ "${nvmet_ctrlkey}" ]] ; then
-		echo "${nvmet_ctrlkey}" > "${host_path}/dhchap_ctrl_key"
-	fi
-}
-
-_remove_nvmet_ns() {
-	local nvmet_subsystem="$1"
-	local nsid=$2
-	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
-	local nvmet_ns_path="${subsys_path}/namespaces/${nsid}"
-
-	echo 0 > "${nvmet_ns_path}/enable"
-	rmdir "${nvmet_ns_path}"
-}
-
-_remove_nvmet_subsystem() {
-	local nvmet_subsystem="$1"
-	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
-
-	_remove_nvmet_ns "${nvmet_subsystem}" "1"
-	rm -f "${subsys_path}"/allowed_hosts/*
-	rmdir "${subsys_path}"
-}
-
-_remove_nvmet_host() {
-	local nvmet_host="$1"
-	local host_path="${NVMET_CFS}/hosts/${nvmet_host}"
-
-	rmdir "${host_path}"
-}
-
 _create_nvmet_passthru() {
 	local nvmet_subsystem="$1"
 	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
@@ -765,34 +256,6 @@ _remove_nvmet_passhtru() {
 	rmdir "${subsys_path}"
 }
 
-_add_nvmet_subsys_to_port() {
-	local port="$1"
-	local nvmet_subsystem="$2"
-
-	ln -s "${NVMET_CFS}/subsystems/${nvmet_subsystem}" \
-		"${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
-}
-
-_remove_nvmet_subsystem_from_port() {
-	local port="$1"
-	local nvmet_subsystem="$2"
-
-	rm "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
-}
-
-_get_nvmet_ports() {
-	local nvmet_subsystem="$1"
-	local -n nvmet_ports="$2"
-	local cfs_path="${NVMET_CFS}/ports"
-	local sarg
-
-	sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p"
-
-	for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do
-		nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")")
-	done
-}
-
 _set_nvmet_hostkey() {
 	local nvmet_hostnqn="$1"
 	local nvmet_hostkey="$2"
@@ -829,20 +292,6 @@ _set_nvmet_dhgroup() {
 	     "${cfs_path}/dhchap_dhgroup"
 }
 
-_find_nvme_dev() {
-	local subsys=$1
-	local subsysnqn
-	local dev
-	for dev in /sys/class/nvme/nvme*; do
-		[ -e "$dev" ] || continue
-		dev="$(basename "$dev")"
-		subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)"
-		if [[ "$subsysnqn" == "$subsys" ]]; then
-			echo "$dev"
-		fi
-	done
-}
-
 _find_nvme_ns() {
 	local subsys_uuid=$1
 	local uuid
@@ -924,44 +373,6 @@ _nvmet_target_setup() {
 			"${hostkey}" "${ctrlkey}"
 }
 
-_nvmet_target_cleanup() {
-	local ports
-	local port
-	local blkdev
-	local subsysnqn="${def_subsysnqn}"
-	local blkdev_type=""
-
-	while [[ $# -gt 0 ]]; do
-		case $1 in
-			--blkdev)
-				blkdev_type="$2"
-				shift 2
-				;;
-			--subsysnqn)
-				subsysnqn="$2"
-				shift 2
-				;;
-			*)
-				echo "WARNING: unknown argument: $1"
-				shift
-				;;
-		esac
-	done
-
-	_get_nvmet_ports "${subsysnqn}" ports
-
-	for port in "${ports[@]}"; do
-		_remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}"
-		_remove_nvmet_port "${port}"
-	done
-	_remove_nvmet_subsystem "${subsysnqn}"
-	_remove_nvmet_host "${def_hostnqn}"
-
-	if [[ "${blkdev_type}" == "device" ]]; then
-		_cleanup_blkdev
-	fi
-}
-
 _nvmet_passthru_target_setup() {
 	local subsysnqn="$def_subsysnqn"
 	local port
-- 
2.45.1

Re: [PATCH blktests v2 1/2] nvme: move helper functions to common/nvme

[Shinichiro Kawasaki]

On Jun 24, 2024 / 13:46, Ofir Gal wrote:
> Move functions from tests/nvme/rc to common/nvme to be able to reuse
> them in other tests groups.
> 
> Signed-off-by: Ofir Gal <ofir.gal@volumez.com>
> ---
> I have moved the function that are nessecary for the regression test I
> add. Let me know if we want to move more or all the functions to
> common/nvme.
> 
> shellcheck detects 2 new warnings:
> common/nvme:35:10: warning: nvme_trtype is referenced but not assigned. [SC2154]
> common/nvme:234:11: warning: nvme_adrfam is referenced but not assigned. [SC2154]
> 
> I have tried to figure out why but I couldn't figure out what is
> different than the upstream version. How can I fix them?

Hi Ofir, thanks for this work. Please find two comments in line below. One of
them suggests a fix for the shellcheck warnings.

> 
> 
>  common/nvme   | 595 ++++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/nvme/rc | 591 +------------------------------------------------
>  2 files changed, 596 insertions(+), 590 deletions(-)
>  create mode 100644 common/nvme
> 
> diff --git a/common/nvme b/common/nvme
> new file mode 100644
> index 0000000..1800263
> --- /dev/null
> +++ b/common/nvme
> @@ -0,0 +1,595 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-3.0+
> +#
> +# nvme helper functions.
> +
> +. common/shellcheck
> +
> +def_traddr="127.0.0.1"
> +def_adrfam="ipv4"
> +def_trsvcid="4420"
> +def_remote_wwnn="0x10001100aa000001"
> +def_remote_wwpn="0x20001100aa000001"
> +def_local_wwnn="0x10001100aa000002"
> +def_local_wwpn="0x20001100aa000002"
> +def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349"
> +def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}"
> +export def_subsysnqn="blktests-subsystem-1"
> +export def_subsys_uuid="91fdba0d-f87b-4c25-b80f-db7be1418b9e"
> +_check_conflict_and_set_default NVMET_TRTYPES nvme_trtype "loop"
> +_check_conflict_and_set_default NVME_IMG_SIZE nvme_img_size 1G
> +_check_conflict_and_set_default NVME_NUM_ITER nvme_num_iter 1000
> +nvmet_blkdev_type=${nvmet_blkdev_type:-"device"}
> +NVMET_BLKDEV_TYPES=${NVMET_BLKDEV_TYPES:-"device file"}
> +NVMET_CFS="/sys/kernel/config/nvmet/"

As for the shellcheck warnings, I suggestto add two lines below here to suppress
them.

nvme_trtype=${nvme_trtype:-}
nvme_adrfam=${nvme_adrfam:-}

> +
> +# TMPDIR can not be referred out of test() or test_device() context. Instead of
> +# global variable def_flie_path, use this getter function.
> +_nvme_def_file_path() {
> +	echo "${TMPDIR}/img"
> +}
> +
> +_require_nvme_trtype() {
> +	local trtype
> +	for trtype in "$@"; do
> +		if [[ "${nvme_trtype}" == "$trtype" ]]; then
> +			return 0
> +		fi
> +	done
> +	SKIP_REASONS+=("nvme_trtype=${nvme_trtype} is not supported in this test")
> +	return 1
> +}
> +
> +_require_nvme_trtype_is_loop() {
> +	if ! _require_nvme_trtype loop; then
> +		return 1
> +	fi
> +	return 0
> +}
> +
> +_require_nvme_trtype_is_fabrics() {
> +	if ! _require_nvme_trtype loop fc rdma tcp; then
> +		return 1
> +	fi
> +	return 0
> +}
> +
> +_nvme_fcloop_add_rport() {
> +	local local_wwnn="$1"
> +	local local_wwpn="$2"
> +	local remote_wwnn="$3"
> +	local remote_wwpn="$4"
> +	local loopctl=/sys/class/fcloop/ctl
> +
> +	echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port
> +}
> +
> +_nvme_fcloop_add_lport() {
> +	local wwnn="$1"
> +	local wwpn="$2"
> +	local loopctl=/sys/class/fcloop/ctl
> +
> +	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port
> +}
> +
> +_nvme_fcloop_add_tport() {
> +	local wwnn="$1"
> +	local wwpn="$2"
> +	local loopctl=/sys/class/fcloop/ctl
> +
> +	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port
> +}
> +
> +_setup_fcloop() {
> +	local local_wwnn="${1:-$def_local_wwnn}"
> +	local local_wwpn="${2:-$def_local_wwpn}"
> +	local remote_wwnn="${3:-$def_remote_wwnn}"
> +	local remote_wwpn="${4:-$def_remote_wwpn}"
> +
> +	_nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}"
> +	_nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}"
> +	_nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \
> +		               "${remote_wwnn}" "${remote_wwpn}"
> +}
> +
> +_nvme_fcloop_del_rport() {
> +	local local_wwnn="$1"
> +	local local_wwpn="$2"
> +	local remote_wwnn="$3"
> +	local remote_wwpn="$4"
> +	local loopctl=/sys/class/fcloop/ctl
> +
> +	if [[ ! -f "${loopctl}/del_remote_port" ]]; then
> +		return
> +	fi
> +	echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > "${loopctl}/del_remote_port"
> +}
> +
> +_nvme_fcloop_del_lport() {
> +	local wwnn="$1"
> +	local wwpn="$2"
> +	local loopctl=/sys/class/fcloop/ctl
> +
> +	if [[ ! -f "${loopctl}/del_local_port" ]]; then
> +		return
> +	fi
> +	echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_local_port"
> +}
> +
> +_nvme_fcloop_del_tport() {
> +	local wwnn="$1"
> +	local wwpn="$2"
> +	local loopctl=/sys/class/fcloop/ctl
> +
> +	if [[ ! -f "${loopctl}/del_target_port" ]]; then
> +		return
> +	fi
> +	echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port"
> +}
> +
> +_cleanup_fcloop() {
> +	local local_wwnn="${1:-$def_local_wwnn}"
> +	local local_wwpn="${2:-$def_local_wwpn}"
> +	local remote_wwnn="${3:-$def_remote_wwnn}"
> +	local remote_wwpn="${4:-$def_remote_wwpn}"
> +
> +	_nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}"
> +	_nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}"
> +	_nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \
> +			       "${remote_wwnn}" "${remote_wwpn}"
> +}
> +
> +_cleanup_blkdev() {
> +	local blkdev
> +	local dev
> +
> +	blkdev="$(losetup -l | awk '$6 == "'"$(_nvme_def_file_path)"'" { print $1 }')"
> +	for dev in ${blkdev}; do
> +		losetup -d "${dev}"
> +	done
> +	rm -f "$(_nvme_def_file_path)"
> +}
> +
> +_cleanup_nvmet() {
> +	local dev
> +	local port
> +	local subsys
> +	local transport
> +	local name
> +
> +	if [[ ! -d "${NVMET_CFS}" ]]; then
> +		return 0
> +	fi
> +
> +	# Don't let successive Ctrl-Cs interrupt the cleanup processes
> +	trap '' SIGINT
> +
> +	shopt -s nullglob
> +
> +	for dev in /sys/class/nvme/nvme*; do
> +		dev="$(basename "$dev")"
> +		transport="$(cat "/sys/class/nvme/${dev}/transport" 2>/dev/null)"
> +		if [[ "$transport" == "${nvme_trtype}" ]]; then
> +			# if udev auto connect is enabled for FC we get false positives
> +			if [[ "$transport" != "fc" ]]; then
> +				echo "WARNING: Test did not clean up ${nvme_trtype} device: ${dev}"
> +			fi
> +			_nvme_disconnect_ctrl "${dev}" 2>/dev/null
> +		fi
> +	done
> +
> +	for port in "${NVMET_CFS}"/ports/*; do
> +		name=$(basename "${port}")
> +		echo "WARNING: Test did not clean up port: ${name}"
> +		rm -f "${port}"/subsystems/*
> +		rmdir "${port}"
> +	done
> +
> +	for subsys in "${NVMET_CFS}"/subsystems/*; do
> +		name=$(basename "${subsys}")
> +		echo "WARNING: Test did not clean up subsystem: ${name}"
> +		for ns in "${subsys}"/namespaces/*; do
> +			rmdir "${ns}"
> +		done
> +		rmdir "${subsys}"
> +	done
> +
> +	for host in "${NVMET_CFS}"/hosts/*; do
> +		name=$(basename "${host}")
> +		echo "WARNING: Test did not clean up host: ${name}"
> +		rmdir "${host}"
> +	done
> +
> +	shopt -u nullglob
> +	trap SIGINT
> +
> +	if [[ "${nvme_trtype}" == "fc" ]]; then
> +		_cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
> +				"${def_remote_wwnn}" "${def_remote_wwpn}"
> +		modprobe -rq nvme-fcloop 2>/dev/null
> +	fi
> +	modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null
> +	if [[ "${nvme_trtype}" != "loop" ]]; then
> +		modprobe -rq nvmet-"${nvme_trtype}" 2>/dev/null
> +	fi
> +	modprobe -rq nvmet 2>/dev/null
> +	if [[ "${nvme_trtype}" == "rdma" ]]; then
> +		stop_soft_rdma
> +	fi
> +
> +	_cleanup_blkdev
> +}
> +
> +_setup_nvmet() {
> +	_register_test_cleanup _cleanup_nvmet
> +	modprobe -q nvmet
> +	if [[ "${nvme_trtype}" != "loop" ]]; then
> +		modprobe -q nvmet-"${nvme_trtype}"
> +	fi
> +	modprobe -q nvme-"${nvme_trtype}"
> +	if [[ "${nvme_trtype}" == "rdma" ]]; then
> +		start_soft_rdma
> +		for i in $(rdma_network_interfaces)
> +		do
> +			if [[ "${nvme_adrfam}" == "ipv6" ]]; then
> +				ipv6_addr=$(get_ipv6_ll_addr "$i")
> +				if [[ -n "${ipv6_addr}" ]]; then
> +					def_traddr=${ipv6_addr}
> +				fi
> +			else
> +				ipv4_addr=$(get_ipv4_addr "$i")
> +				if [[ -n "${ipv4_addr}" ]]; then
> +					def_traddr=${ipv4_addr}
> +				fi
> +			fi
> +		done
> +	fi
> +	if [[ "${nvme_trtype}" = "fc" ]]; then
> +		modprobe -q nvme-fcloop
> +		_setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
> +			      "${def_remote_wwnn}" "${def_remote_wwpn}"
> +
> +		def_traddr=$(printf "nn-%s:pn-%s" \
> +				    "${def_remote_wwnn}" \
> +				    "${def_remote_wwpn}")
> +		def_host_traddr=$(printf "nn-%s:pn-%s" \
> +					 "${def_local_wwnn}" \
> +					 "${def_local_wwpn}")
> +	fi
> +}
> +
> +_nvme_disconnect_ctrl() {
> +	local ctrl="$1"
> +
> +	nvme disconnect --device "${ctrl}"
> +}
> +
> +_nvme_connect_subsys() {
> +	local subsysnqn="$def_subsysnqn"
> +	local hostnqn="$def_hostnqn"
> +	local hostid="$def_hostid"
> +	local hostkey=""
> +	local ctrlkey=""
> +	local nr_io_queues=""
> +	local nr_write_queues=""
> +	local nr_poll_queues=""
> +	local keep_alive_tmo=""
> +	local reconnect_delay=""
> +	local ctrl_loss_tmo=""
> +	local no_wait=false
> +	local i
> +
> +	while [[ $# -gt 0 ]]; do
> +		case $1 in
> +			--subsysnqn)
> +				subsysnqn="$2"
> +				shift 2
> +				;;
> +			--hostnqn)
> +				hostnqn="$2"
> +				shift 2
> +				;;
> +			--hostid)
> +				hostid="$2"
> +				shift 2
> +				;;
> +			--dhchap-secret)
> +				hostkey="$2"
> +				shift 2
> +				;;
> +			--dhchap-ctrl-secret)
> +				ctrlkey="$2"
> +				shift 2
> +				;;
> +			--nr-io-queues)
> +				nr_io_queues="$2"
> +				shift 2
> +				;;
> +			--nr-write-queues)
> +				nr_write_queues="$2"
> +				shift 2
> +				;;
> +			--nr-poll-queues)
> +				nr_poll_queues="$2"
> +				shift 2
> +				;;
> +			--keep-alive-tmo)
> +				keep_alive_tmo="$2"
> +				shift 2
> +				;;
> +			--reconnect-delay)
> +				reconnect_delay="$2"
> +				shift 2
> +				;;
> +			--ctrl-loss-tmo)
> +				ctrl_loss_tmo="$2"
> +				shift 2
> +				;;
> +			--no-wait)
> +				no_wait=true
> +				shift 1
> +				;;
> +			*)
> +				echo "WARNING: unknown argument: $1"
> +				shift
> +				;;
> +		esac
> +	done
> +
> +	ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}")
> +	if [[ "${nvme_trtype}" == "fc" ]] ; then
> +		ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}")
> +	elif [[ "${nvme_trtype}" != "loop" ]]; then
> +		ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}")
> +	fi
> +	ARGS+=(--hostnqn="${hostnqn}")
> +	ARGS+=(--hostid="${hostid}")
> +	if [[ -n "${hostkey}" ]]; then
> +		ARGS+=(--dhchap-secret="${hostkey}")
> +	fi
> +	if [[ -n "${ctrlkey}" ]]; then
> +		ARGS+=(--dhchap-ctrl-secret="${ctrlkey}")
> +	fi
> +	if [[ -n "${nr_io_queues}" ]]; then
> +		ARGS+=(--nr-io-queues="${nr_io_queues}")
> +	fi
> +	if [[ -n "${nr_write_queues}" ]]; then
> +		ARGS+=(--nr-write-queues="${nr_write_queues}")
> +	fi
> +	if [[ -n "${nr_poll_queues}" ]]; then
> +		ARGS+=(--nr-poll-queues="${nr_poll_queues}")
> +	fi
> +	if [[ -n "${keep_alive_tmo}" ]]; then
> +		ARGS+=(--keep-alive-tmo="${keep_alive_tmo}")
> +	fi
> +	if [[ -n "${reconnect_delay}" ]]; then
> +		ARGS+=(--reconnect-delay="${reconnect_delay}")
> +	fi
> +	if [[ -n "${ctrl_loss_tmo}" ]]; then
> +		ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}")
> +	fi
> +
> +	nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:"
> +
> +	# Wait until device file and uuid/wwid sysfs attributes get ready for
> +	# all namespaces.
> +	if [[ ${no_wait} = false ]]; then
> +		udevadm settle
> +		for ((i = 0; i < 10; i++)); do
> +			_nvme_ns_ready "${subsysnqn}" && return
> +			sleep .1
> +		done
> +	fi
> +}
> +
> +_nvme_ns_ready() {
> +	local subsysnqn="${1}"
> +	local ns_path ns_id dev
> +	local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn"
> +
> +	dev=$(_find_nvme_dev "$subsysnqn")
> +	for ns_path in "${cfs_path}/namespaces/"*; do
> +		ns_id=${ns_path##*/}
> +		if [[ ! -b /dev/${dev}n${ns_id} ||
> +			   ! -e /sys/block/${dev}n${ns_id}/uuid ||
> +			   ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then
> +			return 1
> +		fi
> +	done
> +	return 0
> +}
> +
> +_create_nvmet_port() {
> +	local trtype="$1"
> +	local traddr="${2:-$def_traddr}"
> +	local adrfam="${3:-$def_adrfam}"
> +	local trsvcid="${4:-$def_trsvcid}"
> +
> +	local port
> +	for ((port = 0; ; port++)); do
> +		if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then
> +			break
> +		fi
> +	done
> +
> +	mkdir "${NVMET_CFS}/ports/${port}"
> +	echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
> +	echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
> +	echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam"
> +	if [[ "${adrfam}" != "fc" ]]; then
> +		echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid"
> +	fi
> +
> +	echo "${port}"
> +}
> +
> +_remove_nvmet_port() {
> +	local port="$1"
> +	rmdir "${NVMET_CFS}/ports/${port}"
> +}
> +
> +_create_nvmet_ns() {
> +	local nvmet_subsystem="$1"
> +	local nsid="$2"
> +	local blkdev="$3"
> +	local uuid="00000000-0000-0000-0000-000000000000"
> +	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> +	local ns_path="${subsys_path}/namespaces/${nsid}"
> +
> +	if [[ $# -eq 4 ]]; then
> +		uuid="$4"
> +	fi
> +
> +	mkdir "${ns_path}"
> +	printf "%s" "${blkdev}" > "${ns_path}/device_path"
> +	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
> +	printf 1 > "${ns_path}/enable"
> +}
> +
> +_create_nvmet_subsystem() {
> +	local nvmet_subsystem="$1"
> +	local blkdev="$2"
> +	local uuid=$3
> +	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> +
> +	mkdir -p "${cfs_path}"
> +	echo 0 > "${cfs_path}/attr_allow_any_host"
> +	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
> +}
> +
> +_add_nvmet_allow_hosts() {
> +	local nvmet_subsystem="$1"
> +	local nvmet_hostnqn="$2"
> +	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> +	local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
> +
> +	ln -s "${host_path}" "${cfs_path}/allowed_hosts/${nvmet_hostnqn}"
> +}
> +
> +_create_nvmet_host() {
> +	local nvmet_subsystem="$1"
> +	local nvmet_hostnqn="$2"
> +	local nvmet_hostkey="$3"
> +	local nvmet_ctrlkey="$4"
> +	local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
> +
> +	if [[ -d "${host_path}" ]]; then
> +		echo "FAIL target setup failed. stale host configuration found"
> +		return 1;
> +	fi
> +
> +	mkdir "${host_path}"
> +	_add_nvmet_allow_hosts "${nvmet_subsystem}" "${nvmet_hostnqn}"
> +	if [[ "${nvmet_hostkey}" ]] ; then
> +		echo "${nvmet_hostkey}" > "${host_path}/dhchap_key"
> +	fi
> +	if [[ "${nvmet_ctrlkey}" ]] ; then
> +		echo "${nvmet_ctrlkey}" > "${host_path}/dhchap_ctrl_key"
> +	fi
> +}
> +
> +_remove_nvmet_ns() {
> +	local nvmet_subsystem="$1"
> +	local nsid=$2
> +	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> +	local nvmet_ns_path="${subsys_path}/namespaces/${nsid}"
> +
> +	echo 0 > "${nvmet_ns_path}/enable"
> +	rmdir "${nvmet_ns_path}"
> +}
> +
> +_remove_nvmet_subsystem() {
> +	local nvmet_subsystem="$1"
> +	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> +
> +	_remove_nvmet_ns "${nvmet_subsystem}" "1"
> +	rm -f "${subsys_path}"/allowed_hosts/*
> +	rmdir "${subsys_path}"
> +}
> +
> +_remove_nvmet_host() {
> +	local nvmet_host="$1"
> +	local host_path="${NVMET_CFS}/hosts/${nvmet_host}"
> +
> +	rmdir "${host_path}"
> +}
> +
> +_add_nvmet_subsys_to_port() {
> +	local port="$1"
> +	local nvmet_subsystem="$2"
> +
> +	ln -s "${NVMET_CFS}/subsystems/${nvmet_subsystem}" \
> +		"${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
> +}
> +
> +_remove_nvmet_subsystem_from_port() {
> +	local port="$1"
> +	local nvmet_subsystem="$2"
> +
> +	rm "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
> +}
> +
> +_get_nvmet_ports() {
> +	local nvmet_subsystem="$1"
> +	local -n nvmet_ports="$2"
> +	local cfs_path="${NVMET_CFS}/ports"
> +	local sarg
> +
> +	sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p"
> +
> +	for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do
> +		nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")")
> +	done
> +}
> +
> +_find_nvme_dev() {
> +	local subsys=$1
> +	local subsysnqn
> +	local dev
> +	for dev in /sys/class/nvme/nvme*; do
> +		[ -e "$dev" ] || continue
> +		dev="$(basename "$dev")"
> +		subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)"
> +		if [[ "$subsysnqn" == "$subsys" ]]; then
> +			echo "$dev"
> +		fi
> +	done
> +}
> +
> +_nvmet_target_cleanup() {
> +	local ports
> +	local port
> +	local blkdev
> +	local subsysnqn="${def_subsysnqn}"
> +	local blkdev_type=""
> +
> +	while [[ $# -gt 0 ]]; do
> +		case $1 in
> +			--blkdev)
> +				blkdev_type="$2"
> +				shift 2
> +				;;
> +			--subsysnqn)
> +				subsysnqn="$2"
> +				shift 2
> +				;;
> +			*)
> +				echo "WARNING: unknown argument: $1"
> +				shift
> +				;;
> +		esac
> +	done
> +
> +	_get_nvmet_ports "${subsysnqn}" ports
> +
> +	for port in "${ports[@]}"; do
> +		_remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}"
> +		_remove_nvmet_port "${port}"
> +	done
> +	_remove_nvmet_subsystem "${subsysnqn}"
> +	_remove_nvmet_host "${def_hostnqn}"
> +
> +	if [[ "${blkdev_type}" == "device" ]]; then
> +		_cleanup_blkdev
> +	fi
> +}
> diff --git a/tests/nvme/rc b/tests/nvme/rc
> index c1ddf41..3462f2e 100644
> --- a/tests/nvme/rc
> +++ b/tests/nvme/rc
> @@ -5,25 +5,9 @@
>  # Test specific to NVMe devices
>  
>  . common/rc
> +. common/nvme
>  . common/multipath-over-rdma
>  
> -def_traddr="127.0.0.1"
> -def_adrfam="ipv4"
> -def_trsvcid="4420"
> -def_remote_wwnn="0x10001100aa000001"
> -def_remote_wwpn="0x20001100aa000001"
> -def_local_wwnn="0x10001100aa000002"
> -def_local_wwpn="0x20001100aa000002"
> -def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349"
> -def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}"
> -export def_subsysnqn="blktests-subsystem-1"
> -export def_subsys_uuid="91fdba0d-f87b-4c25-b80f-db7be1418b9e"
> -_check_conflict_and_set_default NVMET_TRTYPES nvme_trtype "loop"
> -_check_conflict_and_set_default NVME_IMG_SIZE nvme_img_size 1G
> -_check_conflict_and_set_default NVME_NUM_ITER nvme_num_iter 1000
> -nvmet_blkdev_type=${nvmet_blkdev_type:-"device"}
> -NVMET_BLKDEV_TYPES=${NVMET_BLKDEV_TYPES:-"device file"}
> -
>  _NVMET_TRTYPES_is_valid() {
>  	local type
>  
> @@ -70,12 +54,6 @@ _set_nvmet_blkdev_type() {
>  	COND_DESC="bd=${nvmet_blkdev_type}"
>  }
>  
> -# TMPDIR can not be referred out of test() or test_device() context. Instead of
> -# global variable def_flie_path, use this getter function.
> -_nvme_def_file_path() {
> -	echo "${TMPDIR}/img"
> -}
> -
>  _nvme_requires() {
>  	_have_program nvme
>  	_require_nvme_test_img_size 4m
> @@ -144,8 +122,6 @@ group_device_requires() {
>  	_require_test_dev_is_nvme
>  }
>  
> -NVMET_CFS="/sys/kernel/config/nvmet/"
> -
>  _require_test_dev_is_nvme() {
>  	if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then
>  		SKIP_REASONS+=("$TEST_DEV is not a NVMe device")
> @@ -168,31 +144,6 @@ _require_nvme_test_img_size() {
>  	return 0
>  }
>  
> -_require_nvme_trtype() {
> -	local trtype
> -	for trtype in "$@"; do
> -		if [[ "${nvme_trtype}" == "$trtype" ]]; then
> -			return 0
> -		fi
> -	done
> -	SKIP_REASONS+=("nvme_trtype=${nvme_trtype} is not supported in this test")
> -	return 1
> -}
> -
> -_require_nvme_trtype_is_loop() {
> -	if ! _require_nvme_trtype loop; then
> -		return 1
> -	fi
> -	return 0
> -}
> -
> -_require_nvme_trtype_is_fabrics() {
> -	if ! _require_nvme_trtype loop fc rdma tcp; then
> -		return 1
> -	fi
> -	return 0
> -}
> -
>  _require_nvme_cli_auth() {
>  	if ! nvme gen-dhchap-key --nqn nvmf-test-subsys > /dev/null 2>&1 ; then
>  		SKIP_REASONS+=("nvme gen-dhchap-key command missing")
> @@ -235,216 +186,6 @@ _nvme_calc_rand_io_size() {
>  	echo "${io_size_kb}k"
>  }
>  
> -_nvme_fcloop_add_rport() {
> -	local local_wwnn="$1"
> -	local local_wwpn="$2"
> -	local remote_wwnn="$3"
> -	local remote_wwpn="$4"
> -	local loopctl=/sys/class/fcloop/ctl
> -
> -	echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port
> -}
> -
> -_nvme_fcloop_add_lport() {
> -	local wwnn="$1"
> -	local wwpn="$2"
> -	local loopctl=/sys/class/fcloop/ctl
> -
> -	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port
> -}
> -
> -_nvme_fcloop_add_tport() {
> -	local wwnn="$1"
> -	local wwpn="$2"
> -	local loopctl=/sys/class/fcloop/ctl
> -
> -	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port
> -}
> -
> -_setup_fcloop() {
> -	local local_wwnn="${1:-$def_local_wwnn}"
> -	local local_wwpn="${2:-$def_local_wwpn}"
> -	local remote_wwnn="${3:-$def_remote_wwnn}"
> -	local remote_wwpn="${4:-$def_remote_wwpn}"
> -
> -	_nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}"
> -	_nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}"
> -	_nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \
> -		               "${remote_wwnn}" "${remote_wwpn}"
> -}
> -
> -_nvme_fcloop_del_rport() {
> -	local local_wwnn="$1"
> -	local local_wwpn="$2"
> -	local remote_wwnn="$3"
> -	local remote_wwpn="$4"
> -	local loopctl=/sys/class/fcloop/ctl
> -
> -	if [[ ! -f "${loopctl}/del_remote_port" ]]; then
> -		return
> -	fi
> -	echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > "${loopctl}/del_remote_port"
> -}
> -
> -_nvme_fcloop_del_lport() {
> -	local wwnn="$1"
> -	local wwpn="$2"
> -	local loopctl=/sys/class/fcloop/ctl
> -
> -	if [[ ! -f "${loopctl}/del_local_port" ]]; then
> -		return
> -	fi
> -	echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_local_port"
> -}
> -
> -_nvme_fcloop_del_tport() {
> -	local wwnn="$1"
> -	local wwpn="$2"
> -	local loopctl=/sys/class/fcloop/ctl
> -
> -	if [[ ! -f "${loopctl}/del_target_port" ]]; then
> -		return
> -	fi
> -	echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port"
> -}
> -
> -_cleanup_fcloop() {
> -	local local_wwnn="${1:-$def_local_wwnn}"
> -	local local_wwpn="${2:-$def_local_wwpn}"
> -	local remote_wwnn="${3:-$def_remote_wwnn}"
> -	local remote_wwpn="${4:-$def_remote_wwpn}"
> -
> -	_nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}"
> -	_nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}"
> -	_nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \
> -			       "${remote_wwnn}" "${remote_wwpn}"
> -}
> -
> -_cleanup_blkdev() {
> -	local blkdev
> -	local dev
> -
> -	blkdev="$(losetup -l | awk '$6 == "'"$(_nvme_def_file_path)"'" { print $1 }')"
> -	for dev in ${blkdev}; do
> -		losetup -d "${dev}"
> -	done
> -	rm -f "$(_nvme_def_file_path)"
> -}
> -
> -_cleanup_nvmet() {
> -	local dev
> -	local port
> -	local subsys
> -	local transport
> -	local name
> -
> -	if [[ ! -d "${NVMET_CFS}" ]]; then
> -		return 0
> -	fi
> -
> -	# Don't let successive Ctrl-Cs interrupt the cleanup processes
> -	trap '' SIGINT
> -
> -	shopt -s nullglob
> -
> -	for dev in /sys/class/nvme/nvme*; do
> -		dev="$(basename "$dev")"
> -		transport="$(cat "/sys/class/nvme/${dev}/transport" 2>/dev/null)"
> -		if [[ "$transport" == "${nvme_trtype}" ]]; then
> -			# if udev auto connect is enabled for FC we get false positives
> -			if [[ "$transport" != "fc" ]]; then
> -				echo "WARNING: Test did not clean up ${nvme_trtype} device: ${dev}"
> -			fi
> -			_nvme_disconnect_ctrl "${dev}" 2>/dev/null
> -		fi
> -	done
> -
> -	for port in "${NVMET_CFS}"/ports/*; do
> -		name=$(basename "${port}")
> -		echo "WARNING: Test did not clean up port: ${name}"
> -		rm -f "${port}"/subsystems/*
> -		rmdir "${port}"
> -	done
> -
> -	for subsys in "${NVMET_CFS}"/subsystems/*; do
> -		name=$(basename "${subsys}")
> -		echo "WARNING: Test did not clean up subsystem: ${name}"
> -		for ns in "${subsys}"/namespaces/*; do
> -			rmdir "${ns}"
> -		done
> -		rmdir "${subsys}"
> -	done
> -
> -	for host in "${NVMET_CFS}"/hosts/*; do
> -		name=$(basename "${host}")
> -		echo "WARNING: Test did not clean up host: ${name}"
> -		rmdir "${host}"
> -	done
> -
> -	shopt -u nullglob
> -	trap SIGINT
> -
> -	if [[ "${nvme_trtype}" == "fc" ]]; then
> -		_cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
> -				"${def_remote_wwnn}" "${def_remote_wwpn}"
> -		modprobe -rq nvme-fcloop 2>/dev/null
> -	fi
> -	modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null
> -	if [[ "${nvme_trtype}" != "loop" ]]; then
> -		modprobe -rq nvmet-"${nvme_trtype}" 2>/dev/null
> -	fi
> -	modprobe -rq nvmet 2>/dev/null
> -	if [[ "${nvme_trtype}" == "rdma" ]]; then
> -		stop_soft_rdma
> -	fi
> -
> -	_cleanup_blkdev
> -}
> -
> -_setup_nvmet() {
> -	_register_test_cleanup _cleanup_nvmet
> -	modprobe -q nvmet
> -	if [[ "${nvme_trtype}" != "loop" ]]; then
> -		modprobe -q nvmet-"${nvme_trtype}"
> -	fi
> -	modprobe -q nvme-"${nvme_trtype}"
> -	if [[ "${nvme_trtype}" == "rdma" ]]; then
> -		start_soft_rdma
> -		for i in $(rdma_network_interfaces)
> -		do
> -			if [[ "${nvme_adrfam}" == "ipv6" ]]; then
> -				ipv6_addr=$(get_ipv6_ll_addr "$i")
> -				if [[ -n "${ipv6_addr}" ]]; then
> -					def_traddr=${ipv6_addr}
> -				fi
> -			else
> -				ipv4_addr=$(get_ipv4_addr "$i")
> -				if [[ -n "${ipv4_addr}" ]]; then
> -					def_traddr=${ipv4_addr}
> -				fi
> -			fi
> -		done
> -	fi
> -	if [[ "${nvme_trtype}" = "fc" ]]; then
> -		modprobe -q nvme-fcloop
> -		_setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
> -			      "${def_remote_wwnn}" "${def_remote_wwpn}"
> -
> -		def_traddr=$(printf "nn-%s:pn-%s" \
> -				    "${def_remote_wwnn}" \
> -				    "${def_remote_wwpn}")
> -		def_host_traddr=$(printf "nn-%s:pn-%s" \
> -					 "${def_local_wwnn}" \
> -					 "${def_local_wwpn}")
> -	fi
> -}
> -
> -_nvme_disconnect_ctrl() {
> -	local ctrl="$1"
> -
> -	nvme disconnect --device "${ctrl}"
> -}
> -
>  _nvme_disconnect_subsys() {
>  	local subsysnqn="$def_subsysnqn"
>  
> @@ -465,141 +206,6 @@ _nvme_disconnect_subsys() {
>  		grep -o "disconnected.*"
>  }
>  
> -_nvme_connect_subsys() {
> -	local subsysnqn="$def_subsysnqn"
> -	local hostnqn="$def_hostnqn"

It looks weird that _nvme_connect_subsys() is moved to common/nvme, but
_nvme_disconnect_subsys() stays in tests/nvme/rc. I think it's the better to
move _nvme_disconnect_subsys() also. Currently, md/001 does not use
_nvme_disconnect_subsys(). Isn't it the better to call it in
cleanup_nvme_over_tcp()?

> -	local hostid="$def_hostid"
> -	local hostkey=""
> -	local ctrlkey=""
> -	local nr_io_queues=""
> -	local nr_write_queues=""
> -	local nr_poll_queues=""
> -	local keep_alive_tmo=""
> -	local reconnect_delay=""
> -	local ctrl_loss_tmo=""
> -	local no_wait=false
> -	local i
> -
> -	while [[ $# -gt 0 ]]; do
> -		case $1 in
> -			--subsysnqn)
> -				subsysnqn="$2"
> -				shift 2
> -				;;
> -			--hostnqn)
> -				hostnqn="$2"
> -				shift 2
> -				;;
> -			--hostid)
> -				hostid="$2"
> -				shift 2
> -				;;
> -			--dhchap-secret)
> -				hostkey="$2"
> -				shift 2
> -				;;
> -			--dhchap-ctrl-secret)
> -				ctrlkey="$2"
> -				shift 2
> -				;;
> -			--nr-io-queues)
> -				nr_io_queues="$2"
> -				shift 2
> -				;;
> -			--nr-write-queues)
> -				nr_write_queues="$2"
> -				shift 2
> -				;;
> -			--nr-poll-queues)
> -				nr_poll_queues="$2"
> -				shift 2
> -				;;
> -			--keep-alive-tmo)
> -				keep_alive_tmo="$2"
> -				shift 2
> -				;;
> -			--reconnect-delay)
> -				reconnect_delay="$2"
> -				shift 2
> -				;;
> -			--ctrl-loss-tmo)
> -				ctrl_loss_tmo="$2"
> -				shift 2
> -				;;
> -			--no-wait)
> -				no_wait=true
> -				shift 1
> -				;;
> -			*)
> -				echo "WARNING: unknown argument: $1"
> -				shift
> -				;;
> -		esac
> -	done
> -
> -	ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}")
> -	if [[ "${nvme_trtype}" == "fc" ]] ; then
> -		ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}")
> -	elif [[ "${nvme_trtype}" != "loop" ]]; then
> -		ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}")
> -	fi
> -	ARGS+=(--hostnqn="${hostnqn}")
> -	ARGS+=(--hostid="${hostid}")
> -	if [[ -n "${hostkey}" ]]; then
> -		ARGS+=(--dhchap-secret="${hostkey}")
> -	fi
> -	if [[ -n "${ctrlkey}" ]]; then
> -		ARGS+=(--dhchap-ctrl-secret="${ctrlkey}")
> -	fi
> -	if [[ -n "${nr_io_queues}" ]]; then
> -		ARGS+=(--nr-io-queues="${nr_io_queues}")
> -	fi
> -	if [[ -n "${nr_write_queues}" ]]; then
> -		ARGS+=(--nr-write-queues="${nr_write_queues}")
> -	fi
> -	if [[ -n "${nr_poll_queues}" ]]; then
> -		ARGS+=(--nr-poll-queues="${nr_poll_queues}")
> -	fi
> -	if [[ -n "${keep_alive_tmo}" ]]; then
> -		ARGS+=(--keep-alive-tmo="${keep_alive_tmo}")
> -	fi
> -	if [[ -n "${reconnect_delay}" ]]; then
> -		ARGS+=(--reconnect-delay="${reconnect_delay}")
> -	fi
> -	if [[ -n "${ctrl_loss_tmo}" ]]; then
> -		ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}")
> -	fi
> -
> -	nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:"
> -
> -	# Wait until device file and uuid/wwid sysfs attributes get ready for
> -	# all namespaces.
> -	if [[ ${no_wait} = false ]]; then
> -		udevadm settle
> -		for ((i = 0; i < 10; i++)); do
> -			_nvme_ns_ready "${subsysnqn}" && return
> -			sleep .1
> -		done
> -	fi
> -}
> -
> -_nvme_ns_ready() {
> -	local subsysnqn="${1}"
> -	local ns_path ns_id dev
> -	local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn"
> -
> -	dev=$(_find_nvme_dev "$subsysnqn")
> -	for ns_path in "${cfs_path}/namespaces/"*; do
> -		ns_id=${ns_path##*/}
> -		if [[ ! -b /dev/${dev}n${ns_id} ||
> -			   ! -e /sys/block/${dev}n${ns_id}/uuid ||
> -			   ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then
> -			return 1
> -		fi
> -	done
> -	return 0
> -}
> -
>  _nvme_discover() {
>  	local trtype="$1"
>  	local traddr="${2:-$def_traddr}"
> @@ -617,73 +223,6 @@ _nvme_discover() {
>  	nvme discover "${ARGS[@]}"
>  }
>  
> -_create_nvmet_port() {
> -	local trtype="$1"
> -	local traddr="${2:-$def_traddr}"
> -	local adrfam="${3:-$def_adrfam}"
> -	local trsvcid="${4:-$def_trsvcid}"
> -
> -	local port
> -	for ((port = 0; ; port++)); do
> -		if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then
> -			break
> -		fi
> -	done
> -
> -	mkdir "${NVMET_CFS}/ports/${port}"
> -	echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
> -	echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
> -	echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam"
> -	if [[ "${adrfam}" != "fc" ]]; then
> -		echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid"
> -	fi
> -
> -	echo "${port}"
> -}
> -
> -_remove_nvmet_port() {
> -	local port="$1"
> -	rmdir "${NVMET_CFS}/ports/${port}"
> -}
> -
> -_create_nvmet_ns() {
> -	local nvmet_subsystem="$1"
> -	local nsid="$2"
> -	local blkdev="$3"
> -	local uuid="00000000-0000-0000-0000-000000000000"
> -	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> -	local ns_path="${subsys_path}/namespaces/${nsid}"
> -
> -	if [[ $# -eq 4 ]]; then
> -		uuid="$4"
> -	fi
> -
> -	mkdir "${ns_path}"
> -	printf "%s" "${blkdev}" > "${ns_path}/device_path"
> -	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
> -	printf 1 > "${ns_path}/enable"
> -}
> -
> -_create_nvmet_subsystem() {
> -	local nvmet_subsystem="$1"
> -	local blkdev="$2"
> -	local uuid=$3
> -	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> -
> -	mkdir -p "${cfs_path}"
> -	echo 0 > "${cfs_path}/attr_allow_any_host"
> -	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
> -}
> -
> -_add_nvmet_allow_hosts() {
> -	local nvmet_subsystem="$1"
> -	local nvmet_hostnqn="$2"
> -	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> -	local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
> -
> -	ln -s "${host_path}" "${cfs_path}/allowed_hosts/${nvmet_hostnqn}"
> -}
> -
>  _remove_nvmet_allow_hosts() {
>  	local nvmet_subsystem="$1"
>  	local nvmet_hostnqn="$2"
> @@ -692,54 +231,6 @@ _remove_nvmet_allow_hosts() {
>  	rm "${cfs_path}/allowed_hosts/${nvmet_hostnqn}"
>  }
>  
> -_create_nvmet_host() {
> -	local nvmet_subsystem="$1"
> -	local nvmet_hostnqn="$2"
> -	local nvmet_hostkey="$3"
> -	local nvmet_ctrlkey="$4"
> -	local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
> -
> -	if [[ -d "${host_path}" ]]; then
> -		echo "FAIL target setup failed. stale host configuration found"
> -		return 1;
> -	fi
> -
> -	mkdir "${host_path}"
> -	_add_nvmet_allow_hosts "${nvmet_subsystem}" "${nvmet_hostnqn}"
> -	if [[ "${nvmet_hostkey}" ]] ; then
> -		echo "${nvmet_hostkey}" > "${host_path}/dhchap_key"
> -	fi
> -	if [[ "${nvmet_ctrlkey}" ]] ; then
> -		echo "${nvmet_ctrlkey}" > "${host_path}/dhchap_ctrl_key"
> -	fi
> -}
> -
> -_remove_nvmet_ns() {
> -	local nvmet_subsystem="$1"
> -	local nsid=$2
> -	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> -	local nvmet_ns_path="${subsys_path}/namespaces/${nsid}"
> -
> -	echo 0 > "${nvmet_ns_path}/enable"
> -	rmdir "${nvmet_ns_path}"
> -}
> -
> -_remove_nvmet_subsystem() {
> -	local nvmet_subsystem="$1"
> -	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> -
> -	_remove_nvmet_ns "${nvmet_subsystem}" "1"
> -	rm -f "${subsys_path}"/allowed_hosts/*
> -	rmdir "${subsys_path}"
> -}
> -
> -_remove_nvmet_host() {
> -	local nvmet_host="$1"
> -	local host_path="${NVMET_CFS}/hosts/${nvmet_host}"
> -
> -	rmdir "${host_path}"
> -}
> -
>  _create_nvmet_passthru() {
>  	local nvmet_subsystem="$1"
>  	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> @@ -765,34 +256,6 @@ _remove_nvmet_passhtru() {
>  	rmdir "${subsys_path}"
>  }
>  
> -_add_nvmet_subsys_to_port() {
> -	local port="$1"
> -	local nvmet_subsystem="$2"
> -
> -	ln -s "${NVMET_CFS}/subsystems/${nvmet_subsystem}" \
> -		"${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
> -}
> -
> -_remove_nvmet_subsystem_from_port() {
> -	local port="$1"
> -	local nvmet_subsystem="$2"
> -
> -	rm "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
> -}
> -
> -_get_nvmet_ports() {
> -	local nvmet_subsystem="$1"
> -	local -n nvmet_ports="$2"
> -	local cfs_path="${NVMET_CFS}/ports"
> -	local sarg
> -
> -	sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p"
> -
> -	for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do
> -		nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")")
> -	done
> -}
> -
>  _set_nvmet_hostkey() {
>  	local nvmet_hostnqn="$1"
>  	local nvmet_hostkey="$2"
> @@ -829,20 +292,6 @@ _set_nvmet_dhgroup() {
>  	     "${cfs_path}/dhchap_dhgroup"
>  }
>  
> -_find_nvme_dev() {
> -	local subsys=$1
> -	local subsysnqn
> -	local dev
> -	for dev in /sys/class/nvme/nvme*; do
> -		[ -e "$dev" ] || continue
> -		dev="$(basename "$dev")"
> -		subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)"
> -		if [[ "$subsysnqn" == "$subsys" ]]; then
> -			echo "$dev"
> -		fi
> -	done
> -}
> -
>  _find_nvme_ns() {
>  	local subsys_uuid=$1
>  	local uuid
> @@ -924,44 +373,6 @@ _nvmet_target_setup() {
>  			"${hostkey}" "${ctrlkey}"
>  }
>  
> -_nvmet_target_cleanup() {
> -	local ports
> -	local port
> -	local blkdev
> -	local subsysnqn="${def_subsysnqn}"
> -	local blkdev_type=""
> -
> -	while [[ $# -gt 0 ]]; do
> -		case $1 in
> -			--blkdev)
> -				blkdev_type="$2"
> -				shift 2
> -				;;
> -			--subsysnqn)
> -				subsysnqn="$2"
> -				shift 2
> -				;;
> -			*)
> -				echo "WARNING: unknown argument: $1"
> -				shift
> -				;;
> -		esac
> -	done
> -
> -	_get_nvmet_ports "${subsysnqn}" ports
> -
> -	for port in "${ports[@]}"; do
> -		_remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}"
> -		_remove_nvmet_port "${port}"
> -	done
> -	_remove_nvmet_subsystem "${subsysnqn}"
> -	_remove_nvmet_host "${def_hostnqn}"
> -
> -	if [[ "${blkdev_type}" == "device" ]]; then
> -		_cleanup_blkdev
> -	fi
> -}
> -
>  _nvmet_passthru_target_setup() {
>  	local subsysnqn="$def_subsysnqn"
>  	local port
> -- 
> 2.45.1
> 

Re: [PATCH blktests v2 1/2] nvme: move helper functions to common/nvme

[Ofir Gal]

On 6/26/24 14:18, Shinichiro Kawasaki wrote:
> On Jun 24, 2024 / 13:46, Ofir Gal wrote:
>> Move functions from tests/nvme/rc to common/nvme to be able to reuse
>> them in other tests groups.
>>
>> Signed-off-by: Ofir Gal <ofir.gal@volumez.com>
>> ---
>> I have moved the function that are nessecary for the regression test I
>> add. Let me know if we want to move more or all the functions to
>> common/nvme.
>>
>> shellcheck detects 2 new warnings:
>> common/nvme:35:10: warning: nvme_trtype is referenced but not assigned. [SC2154]
>> common/nvme:234:11: warning: nvme_adrfam is referenced but not assigned. [SC2154]
>>
>> I have tried to figure out why but I couldn't figure out what is
>> different than the upstream version. How can I fix them?
>
> Hi Ofir, thanks for this work. Please find two comments in line below. One of
> them suggests a fix for the shellcheck warnings.
>
Applied to v3

>>
>>
>>  common/nvme   | 595 ++++++++++++++++++++++++++++++++++++++++++++++++++
>>  tests/nvme/rc | 591 +------------------------------------------------
>>  2 files changed, 596 insertions(+), 590 deletions(-)
>>  create mode 100644 common/nvme
>>
>> diff --git a/common/nvme b/common/nvme
>> new file mode 100644
>> index 0000000..1800263
>> --- /dev/null
>> +++ b/common/nvme
>> @@ -0,0 +1,595 @@
>> +#!/bin/bash
>> +# SPDX-License-Identifier: GPL-3.0+
>> +#
>> +# nvme helper functions.
>> +
>> +. common/shellcheck
>> +
>> +def_traddr="127.0.0.1"
>> +def_adrfam="ipv4"
>> +def_trsvcid="4420"
>> +def_remote_wwnn="0x10001100aa000001"
>> +def_remote_wwpn="0x20001100aa000001"
>> +def_local_wwnn="0x10001100aa000002"
>> +def_local_wwpn="0x20001100aa000002"
>> +def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349"
>> +def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}"
>> +export def_subsysnqn="blktests-subsystem-1"
>> +export def_subsys_uuid="91fdba0d-f87b-4c25-b80f-db7be1418b9e"
>> +_check_conflict_and_set_default NVMET_TRTYPES nvme_trtype "loop"
>> +_check_conflict_and_set_default NVME_IMG_SIZE nvme_img_size 1G
>> +_check_conflict_and_set_default NVME_NUM_ITER nvme_num_iter 1000
>> +nvmet_blkdev_type=${nvmet_blkdev_type:-"device"}
>> +NVMET_BLKDEV_TYPES=${NVMET_BLKDEV_TYPES:-"device file"}
>> +NVMET_CFS="/sys/kernel/config/nvmet/"
>
> As for the shellcheck warnings, I suggestto add two lines below here to suppress
> them.
>
> nvme_trtype=${nvme_trtype:-}
> nvme_adrfam=${nvme_adrfam:-}
>
Thanks, applied to v3

>> +
>> +# TMPDIR can not be referred out of test() or test_device() context. Instead of
>> +# global variable def_flie_path, use this getter function.
>> +_nvme_def_file_path() {
>> +    echo "${TMPDIR}/img"
>> +}
>> +
>> +_require_nvme_trtype() {
>> +    local trtype
>> +    for trtype in "$@"; do
>> +        if [[ "${nvme_trtype}" == "$trtype" ]]; then
>> +            return 0
>> +        fi
>> +    done
>> +    SKIP_REASONS+=("nvme_trtype=${nvme_trtype} is not supported in this test")
>> +    return 1
>> +}
>> +
>> +_require_nvme_trtype_is_loop() {
>> +    if ! _require_nvme_trtype loop; then
>> +        return 1
>> +    fi
>> +    return 0
>> +}
>> +
>> +_require_nvme_trtype_is_fabrics() {
>> +    if ! _require_nvme_trtype loop fc rdma tcp; then
>> +        return 1
>> +    fi
>> +    return 0
>> +}
>> +
>> +_nvme_fcloop_add_rport() {
>> +    local local_wwnn="$1"
>> +    local local_wwpn="$2"
>> +    local remote_wwnn="$3"
>> +    local remote_wwpn="$4"
>> +    local loopctl=/sys/class/fcloop/ctl
>> +
>> +    echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port
>> +}
>> +
>> +_nvme_fcloop_add_lport() {
>> +    local wwnn="$1"
>> +    local wwpn="$2"
>> +    local loopctl=/sys/class/fcloop/ctl
>> +
>> +    echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port
>> +}
>> +
>> +_nvme_fcloop_add_tport() {
>> +    local wwnn="$1"
>> +    local wwpn="$2"
>> +    local loopctl=/sys/class/fcloop/ctl
>> +
>> +    echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port
>> +}
>> +
>> +_setup_fcloop() {
>> +    local local_wwnn="${1:-$def_local_wwnn}"
>> +    local local_wwpn="${2:-$def_local_wwpn}"
>> +    local remote_wwnn="${3:-$def_remote_wwnn}"
>> +    local remote_wwpn="${4:-$def_remote_wwpn}"
>> +
>> +    _nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}"
>> +    _nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}"
>> +    _nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \
>> +                       "${remote_wwnn}" "${remote_wwpn}"
>> +}
>> +
>> +_nvme_fcloop_del_rport() {
>> +    local local_wwnn="$1"
>> +    local local_wwpn="$2"
>> +    local remote_wwnn="$3"
>> +    local remote_wwpn="$4"
>> +    local loopctl=/sys/class/fcloop/ctl
>> +
>> +    if [[ ! -f "${loopctl}/del_remote_port" ]]; then
>> +        return
>> +    fi
>> +    echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > "${loopctl}/del_remote_port"
>> +}
>> +
>> +_nvme_fcloop_del_lport() {
>> +    local wwnn="$1"
>> +    local wwpn="$2"
>> +    local loopctl=/sys/class/fcloop/ctl
>> +
>> +    if [[ ! -f "${loopctl}/del_local_port" ]]; then
>> +        return
>> +    fi
>> +    echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_local_port"
>> +}
>> +
>> +_nvme_fcloop_del_tport() {
>> +    local wwnn="$1"
>> +    local wwpn="$2"
>> +    local loopctl=/sys/class/fcloop/ctl
>> +
>> +    if [[ ! -f "${loopctl}/del_target_port" ]]; then
>> +        return
>> +    fi
>> +    echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port"
>> +}
>> +
>> +_cleanup_fcloop() {
>> +    local local_wwnn="${1:-$def_local_wwnn}"
>> +    local local_wwpn="${2:-$def_local_wwpn}"
>> +    local remote_wwnn="${3:-$def_remote_wwnn}"
>> +    local remote_wwpn="${4:-$def_remote_wwpn}"
>> +
>> +    _nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}"
>> +    _nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}"
>> +    _nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \
>> +                   "${remote_wwnn}" "${remote_wwpn}"
>> +}
>> +
>> +_cleanup_blkdev() {
>> +    local blkdev
>> +    local dev
>> +
>> +    blkdev="$(losetup -l | awk '$6 == "'"$(_nvme_def_file_path)"'" { print $1 }')"
>> +    for dev in ${blkdev}; do
>> +        losetup -d "${dev}"
>> +    done
>> +    rm -f "$(_nvme_def_file_path)"
>> +}
>> +
>> +_cleanup_nvmet() {
>> +    local dev
>> +    local port
>> +    local subsys
>> +    local transport
>> +    local name
>> +
>> +    if [[ ! -d "${NVMET_CFS}" ]]; then
>> +        return 0
>> +    fi
>> +
>> +    # Don't let successive Ctrl-Cs interrupt the cleanup processes
>> +    trap '' SIGINT
>> +
>> +    shopt -s nullglob
>> +
>> +    for dev in /sys/class/nvme/nvme*; do
>> +        dev="$(basename "$dev")"
>> +        transport="$(cat "/sys/class/nvme/${dev}/transport" 2>/dev/null)"
>> +        if [[ "$transport" == "${nvme_trtype}" ]]; then
>> +            # if udev auto connect is enabled for FC we get false positives
>> +            if [[ "$transport" != "fc" ]]; then
>> +                echo "WARNING: Test did not clean up ${nvme_trtype} device: ${dev}"
>> +            fi
>> +            _nvme_disconnect_ctrl "${dev}" 2>/dev/null
>> +        fi
>> +    done
>> +
>> +    for port in "${NVMET_CFS}"/ports/*; do
>> +        name=$(basename "${port}")
>> +        echo "WARNING: Test did not clean up port: ${name}"
>> +        rm -f "${port}"/subsystems/*
>> +        rmdir "${port}"
>> +    done
>> +
>> +    for subsys in "${NVMET_CFS}"/subsystems/*; do
>> +        name=$(basename "${subsys}")
>> +        echo "WARNING: Test did not clean up subsystem: ${name}"
>> +        for ns in "${subsys}"/namespaces/*; do
>> +            rmdir "${ns}"
>> +        done
>> +        rmdir "${subsys}"
>> +    done
>> +
>> +    for host in "${NVMET_CFS}"/hosts/*; do
>> +        name=$(basename "${host}")
>> +        echo "WARNING: Test did not clean up host: ${name}"
>> +        rmdir "${host}"
>> +    done
>> +
>> +    shopt -u nullglob
>> +    trap SIGINT
>> +
>> +    if [[ "${nvme_trtype}" == "fc" ]]; then
>> +        _cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
>> +                "${def_remote_wwnn}" "${def_remote_wwpn}"
>> +        modprobe -rq nvme-fcloop 2>/dev/null
>> +    fi
>> +    modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null
>> +    if [[ "${nvme_trtype}" != "loop" ]]; then
>> +        modprobe -rq nvmet-"${nvme_trtype}" 2>/dev/null
>> +    fi
>> +    modprobe -rq nvmet 2>/dev/null
>> +    if [[ "${nvme_trtype}" == "rdma" ]]; then
>> +        stop_soft_rdma
>> +    fi
>> +
>> +    _cleanup_blkdev
>> +}
>> +
>> +_setup_nvmet() {
>> +    _register_test_cleanup _cleanup_nvmet
>> +    modprobe -q nvmet
>> +    if [[ "${nvme_trtype}" != "loop" ]]; then
>> +        modprobe -q nvmet-"${nvme_trtype}"
>> +    fi
>> +    modprobe -q nvme-"${nvme_trtype}"
>> +    if [[ "${nvme_trtype}" == "rdma" ]]; then
>> +        start_soft_rdma
>> +        for i in $(rdma_network_interfaces)
>> +        do
>> +            if [[ "${nvme_adrfam}" == "ipv6" ]]; then
>> +                ipv6_addr=$(get_ipv6_ll_addr "$i")
>> +                if [[ -n "${ipv6_addr}" ]]; then
>> +                    def_traddr=${ipv6_addr}
>> +                fi
>> +            else
>> +                ipv4_addr=$(get_ipv4_addr "$i")
>> +                if [[ -n "${ipv4_addr}" ]]; then
>> +                    def_traddr=${ipv4_addr}
>> +                fi
>> +            fi
>> +        done
>> +    fi
>> +    if [[ "${nvme_trtype}" = "fc" ]]; then
>> +        modprobe -q nvme-fcloop
>> +        _setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
>> +                  "${def_remote_wwnn}" "${def_remote_wwpn}"
>> +
>> +        def_traddr=$(printf "nn-%s:pn-%s" \
>> +                    "${def_remote_wwnn}" \
>> +                    "${def_remote_wwpn}")
>> +        def_host_traddr=$(printf "nn-%s:pn-%s" \
>> +                     "${def_local_wwnn}" \
>> +                     "${def_local_wwpn}")
>> +    fi
>> +}
>> +
>> +_nvme_disconnect_ctrl() {
>> +    local ctrl="$1"
>> +
>> +    nvme disconnect --device "${ctrl}"
>> +}
>> +
>> +_nvme_connect_subsys() {
>> +    local subsysnqn="$def_subsysnqn"
>> +    local hostnqn="$def_hostnqn"
>> +    local hostid="$def_hostid"
>> +    local hostkey=""
>> +    local ctrlkey=""
>> +    local nr_io_queues=""
>> +    local nr_write_queues=""
>> +    local nr_poll_queues=""
>> +    local keep_alive_tmo=""
>> +    local reconnect_delay=""
>> +    local ctrl_loss_tmo=""
>> +    local no_wait=false
>> +    local i
>> +
>> +    while [[ $# -gt 0 ]]; do
>> +        case $1 in
>> +            --subsysnqn)
>> +                subsysnqn="$2"
>> +                shift 2
>> +                ;;
>> +            --hostnqn)
>> +                hostnqn="$2"
>> +                shift 2
>> +                ;;
>> +            --hostid)
>> +                hostid="$2"
>> +                shift 2
>> +                ;;
>> +            --dhchap-secret)
>> +                hostkey="$2"
>> +                shift 2
>> +                ;;
>> +            --dhchap-ctrl-secret)
>> +                ctrlkey="$2"
>> +                shift 2
>> +                ;;
>> +            --nr-io-queues)
>> +                nr_io_queues="$2"
>> +                shift 2
>> +                ;;
>> +            --nr-write-queues)
>> +                nr_write_queues="$2"
>> +                shift 2
>> +                ;;
>> +            --nr-poll-queues)
>> +                nr_poll_queues="$2"
>> +                shift 2
>> +                ;;
>> +            --keep-alive-tmo)
>> +                keep_alive_tmo="$2"
>> +                shift 2
>> +                ;;
>> +            --reconnect-delay)
>> +                reconnect_delay="$2"
>> +                shift 2
>> +                ;;
>> +            --ctrl-loss-tmo)
>> +                ctrl_loss_tmo="$2"
>> +                shift 2
>> +                ;;
>> +            --no-wait)
>> +                no_wait=true
>> +                shift 1
>> +                ;;
>> +            *)
>> +                echo "WARNING: unknown argument: $1"
>> +                shift
>> +                ;;
>> +        esac
>> +    done
>> +
>> +    ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}")
>> +    if [[ "${nvme_trtype}" == "fc" ]] ; then
>> +        ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}")
>> +    elif [[ "${nvme_trtype}" != "loop" ]]; then
>> +        ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}")
>> +    fi
>> +    ARGS+=(--hostnqn="${hostnqn}")
>> +    ARGS+=(--hostid="${hostid}")
>> +    if [[ -n "${hostkey}" ]]; then
>> +        ARGS+=(--dhchap-secret="${hostkey}")
>> +    fi
>> +    if [[ -n "${ctrlkey}" ]]; then
>> +        ARGS+=(--dhchap-ctrl-secret="${ctrlkey}")
>> +    fi
>> +    if [[ -n "${nr_io_queues}" ]]; then
>> +        ARGS+=(--nr-io-queues="${nr_io_queues}")
>> +    fi
>> +    if [[ -n "${nr_write_queues}" ]]; then
>> +        ARGS+=(--nr-write-queues="${nr_write_queues}")
>> +    fi
>> +    if [[ -n "${nr_poll_queues}" ]]; then
>> +        ARGS+=(--nr-poll-queues="${nr_poll_queues}")
>> +    fi
>> +    if [[ -n "${keep_alive_tmo}" ]]; then
>> +        ARGS+=(--keep-alive-tmo="${keep_alive_tmo}")
>> +    fi
>> +    if [[ -n "${reconnect_delay}" ]]; then
>> +        ARGS+=(--reconnect-delay="${reconnect_delay}")
>> +    fi
>> +    if [[ -n "${ctrl_loss_tmo}" ]]; then
>> +        ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}")
>> +    fi
>> +
>> +    nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:"
>> +
>> +    # Wait until device file and uuid/wwid sysfs attributes get ready for
>> +    # all namespaces.
>> +    if [[ ${no_wait} = false ]]; then
>> +        udevadm settle
>> +        for ((i = 0; i < 10; i++)); do
>> +            _nvme_ns_ready "${subsysnqn}" && return
>> +            sleep .1
>> +        done
>> +    fi
>> +}
>> +
>> +_nvme_ns_ready() {
>> +    local subsysnqn="${1}"
>> +    local ns_path ns_id dev
>> +    local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn"
>> +
>> +    dev=$(_find_nvme_dev "$subsysnqn")
>> +    for ns_path in "${cfs_path}/namespaces/"*; do
>> +        ns_id=${ns_path##*/}
>> +        if [[ ! -b /dev/${dev}n${ns_id} ||
>> +               ! -e /sys/block/${dev}n${ns_id}/uuid ||
>> +               ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then
>> +            return 1
>> +        fi
>> +    done
>> +    return 0
>> +}
>> +
>> +_create_nvmet_port() {
>> +    local trtype="$1"
>> +    local traddr="${2:-$def_traddr}"
>> +    local adrfam="${3:-$def_adrfam}"
>> +    local trsvcid="${4:-$def_trsvcid}"
>> +
>> +    local port
>> +    for ((port = 0; ; port++)); do
>> +        if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then
>> +            break
>> +        fi
>> +    done
>> +
>> +    mkdir "${NVMET_CFS}/ports/${port}"
>> +    echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
>> +    echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
>> +    echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam"
>> +    if [[ "${adrfam}" != "fc" ]]; then
>> +        echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid"
>> +    fi
>> +
>> +    echo "${port}"
>> +}
>> +
>> +_remove_nvmet_port() {
>> +    local port="$1"
>> +    rmdir "${NVMET_CFS}/ports/${port}"
>> +}
>> +
>> +_create_nvmet_ns() {
>> +    local nvmet_subsystem="$1"
>> +    local nsid="$2"
>> +    local blkdev="$3"
>> +    local uuid="00000000-0000-0000-0000-000000000000"
>> +    local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>> +    local ns_path="${subsys_path}/namespaces/${nsid}"
>> +
>> +    if [[ $# -eq 4 ]]; then
>> +        uuid="$4"
>> +    fi
>> +
>> +    mkdir "${ns_path}"
>> +    printf "%s" "${blkdev}" > "${ns_path}/device_path"
>> +    printf "%s" "${uuid}" > "${ns_path}/device_uuid"
>> +    printf 1 > "${ns_path}/enable"
>> +}
>> +
>> +_create_nvmet_subsystem() {
>> +    local nvmet_subsystem="$1"
>> +    local blkdev="$2"
>> +    local uuid=$3
>> +    local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>> +
>> +    mkdir -p "${cfs_path}"
>> +    echo 0 > "${cfs_path}/attr_allow_any_host"
>> +    _create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
>> +}
>> +
>> +_add_nvmet_allow_hosts() {
>> +    local nvmet_subsystem="$1"
>> +    local nvmet_hostnqn="$2"
>> +    local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>> +    local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
>> +
>> +    ln -s "${host_path}" "${cfs_path}/allowed_hosts/${nvmet_hostnqn}"
>> +}
>> +
>> +_create_nvmet_host() {
>> +    local nvmet_subsystem="$1"
>> +    local nvmet_hostnqn="$2"
>> +    local nvmet_hostkey="$3"
>> +    local nvmet_ctrlkey="$4"
>> +    local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
>> +
>> +    if [[ -d "${host_path}" ]]; then
>> +        echo "FAIL target setup failed. stale host configuration found"
>> +        return 1;
>> +    fi
>> +
>> +    mkdir "${host_path}"
>> +    _add_nvmet_allow_hosts "${nvmet_subsystem}" "${nvmet_hostnqn}"
>> +    if [[ "${nvmet_hostkey}" ]] ; then
>> +        echo "${nvmet_hostkey}" > "${host_path}/dhchap_key"
>> +    fi
>> +    if [[ "${nvmet_ctrlkey}" ]] ; then
>> +        echo "${nvmet_ctrlkey}" > "${host_path}/dhchap_ctrl_key"
>> +    fi
>> +}
>> +
>> +_remove_nvmet_ns() {
>> +    local nvmet_subsystem="$1"
>> +    local nsid=$2
>> +    local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>> +    local nvmet_ns_path="${subsys_path}/namespaces/${nsid}"
>> +
>> +    echo 0 > "${nvmet_ns_path}/enable"
>> +    rmdir "${nvmet_ns_path}"
>> +}
>> +
>> +_remove_nvmet_subsystem() {
>> +    local nvmet_subsystem="$1"
>> +    local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>> +
>> +    _remove_nvmet_ns "${nvmet_subsystem}" "1"
>> +    rm -f "${subsys_path}"/allowed_hosts/*
>> +    rmdir "${subsys_path}"
>> +}
>> +
>> +_remove_nvmet_host() {
>> +    local nvmet_host="$1"
>> +    local host_path="${NVMET_CFS}/hosts/${nvmet_host}"
>> +
>> +    rmdir "${host_path}"
>> +}
>> +
>> +_add_nvmet_subsys_to_port() {
>> +    local port="$1"
>> +    local nvmet_subsystem="$2"
>> +
>> +    ln -s "${NVMET_CFS}/subsystems/${nvmet_subsystem}" \
>> +        "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
>> +}
>> +
>> +_remove_nvmet_subsystem_from_port() {
>> +    local port="$1"
>> +    local nvmet_subsystem="$2"
>> +
>> +    rm "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
>> +}
>> +
>> +_get_nvmet_ports() {
>> +    local nvmet_subsystem="$1"
>> +    local -n nvmet_ports="$2"
>> +    local cfs_path="${NVMET_CFS}/ports"
>> +    local sarg
>> +
>> +    sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p"
>> +
>> +    for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do
>> +        nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")")
>> +    done
>> +}
>> +
>> +_find_nvme_dev() {
>> +    local subsys=$1
>> +    local subsysnqn
>> +    local dev
>> +    for dev in /sys/class/nvme/nvme*; do
>> +        [ -e "$dev" ] || continue
>> +        dev="$(basename "$dev")"
>> +        subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)"
>> +        if [[ "$subsysnqn" == "$subsys" ]]; then
>> +            echo "$dev"
>> +        fi
>> +    done
>> +}
>> +
>> +_nvmet_target_cleanup() {
>> +    local ports
>> +    local port
>> +    local blkdev
>> +    local subsysnqn="${def_subsysnqn}"
>> +    local blkdev_type=""
>> +
>> +    while [[ $# -gt 0 ]]; do
>> +        case $1 in
>> +            --blkdev)
>> +                blkdev_type="$2"
>> +                shift 2
>> +                ;;
>> +            --subsysnqn)
>> +                subsysnqn="$2"
>> +                shift 2
>> +                ;;
>> +            *)
>> +                echo "WARNING: unknown argument: $1"
>> +                shift
>> +                ;;
>> +        esac
>> +    done
>> +
>> +    _get_nvmet_ports "${subsysnqn}" ports
>> +
>> +    for port in "${ports[@]}"; do
>> +        _remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}"
>> +        _remove_nvmet_port "${port}"
>> +    done
>> +    _remove_nvmet_subsystem "${subsysnqn}"
>> +    _remove_nvmet_host "${def_hostnqn}"
>> +
>> +    if [[ "${blkdev_type}" == "device" ]]; then
>> +        _cleanup_blkdev
>> +    fi
>> +}
>> diff --git a/tests/nvme/rc b/tests/nvme/rc
>> index c1ddf41..3462f2e 100644
>> --- a/tests/nvme/rc
>> +++ b/tests/nvme/rc
>> @@ -5,25 +5,9 @@
>>  # Test specific to NVMe devices
>>  
>>  . common/rc
>> +. common/nvme
>>  . common/multipath-over-rdma
>>  
>> -def_traddr="127.0.0.1"
>> -def_adrfam="ipv4"
>> -def_trsvcid="4420"
>> -def_remote_wwnn="0x10001100aa000001"
>> -def_remote_wwpn="0x20001100aa000001"
>> -def_local_wwnn="0x10001100aa000002"
>> -def_local_wwpn="0x20001100aa000002"
>> -def_hostid="0f01fb42-9f7f-4856-b0b3-51e60b8de349"
>> -def_hostnqn="nqn.2014-08.org.nvmexpress:uuid:${def_hostid}"
>> -export def_subsysnqn="blktests-subsystem-1"
>> -export def_subsys_uuid="91fdba0d-f87b-4c25-b80f-db7be1418b9e"
>> -_check_conflict_and_set_default NVMET_TRTYPES nvme_trtype "loop"
>> -_check_conflict_and_set_default NVME_IMG_SIZE nvme_img_size 1G
>> -_check_conflict_and_set_default NVME_NUM_ITER nvme_num_iter 1000
>> -nvmet_blkdev_type=${nvmet_blkdev_type:-"device"}
>> -NVMET_BLKDEV_TYPES=${NVMET_BLKDEV_TYPES:-"device file"}
>> -
>>  _NVMET_TRTYPES_is_valid() {
>>      local type
>>  
>> @@ -70,12 +54,6 @@ _set_nvmet_blkdev_type() {
>>      COND_DESC="bd=${nvmet_blkdev_type}"
>>  }
>>  
>> -# TMPDIR can not be referred out of test() or test_device() context. Instead of
>> -# global variable def_flie_path, use this getter function.
>> -_nvme_def_file_path() {
>> -    echo "${TMPDIR}/img"
>> -}
>> -
>>  _nvme_requires() {
>>      _have_program nvme
>>      _require_nvme_test_img_size 4m
>> @@ -144,8 +122,6 @@ group_device_requires() {
>>      _require_test_dev_is_nvme
>>  }
>>  
>> -NVMET_CFS="/sys/kernel/config/nvmet/"
>> -
>>  _require_test_dev_is_nvme() {
>>      if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then
>>          SKIP_REASONS+=("$TEST_DEV is not a NVMe device")
>> @@ -168,31 +144,6 @@ _require_nvme_test_img_size() {
>>      return 0
>>  }
>>  
>> -_require_nvme_trtype() {
>> -    local trtype
>> -    for trtype in "$@"; do
>> -        if [[ "${nvme_trtype}" == "$trtype" ]]; then
>> -            return 0
>> -        fi
>> -    done
>> -    SKIP_REASONS+=("nvme_trtype=${nvme_trtype} is not supported in this test")
>> -    return 1
>> -}
>> -
>> -_require_nvme_trtype_is_loop() {
>> -    if ! _require_nvme_trtype loop; then
>> -        return 1
>> -    fi
>> -    return 0
>> -}
>> -
>> -_require_nvme_trtype_is_fabrics() {
>> -    if ! _require_nvme_trtype loop fc rdma tcp; then
>> -        return 1
>> -    fi
>> -    return 0
>> -}
>> -
>>  _require_nvme_cli_auth() {
>>      if ! nvme gen-dhchap-key --nqn nvmf-test-subsys > /dev/null 2>&1 ; then
>>          SKIP_REASONS+=("nvme gen-dhchap-key command missing")
>> @@ -235,216 +186,6 @@ _nvme_calc_rand_io_size() {
>>      echo "${io_size_kb}k"
>>  }
>>  
>> -_nvme_fcloop_add_rport() {
>> -    local local_wwnn="$1"
>> -    local local_wwpn="$2"
>> -    local remote_wwnn="$3"
>> -    local remote_wwpn="$4"
>> -    local loopctl=/sys/class/fcloop/ctl
>> -
>> -    echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port
>> -}
>> -
>> -_nvme_fcloop_add_lport() {
>> -    local wwnn="$1"
>> -    local wwpn="$2"
>> -    local loopctl=/sys/class/fcloop/ctl
>> -
>> -    echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port
>> -}
>> -
>> -_nvme_fcloop_add_tport() {
>> -    local wwnn="$1"
>> -    local wwpn="$2"
>> -    local loopctl=/sys/class/fcloop/ctl
>> -
>> -    echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port
>> -}
>> -
>> -_setup_fcloop() {
>> -    local local_wwnn="${1:-$def_local_wwnn}"
>> -    local local_wwpn="${2:-$def_local_wwpn}"
>> -    local remote_wwnn="${3:-$def_remote_wwnn}"
>> -    local remote_wwpn="${4:-$def_remote_wwpn}"
>> -
>> -    _nvme_fcloop_add_tport "${remote_wwnn}" "${remote_wwpn}"
>> -    _nvme_fcloop_add_lport "${local_wwnn}" "${local_wwpn}"
>> -    _nvme_fcloop_add_rport "${local_wwnn}" "${local_wwpn}" \
>> -                       "${remote_wwnn}" "${remote_wwpn}"
>> -}
>> -
>> -_nvme_fcloop_del_rport() {
>> -    local local_wwnn="$1"
>> -    local local_wwpn="$2"
>> -    local remote_wwnn="$3"
>> -    local remote_wwpn="$4"
>> -    local loopctl=/sys/class/fcloop/ctl
>> -
>> -    if [[ ! -f "${loopctl}/del_remote_port" ]]; then
>> -        return
>> -    fi
>> -    echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > "${loopctl}/del_remote_port"
>> -}
>> -
>> -_nvme_fcloop_del_lport() {
>> -    local wwnn="$1"
>> -    local wwpn="$2"
>> -    local loopctl=/sys/class/fcloop/ctl
>> -
>> -    if [[ ! -f "${loopctl}/del_local_port" ]]; then
>> -        return
>> -    fi
>> -    echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_local_port"
>> -}
>> -
>> -_nvme_fcloop_del_tport() {
>> -    local wwnn="$1"
>> -    local wwpn="$2"
>> -    local loopctl=/sys/class/fcloop/ctl
>> -
>> -    if [[ ! -f "${loopctl}/del_target_port" ]]; then
>> -        return
>> -    fi
>> -    echo "wwnn=${wwnn},wwpn=${wwpn}" > "${loopctl}/del_target_port"
>> -}
>> -
>> -_cleanup_fcloop() {
>> -    local local_wwnn="${1:-$def_local_wwnn}"
>> -    local local_wwpn="${2:-$def_local_wwpn}"
>> -    local remote_wwnn="${3:-$def_remote_wwnn}"
>> -    local remote_wwpn="${4:-$def_remote_wwpn}"
>> -
>> -    _nvme_fcloop_del_tport "${remote_wwnn}" "${remote_wwpn}"
>> -    _nvme_fcloop_del_lport "${local_wwnn}" "${local_wwpn}"
>> -    _nvme_fcloop_del_rport "${local_wwnn}" "${local_wwpn}" \
>> -                   "${remote_wwnn}" "${remote_wwpn}"
>> -}
>> -
>> -_cleanup_blkdev() {
>> -    local blkdev
>> -    local dev
>> -
>> -    blkdev="$(losetup -l | awk '$6 == "'"$(_nvme_def_file_path)"'" { print $1 }')"
>> -    for dev in ${blkdev}; do
>> -        losetup -d "${dev}"
>> -    done
>> -    rm -f "$(_nvme_def_file_path)"
>> -}
>> -
>> -_cleanup_nvmet() {
>> -    local dev
>> -    local port
>> -    local subsys
>> -    local transport
>> -    local name
>> -
>> -    if [[ ! -d "${NVMET_CFS}" ]]; then
>> -        return 0
>> -    fi
>> -
>> -    # Don't let successive Ctrl-Cs interrupt the cleanup processes
>> -    trap '' SIGINT
>> -
>> -    shopt -s nullglob
>> -
>> -    for dev in /sys/class/nvme/nvme*; do
>> -        dev="$(basename "$dev")"
>> -        transport="$(cat "/sys/class/nvme/${dev}/transport" 2>/dev/null)"
>> -        if [[ "$transport" == "${nvme_trtype}" ]]; then
>> -            # if udev auto connect is enabled for FC we get false positives
>> -            if [[ "$transport" != "fc" ]]; then
>> -                echo "WARNING: Test did not clean up ${nvme_trtype} device: ${dev}"
>> -            fi
>> -            _nvme_disconnect_ctrl "${dev}" 2>/dev/null
>> -        fi
>> -    done
>> -
>> -    for port in "${NVMET_CFS}"/ports/*; do
>> -        name=$(basename "${port}")
>> -        echo "WARNING: Test did not clean up port: ${name}"
>> -        rm -f "${port}"/subsystems/*
>> -        rmdir "${port}"
>> -    done
>> -
>> -    for subsys in "${NVMET_CFS}"/subsystems/*; do
>> -        name=$(basename "${subsys}")
>> -        echo "WARNING: Test did not clean up subsystem: ${name}"
>> -        for ns in "${subsys}"/namespaces/*; do
>> -            rmdir "${ns}"
>> -        done
>> -        rmdir "${subsys}"
>> -    done
>> -
>> -    for host in "${NVMET_CFS}"/hosts/*; do
>> -        name=$(basename "${host}")
>> -        echo "WARNING: Test did not clean up host: ${name}"
>> -        rmdir "${host}"
>> -    done
>> -
>> -    shopt -u nullglob
>> -    trap SIGINT
>> -
>> -    if [[ "${nvme_trtype}" == "fc" ]]; then
>> -        _cleanup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
>> -                "${def_remote_wwnn}" "${def_remote_wwpn}"
>> -        modprobe -rq nvme-fcloop 2>/dev/null
>> -    fi
>> -    modprobe -rq nvme-"${nvme_trtype}" 2>/dev/null
>> -    if [[ "${nvme_trtype}" != "loop" ]]; then
>> -        modprobe -rq nvmet-"${nvme_trtype}" 2>/dev/null
>> -    fi
>> -    modprobe -rq nvmet 2>/dev/null
>> -    if [[ "${nvme_trtype}" == "rdma" ]]; then
>> -        stop_soft_rdma
>> -    fi
>> -
>> -    _cleanup_blkdev
>> -}
>> -
>> -_setup_nvmet() {
>> -    _register_test_cleanup _cleanup_nvmet
>> -    modprobe -q nvmet
>> -    if [[ "${nvme_trtype}" != "loop" ]]; then
>> -        modprobe -q nvmet-"${nvme_trtype}"
>> -    fi
>> -    modprobe -q nvme-"${nvme_trtype}"
>> -    if [[ "${nvme_trtype}" == "rdma" ]]; then
>> -        start_soft_rdma
>> -        for i in $(rdma_network_interfaces)
>> -        do
>> -            if [[ "${nvme_adrfam}" == "ipv6" ]]; then
>> -                ipv6_addr=$(get_ipv6_ll_addr "$i")
>> -                if [[ -n "${ipv6_addr}" ]]; then
>> -                    def_traddr=${ipv6_addr}
>> -                fi
>> -            else
>> -                ipv4_addr=$(get_ipv4_addr "$i")
>> -                if [[ -n "${ipv4_addr}" ]]; then
>> -                    def_traddr=${ipv4_addr}
>> -                fi
>> -            fi
>> -        done
>> -    fi
>> -    if [[ "${nvme_trtype}" = "fc" ]]; then
>> -        modprobe -q nvme-fcloop
>> -        _setup_fcloop "${def_local_wwnn}" "${def_local_wwpn}" \
>> -                  "${def_remote_wwnn}" "${def_remote_wwpn}"
>> -
>> -        def_traddr=$(printf "nn-%s:pn-%s" \
>> -                    "${def_remote_wwnn}" \
>> -                    "${def_remote_wwpn}")
>> -        def_host_traddr=$(printf "nn-%s:pn-%s" \
>> -                     "${def_local_wwnn}" \
>> -                     "${def_local_wwpn}")
>> -    fi
>> -}
>> -
>> -_nvme_disconnect_ctrl() {
>> -    local ctrl="$1"
>> -
>> -    nvme disconnect --device "${ctrl}"
>> -}
>> -
>>  _nvme_disconnect_subsys() {
>>      local subsysnqn="$def_subsysnqn"
>>  
>> @@ -465,141 +206,6 @@ _nvme_disconnect_subsys() {
>>          grep -o "disconnected.*"
>>  }
>>  
>> -_nvme_connect_subsys() {
>> -    local subsysnqn="$def_subsysnqn"
>> -    local hostnqn="$def_hostnqn"
>
> It looks weird that _nvme_connect_subsys() is moved to common/nvme, but
> _nvme_disconnect_subsys() stays in tests/nvme/rc. I think it's the better to
> move _nvme_disconnect_subsys() also. Currently, md/001 does not use
> _nvme_disconnect_subsys(). Isn't it the better to call it in
> cleanup_nvme_over_tcp()?
>
I agree, I would move _nvme_disconnect_subsys() in v3.
cleanup_nvme_over_tcp() use _nvme_disconnect_ctrl() to disconnect the
controller by the controller name rather the subsys name. I can change
it to _nvme_disconnect_subsys() if it's more appropriate.

>> -    local hostid="$def_hostid"
>> -    local hostkey=""
>> -    local ctrlkey=""
>> -    local nr_io_queues=""
>> -    local nr_write_queues=""
>> -    local nr_poll_queues=""
>> -    local keep_alive_tmo=""
>> -    local reconnect_delay=""
>> -    local ctrl_loss_tmo=""
>> -    local no_wait=false
>> -    local i
>> -
>> -    while [[ $# -gt 0 ]]; do
>> -        case $1 in
>> -            --subsysnqn)
>> -                subsysnqn="$2"
>> -                shift 2
>> -                ;;
>> -            --hostnqn)
>> -                hostnqn="$2"
>> -                shift 2
>> -                ;;
>> -            --hostid)
>> -                hostid="$2"
>> -                shift 2
>> -                ;;
>> -            --dhchap-secret)
>> -                hostkey="$2"
>> -                shift 2
>> -                ;;
>> -            --dhchap-ctrl-secret)
>> -                ctrlkey="$2"
>> -                shift 2
>> -                ;;
>> -            --nr-io-queues)
>> -                nr_io_queues="$2"
>> -                shift 2
>> -                ;;
>> -            --nr-write-queues)
>> -                nr_write_queues="$2"
>> -                shift 2
>> -                ;;
>> -            --nr-poll-queues)
>> -                nr_poll_queues="$2"
>> -                shift 2
>> -                ;;
>> -            --keep-alive-tmo)
>> -                keep_alive_tmo="$2"
>> -                shift 2
>> -                ;;
>> -            --reconnect-delay)
>> -                reconnect_delay="$2"
>> -                shift 2
>> -                ;;
>> -            --ctrl-loss-tmo)
>> -                ctrl_loss_tmo="$2"
>> -                shift 2
>> -                ;;
>> -            --no-wait)
>> -                no_wait=true
>> -                shift 1
>> -                ;;
>> -            *)
>> -                echo "WARNING: unknown argument: $1"
>> -                shift
>> -                ;;
>> -        esac
>> -    done
>> -
>> -    ARGS=(--transport "${nvme_trtype}" --nqn "${subsysnqn}")
>> -    if [[ "${nvme_trtype}" == "fc" ]] ; then
>> -        ARGS+=(--traddr "${def_traddr}" --host-traddr "${def_host_traddr}")
>> -    elif [[ "${nvme_trtype}" != "loop" ]]; then
>> -        ARGS+=(--traddr "${def_traddr}" --trsvcid "${def_trsvcid}")
>> -    fi
>> -    ARGS+=(--hostnqn="${hostnqn}")
>> -    ARGS+=(--hostid="${hostid}")
>> -    if [[ -n "${hostkey}" ]]; then
>> -        ARGS+=(--dhchap-secret="${hostkey}")
>> -    fi
>> -    if [[ -n "${ctrlkey}" ]]; then
>> -        ARGS+=(--dhchap-ctrl-secret="${ctrlkey}")
>> -    fi
>> -    if [[ -n "${nr_io_queues}" ]]; then
>> -        ARGS+=(--nr-io-queues="${nr_io_queues}")
>> -    fi
>> -    if [[ -n "${nr_write_queues}" ]]; then
>> -        ARGS+=(--nr-write-queues="${nr_write_queues}")
>> -    fi
>> -    if [[ -n "${nr_poll_queues}" ]]; then
>> -        ARGS+=(--nr-poll-queues="${nr_poll_queues}")
>> -    fi
>> -    if [[ -n "${keep_alive_tmo}" ]]; then
>> -        ARGS+=(--keep-alive-tmo="${keep_alive_tmo}")
>> -    fi
>> -    if [[ -n "${reconnect_delay}" ]]; then
>> -        ARGS+=(--reconnect-delay="${reconnect_delay}")
>> -    fi
>> -    if [[ -n "${ctrl_loss_tmo}" ]]; then
>> -        ARGS+=(--ctrl-loss-tmo="${ctrl_loss_tmo}")
>> -    fi
>> -
>> -    nvme connect "${ARGS[@]}" 2> /dev/null | grep -v "connecting to device:"
>> -
>> -    # Wait until device file and uuid/wwid sysfs attributes get ready for
>> -    # all namespaces.
>> -    if [[ ${no_wait} = false ]]; then
>> -        udevadm settle
>> -        for ((i = 0; i < 10; i++)); do
>> -            _nvme_ns_ready "${subsysnqn}" && return
>> -            sleep .1
>> -        done
>> -    fi
>> -}
>> -
>> -_nvme_ns_ready() {
>> -    local subsysnqn="${1}"
>> -    local ns_path ns_id dev
>> -    local cfs_path="${NVMET_CFS}/subsystems/$subsysnqn"
>> -
>> -    dev=$(_find_nvme_dev "$subsysnqn")
>> -    for ns_path in "${cfs_path}/namespaces/"*; do
>> -        ns_id=${ns_path##*/}
>> -        if [[ ! -b /dev/${dev}n${ns_id} ||
>> -               ! -e /sys/block/${dev}n${ns_id}/uuid ||
>> -               ! -e /sys/block/${dev}n${ns_id}/wwid ]]; then
>> -            return 1
>> -        fi
>> -    done
>> -    return 0
>> -}
>> -
>>  _nvme_discover() {
>>      local trtype="$1"
>>      local traddr="${2:-$def_traddr}"
>> @@ -617,73 +223,6 @@ _nvme_discover() {
>>      nvme discover "${ARGS[@]}"
>>  }
>>  
>> -_create_nvmet_port() {
>> -    local trtype="$1"
>> -    local traddr="${2:-$def_traddr}"
>> -    local adrfam="${3:-$def_adrfam}"
>> -    local trsvcid="${4:-$def_trsvcid}"
>> -
>> -    local port
>> -    for ((port = 0; ; port++)); do
>> -        if [[ ! -e "${NVMET_CFS}/ports/${port}" ]]; then
>> -            break
>> -        fi
>> -    done
>> -
>> -    mkdir "${NVMET_CFS}/ports/${port}"
>> -    echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
>> -    echo "${traddr}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
>> -    echo "${adrfam}" > "${NVMET_CFS}/ports/${port}/addr_adrfam"
>> -    if [[ "${adrfam}" != "fc" ]]; then
>> -        echo "${trsvcid}" > "${NVMET_CFS}/ports/${port}/addr_trsvcid"
>> -    fi
>> -
>> -    echo "${port}"
>> -}
>> -
>> -_remove_nvmet_port() {
>> -    local port="$1"
>> -    rmdir "${NVMET_CFS}/ports/${port}"
>> -}
>> -
>> -_create_nvmet_ns() {
>> -    local nvmet_subsystem="$1"
>> -    local nsid="$2"
>> -    local blkdev="$3"
>> -    local uuid="00000000-0000-0000-0000-000000000000"
>> -    local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>> -    local ns_path="${subsys_path}/namespaces/${nsid}"
>> -
>> -    if [[ $# -eq 4 ]]; then
>> -        uuid="$4"
>> -    fi
>> -
>> -    mkdir "${ns_path}"
>> -    printf "%s" "${blkdev}" > "${ns_path}/device_path"
>> -    printf "%s" "${uuid}" > "${ns_path}/device_uuid"
>> -    printf 1 > "${ns_path}/enable"
>> -}
>> -
>> -_create_nvmet_subsystem() {
>> -    local nvmet_subsystem="$1"
>> -    local blkdev="$2"
>> -    local uuid=$3
>> -    local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>> -
>> -    mkdir -p "${cfs_path}"
>> -    echo 0 > "${cfs_path}/attr_allow_any_host"
>> -    _create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
>> -}
>> -
>> -_add_nvmet_allow_hosts() {
>> -    local nvmet_subsystem="$1"
>> -    local nvmet_hostnqn="$2"
>> -    local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>> -    local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
>> -
>> -    ln -s "${host_path}" "${cfs_path}/allowed_hosts/${nvmet_hostnqn}"
>> -}
>> -
>>  _remove_nvmet_allow_hosts() {
>>      local nvmet_subsystem="$1"
>>      local nvmet_hostnqn="$2"
>> @@ -692,54 +231,6 @@ _remove_nvmet_allow_hosts() {
>>      rm "${cfs_path}/allowed_hosts/${nvmet_hostnqn}"
>>  }
>>  
>> -_create_nvmet_host() {
>> -    local nvmet_subsystem="$1"
>> -    local nvmet_hostnqn="$2"
>> -    local nvmet_hostkey="$3"
>> -    local nvmet_ctrlkey="$4"
>> -    local host_path="${NVMET_CFS}/hosts/${nvmet_hostnqn}"
>> -
>> -    if [[ -d "${host_path}" ]]; then
>> -        echo "FAIL target setup failed. stale host configuration found"
>> -        return 1;
>> -    fi
>> -
>> -    mkdir "${host_path}"
>> -    _add_nvmet_allow_hosts "${nvmet_subsystem}" "${nvmet_hostnqn}"
>> -    if [[ "${nvmet_hostkey}" ]] ; then
>> -        echo "${nvmet_hostkey}" > "${host_path}/dhchap_key"
>> -    fi
>> -    if [[ "${nvmet_ctrlkey}" ]] ; then
>> -        echo "${nvmet_ctrlkey}" > "${host_path}/dhchap_ctrl_key"
>> -    fi
>> -}
>> -
>> -_remove_nvmet_ns() {
>> -    local nvmet_subsystem="$1"
>> -    local nsid=$2
>> -    local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>> -    local nvmet_ns_path="${subsys_path}/namespaces/${nsid}"
>> -
>> -    echo 0 > "${nvmet_ns_path}/enable"
>> -    rmdir "${nvmet_ns_path}"
>> -}
>> -
>> -_remove_nvmet_subsystem() {
>> -    local nvmet_subsystem="$1"
>> -    local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>> -
>> -    _remove_nvmet_ns "${nvmet_subsystem}" "1"
>> -    rm -f "${subsys_path}"/allowed_hosts/*
>> -    rmdir "${subsys_path}"
>> -}
>> -
>> -_remove_nvmet_host() {
>> -    local nvmet_host="$1"
>> -    local host_path="${NVMET_CFS}/hosts/${nvmet_host}"
>> -
>> -    rmdir "${host_path}"
>> -}
>> -
>>  _create_nvmet_passthru() {
>>      local nvmet_subsystem="$1"
>>      local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>> @@ -765,34 +256,6 @@ _remove_nvmet_passhtru() {
>>      rmdir "${subsys_path}"
>>  }
>>  
>> -_add_nvmet_subsys_to_port() {
>> -    local port="$1"
>> -    local nvmet_subsystem="$2"
>> -
>> -    ln -s "${NVMET_CFS}/subsystems/${nvmet_subsystem}" \
>> -        "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
>> -}
>> -
>> -_remove_nvmet_subsystem_from_port() {
>> -    local port="$1"
>> -    local nvmet_subsystem="$2"
>> -
>> -    rm "${NVMET_CFS}/ports/${port}/subsystems/${nvmet_subsystem}"
>> -}
>> -
>> -_get_nvmet_ports() {
>> -    local nvmet_subsystem="$1"
>> -    local -n nvmet_ports="$2"
>> -    local cfs_path="${NVMET_CFS}/ports"
>> -    local sarg
>> -
>> -    sarg="s;^${cfs_path}/\([0-9]\+\)/subsystems/${nvmet_subsystem}$;\1;p"
>> -
>> -    for path in "${cfs_path}/"*"/subsystems/${nvmet_subsystem}"; do
>> -        nvmet_ports+=("$(echo "${path}" | sed -n -s "${sarg}")")
>> -    done
>> -}
>> -
>>  _set_nvmet_hostkey() {
>>      local nvmet_hostnqn="$1"
>>      local nvmet_hostkey="$2"
>> @@ -829,20 +292,6 @@ _set_nvmet_dhgroup() {
>>           "${cfs_path}/dhchap_dhgroup"
>>  }
>>  
>> -_find_nvme_dev() {
>> -    local subsys=$1
>> -    local subsysnqn
>> -    local dev
>> -    for dev in /sys/class/nvme/nvme*; do
>> -        [ -e "$dev" ] || continue
>> -        dev="$(basename "$dev")"
>> -        subsysnqn="$(cat "/sys/class/nvme/${dev}/subsysnqn" 2>/dev/null)"
>> -        if [[ "$subsysnqn" == "$subsys" ]]; then
>> -            echo "$dev"
>> -        fi
>> -    done
>> -}
>> -
>>  _find_nvme_ns() {
>>      local subsys_uuid=$1
>>      local uuid
>> @@ -924,44 +373,6 @@ _nvmet_target_setup() {
>>              "${hostkey}" "${ctrlkey}"
>>  }
>>  
>> -_nvmet_target_cleanup() {
>> -    local ports
>> -    local port
>> -    local blkdev
>> -    local subsysnqn="${def_subsysnqn}"
>> -    local blkdev_type=""
>> -
>> -    while [[ $# -gt 0 ]]; do
>> -        case $1 in
>> -            --blkdev)
>> -                blkdev_type="$2"
>> -                shift 2
>> -                ;;
>> -            --subsysnqn)
>> -                subsysnqn="$2"
>> -                shift 2
>> -                ;;
>> -            *)
>> -                echo "WARNING: unknown argument: $1"
>> -                shift
>> -                ;;
>> -        esac
>> -    done
>> -
>> -    _get_nvmet_ports "${subsysnqn}" ports
>> -
>> -    for port in "${ports[@]}"; do
>> -        _remove_nvmet_subsystem_from_port "${port}" "${subsysnqn}"
>> -        _remove_nvmet_port "${port}"
>> -    done
>> -    _remove_nvmet_subsystem "${subsysnqn}"
>> -    _remove_nvmet_host "${def_hostnqn}"
>> -
>> -    if [[ "${blkdev_type}" == "device" ]]; then
>> -        _cleanup_blkdev
>> -    fi
>> -}
>> -
>>  _nvmet_passthru_target_setup() {
>>      local subsysnqn="$def_subsysnqn"
>>      local port
>> --
>> 2.45.1

Re: [PATCH blktests v2 1/2] nvme: move helper functions to common/nvme

[Ofir Gal]

On 7/16/24 12:23, Ofir Gal wrote:
>>> -_nvme_disconnect_ctrl() {
>>> -    local ctrl="$1"
>>> -
>>> -    nvme disconnect --device "${ctrl}"
>>> -}
>>> -
>>>   _nvme_disconnect_subsys() {
>>>      local subsysnqn="$def_subsysnqn"
>>>  
>>> @@ -465,141 +206,6 @@ _nvme_disconnect_subsys() {
>>>          grep -o "disconnected.*"
>>>   }
>>>  
>>> -_nvme_connect_subsys() {
>>> -    local subsysnqn="$def_subsysnqn"
>>> -    local hostnqn="$def_hostnqn"
>>
>> It looks weird that _nvme_connect_subsys() is moved to common/nvme, but
>> _nvme_disconnect_subsys() stays in tests/nvme/rc. I think it's the better to
>> move _nvme_disconnect_subsys() also. Currently, md/001 does not use
>> _nvme_disconnect_subsys(). Isn't it the better to call it in
>> cleanup_nvme_over_tcp()?
>>
> I agree, I would move _nvme_disconnect_subsys() in v3.
> cleanup_nvme_over_tcp() use _nvme_disconnect_ctrl() to disconnect the
> controller by the controller name rather the subsys name. I can change
> it to _nvme_disconnect_subsys() if it's more appropriate.
>
I changed it to _nvme_disconnect_subsys() after using _find_nvme_ns() as
Daniel suggested, will be in v3.

[PATCH blktests v2 2/2] md: add regression test for "md/md-bitmap: fix writing non bitmap pages"

[Ofir Gal]

A bug in md-bitmap has been discovered by setting up a md raid on top of
nvme-tcp devices that has optimal io size larger than the allocated
bitmap.

The following test reproduce the bug by setting up a md-raid on top of
nvme-tcp device over ram device that sets the optimal io size by using
dm-stripe.

Signed-off-by: Ofir Gal <ofir.gal@volumez.com>
---
 common/brd       | 28 +++++++++++++++
 tests/md/001     | 88 ++++++++++++++++++++++++++++++++++++++++++++++++
 tests/md/001.out |  2 ++
 tests/md/rc      | 12 +++++++
 4 files changed, 130 insertions(+)
 create mode 100644 common/brd
 create mode 100755 tests/md/001
 create mode 100644 tests/md/001.out
 create mode 100644 tests/md/rc

diff --git a/common/brd b/common/brd
new file mode 100644
index 0000000..31e964f
--- /dev/null
+++ b/common/brd
@@ -0,0 +1,28 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2024 Ofir Gal
+#
+# brd helper functions
+
+. common/shellcheck
+
+_have_brd() {
+	_have_module brd
+}
+
+_init_brd() {
+	# _have_brd loads brd, we need to wait a bit for brd to be not in use in
+	# order to reload it
+	sleep 0.2
+
+	if ! modprobe -r brd || ! modprobe brd "$@" ; then
+		echo "failed to reload brd with args: $*"
+		return 1
+	fi
+
+	return 0
+}
+
+_cleanup_brd() {
+	modprobe -r brd
+}
diff --git a/tests/md/001 b/tests/md/001
new file mode 100755
index 0000000..fc23ff7
--- /dev/null
+++ b/tests/md/001
@@ -0,0 +1,88 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2024 Ofir Gal
+#
+# The bug is "visible" only when the underlying device of the raid is a network
+# block device that utilize MSG_SPLICE_PAGES. nvme-tcp is used as the network device.
+#
+# Regression test for patch "md/md-bitmap: fix writing non bitmap pages" and
+# for patch "nvme-tcp: use sendpages_ok() instead of sendpage_ok()"
+
+. tests/md/rc
+. common/brd
+. common/nvme
+
+DESCRIPTION="Raid with bitmap on tcp nvmet with opt-io-size over bitmap size"
+QUICK=1
+
+#restrict test to nvme-tcp only
+nvme_trtype=tcp
+nvmet_blkdev_type="device"
+
+requires() {
+	# Require dm-stripe
+	_have_program dmsetup
+	_have_driver dm-mod
+
+	_require_nvme_trtype tcp
+	_have_brd
+}
+
+# Sets up a brd device of 1G with optimal-io-size of 256K
+setup_underlying_device() {
+	if ! _init_brd rd_size=1048576 rd_nr=1; then
+		return 1
+	fi
+
+	dmsetup create ram0_big_optio --table \
+		"0 $(blockdev --getsz /dev/ram0) striped 1 512 /dev/ram0 0"
+}
+
+cleanup_underlying_device() {
+	dmsetup remove ram0_big_optio
+	_cleanup_brd
+}
+
+# Sets up a local host nvme over tcp
+setup_nvme_over_tcp() {
+	_setup_nvmet
+
+	local port
+	port="$(_create_nvmet_port "${nvme_trtype}")"
+
+	_create_nvmet_subsystem "blktests-subsystem-0" "/dev/mapper/ram0_big_optio"
+	_add_nvmet_subsys_to_port "${port}" "blktests-subsystem-0"
+
+	_create_nvmet_host "blktests-subsystem-0" "${def_hostnqn}"
+
+	_nvme_connect_subsys --subsysnqn "blktests-subsystem-0"
+
+	local nvmedev
+	nvmedev=$(_find_nvme_dev "blktests-subsystem-0")
+	echo "${nvmedev}"
+}
+
+cleanup_nvme_over_tcp() {
+	local nvmedev=$1
+	_nvme_disconnect_ctrl "${nvmedev}"
+	_nvmet_target_cleanup --subsysnqn "blktests-subsystem-0"
+}
+
+test() {
+	echo "Running ${TEST_NAME}"
+
+	setup_underlying_device
+	local nvmedev
+	nvmedev=$(setup_nvme_over_tcp)
+
+	# Hangs here without the fix
+	mdadm --quiet --create /dev/md/blktests_md --level=1 --bitmap=internal \
+		--bitmap-chunk=1024K --assume-clean --run --raid-devices=2 \
+		/dev/"${nvmedev}"n1 missing
+
+	mdadm --quiet --stop /dev/md/blktests_md
+	cleanup_nvme_over_tcp "${nvmedev}"
+	cleanup_underlying_device
+
+	echo "Test complete"
+}
diff --git a/tests/md/001.out b/tests/md/001.out
new file mode 100644
index 0000000..edd2ced
--- /dev/null
+++ b/tests/md/001.out
@@ -0,0 +1,2 @@
+Running md/001
+Test complete
diff --git a/tests/md/rc b/tests/md/rc
new file mode 100644
index 0000000..d492579
--- /dev/null
+++ b/tests/md/rc
@@ -0,0 +1,12 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-3.0+
+# Copyright (C) 2024 Ofir Gal
+#
+# Tests for md raid
+
+. common/rc
+
+group_requires() {
+	_have_root
+	_have_program mdadm
+}
-- 
2.45.1

Re: [PATCH blktests v2 2/2] md: add regression test for "md/md-bitmap: fix writing non bitmap pages"

[Daniel Wagner]

On Mon, Jun 24, 2024 at 01:46:18PM GMT, Ofir Gal wrote:
> +#restrict test to nvme-tcp only
> +nvme_trtype=tcp
> +nvmet_blkdev_type="device"
> +
> +requires() {
> +	# Require dm-stripe
> +	_have_program dmsetup
> +	_have_driver dm-mod
> +
> +	_require_nvme_trtype tcp
> +	_have_brd
> +}
> +
> +# Sets up a brd device of 1G with optimal-io-size of 256K
> +setup_underlying_device() {
> +	if ! _init_brd rd_size=1048576 rd_nr=1; then
> +		return 1
> +	fi
> +
> +	dmsetup create ram0_big_optio --table \
> +		"0 $(blockdev --getsz /dev/ram0) striped 1 512 /dev/ram0 0"
> +}
> +
> +cleanup_underlying_device() {
> +	dmsetup remove ram0_big_optio
> +	_cleanup_brd
> +}
> +
> +# Sets up a local host nvme over tcp
> +setup_nvme_over_tcp() {
> +	_setup_nvmet
> +
> +	local port
> +	port="$(_create_nvmet_port "${nvme_trtype}")"
> +
> +	_create_nvmet_subsystem "blktests-subsystem-0" "/dev/mapper/ram0_big_optio"
> +	_add_nvmet_subsys_to_port "${port}" "blktests-subsystem-0"

Use the defaults from blktests, e.g. ${def_subsysnqn}"

> +
> +	_create_nvmet_host "blktests-subsystem-0" "${def_hostnqn}"
> +
> +	_nvme_connect_subsys --subsysnqn "blktests-subsystem-0"
> +
> +	local nvmedev
> +	nvmedev=$(_find_nvme_dev "blktests-subsystem-0")

here too

> +	echo "${nvmedev}"
> +}
> +
> +cleanup_nvme_over_tcp() {
> +	local nvmedev=$1
> +	_nvme_disconnect_ctrl "${nvmedev}"
> +	_nvmet_target_cleanup --subsysnqn "blktests-subsystem-0"

same here

> +}
> +
> +test() {
> +	echo "Running ${TEST_NAME}"
> +
> +	setup_underlying_device
> +	local nvmedev
> +	nvmedev=$(setup_nvme_over_tcp)
> +
> +	# Hangs here without the fix
> +	mdadm --quiet --create /dev/md/blktests_md --level=1 --bitmap=internal \
> +		--bitmap-chunk=1024K --assume-clean --run --raid-devices=2 \
> +		/dev/"${nvmedev}"n1 missing

Instead hard coding the namespace ID, this should be made a bit more
robust by looking it up with _find_nvme_ns.

Re: [PATCH blktests v2 2/2] md: add regression test for "md/md-bitmap: fix writing non bitmap pages"

[Ofir Gal]

On 6/25/24 18:01, Daniel Wagner wrote:
> On Mon, Jun 24, 2024 at 01:46:18PM GMT, Ofir Gal wrote:
>> +#restrict test to nvme-tcp only
>> +nvme_trtype=tcp
>> +nvmet_blkdev_type="device"
>> +
>> +requires() {
>> +	# Require dm-stripe
>> +	_have_program dmsetup
>> +	_have_driver dm-mod
>> +
>> +	_require_nvme_trtype tcp
>> +	_have_brd
>> +}
>> +
>> +# Sets up a brd device of 1G with optimal-io-size of 256K
>> +setup_underlying_device() {
>> +	if ! _init_brd rd_size=1048576 rd_nr=1; then
>> +		return 1
>> +	fi
>> +
>> +	dmsetup create ram0_big_optio --table \
>> +		"0 $(blockdev --getsz /dev/ram0) striped 1 512 /dev/ram0 0"
>> +}
>> +
>> +cleanup_underlying_device() {
>> +	dmsetup remove ram0_big_optio
>> +	_cleanup_brd
>> +}
>> +
>> +# Sets up a local host nvme over tcp
>> +setup_nvme_over_tcp() {
>> +	_setup_nvmet
>> +
>> +	local port
>> +	port="$(_create_nvmet_port "${nvme_trtype}")"
>> +
>> +	_create_nvmet_subsystem "blktests-subsystem-0" "/dev/mapper/ram0_big_optio"
>> +	_add_nvmet_subsys_to_port "${port}" "blktests-subsystem-0"
> Use the defaults from blktests, e.g. ${def_subsysnqn}"
>
>> +
>> +	_create_nvmet_host "blktests-subsystem-0" "${def_hostnqn}"
>> +
>> +	_nvme_connect_subsys --subsysnqn "blktests-subsystem-0"
>> +
>> +	local nvmedev
>> +	nvmedev=$(_find_nvme_dev "blktests-subsystem-0")
> here too
>
>> +	echo "${nvmedev}"
>> +}
>> +
>> +cleanup_nvme_over_tcp() {
>> +	local nvmedev=$1
>> +	_nvme_disconnect_ctrl "${nvmedev}"
>> +	_nvmet_target_cleanup --subsysnqn "blktests-subsystem-0"
> same here
>
>> +}
>> +
>> +test() {
>> +	echo "Running ${TEST_NAME}"
>> +
>> +	setup_underlying_device
>> +	local nvmedev
>> +	nvmedev=$(setup_nvme_over_tcp)
>> +
>> +	# Hangs here without the fix
>> +	mdadm --quiet --create /dev/md/blktests_md --level=1 --bitmap=internal \
>> +		--bitmap-chunk=1024K --assume-clean --run --raid-devices=2 \
>> +		/dev/"${nvmedev}"n1 missing
> Instead hard coding the namespace ID, this should be made a bit more
> robust by looking it up with _find_nvme_ns.
Thanks, applied comments to v3.