[PATCH 0/3] blktests: ANA and FC-Loop support

[PATCH 0/3] blktests: ANA and FC-Loop support

[Hannes Reinecke]

Hi all,

here are two new tests for ANA support and FC-Loop; the latter more
as a showcase how fcloop is supposed to be setup.
But also serves as a nice reproducer for fcloop failing to be removed
properly.

As usual, comments and reviews are welcome.

Hannes Reinecke (3):
  nvme: enable ANA support
  nvme: add test for ANA state transition
  nvme: Add test for fcloop

 tests/nvme/018     | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/018.out |  18 +++++++
 tests/nvme/029     | 123 ++++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/029.out |   2 +
 tests/nvme/rc      | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++--
 5 files changed, 422 insertions(+), 3 deletions(-)
 create mode 100755 tests/nvme/018
 create mode 100644 tests/nvme/018.out
 create mode 100755 tests/nvme/029
 create mode 100644 tests/nvme/029.out

-- 
2.13.7

[PATCH 1/3] nvme: enable ANA support

[Hannes Reinecke]

Update functions to support ANA.

Signed-off-by: Hannes Reinecke <hare at suse.com>
---
 tests/nvme/rc | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 86 insertions(+), 3 deletions(-)

diff --git a/tests/nvme/rc b/tests/nvme/rc
index eff1dd9..aaf438b 100644
--- a/tests/nvme/rc
+++ b/tests/nvme/rc
@@ -14,7 +14,7 @@ group_device_requires() {
 	_test_dev_is_nvme
 }
 
-NVMET_CFS="/sys/kernel/config/nvmet/"
+NVMET_CFS="/sys/kernel/config/nvmet"
 
 _test_dev_is_nvme() {
 	if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then
@@ -36,6 +36,7 @@ _create_nvmet_port() {
 
 	mkdir "${NVMET_CFS}/ports/${port}"
 	echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
+	echo "${port}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
 
 	echo "${port}"
 }
@@ -45,6 +46,62 @@ _remove_nvmet_port() {
 	rmdir "${NVMET_CFS}/ports/${port}"
 }
 
+_create_nvmet_anagroup() {
+	local port="$1"
+	local port_cfs="${NVMET_CFS}/ports/${port}"
+	local grpid
+
+	for ((grpid = 1; ; grpid++)); do
+		if [[ ! -e "${port_cfs}/ana_groups/${grpid}" ]]; then
+			break
+		fi
+	done
+
+	mkdir "${port_cfs}/ana_groups/${grpid}"
+
+	echo "${grpid}"
+}
+
+_remove_nvmet_anagroup() {
+	local port="$1"
+	local grpid="$2"
+	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
+
+	rmdir "${ana_cfs}"
+}
+
+_set_nvmet_anagroup_state() {
+	local port="$1"
+	local grpid="$2"
+	local state="$3"
+	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
+
+	echo "${state}" > "${ana_cfs}/ana_state"
+}
+
+_display_ana_state() {
+	local nvmet_subsystem="$1"
+	local grpid state cntlid
+
+	for nvme in /sys/class/nvme/* ; do
+		[ -f ${nvme}/subsysnqn ] || continue
+		subsys="$(cat "${nvme}/subsysnqn")"
+		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
+			continue
+		fi
+		cntlid="$(cat "${nvme}/cntlid")"
+		for c in ${nvme}/nvme* ; do
+			if [ ! -d ${c} ] ; then
+				echo "${cntlid}: ANA disabled"
+				continue
+			fi
+			grpid="$(cat "${c}/ana_grpid")"
+			state="$(cat "${c}/ana_state")"
+			echo "${cntlid}: grpid ${grpid} state ${state}"
+		done
+	done
+}
+
 _create_nvmet_ns() {
 	local nvmet_subsystem="$1"
 	local nsid="$2"
@@ -52,14 +109,22 @@ _create_nvmet_ns() {
 	local uuid="00000000-0000-0000-0000-000000000000"
 	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
 	local ns_path="${subsys_path}/namespaces/${nsid}"
+	local ana_grpid
 
-	if [[ $# -eq 4 ]]; then
+	if [[ $# -ge 4 ]]; then
 		uuid="$4"
 	fi
 
+	if [[ $# -eq 5 ]]; then
+		ana_grpid="$5"
+	fi
+
 	mkdir "${ns_path}"
 	printf "%s" "${blkdev}" > "${ns_path}/device_path"
 	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
+	if [ -n "${ana_grpid}" ] ; then
+		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
+	fi
 	printf 1 > "${ns_path}/enable"
 }
 
@@ -67,11 +132,12 @@ _create_nvmet_subsystem() {
 	local nvmet_subsystem="$1"
 	local blkdev="$2"
 	local uuid=$3
+	local ana_grpid=$4
 	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
 
 	mkdir -p "${cfs_path}"
 	echo 1 > "${cfs_path}/attr_allow_any_host"
-	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
+	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" "${ana_grpid}"
 }
 
 _remove_nvmet_ns() {
@@ -115,6 +181,23 @@ _find_nvme_loop_dev() {
 		transport="$(cat "/sys/class/nvme/${dev}/transport")"
 		if [[ "$transport" == "loop" ]]; then
 			echo "$dev"
+			break
+		fi
+	done
+}
+
+_find_nvme_ns() {
+	local uuid=$1
+	local dev
+	local hidden
+
+	for dev in /sys/block/nvme*; do
+		hidden="$(cat "${dev}/hidden")"
+		[[ "$hidden" -eq 1 ]] && continue
+		u="$(cat "${dev}/uuid")"
+		if [[ "$uuid" = "$u" ]] ; then
+			echo "$(basename "${dev}")"
+			break
 		fi
 	done
 }
-- 
2.13.7

[PATCH 1/3] nvme: enable ANA support

[Omar Sandoval]

On Mon, Oct 15, 2018@09:01:28AM +0200, Hannes Reinecke wrote:
> Update functions to support ANA.
> 
> Signed-off-by: Hannes Reinecke <hare at suse.com>
> ---
>  tests/nvme/rc | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
>  1 file changed, 86 insertions(+), 3 deletions(-)
> 
> diff --git a/tests/nvme/rc b/tests/nvme/rc
> index eff1dd9..aaf438b 100644
> --- a/tests/nvme/rc
> +++ b/tests/nvme/rc
> @@ -14,7 +14,7 @@ group_device_requires() {
>  	_test_dev_is_nvme
>  }
>  
> -NVMET_CFS="/sys/kernel/config/nvmet/"
> +NVMET_CFS="/sys/kernel/config/nvmet"
>  
>  _test_dev_is_nvme() {
>  	if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then
> @@ -36,6 +36,7 @@ _create_nvmet_port() {
>  
>  	mkdir "${NVMET_CFS}/ports/${port}"
>  	echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
> +	echo "${port}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
>  
>  	echo "${port}"
>  }
> @@ -45,6 +46,62 @@ _remove_nvmet_port() {
>  	rmdir "${NVMET_CFS}/ports/${port}"
>  }
>  
> +_create_nvmet_anagroup() {
> +	local port="$1"
> +	local port_cfs="${NVMET_CFS}/ports/${port}"
> +	local grpid
> +
> +	for ((grpid = 1; ; grpid++)); do
> +		if [[ ! -e "${port_cfs}/ana_groups/${grpid}" ]]; then
> +			break
> +		fi
> +	done
> +
> +	mkdir "${port_cfs}/ana_groups/${grpid}"
> +
> +	echo "${grpid}"
> +}
> +
> +_remove_nvmet_anagroup() {
> +	local port="$1"
> +	local grpid="$2"
> +	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
> +
> +	rmdir "${ana_cfs}"
> +}
> +
> +_set_nvmet_anagroup_state() {
> +	local port="$1"
> +	local grpid="$2"
> +	local state="$3"
> +	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
> +
> +	echo "${state}" > "${ana_cfs}/ana_state"
> +}
> +
> +_display_ana_state() {
> +	local nvmet_subsystem="$1"
> +	local grpid state cntlid
> +
> +	for nvme in /sys/class/nvme/* ; do
> +		[ -f ${nvme}/subsysnqn ] || continue
> +		subsys="$(cat "${nvme}/subsysnqn")"
> +		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
> +			continue
> +		fi
> +		cntlid="$(cat "${nvme}/cntlid")"
> +		for c in ${nvme}/nvme* ; do
> +			if [ ! -d ${c} ] ; then
> +				echo "${cntlid}: ANA disabled"
> +				continue
> +			fi
> +			grpid="$(cat "${c}/ana_grpid")"
> +			state="$(cat "${c}/ana_state")"
> +			echo "${cntlid}: grpid ${grpid} state ${state}"
> +		done
> +	done
> +}
> +
>  _create_nvmet_ns() {
>  	local nvmet_subsystem="$1"
>  	local nsid="$2"
> @@ -52,14 +109,22 @@ _create_nvmet_ns() {
>  	local uuid="00000000-0000-0000-0000-000000000000"
>  	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>  	local ns_path="${subsys_path}/namespaces/${nsid}"
> +	local ana_grpid
>  
> -	if [[ $# -eq 4 ]]; then
> +	if [[ $# -ge 4 ]]; then
>  		uuid="$4"
>  	fi
>  
> +	if [[ $# -eq 5 ]]; then
> +		ana_grpid="$5"
> +	fi
> +
>  	mkdir "${ns_path}"
>  	printf "%s" "${blkdev}" > "${ns_path}/device_path"
>  	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
> +	if [ -n "${ana_grpid}" ] ; then
> +		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
> +	fi
>  	printf 1 > "${ns_path}/enable"
>  }
>  
> @@ -67,11 +132,12 @@ _create_nvmet_subsystem() {
>  	local nvmet_subsystem="$1"
>  	local blkdev="$2"
>  	local uuid=$3
> +	local ana_grpid=$4
>  	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>  
>  	mkdir -p "${cfs_path}"
>  	echo 1 > "${cfs_path}/attr_allow_any_host"
> -	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
> +	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" "${ana_grpid}"
>  }
>  
>  _remove_nvmet_ns() {
> @@ -115,6 +181,23 @@ _find_nvme_loop_dev() {
>  		transport="$(cat "/sys/class/nvme/${dev}/transport")"
>  		if [[ "$transport" == "loop" ]]; then
>  			echo "$dev"
> +			break
> +		fi
> +	done
> +}
> +
> +_find_nvme_ns() {
> +	local uuid=$1
> +	local dev
> +	local hidden
> +
> +	for dev in /sys/block/nvme*; do
> +		hidden="$(cat "${dev}/hidden")"
> +		[[ "$hidden" -eq 1 ]] && continue
> +		u="$(cat "${dev}/uuid")"
> +		if [[ "$uuid" = "$u" ]] ; then
> +			echo "$(basename "${dev}")"
> +			break

This fails if I have another NVMe device:

cat: /sys/block/nvme0n1/uuid: No such file or directory

And there are a bunch of shellcheck errors. The following fixes all of
that:

diff --git a/tests/nvme/rc b/tests/nvme/rc
index 3b3d852..0c8fc62 100644
--- a/tests/nvme/rc
+++ b/tests/nvme/rc
@@ -84,14 +84,14 @@ _display_ana_state() {
 	local grpid state cntlid
 
 	for nvme in /sys/class/nvme/* ; do
-		[ -f ${nvme}/subsysnqn ] || continue
+		[[ -f ${nvme}/subsysnqn ]] || continue
 		subsys="$(cat "${nvme}/subsysnqn")"
-		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
+		if [[ "${subsys}" != "${nvmet_subsystem}" ]] ; then
 			continue
 		fi
 		cntlid="$(cat "${nvme}/cntlid")"
-		for c in ${nvme}/nvme* ; do
-			if [ ! -d ${c} ] ; then
+		for c in "${nvme}/nvme"* ; do
+			if [[ ! -d ${c} ]] ; then
 				echo "${cntlid}: ANA disabled"
 				continue
 			fi
@@ -122,7 +122,7 @@ _create_nvmet_ns() {
 	mkdir "${ns_path}"
 	printf "%s" "${blkdev}" > "${ns_path}/device_path"
 	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
-	if [ -n "${ana_grpid}" ] ; then
+	if [[ -n ${ana_grpid} ]] ; then
 		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
 	fi
 	printf 1 > "${ns_path}/enable"
@@ -194,9 +194,8 @@ _find_nvme_ns() {
 	for dev in /sys/block/nvme*; do
 		hidden="$(cat "${dev}/hidden")"
 		[[ "$hidden" -eq 1 ]] && continue
-		u="$(cat "${dev}/uuid")"
-		if [[ "$uuid" = "$u" ]] ; then
-			echo "$(basename "${dev}")"
+		if [[ -e "${dev}/uuid" && "$(cat "${dev}/uuid")" = "$uuid" ]] ; then
+			basename "${dev}"
 			break
 		fi
 	done

[PATCH 1/3] nvme: enable ANA support

[Hannes Reinecke]

On 10/15/18 11:13 PM, Omar Sandoval wrote:
> On Mon, Oct 15, 2018@09:01:28AM +0200, Hannes Reinecke wrote:
>> Update functions to support ANA.
>>
>> Signed-off-by: Hannes Reinecke <hare at suse.com>
>> ---
>>   tests/nvme/rc | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
>>   1 file changed, 86 insertions(+), 3 deletions(-)
>>
>> diff --git a/tests/nvme/rc b/tests/nvme/rc
>> index eff1dd9..aaf438b 100644
>> --- a/tests/nvme/rc
>> +++ b/tests/nvme/rc
>> @@ -14,7 +14,7 @@ group_device_requires() {
>>   	_test_dev_is_nvme
>>   }
>>   
>> -NVMET_CFS="/sys/kernel/config/nvmet/"
>> +NVMET_CFS="/sys/kernel/config/nvmet"
>>   
>>   _test_dev_is_nvme() {
>>   	if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then
>> @@ -36,6 +36,7 @@ _create_nvmet_port() {
>>   
>>   	mkdir "${NVMET_CFS}/ports/${port}"
>>   	echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
>> +	echo "${port}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
>>   
>>   	echo "${port}"
>>   }
>> @@ -45,6 +46,62 @@ _remove_nvmet_port() {
>>   	rmdir "${NVMET_CFS}/ports/${port}"
>>   }
>>   
>> +_create_nvmet_anagroup() {
>> +	local port="$1"
>> +	local port_cfs="${NVMET_CFS}/ports/${port}"
>> +	local grpid
>> +
>> +	for ((grpid = 1; ; grpid++)); do
>> +		if [[ ! -e "${port_cfs}/ana_groups/${grpid}" ]]; then
>> +			break
>> +		fi
>> +	done
>> +
>> +	mkdir "${port_cfs}/ana_groups/${grpid}"
>> +
>> +	echo "${grpid}"
>> +}
>> +
>> +_remove_nvmet_anagroup() {
>> +	local port="$1"
>> +	local grpid="$2"
>> +	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
>> +
>> +	rmdir "${ana_cfs}"
>> +}
>> +
>> +_set_nvmet_anagroup_state() {
>> +	local port="$1"
>> +	local grpid="$2"
>> +	local state="$3"
>> +	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
>> +
>> +	echo "${state}" > "${ana_cfs}/ana_state"
>> +}
>> +
>> +_display_ana_state() {
>> +	local nvmet_subsystem="$1"
>> +	local grpid state cntlid
>> +
>> +	for nvme in /sys/class/nvme/* ; do
>> +		[ -f ${nvme}/subsysnqn ] || continue
>> +		subsys="$(cat "${nvme}/subsysnqn")"
>> +		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
>> +			continue
>> +		fi
>> +		cntlid="$(cat "${nvme}/cntlid")"
>> +		for c in ${nvme}/nvme* ; do
>> +			if [ ! -d ${c} ] ; then
>> +				echo "${cntlid}: ANA disabled"
>> +				continue
>> +			fi
>> +			grpid="$(cat "${c}/ana_grpid")"
>> +			state="$(cat "${c}/ana_state")"
>> +			echo "${cntlid}: grpid ${grpid} state ${state}"
>> +		done
>> +	done
>> +}
>> +
>>   _create_nvmet_ns() {
>>   	local nvmet_subsystem="$1"
>>   	local nsid="$2"
>> @@ -52,14 +109,22 @@ _create_nvmet_ns() {
>>   	local uuid="00000000-0000-0000-0000-000000000000"
>>   	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>>   	local ns_path="${subsys_path}/namespaces/${nsid}"
>> +	local ana_grpid
>>   
>> -	if [[ $# -eq 4 ]]; then
>> +	if [[ $# -ge 4 ]]; then
>>   		uuid="$4"
>>   	fi
>>   
>> +	if [[ $# -eq 5 ]]; then
>> +		ana_grpid="$5"
>> +	fi
>> +
>>   	mkdir "${ns_path}"
>>   	printf "%s" "${blkdev}" > "${ns_path}/device_path"
>>   	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
>> +	if [ -n "${ana_grpid}" ] ; then
>> +		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
>> +	fi
>>   	printf 1 > "${ns_path}/enable"
>>   }
>>   
>> @@ -67,11 +132,12 @@ _create_nvmet_subsystem() {
>>   	local nvmet_subsystem="$1"
>>   	local blkdev="$2"
>>   	local uuid=$3
>> +	local ana_grpid=$4
>>   	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>>   
>>   	mkdir -p "${cfs_path}"
>>   	echo 1 > "${cfs_path}/attr_allow_any_host"
>> -	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
>> +	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" "${ana_grpid}"
>>   }
>>   
>>   _remove_nvmet_ns() {
>> @@ -115,6 +181,23 @@ _find_nvme_loop_dev() {
>>   		transport="$(cat "/sys/class/nvme/${dev}/transport")"
>>   		if [[ "$transport" == "loop" ]]; then
>>   			echo "$dev"
>> +			break
>> +		fi
>> +	done
>> +}
>> +
>> +_find_nvme_ns() {
>> +	local uuid=$1
>> +	local dev
>> +	local hidden
>> +
>> +	for dev in /sys/block/nvme*; do
>> +		hidden="$(cat "${dev}/hidden")"
>> +		[[ "$hidden" -eq 1 ]] && continue
>> +		u="$(cat "${dev}/uuid")"
>> +		if [[ "$uuid" = "$u" ]] ; then
>> +			echo "$(basename "${dev}")"
>> +			break
> 
> This fails if I have another NVMe device:
> 
> cat: /sys/block/nvme0n1/uuid: No such file or directory
> 
> And there are a bunch of shellcheck errors. The following fixes all of
> that:
> 
> diff --git a/tests/nvme/rc b/tests/nvme/rc
> index 3b3d852..0c8fc62 100644
> --- a/tests/nvme/rc
> +++ b/tests/nvme/rc
> @@ -84,14 +84,14 @@ _display_ana_state() {
>   	local grpid state cntlid
>   
>   	for nvme in /sys/class/nvme/* ; do
> -		[ -f ${nvme}/subsysnqn ] || continue
> +		[[ -f ${nvme}/subsysnqn ]] || continue
>   		subsys="$(cat "${nvme}/subsysnqn")"
> -		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
> +		if [[ "${subsys}" != "${nvmet_subsystem}" ]] ; then
>   			continue
>   		fi
>   		cntlid="$(cat "${nvme}/cntlid")"
> -		for c in ${nvme}/nvme* ; do
> -			if [ ! -d ${c} ] ; then
> +		for c in "${nvme}/nvme"* ; do
> +			if [[ ! -d ${c} ]] ; then
>   				echo "${cntlid}: ANA disabled"
>   				continue
>   			fi
> @@ -122,7 +122,7 @@ _create_nvmet_ns() {
>   	mkdir "${ns_path}"
>   	printf "%s" "${blkdev}" > "${ns_path}/device_path"
>   	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
> -	if [ -n "${ana_grpid}" ] ; then
> +	if [[ -n ${ana_grpid} ]] ; then
>   		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
>   	fi
>   	printf 1 > "${ns_path}/enable"
> @@ -194,9 +194,8 @@ _find_nvme_ns() {
>   	for dev in /sys/block/nvme*; do
>   		hidden="$(cat "${dev}/hidden")"
>   		[[ "$hidden" -eq 1 ]] && continue
> -		u="$(cat "${dev}/uuid")"
> -		if [[ "$uuid" = "$u" ]] ; then
> -			echo "$(basename "${dev}")"
> +		if [[ -e "${dev}/uuid" && "$(cat "${dev}/uuid")" = "$uuid" ]] ; then
> +			basename "${dev}"
>   			break
>   		fi
>   	done
> 
Thanks.

Should I resend my patchset?

Cheers,

Hannes
-- 
Dr. Hannes Reinecke		   Teamlead Storage & Networking
hare at suse.de			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: F. Imend?rffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton
HRB 21284 (AG N?rnberg)

[PATCH 1/3] nvme: enable ANA support

[Omar Sandoval]

On Tue, Oct 16, 2018@07:49:20AM +0200, Hannes Reinecke wrote:
> On 10/15/18 11:13 PM, Omar Sandoval wrote:
> > On Mon, Oct 15, 2018@09:01:28AM +0200, Hannes Reinecke wrote:
> > > Update functions to support ANA.
> > > 
> > > Signed-off-by: Hannes Reinecke <hare at suse.com>
> > > ---
> > >   tests/nvme/rc | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
> > >   1 file changed, 86 insertions(+), 3 deletions(-)
> > > 
> > > diff --git a/tests/nvme/rc b/tests/nvme/rc
> > > index eff1dd9..aaf438b 100644
> > > --- a/tests/nvme/rc
> > > +++ b/tests/nvme/rc
> > > @@ -14,7 +14,7 @@ group_device_requires() {
> > >   	_test_dev_is_nvme
> > >   }
> > > -NVMET_CFS="/sys/kernel/config/nvmet/"
> > > +NVMET_CFS="/sys/kernel/config/nvmet"
> > >   _test_dev_is_nvme() {
> > >   	if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then
> > > @@ -36,6 +36,7 @@ _create_nvmet_port() {
> > >   	mkdir "${NVMET_CFS}/ports/${port}"
> > >   	echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
> > > +	echo "${port}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
> > >   	echo "${port}"
> > >   }
> > > @@ -45,6 +46,62 @@ _remove_nvmet_port() {
> > >   	rmdir "${NVMET_CFS}/ports/${port}"
> > >   }
> > > +_create_nvmet_anagroup() {
> > > +	local port="$1"
> > > +	local port_cfs="${NVMET_CFS}/ports/${port}"
> > > +	local grpid
> > > +
> > > +	for ((grpid = 1; ; grpid++)); do
> > > +		if [[ ! -e "${port_cfs}/ana_groups/${grpid}" ]]; then
> > > +			break
> > > +		fi
> > > +	done
> > > +
> > > +	mkdir "${port_cfs}/ana_groups/${grpid}"
> > > +
> > > +	echo "${grpid}"
> > > +}
> > > +
> > > +_remove_nvmet_anagroup() {
> > > +	local port="$1"
> > > +	local grpid="$2"
> > > +	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
> > > +
> > > +	rmdir "${ana_cfs}"
> > > +}
> > > +
> > > +_set_nvmet_anagroup_state() {
> > > +	local port="$1"
> > > +	local grpid="$2"
> > > +	local state="$3"
> > > +	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
> > > +
> > > +	echo "${state}" > "${ana_cfs}/ana_state"
> > > +}
> > > +
> > > +_display_ana_state() {
> > > +	local nvmet_subsystem="$1"
> > > +	local grpid state cntlid
> > > +
> > > +	for nvme in /sys/class/nvme/* ; do
> > > +		[ -f ${nvme}/subsysnqn ] || continue
> > > +		subsys="$(cat "${nvme}/subsysnqn")"
> > > +		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
> > > +			continue
> > > +		fi
> > > +		cntlid="$(cat "${nvme}/cntlid")"
> > > +		for c in ${nvme}/nvme* ; do
> > > +			if [ ! -d ${c} ] ; then
> > > +				echo "${cntlid}: ANA disabled"
> > > +				continue
> > > +			fi
> > > +			grpid="$(cat "${c}/ana_grpid")"
> > > +			state="$(cat "${c}/ana_state")"
> > > +			echo "${cntlid}: grpid ${grpid} state ${state}"
> > > +		done
> > > +	done
> > > +}
> > > +
> > >   _create_nvmet_ns() {
> > >   	local nvmet_subsystem="$1"
> > >   	local nsid="$2"
> > > @@ -52,14 +109,22 @@ _create_nvmet_ns() {
> > >   	local uuid="00000000-0000-0000-0000-000000000000"
> > >   	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> > >   	local ns_path="${subsys_path}/namespaces/${nsid}"
> > > +	local ana_grpid
> > > -	if [[ $# -eq 4 ]]; then
> > > +	if [[ $# -ge 4 ]]; then
> > >   		uuid="$4"
> > >   	fi
> > > +	if [[ $# -eq 5 ]]; then
> > > +		ana_grpid="$5"
> > > +	fi
> > > +
> > >   	mkdir "${ns_path}"
> > >   	printf "%s" "${blkdev}" > "${ns_path}/device_path"
> > >   	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
> > > +	if [ -n "${ana_grpid}" ] ; then
> > > +		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
> > > +	fi
> > >   	printf 1 > "${ns_path}/enable"
> > >   }
> > > @@ -67,11 +132,12 @@ _create_nvmet_subsystem() {
> > >   	local nvmet_subsystem="$1"
> > >   	local blkdev="$2"
> > >   	local uuid=$3
> > > +	local ana_grpid=$4
> > >   	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> > >   	mkdir -p "${cfs_path}"
> > >   	echo 1 > "${cfs_path}/attr_allow_any_host"
> > > -	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
> > > +	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" "${ana_grpid}"
> > >   }
> > >   _remove_nvmet_ns() {
> > > @@ -115,6 +181,23 @@ _find_nvme_loop_dev() {
> > >   		transport="$(cat "/sys/class/nvme/${dev}/transport")"
> > >   		if [[ "$transport" == "loop" ]]; then
> > >   			echo "$dev"
> > > +			break
> > > +		fi
> > > +	done
> > > +}
> > > +
> > > +_find_nvme_ns() {
> > > +	local uuid=$1
> > > +	local dev
> > > +	local hidden
> > > +
> > > +	for dev in /sys/block/nvme*; do
> > > +		hidden="$(cat "${dev}/hidden")"
> > > +		[[ "$hidden" -eq 1 ]] && continue
> > > +		u="$(cat "${dev}/uuid")"
> > > +		if [[ "$uuid" = "$u" ]] ; then
> > > +			echo "$(basename "${dev}")"
> > > +			break
> > 
> > This fails if I have another NVMe device:
> > 
> > cat: /sys/block/nvme0n1/uuid: No such file or directory
> > 
> > And there are a bunch of shellcheck errors. The following fixes all of
> > that:
> > 
> > diff --git a/tests/nvme/rc b/tests/nvme/rc
> > index 3b3d852..0c8fc62 100644
> > --- a/tests/nvme/rc
> > +++ b/tests/nvme/rc
> > @@ -84,14 +84,14 @@ _display_ana_state() {
> >   	local grpid state cntlid
> >   	for nvme in /sys/class/nvme/* ; do
> > -		[ -f ${nvme}/subsysnqn ] || continue
> > +		[[ -f ${nvme}/subsysnqn ]] || continue
> >   		subsys="$(cat "${nvme}/subsysnqn")"
> > -		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
> > +		if [[ "${subsys}" != "${nvmet_subsystem}" ]] ; then
> >   			continue
> >   		fi
> >   		cntlid="$(cat "${nvme}/cntlid")"
> > -		for c in ${nvme}/nvme* ; do
> > -			if [ ! -d ${c} ] ; then
> > +		for c in "${nvme}/nvme"* ; do
> > +			if [[ ! -d ${c} ]] ; then
> >   				echo "${cntlid}: ANA disabled"
> >   				continue
> >   			fi
> > @@ -122,7 +122,7 @@ _create_nvmet_ns() {
> >   	mkdir "${ns_path}"
> >   	printf "%s" "${blkdev}" > "${ns_path}/device_path"
> >   	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
> > -	if [ -n "${ana_grpid}" ] ; then
> > +	if [[ -n ${ana_grpid} ]] ; then
> >   		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
> >   	fi
> >   	printf 1 > "${ns_path}/enable"
> > @@ -194,9 +194,8 @@ _find_nvme_ns() {
> >   	for dev in /sys/block/nvme*; do
> >   		hidden="$(cat "${dev}/hidden")"
> >   		[[ "$hidden" -eq 1 ]] && continue
> > -		u="$(cat "${dev}/uuid")"
> > -		if [[ "$uuid" = "$u" ]] ; then
> > -			echo "$(basename "${dev}")"
> > +		if [[ -e "${dev}/uuid" && "$(cat "${dev}/uuid")" = "$uuid" ]] ; then
> > +			basename "${dev}"
> >   			break
> >   		fi
> >   	done
> > 
> Thanks.
> 
> Should I resend my patchset?

Yes, please, with the other mentioned issues addressed.

[PATCH 2/3] nvme: add test for ANA state transition

[Hannes Reinecke]

Signed-off-by: Hannes Reinecke <hare at suse.com>
---
 tests/nvme/018     | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/018.out |  18 +++++++
 2 files changed, 159 insertions(+)
 create mode 100755 tests/nvme/018
 create mode 100644 tests/nvme/018.out

diff --git a/tests/nvme/018 b/tests/nvme/018
new file mode 100755
index 0000000..4d9b6c7
--- /dev/null
+++ b/tests/nvme/018
@@ -0,0 +1,141 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Regression test for ANA base support
+#
+# Copyright (C) 2018 Hannes Reinecke, SUSE Linux GmbH
+#
+
+. tests/nvme/rc
+
+DESCRIPTION="test ANA optimized/transitioning/inaccessible support"
+QUICK=1
+
+switch_nvmet_anagroup() {
+	local port1="$1"
+	local port2="$2"
+	local mode="$3"
+
+	echo "ANA state ${mode}"
+
+	if [ "${mode}" = "change" ] ; then
+		_set_nvmet_anagroup_state "${port1}" "1" "change"
+		_set_nvmet_anagroup_state "${port1}" "2" "change"
+		_set_nvmet_anagroup_state "${port2}" "1" "change"
+		_set_nvmet_anagroup_state "${port2}" "2" "change"
+	elif [ "${mode}" = "failover" ] ; then
+		_set_nvmet_anagroup_state "${port1}" "1" "inaccessible"
+		_set_nvmet_anagroup_state "${port1}" "2" "optimized"
+		_set_nvmet_anagroup_state "${port2}" "1" "optimized"
+		_set_nvmet_anagroup_state "${port2}" "2" "inaccessible"
+	else
+		_set_nvmet_anagroup_state "${port1}" "1" "optimized"
+		_set_nvmet_anagroup_state "${port1}" "2" "inaccessible"
+		_set_nvmet_anagroup_state "${port2}" "1" "inaccessible"
+		_set_nvmet_anagroup_state "${port2}" "2" "optimized"
+	fi
+}
+
+requires() {
+	_have_program nvme && _have_module nvme-loop && _have_module loop && \
+		_have_configfs && _have_fio
+}
+
+test() {
+	local mount_dir="/mnt/blktests"
+	local subsys="blktests-subsystem-1"
+	local port1
+	local port2
+	local loop_dev1
+	local loop_dev2
+	local nvmedev
+
+	echo "Running ${TEST_NAME}"
+
+	modprobe nvmet
+	modprobe nvme-loop
+
+	mkdir -p "${mount_dir}" > /dev/null 2>&1
+
+	port1="$(_create_nvmet_port "loop")"
+	ag1="$(_create_nvmet_anagroup "${port1}")"
+
+	port2="$(_create_nvmet_port "loop")"
+	ag2="$(_create_nvmet_anagroup "${port2}")"
+
+	truncate -s 1G "$TMPDIR/img1"
+
+	loop_dev1="$(losetup -f --show "$TMPDIR/img1")"
+
+	_create_nvmet_subsystem "${subsys}" "${loop_dev1}" \
+		"91fdba0d-f87b-4c25-b80f-db7be1418b9e" "1"
+
+	truncate -s 1G "$TMPDIR/img2"
+
+	loop_dev2="$(losetup -f --show "$TMPDIR/img2")"
+
+	_create_nvmet_ns "${subsys}" "2" "${loop_dev2}" \
+		"9aed0138-bfd9-46f5-92ac-24c70377fd49" "2"
+
+	_add_nvmet_subsys_to_port "${port1}" "${subsys}"
+	_add_nvmet_subsys_to_port "${port2}" "${subsys}"
+
+	switch_nvmet_anagroup "${port1}" "${port2}" failback
+
+	nvme connect -t loop -a "${port1}" -n "${subsys}"
+	nvme connect -t loop -a "${port2}" -n "${subsys}"
+
+	sleep 1
+
+	_display_ana_state "${subsys}"
+
+	nvmedev="$(_find_nvme_ns 91fdba0d-f87b-4c25-b80f-db7be1418b9e)"
+
+	mkfs.xfs -f /dev/"${nvmedev}" > /dev/null 2>&1
+
+	mount /dev/"${nvmedev}" "${mount_dir}"
+
+	_run_fio_rand_io --size=63m --directory="${mount_dir}" --runtime=20s --time_based &
+	trap "kill $!" EXIT
+
+	sleep 10
+
+	switch_nvmet_anagroup "${port1}" "${port2}" "change"
+
+	# Insert a delay to allow the AEN to be processed
+	sleep 1
+
+	_display_ana_state "${subsys}"
+
+	sleep 6
+
+	switch_nvmet_anagroup "${port1}" "${port2}" "failover"
+
+	# Insert a delay to allow the AEN to be processed
+	sleep 1
+
+	_display_ana_state "${subsys}"
+
+	wait
+	trap - EXIT
+
+	umount "${mount_dir}" > /dev/null 2>&1
+
+	nvme disconnect -n "${subsys}"
+
+	_remove_nvmet_subsystem_from_port "${port1}" "${subsys}"
+	_remove_nvmet_subsystem_from_port "${port2}" "${subsys}"
+	_remove_nvmet_ns "${subsys}" "2"
+	_remove_nvmet_subsystem "${subsys}"
+	_remove_nvmet_anagroup "${port1}" "${ag1}"
+	_remove_nvmet_port "${port1}"
+	_remove_nvmet_anagroup "${port2}" "${ag2}"
+	_remove_nvmet_port "${port2}"
+	losetup -d "${loop_dev2}"
+	rm "$TMPDIR/img2"
+	losetup -d "${loop_dev1}"
+	rm "$TMPDIR/img1"
+
+	modprobe -r nvme-loop
+	modprobe -r nvmet
+	echo "Test complete"
+}
diff --git a/tests/nvme/018.out b/tests/nvme/018.out
new file mode 100644
index 0000000..77d52d2
--- /dev/null
+++ b/tests/nvme/018.out
@@ -0,0 +1,18 @@
+Running nvme/018
+ANA state failback
+1: grpid 1 state optimized
+1: grpid 2 state inaccessible
+2: grpid 1 state inaccessible
+2: grpid 2 state optimized
+ANA state change
+1: grpid 1 state change
+1: grpid 2 state change
+2: grpid 1 state change
+2: grpid 2 state change
+ANA state failover
+1: grpid 1 state inaccessible
+1: grpid 2 state optimized
+2: grpid 1 state optimized
+2: grpid 2 state inaccessible
+NQN:blktests-subsystem-1 disconnected 2 controller(s)
+Test complete
-- 
2.13.7

[PATCH 2/3] nvme: add test for ANA state transition

[Omar Sandoval]

On Mon, Oct 15, 2018@09:01:29AM +0200, Hannes Reinecke wrote:
> Signed-off-by: Hannes Reinecke <hare at suse.com>
> ---
>  tests/nvme/018     | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/nvme/018.out |  18 +++++++
>  2 files changed, 159 insertions(+)
>  create mode 100755 tests/nvme/018
>  create mode 100644 tests/nvme/018.out
> 
> diff --git a/tests/nvme/018 b/tests/nvme/018
> new file mode 100755
> index 0000000..4d9b6c7
> --- /dev/null
> +++ b/tests/nvme/018
> @@ -0,0 +1,141 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-2.0+
> +# Regression test for ANA base support
> +#
> +# Copyright (C) 2018 Hannes Reinecke, SUSE Linux GmbH
> +#
> +
> +. tests/nvme/rc
> +
> +DESCRIPTION="test ANA optimized/transitioning/inaccessible support"
> +QUICK=1
> +
> +switch_nvmet_anagroup() {
> +	local port1="$1"
> +	local port2="$2"
> +	local mode="$3"
> +
> +	echo "ANA state ${mode}"
> +
> +	if [ "${mode}" = "change" ] ; then
> +		_set_nvmet_anagroup_state "${port1}" "1" "change"
> +		_set_nvmet_anagroup_state "${port1}" "2" "change"
> +		_set_nvmet_anagroup_state "${port2}" "1" "change"
> +		_set_nvmet_anagroup_state "${port2}" "2" "change"
> +	elif [ "${mode}" = "failover" ] ; then
> +		_set_nvmet_anagroup_state "${port1}" "1" "inaccessible"
> +		_set_nvmet_anagroup_state "${port1}" "2" "optimized"
> +		_set_nvmet_anagroup_state "${port2}" "1" "optimized"
> +		_set_nvmet_anagroup_state "${port2}" "2" "inaccessible"
> +	else
> +		_set_nvmet_anagroup_state "${port1}" "1" "optimized"
> +		_set_nvmet_anagroup_state "${port1}" "2" "inaccessible"
> +		_set_nvmet_anagroup_state "${port2}" "1" "inaccessible"
> +		_set_nvmet_anagroup_state "${port2}" "2" "optimized"
> +	fi
> +}
> +
> +requires() {
> +	_have_program nvme && _have_module nvme-loop && _have_module loop && \
> +		_have_configfs && _have_fio

You didn't address the comments from last time (this needs _have_program
mkfs.xfs and _have_module nvmet, IIRC). You also didn't respond to my
rqeuest that this detects if ANA support is available in the kernel. I
had to figure out that I needed CONFIG_NVME_MULTIPATH=y.

[PATCH 2/3] nvme: add test for ANA state transition

[Hannes Reinecke]

On 10/15/18 11:26 PM, Omar Sandoval wrote:
> On Mon, Oct 15, 2018@09:01:29AM +0200, Hannes Reinecke wrote:
>> Signed-off-by: Hannes Reinecke <hare at suse.com>
>> ---
>>   tests/nvme/018     | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>>   tests/nvme/018.out |  18 +++++++
>>   2 files changed, 159 insertions(+)
>>   create mode 100755 tests/nvme/018
>>   create mode 100644 tests/nvme/018.out
>>
>> diff --git a/tests/nvme/018 b/tests/nvme/018
>> new file mode 100755
>> index 0000000..4d9b6c7
>> --- /dev/null
>> +++ b/tests/nvme/018
>> @@ -0,0 +1,141 @@
>> +#!/bin/bash
>> +# SPDX-License-Identifier: GPL-2.0+
>> +# Regression test for ANA base support
>> +#
>> +# Copyright (C) 2018 Hannes Reinecke, SUSE Linux GmbH
>> +#
>> +
>> +. tests/nvme/rc
>> +
>> +DESCRIPTION="test ANA optimized/transitioning/inaccessible support"
>> +QUICK=1
>> +
>> +switch_nvmet_anagroup() {
>> +	local port1="$1"
>> +	local port2="$2"
>> +	local mode="$3"
>> +
>> +	echo "ANA state ${mode}"
>> +
>> +	if [ "${mode}" = "change" ] ; then
>> +		_set_nvmet_anagroup_state "${port1}" "1" "change"
>> +		_set_nvmet_anagroup_state "${port1}" "2" "change"
>> +		_set_nvmet_anagroup_state "${port2}" "1" "change"
>> +		_set_nvmet_anagroup_state "${port2}" "2" "change"
>> +	elif [ "${mode}" = "failover" ] ; then
>> +		_set_nvmet_anagroup_state "${port1}" "1" "inaccessible"
>> +		_set_nvmet_anagroup_state "${port1}" "2" "optimized"
>> +		_set_nvmet_anagroup_state "${port2}" "1" "optimized"
>> +		_set_nvmet_anagroup_state "${port2}" "2" "inaccessible"
>> +	else
>> +		_set_nvmet_anagroup_state "${port1}" "1" "optimized"
>> +		_set_nvmet_anagroup_state "${port1}" "2" "inaccessible"
>> +		_set_nvmet_anagroup_state "${port2}" "1" "inaccessible"
>> +		_set_nvmet_anagroup_state "${port2}" "2" "optimized"
>> +	fi
>> +}
>> +
>> +requires() {
>> +	_have_program nvme && _have_module nvme-loop && _have_module loop && \
>> +		_have_configfs && _have_fio
> 
> You didn't address the comments from last time (this needs _have_program
> mkfs.xfs and _have_module nvmet, IIRC). You also didn't respond to my
> rqeuest that this detects if ANA support is available in the kernel. I
> had to figure out that I needed CONFIG_NVME_MULTIPATH=y.
> 
And even that is not sufficient; the user might have disabled the use of 
native nvme multipathing on the kernel commandline.
I see what I can do to fix it up.

Cheers,

Hannes
-- 
Dr. Hannes Reinecke		   Teamlead Storage & Networking
hare at suse.de			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: F. Imend?rffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton
HRB 21284 (AG N?rnberg)

[PATCH 3/3] nvme: Add test for fcloop

[Hannes Reinecke]

Signed-off-by: Hannes Reinecke <hare at suse.com>
---
 tests/nvme/029     | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/029.out |   2 +
 tests/nvme/rc      |  52 ++++++++++++++++++++++
 3 files changed, 177 insertions(+)
 create mode 100755 tests/nvme/029
 create mode 100644 tests/nvme/029.out

diff --git a/tests/nvme/029 b/tests/nvme/029
new file mode 100755
index 0000000..f8ed9b5
--- /dev/null
+++ b/tests/nvme/029
@@ -0,0 +1,123 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (C) 2018 Johannes Thumshirn, Hannes Reinecke, SUSE Linux GmbH
+#
+# Reproducer for nvme-fcloop
+#
+
+. tests/nvme/rc
+
+DESCRIPTION="create nvme-fcloop with two ports and connect/disconnect"
+
+requires() {
+	_have_module nvme-fcloop && _have_module nvmet && _have_module loop && \
+		_have_configfs && _have_module nvme-core && \
+		_have_module nvmet-fc
+}
+
+test() {
+	local subsys="blktests-subsystem-1"
+	local remote_wwnn1="0x100140111111dbcc"
+	local remote_wwpn1="0x200140111111dbcc"
+	local remote_wwnn2="0x100140111111dbcd"
+	local remote_wwpn2="0x200140111111dbcd"
+	local host_wwnn1="0x100140111111dac8"
+	local host_wwpn1="0x200140111111dac8"
+	local host_wwnn2="0x100140111111dac9"
+	local host_wwpn2="0x200140111111dac9"
+
+	echo "Running ${TEST_NAME}"
+
+	modprobe nvmet-fc
+	modprobe nvme-fcloop
+
+	_nvme_fcloop_add_tport ${remote_wwnn1} ${remote_wwpn1}
+	_nvme_fcloop_add_tport ${remote_wwnn2} ${remote_wwpn2}
+
+	_nvme_fcloop_add_lport ${host_wwnn1} ${host_wwpn1}
+	_nvme_fcloop_add_lport ${host_wwnn2} ${host_wwpn2}
+
+	_nvme_fcloop_add_rport ${host_wwnn1} ${host_wwpn1} \
+		${remote_wwnn1} ${remote_wwpn1}
+	_nvme_fcloop_add_rport ${host_wwnn2} ${host_wwpn2} \
+		${remote_wwnn2} ${remote_wwpn2}
+
+	local port1
+	port1=$(_create_nvmet_port "fc")
+	ag1="$(_create_nvmet_anagroup "${port1}")"
+
+	local port2
+	port2=$(_create_nvmet_port "fc")
+	ag2="$(_create_nvmet_anagroup "${port2}")"
+
+	local remote_traddr1
+	remote_traddr1=$(printf "nn-%s:pn-%s" "${remote_wwnn1}" "${remote_wwpn1}")
+	echo fc > /sys/kernel/config/nvmet/ports/${port1}/addr_adrfam
+	echo "${remote_traddr1}" > /sys/kernel/config/nvmet/ports/${port1}/addr_traddr
+
+	local remote_traddr2
+	remote_traddr2=$(printf "nn-%s:pn-%s" "${remote_wwnn2}" "${remote_wwpn2}")
+	echo fc > /sys/kernel/config/nvmet/ports/${port2}/addr_adrfam
+	echo "${remote_traddr2}" > /sys/kernel/config/nvmet/ports/${port2}/addr_traddr
+
+	truncate -s 1G "$TMPDIR/img"
+
+	local loop_dev
+	loop_dev="$(losetup -f --show "$TMPDIR/img")"
+
+	_create_nvmet_subsystem "${subsys}" "${loop_dev}" \
+		"91fdba0d-f87b-4c25-b80f-db7be1418b9e" "1"
+	_add_nvmet_subsys_to_port "${port1}" "${subsys}"
+	_add_nvmet_subsys_to_port "${port2}" "${subsys}"
+
+	_set_nvmet_anagroup_state "${port2}" "1" "inaccessible"
+	_set_nvmet_anagroup_state "${port2}" "2" "optimized"
+
+	local host_traddr1
+	host_traddr1=$(printf "nn-%s:pn-%s" "${host_wwnn1}" "${host_wwpn1}")
+
+	nvme connect -t fc -a "${remote_traddr1}"  -w "${host_traddr1}" \
+		-n "${subsys}"
+
+	local host_traddr2
+	host_traddr2=$(printf "nn-%s:pn-%s" "${host_wwnn2}" "${host_wwpn2}")
+
+	nvme connect -t fc -a "${remote_traddr2}"  -w "${host_traddr2}" \
+		-n "${subsys}"
+
+	nvmedev="$(_find_nvme_ns 91fdba0d-f87b-4c25-b80f-db7be1418b9e)"
+
+	# Allow I/O to settle
+	udevadm settle
+
+	nvme disconnect -n "${subsys}"
+
+	_remove_nvmet_subsystem_from_port "${port1}" "${subsys}"
+	_remove_nvmet_subsystem_from_port "${port2}" "${subsys}"
+	_remove_nvmet_subsystem "${subsys}"
+	_remove_nvmet_anagroup "${port1}" "${ag1}"
+	_remove_nvmet_port "${port1}"
+	_remove_nvmet_anagroup "${port2}" "${ag2}"
+	_remove_nvmet_port "${port2}"
+
+	losetup -d "$loop_dev"
+	rm "$TMPDIR/img"
+
+	_nvme_fcloop_del_rport "${host_wwnn1}" "${host_wwpn1}" \
+		"${remote_wwnn1}" "${remote_wwpn1}"
+	_nvme_fcloop_del_rport "${host_wwnn2}" "${host_wwpn2}" \
+		"${remote_wwnn2}" "${remote_wwpn2}"
+
+	_nvme_fcloop_del_tport "${remote_wwnn1}" "${remote_wwpn1}"
+	_nvme_fcloop_del_tport "${remote_wwnn2}" "${remote_wwpn2}"
+
+	_nvme_fcloop_del_lport "${host_wwnn1}" "${host_wwpn1}"
+	_nvme_fcloop_del_lport "${host_wwnn2}" "${host_wwpn2}"
+
+	modprobe -r nvme-fcloop
+	modprobe -r nvmet-fc
+	modprobe -r nvmet
+	modprobe -r loop
+
+	echo "Test complete"
+}
diff --git a/tests/nvme/029.out b/tests/nvme/029.out
new file mode 100644
index 0000000..745f142
--- /dev/null
+++ b/tests/nvme/029.out
@@ -0,0 +1,2 @@
+Running nvme/029
+Test complete
diff --git a/tests/nvme/rc b/tests/nvme/rc
index aaf438b..3b3d852 100644
--- a/tests/nvme/rc
+++ b/tests/nvme/rc
@@ -201,3 +201,55 @@ _find_nvme_ns() {
 		fi
 	done
 }
+
+_nvme_fcloop_add_rport() {
+	local local_wwnn="$1"
+	local local_wwpn="$2"
+	local remote_wwnn="$3"
+	local remote_wwpn="$4"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port
+}
+
+_nvme_fcloop_del_rport() {
+	local local_wwnn="$1"
+	local local_wwpn="$2"
+	local remote_wwnn="$3"
+	local remote_wwpn="$4"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > ${loopctl}/del_remote_port
+}
+
+_nvme_fcloop_add_lport() {
+	local wwnn="$1"
+	local wwpn="$2"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port
+}
+
+_nvme_fcloop_del_lport() {
+	local wwnn="$1"
+	local wwpn="$2"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/del_local_port
+}
+
+_nvme_fcloop_add_tport() {
+	local wwnn="$1"
+	local wwpn="$2"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port
+}
+
+_nvme_fcloop_del_tport() {
+	local wwnn="$1"
+	local wwpn="$2"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/del_target_port
+}
-- 
2.13.7

[PATCH 3/3] nvme: Add test for fcloop

[Omar Sandoval]

On Mon, Oct 15, 2018@09:01:30AM +0200, Hannes Reinecke wrote:
> Signed-off-by: Hannes Reinecke <hare at suse.com>
> ---
>  tests/nvme/029     | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/nvme/029.out |   2 +
>  tests/nvme/rc      |  52 ++++++++++++++++++++++
>  3 files changed, 177 insertions(+)
>  create mode 100755 tests/nvme/029
>  create mode 100644 tests/nvme/029.out
> 
> diff --git a/tests/nvme/029 b/tests/nvme/029
> new file mode 100755
> index 0000000..f8ed9b5
> --- /dev/null
> +++ b/tests/nvme/029
> @@ -0,0 +1,123 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-2.0+
> +# Copyright (C) 2018 Johannes Thumshirn, Hannes Reinecke, SUSE Linux GmbH
> +#
> +# Reproducer for nvme-fcloop
> +#
> +
> +. tests/nvme/rc
> +
> +DESCRIPTION="create nvme-fcloop with two ports and connect/disconnect"
> +
> +requires() {
> +	_have_module nvme-fcloop && _have_module nvmet && _have_module loop && \
> +		_have_configfs && _have_module nvme-core && \
> +		_have_module nvmet-fc
> +}
> +
> +test() {
> +	local subsys="blktests-subsystem-1"
> +	local remote_wwnn1="0x100140111111dbcc"
> +	local remote_wwpn1="0x200140111111dbcc"
> +	local remote_wwnn2="0x100140111111dbcd"
> +	local remote_wwpn2="0x200140111111dbcd"
> +	local host_wwnn1="0x100140111111dac8"
> +	local host_wwpn1="0x200140111111dac8"
> +	local host_wwnn2="0x100140111111dac9"
> +	local host_wwpn2="0x200140111111dac9"
> +
> +	echo "Running ${TEST_NAME}"
> +
> +	modprobe nvmet-fc
> +	modprobe nvme-fcloop
> +
> +	_nvme_fcloop_add_tport ${remote_wwnn1} ${remote_wwpn1}
> +	_nvme_fcloop_add_tport ${remote_wwnn2} ${remote_wwpn2}
> +
> +	_nvme_fcloop_add_lport ${host_wwnn1} ${host_wwpn1}
> +	_nvme_fcloop_add_lport ${host_wwnn2} ${host_wwpn2}
> +
> +	_nvme_fcloop_add_rport ${host_wwnn1} ${host_wwpn1} \
> +		${remote_wwnn1} ${remote_wwpn1}
> +	_nvme_fcloop_add_rport ${host_wwnn2} ${host_wwpn2} \
> +		${remote_wwnn2} ${remote_wwpn2}
> +
> +	local port1
> +	port1=$(_create_nvmet_port "fc")
> +	ag1="$(_create_nvmet_anagroup "${port1}")"
> +
> +	local port2
> +	port2=$(_create_nvmet_port "fc")
> +	ag2="$(_create_nvmet_anagroup "${port2}")"
> +
> +	local remote_traddr1
> +	remote_traddr1=$(printf "nn-%s:pn-%s" "${remote_wwnn1}" "${remote_wwpn1}")
> +	echo fc > /sys/kernel/config/nvmet/ports/${port1}/addr_adrfam
> +	echo "${remote_traddr1}" > /sys/kernel/config/nvmet/ports/${port1}/addr_traddr
> +
> +	local remote_traddr2
> +	remote_traddr2=$(printf "nn-%s:pn-%s" "${remote_wwnn2}" "${remote_wwpn2}")
> +	echo fc > /sys/kernel/config/nvmet/ports/${port2}/addr_adrfam
> +	echo "${remote_traddr2}" > /sys/kernel/config/nvmet/ports/${port2}/addr_traddr
> +
> +	truncate -s 1G "$TMPDIR/img"
> +
> +	local loop_dev
> +	loop_dev="$(losetup -f --show "$TMPDIR/img")"
> +
> +	_create_nvmet_subsystem "${subsys}" "${loop_dev}" \
> +		"91fdba0d-f87b-4c25-b80f-db7be1418b9e" "1"
> +	_add_nvmet_subsys_to_port "${port1}" "${subsys}"
> +	_add_nvmet_subsys_to_port "${port2}" "${subsys}"
> +
> +	_set_nvmet_anagroup_state "${port2}" "1" "inaccessible"
> +	_set_nvmet_anagroup_state "${port2}" "2" "optimized"
> +
> +	local host_traddr1
> +	host_traddr1=$(printf "nn-%s:pn-%s" "${host_wwnn1}" "${host_wwpn1}")
> +
> +	nvme connect -t fc -a "${remote_traddr1}"  -w "${host_traddr1}" \
> +		-n "${subsys}"
> +
> +	local host_traddr2
> +	host_traddr2=$(printf "nn-%s:pn-%s" "${host_wwnn2}" "${host_wwpn2}")
> +
> +	nvme connect -t fc -a "${remote_traddr2}"  -w "${host_traddr2}" \
> +		-n "${subsys}"
> +
> +	nvmedev="$(_find_nvme_ns 91fdba0d-f87b-4c25-b80f-db7be1418b9e)"

nvmedev doesn't seem to be used anywhere (shellcheck warns about this).

> +	# Allow I/O to settle
> +	udevadm settle
> +
> +	nvme disconnect -n "${subsys}"

This outputs the following with my version (1.6) of nvme:

NQN:blktests-subsystem-1 disconnected 2 controller(s)

> +	_remove_nvmet_subsystem_from_port "${port1}" "${subsys}"
> +	_remove_nvmet_subsystem_from_port "${port2}" "${subsys}"
> +	_remove_nvmet_subsystem "${subsys}"
> +	_remove_nvmet_anagroup "${port1}" "${ag1}"
> +	_remove_nvmet_port "${port1}"
> +	_remove_nvmet_anagroup "${port2}" "${ag2}"
> +	_remove_nvmet_port "${port2}"
> +
> +	losetup -d "$loop_dev"
> +	rm "$TMPDIR/img"
> +
> +	_nvme_fcloop_del_rport "${host_wwnn1}" "${host_wwpn1}" \
> +		"${remote_wwnn1}" "${remote_wwpn1}"
> +	_nvme_fcloop_del_rport "${host_wwnn2}" "${host_wwpn2}" \
> +		"${remote_wwnn2}" "${remote_wwpn2}"
> +
> +	_nvme_fcloop_del_tport "${remote_wwnn1}" "${remote_wwpn1}"
> +	_nvme_fcloop_del_tport "${remote_wwnn2}" "${remote_wwpn2}"
> +
> +	_nvme_fcloop_del_lport "${host_wwnn1}" "${host_wwpn1}"
> +	_nvme_fcloop_del_lport "${host_wwnn2}" "${host_wwpn2}"
> +
> +	modprobe -r nvme-fcloop
> +	modprobe -r nvmet-fc
> +	modprobe -r nvmet
> +	modprobe -r loop

We don't want to try to remove loop, that's a generic enough module that
it might be in use elsewhere on the system.

[PATCH 3/3] nvme: Add test for fcloop

[Hannes Reinecke]

On 10/15/18 11:17 PM, Omar Sandoval wrote:
> On Mon, Oct 15, 2018@09:01:30AM +0200, Hannes Reinecke wrote:
>> Signed-off-by: Hannes Reinecke <hare at suse.com>
>> ---
>>   tests/nvme/029     | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>>   tests/nvme/029.out |   2 +
>>   tests/nvme/rc      |  52 ++++++++++++++++++++++
>>   3 files changed, 177 insertions(+)
>>   create mode 100755 tests/nvme/029
>>   create mode 100644 tests/nvme/029.out
>>
>> diff --git a/tests/nvme/029 b/tests/nvme/029
>> new file mode 100755
>> index 0000000..f8ed9b5
>> --- /dev/null
>> +++ b/tests/nvme/029
>> @@ -0,0 +1,123 @@
>> +#!/bin/bash
>> +# SPDX-License-Identifier: GPL-2.0+
>> +# Copyright (C) 2018 Johannes Thumshirn, Hannes Reinecke, SUSE Linux GmbH
>> +#
>> +# Reproducer for nvme-fcloop
>> +#
>> +
>> +. tests/nvme/rc
>> +
>> +DESCRIPTION="create nvme-fcloop with two ports and connect/disconnect"
>> +
>> +requires() {
>> +	_have_module nvme-fcloop && _have_module nvmet && _have_module loop && \
>> +		_have_configfs && _have_module nvme-core && \
>> +		_have_module nvmet-fc
>> +}
>> +
>> +test() {
>> +	local subsys="blktests-subsystem-1"
>> +	local remote_wwnn1="0x100140111111dbcc"
>> +	local remote_wwpn1="0x200140111111dbcc"
>> +	local remote_wwnn2="0x100140111111dbcd"
>> +	local remote_wwpn2="0x200140111111dbcd"
>> +	local host_wwnn1="0x100140111111dac8"
>> +	local host_wwpn1="0x200140111111dac8"
>> +	local host_wwnn2="0x100140111111dac9"
>> +	local host_wwpn2="0x200140111111dac9"
>> +
>> +	echo "Running ${TEST_NAME}"
>> +
>> +	modprobe nvmet-fc
>> +	modprobe nvme-fcloop
>> +
>> +	_nvme_fcloop_add_tport ${remote_wwnn1} ${remote_wwpn1}
>> +	_nvme_fcloop_add_tport ${remote_wwnn2} ${remote_wwpn2}
>> +
>> +	_nvme_fcloop_add_lport ${host_wwnn1} ${host_wwpn1}
>> +	_nvme_fcloop_add_lport ${host_wwnn2} ${host_wwpn2}
>> +
>> +	_nvme_fcloop_add_rport ${host_wwnn1} ${host_wwpn1} \
>> +		${remote_wwnn1} ${remote_wwpn1}
>> +	_nvme_fcloop_add_rport ${host_wwnn2} ${host_wwpn2} \
>> +		${remote_wwnn2} ${remote_wwpn2}
>> +
>> +	local port1
>> +	port1=$(_create_nvmet_port "fc")
>> +	ag1="$(_create_nvmet_anagroup "${port1}")"
>> +
>> +	local port2
>> +	port2=$(_create_nvmet_port "fc")
>> +	ag2="$(_create_nvmet_anagroup "${port2}")"
>> +
>> +	local remote_traddr1
>> +	remote_traddr1=$(printf "nn-%s:pn-%s" "${remote_wwnn1}" "${remote_wwpn1}")
>> +	echo fc > /sys/kernel/config/nvmet/ports/${port1}/addr_adrfam
>> +	echo "${remote_traddr1}" > /sys/kernel/config/nvmet/ports/${port1}/addr_traddr
>> +
>> +	local remote_traddr2
>> +	remote_traddr2=$(printf "nn-%s:pn-%s" "${remote_wwnn2}" "${remote_wwpn2}")
>> +	echo fc > /sys/kernel/config/nvmet/ports/${port2}/addr_adrfam
>> +	echo "${remote_traddr2}" > /sys/kernel/config/nvmet/ports/${port2}/addr_traddr
>> +
>> +	truncate -s 1G "$TMPDIR/img"
>> +
>> +	local loop_dev
>> +	loop_dev="$(losetup -f --show "$TMPDIR/img")"
>> +
>> +	_create_nvmet_subsystem "${subsys}" "${loop_dev}" \
>> +		"91fdba0d-f87b-4c25-b80f-db7be1418b9e" "1"
>> +	_add_nvmet_subsys_to_port "${port1}" "${subsys}"
>> +	_add_nvmet_subsys_to_port "${port2}" "${subsys}"
>> +
>> +	_set_nvmet_anagroup_state "${port2}" "1" "inaccessible"
>> +	_set_nvmet_anagroup_state "${port2}" "2" "optimized"
>> +
>> +	local host_traddr1
>> +	host_traddr1=$(printf "nn-%s:pn-%s" "${host_wwnn1}" "${host_wwpn1}")
>> +
>> +	nvme connect -t fc -a "${remote_traddr1}"  -w "${host_traddr1}" \
>> +		-n "${subsys}"
>> +
>> +	local host_traddr2
>> +	host_traddr2=$(printf "nn-%s:pn-%s" "${host_wwnn2}" "${host_wwpn2}")
>> +
>> +	nvme connect -t fc -a "${remote_traddr2}"  -w "${host_traddr2}" \
>> +		-n "${subsys}"
>> +
>> +	nvmedev="$(_find_nvme_ns 91fdba0d-f87b-4c25-b80f-db7be1418b9e)"
> 
> nvmedev doesn't seem to be used anywhere (shellcheck warns about this).
> 
Okay, will be removing the variable

>> +	# Allow I/O to settle
>> +	udevadm settle
>> +
>> +	nvme disconnect -n "${subsys}"
> 
> This outputs the following with my version (1.6) of nvme:
> 
> NQN:blktests-subsystem-1 disconnected 2 controller(s)
> 
And? That's what I expected, so where's the rub?

>> +	_remove_nvmet_subsystem_from_port "${port1}" "${subsys}"
>> +	_remove_nvmet_subsystem_from_port "${port2}" "${subsys}"
>> +	_remove_nvmet_subsystem "${subsys}"
>> +	_remove_nvmet_anagroup "${port1}" "${ag1}"
>> +	_remove_nvmet_port "${port1}"
>> +	_remove_nvmet_anagroup "${port2}" "${ag2}"
>> +	_remove_nvmet_port "${port2}"
>> +
>> +	losetup -d "$loop_dev"
>> +	rm "$TMPDIR/img"
>> +
>> +	_nvme_fcloop_del_rport "${host_wwnn1}" "${host_wwpn1}" \
>> +		"${remote_wwnn1}" "${remote_wwpn1}"
>> +	_nvme_fcloop_del_rport "${host_wwnn2}" "${host_wwpn2}" \
>> +		"${remote_wwnn2}" "${remote_wwpn2}"
>> +
>> +	_nvme_fcloop_del_tport "${remote_wwnn1}" "${remote_wwpn1}"
>> +	_nvme_fcloop_del_tport "${remote_wwnn2}" "${remote_wwpn2}"
>> +
>> +	_nvme_fcloop_del_lport "${host_wwnn1}" "${host_wwpn1}"
>> +	_nvme_fcloop_del_lport "${host_wwnn2}" "${host_wwpn2}"
>> +
>> +	modprobe -r nvme-fcloop
>> +	modprobe -r nvmet-fc
>> +	modprobe -r nvmet
>> +	modprobe -r loop
> 
> We don't want to try to remove loop, that's a generic enough module that
> it might be in use elsewhere on the system.
> 
Good point.

Will be fixing it up.

Cheers,

Hannes
-- 
Dr. Hannes Reinecke		   Teamlead Storage & Networking
hare at suse.de			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: F. Imend?rffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton
HRB 21284 (AG N?rnberg)

[PATCH 3/3] nvme: Add test for fcloop

[Omar Sandoval]

On Tue, Oct 16, 2018@07:50:37AM +0200, Hannes Reinecke wrote:
> On 10/15/18 11:17 PM, Omar Sandoval wrote:
> > On Mon, Oct 15, 2018@09:01:30AM +0200, Hannes Reinecke wrote:
> > > Signed-off-by: Hannes Reinecke <hare at suse.com>
> > > ---
> > >   tests/nvme/029     | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++
> > >   tests/nvme/029.out |   2 +
> > >   tests/nvme/rc      |  52 ++++++++++++++++++++++
> > >   3 files changed, 177 insertions(+)
> > >   create mode 100755 tests/nvme/029
> > >   create mode 100644 tests/nvme/029.out
> > > 
> > > diff --git a/tests/nvme/029 b/tests/nvme/029
> > > new file mode 100755
> > > index 0000000..f8ed9b5
> > > --- /dev/null
> > > +++ b/tests/nvme/029
> > > @@ -0,0 +1,123 @@
> > > +#!/bin/bash
> > > +# SPDX-License-Identifier: GPL-2.0+
> > > +# Copyright (C) 2018 Johannes Thumshirn, Hannes Reinecke, SUSE Linux GmbH
> > > +#
> > > +# Reproducer for nvme-fcloop
> > > +#
> > > +
> > > +. tests/nvme/rc
> > > +
> > > +DESCRIPTION="create nvme-fcloop with two ports and connect/disconnect"
> > > +
> > > +requires() {
> > > +	_have_module nvme-fcloop && _have_module nvmet && _have_module loop && \
> > > +		_have_configfs && _have_module nvme-core && \
> > > +		_have_module nvmet-fc
> > > +}
> > > +
> > > +test() {
> > > +	local subsys="blktests-subsystem-1"
> > > +	local remote_wwnn1="0x100140111111dbcc"
> > > +	local remote_wwpn1="0x200140111111dbcc"
> > > +	local remote_wwnn2="0x100140111111dbcd"
> > > +	local remote_wwpn2="0x200140111111dbcd"
> > > +	local host_wwnn1="0x100140111111dac8"
> > > +	local host_wwpn1="0x200140111111dac8"
> > > +	local host_wwnn2="0x100140111111dac9"
> > > +	local host_wwpn2="0x200140111111dac9"
> > > +
> > > +	echo "Running ${TEST_NAME}"
> > > +
> > > +	modprobe nvmet-fc
> > > +	modprobe nvme-fcloop
> > > +
> > > +	_nvme_fcloop_add_tport ${remote_wwnn1} ${remote_wwpn1}
> > > +	_nvme_fcloop_add_tport ${remote_wwnn2} ${remote_wwpn2}
> > > +
> > > +	_nvme_fcloop_add_lport ${host_wwnn1} ${host_wwpn1}
> > > +	_nvme_fcloop_add_lport ${host_wwnn2} ${host_wwpn2}
> > > +
> > > +	_nvme_fcloop_add_rport ${host_wwnn1} ${host_wwpn1} \
> > > +		${remote_wwnn1} ${remote_wwpn1}
> > > +	_nvme_fcloop_add_rport ${host_wwnn2} ${host_wwpn2} \
> > > +		${remote_wwnn2} ${remote_wwpn2}
> > > +
> > > +	local port1
> > > +	port1=$(_create_nvmet_port "fc")
> > > +	ag1="$(_create_nvmet_anagroup "${port1}")"
> > > +
> > > +	local port2
> > > +	port2=$(_create_nvmet_port "fc")
> > > +	ag2="$(_create_nvmet_anagroup "${port2}")"
> > > +
> > > +	local remote_traddr1
> > > +	remote_traddr1=$(printf "nn-%s:pn-%s" "${remote_wwnn1}" "${remote_wwpn1}")
> > > +	echo fc > /sys/kernel/config/nvmet/ports/${port1}/addr_adrfam
> > > +	echo "${remote_traddr1}" > /sys/kernel/config/nvmet/ports/${port1}/addr_traddr
> > > +
> > > +	local remote_traddr2
> > > +	remote_traddr2=$(printf "nn-%s:pn-%s" "${remote_wwnn2}" "${remote_wwpn2}")
> > > +	echo fc > /sys/kernel/config/nvmet/ports/${port2}/addr_adrfam
> > > +	echo "${remote_traddr2}" > /sys/kernel/config/nvmet/ports/${port2}/addr_traddr
> > > +
> > > +	truncate -s 1G "$TMPDIR/img"
> > > +
> > > +	local loop_dev
> > > +	loop_dev="$(losetup -f --show "$TMPDIR/img")"
> > > +
> > > +	_create_nvmet_subsystem "${subsys}" "${loop_dev}" \
> > > +		"91fdba0d-f87b-4c25-b80f-db7be1418b9e" "1"
> > > +	_add_nvmet_subsys_to_port "${port1}" "${subsys}"
> > > +	_add_nvmet_subsys_to_port "${port2}" "${subsys}"
> > > +
> > > +	_set_nvmet_anagroup_state "${port2}" "1" "inaccessible"
> > > +	_set_nvmet_anagroup_state "${port2}" "2" "optimized"
> > > +
> > > +	local host_traddr1
> > > +	host_traddr1=$(printf "nn-%s:pn-%s" "${host_wwnn1}" "${host_wwpn1}")
> > > +
> > > +	nvme connect -t fc -a "${remote_traddr1}"  -w "${host_traddr1}" \
> > > +		-n "${subsys}"
> > > +
> > > +	local host_traddr2
> > > +	host_traddr2=$(printf "nn-%s:pn-%s" "${host_wwnn2}" "${host_wwpn2}")
> > > +
> > > +	nvme connect -t fc -a "${remote_traddr2}"  -w "${host_traddr2}" \
> > > +		-n "${subsys}"
> > > +
> > > +	nvmedev="$(_find_nvme_ns 91fdba0d-f87b-4c25-b80f-db7be1418b9e)"
> > 
> > nvmedev doesn't seem to be used anywhere (shellcheck warns about this).
> > 
> Okay, will be removing the variable
> 
> > > +	# Allow I/O to settle
> > > +	udevadm settle
> > > +
> > > +	nvme disconnect -n "${subsys}"
> > 
> > This outputs the following with my version (1.6) of nvme:
> > 
> > NQN:blktests-subsystem-1 disconnected 2 controller(s)
> > 
> And? That's what I expected, so where's the rub?

The expected test output does not include it.