[PATCH 0/3] blktests: ANA and FC-Loop support

[PATCH 0/3] blktests: ANA and FC-Loop support

[Hannes Reinecke]

Hi all,

here are two new tests for ANA support and FC-Loop; the latter more
as a showcase how fcloop is supposed to be setup.
But also serves as a nice reproducer for fcloop failing to be removed
properly.

As usual, comments and reviews are welcome.

Hannes Reinecke (3):
  nvme: enable ANA support
  nvme: add test for ANA state transition
  nvme: Add test for fcloop

 tests/nvme/018     | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/018.out |  18 +++++++
 tests/nvme/029     | 123 ++++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/029.out |   2 +
 tests/nvme/rc      | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++--
 5 files changed, 422 insertions(+), 3 deletions(-)
 create mode 100755 tests/nvme/018
 create mode 100644 tests/nvme/018.out
 create mode 100755 tests/nvme/029
 create mode 100644 tests/nvme/029.out

-- 
2.13.7

[PATCH 1/3] nvme: enable ANA support

[Hannes Reinecke]

Update functions to support ANA.

Signed-off-by: Hannes Reinecke <hare at suse.com>
---
 tests/nvme/rc | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 86 insertions(+), 3 deletions(-)

diff --git a/tests/nvme/rc b/tests/nvme/rc
index eff1dd9..aaf438b 100644
--- a/tests/nvme/rc
+++ b/tests/nvme/rc
@@ -14,7 +14,7 @@ group_device_requires() {
 	_test_dev_is_nvme
 }
 
-NVMET_CFS="/sys/kernel/config/nvmet/"
+NVMET_CFS="/sys/kernel/config/nvmet"
 
 _test_dev_is_nvme() {
 	if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then
@@ -36,6 +36,7 @@ _create_nvmet_port() {
 
 	mkdir "${NVMET_CFS}/ports/${port}"
 	echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
+	echo "${port}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
 
 	echo "${port}"
 }
@@ -45,6 +46,62 @@ _remove_nvmet_port() {
 	rmdir "${NVMET_CFS}/ports/${port}"
 }
 
+_create_nvmet_anagroup() {
+	local port="$1"
+	local port_cfs="${NVMET_CFS}/ports/${port}"
+	local grpid
+
+	for ((grpid = 1; ; grpid++)); do
+		if [[ ! -e "${port_cfs}/ana_groups/${grpid}" ]]; then
+			break
+		fi
+	done
+
+	mkdir "${port_cfs}/ana_groups/${grpid}"
+
+	echo "${grpid}"
+}
+
+_remove_nvmet_anagroup() {
+	local port="$1"
+	local grpid="$2"
+	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
+
+	rmdir "${ana_cfs}"
+}
+
+_set_nvmet_anagroup_state() {
+	local port="$1"
+	local grpid="$2"
+	local state="$3"
+	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
+
+	echo "${state}" > "${ana_cfs}/ana_state"
+}
+
+_display_ana_state() {
+	local nvmet_subsystem="$1"
+	local grpid state cntlid
+
+	for nvme in /sys/class/nvme/* ; do
+		[ -f ${nvme}/subsysnqn ] || continue
+		subsys="$(cat "${nvme}/subsysnqn")"
+		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
+			continue
+		fi
+		cntlid="$(cat "${nvme}/cntlid")"
+		for c in ${nvme}/nvme* ; do
+			if [ ! -d ${c} ] ; then
+				echo "${cntlid}: ANA disabled"
+				continue
+			fi
+			grpid="$(cat "${c}/ana_grpid")"
+			state="$(cat "${c}/ana_state")"
+			echo "${cntlid}: grpid ${grpid} state ${state}"
+		done
+	done
+}
+
 _create_nvmet_ns() {
 	local nvmet_subsystem="$1"
 	local nsid="$2"
@@ -52,14 +109,22 @@ _create_nvmet_ns() {
 	local uuid="00000000-0000-0000-0000-000000000000"
 	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
 	local ns_path="${subsys_path}/namespaces/${nsid}"
+	local ana_grpid
 
-	if [[ $# -eq 4 ]]; then
+	if [[ $# -ge 4 ]]; then
 		uuid="$4"
 	fi
 
+	if [[ $# -eq 5 ]]; then
+		ana_grpid="$5"
+	fi
+
 	mkdir "${ns_path}"
 	printf "%s" "${blkdev}" > "${ns_path}/device_path"
 	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
+	if [ -n "${ana_grpid}" ] ; then
+		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
+	fi
 	printf 1 > "${ns_path}/enable"
 }
 
@@ -67,11 +132,12 @@ _create_nvmet_subsystem() {
 	local nvmet_subsystem="$1"
 	local blkdev="$2"
 	local uuid=$3
+	local ana_grpid=$4
 	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
 
 	mkdir -p "${cfs_path}"
 	echo 1 > "${cfs_path}/attr_allow_any_host"
-	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
+	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" "${ana_grpid}"
 }
 
 _remove_nvmet_ns() {
@@ -115,6 +181,23 @@ _find_nvme_loop_dev() {
 		transport="$(cat "/sys/class/nvme/${dev}/transport")"
 		if [[ "$transport" == "loop" ]]; then
 			echo "$dev"
+			break
+		fi
+	done
+}
+
+_find_nvme_ns() {
+	local uuid=$1
+	local dev
+	local hidden
+
+	for dev in /sys/block/nvme*; do
+		hidden="$(cat "${dev}/hidden")"
+		[[ "$hidden" -eq 1 ]] && continue
+		u="$(cat "${dev}/uuid")"
+		if [[ "$uuid" = "$u" ]] ; then
+			echo "$(basename "${dev}")"
+			break
 		fi
 	done
 }
-- 
2.13.7

[PATCH 1/3] nvme: enable ANA support

[Omar Sandoval]

On Mon, Oct 15, 2018@09:01:28AM +0200, Hannes Reinecke wrote:
> Update functions to support ANA.
> 
> Signed-off-by: Hannes Reinecke <hare at suse.com>
> ---
>  tests/nvme/rc | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
>  1 file changed, 86 insertions(+), 3 deletions(-)
> 
> diff --git a/tests/nvme/rc b/tests/nvme/rc
> index eff1dd9..aaf438b 100644
> --- a/tests/nvme/rc
> +++ b/tests/nvme/rc
> @@ -14,7 +14,7 @@ group_device_requires() {
>  	_test_dev_is_nvme
>  }
>  
> -NVMET_CFS="/sys/kernel/config/nvmet/"
> +NVMET_CFS="/sys/kernel/config/nvmet"
>  
>  _test_dev_is_nvme() {
>  	if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then
> @@ -36,6 +36,7 @@ _create_nvmet_port() {
>  
>  	mkdir "${NVMET_CFS}/ports/${port}"
>  	echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
> +	echo "${port}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
>  
>  	echo "${port}"
>  }
> @@ -45,6 +46,62 @@ _remove_nvmet_port() {
>  	rmdir "${NVMET_CFS}/ports/${port}"
>  }
>  
> +_create_nvmet_anagroup() {
> +	local port="$1"
> +	local port_cfs="${NVMET_CFS}/ports/${port}"
> +	local grpid
> +
> +	for ((grpid = 1; ; grpid++)); do
> +		if [[ ! -e "${port_cfs}/ana_groups/${grpid}" ]]; then
> +			break
> +		fi
> +	done
> +
> +	mkdir "${port_cfs}/ana_groups/${grpid}"
> +
> +	echo "${grpid}"
> +}
> +
> +_remove_nvmet_anagroup() {
> +	local port="$1"
> +	local grpid="$2"
> +	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
> +
> +	rmdir "${ana_cfs}"
> +}
> +
> +_set_nvmet_anagroup_state() {
> +	local port="$1"
> +	local grpid="$2"
> +	local state="$3"
> +	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
> +
> +	echo "${state}" > "${ana_cfs}/ana_state"
> +}
> +
> +_display_ana_state() {
> +	local nvmet_subsystem="$1"
> +	local grpid state cntlid
> +
> +	for nvme in /sys/class/nvme/* ; do
> +		[ -f ${nvme}/subsysnqn ] || continue
> +		subsys="$(cat "${nvme}/subsysnqn")"
> +		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
> +			continue
> +		fi
> +		cntlid="$(cat "${nvme}/cntlid")"
> +		for c in ${nvme}/nvme* ; do
> +			if [ ! -d ${c} ] ; then
> +				echo "${cntlid}: ANA disabled"
> +				continue
> +			fi
> +			grpid="$(cat "${c}/ana_grpid")"
> +			state="$(cat "${c}/ana_state")"
> +			echo "${cntlid}: grpid ${grpid} state ${state}"
> +		done
> +	done
> +}
> +
>  _create_nvmet_ns() {
>  	local nvmet_subsystem="$1"
>  	local nsid="$2"
> @@ -52,14 +109,22 @@ _create_nvmet_ns() {
>  	local uuid="00000000-0000-0000-0000-000000000000"
>  	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>  	local ns_path="${subsys_path}/namespaces/${nsid}"
> +	local ana_grpid
>  
> -	if [[ $# -eq 4 ]]; then
> +	if [[ $# -ge 4 ]]; then
>  		uuid="$4"
>  	fi
>  
> +	if [[ $# -eq 5 ]]; then
> +		ana_grpid="$5"
> +	fi
> +
>  	mkdir "${ns_path}"
>  	printf "%s" "${blkdev}" > "${ns_path}/device_path"
>  	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
> +	if [ -n "${ana_grpid}" ] ; then
> +		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
> +	fi
>  	printf 1 > "${ns_path}/enable"
>  }
>  
> @@ -67,11 +132,12 @@ _create_nvmet_subsystem() {
>  	local nvmet_subsystem="$1"
>  	local blkdev="$2"
>  	local uuid=$3
> +	local ana_grpid=$4
>  	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>  
>  	mkdir -p "${cfs_path}"
>  	echo 1 > "${cfs_path}/attr_allow_any_host"
> -	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
> +	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" "${ana_grpid}"
>  }
>  
>  _remove_nvmet_ns() {
> @@ -115,6 +181,23 @@ _find_nvme_loop_dev() {
>  		transport="$(cat "/sys/class/nvme/${dev}/transport")"
>  		if [[ "$transport" == "loop" ]]; then
>  			echo "$dev"
> +			break
> +		fi
> +	done
> +}
> +
> +_find_nvme_ns() {
> +	local uuid=$1
> +	local dev
> +	local hidden
> +
> +	for dev in /sys/block/nvme*; do
> +		hidden="$(cat "${dev}/hidden")"
> +		[[ "$hidden" -eq 1 ]] && continue
> +		u="$(cat "${dev}/uuid")"
> +		if [[ "$uuid" = "$u" ]] ; then
> +			echo "$(basename "${dev}")"
> +			break

This fails if I have another NVMe device:

cat: /sys/block/nvme0n1/uuid: No such file or directory

And there are a bunch of shellcheck errors. The following fixes all of
that:

diff --git a/tests/nvme/rc b/tests/nvme/rc
index 3b3d852..0c8fc62 100644
--- a/tests/nvme/rc
+++ b/tests/nvme/rc
@@ -84,14 +84,14 @@ _display_ana_state() {
 	local grpid state cntlid
 
 	for nvme in /sys/class/nvme/* ; do
-		[ -f ${nvme}/subsysnqn ] || continue
+		[[ -f ${nvme}/subsysnqn ]] || continue
 		subsys="$(cat "${nvme}/subsysnqn")"
-		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
+		if [[ "${subsys}" != "${nvmet_subsystem}" ]] ; then
 			continue
 		fi
 		cntlid="$(cat "${nvme}/cntlid")"
-		for c in ${nvme}/nvme* ; do
-			if [ ! -d ${c} ] ; then
+		for c in "${nvme}/nvme"* ; do
+			if [[ ! -d ${c} ]] ; then
 				echo "${cntlid}: ANA disabled"
 				continue
 			fi
@@ -122,7 +122,7 @@ _create_nvmet_ns() {
 	mkdir "${ns_path}"
 	printf "%s" "${blkdev}" > "${ns_path}/device_path"
 	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
-	if [ -n "${ana_grpid}" ] ; then
+	if [[ -n ${ana_grpid} ]] ; then
 		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
 	fi
 	printf 1 > "${ns_path}/enable"
@@ -194,9 +194,8 @@ _find_nvme_ns() {
 	for dev in /sys/block/nvme*; do
 		hidden="$(cat "${dev}/hidden")"
 		[[ "$hidden" -eq 1 ]] && continue
-		u="$(cat "${dev}/uuid")"
-		if [[ "$uuid" = "$u" ]] ; then
-			echo "$(basename "${dev}")"
+		if [[ -e "${dev}/uuid" && "$(cat "${dev}/uuid")" = "$uuid" ]] ; then
+			basename "${dev}"
 			break
 		fi
 	done

[PATCH 1/3] nvme: enable ANA support

[Hannes Reinecke]

On 10/15/18 11:13 PM, Omar Sandoval wrote:
> On Mon, Oct 15, 2018@09:01:28AM +0200, Hannes Reinecke wrote:
>> Update functions to support ANA.
>>
>> Signed-off-by: Hannes Reinecke <hare at suse.com>
>> ---
>>   tests/nvme/rc | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
>>   1 file changed, 86 insertions(+), 3 deletions(-)
>>
>> diff --git a/tests/nvme/rc b/tests/nvme/rc
>> index eff1dd9..aaf438b 100644
>> --- a/tests/nvme/rc
>> +++ b/tests/nvme/rc
>> @@ -14,7 +14,7 @@ group_device_requires() {
>>   	_test_dev_is_nvme
>>   }
>>   
>> -NVMET_CFS="/sys/kernel/config/nvmet/"
>> +NVMET_CFS="/sys/kernel/config/nvmet"
>>   
>>   _test_dev_is_nvme() {
>>   	if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then
>> @@ -36,6 +36,7 @@ _create_nvmet_port() {
>>   
>>   	mkdir "${NVMET_CFS}/ports/${port}"
>>   	echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
>> +	echo "${port}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
>>   
>>   	echo "${port}"
>>   }
>> @@ -45,6 +46,62 @@ _remove_nvmet_port() {
>>   	rmdir "${NVMET_CFS}/ports/${port}"
>>   }
>>   
>> +_create_nvmet_anagroup() {
>> +	local port="$1"
>> +	local port_cfs="${NVMET_CFS}/ports/${port}"
>> +	local grpid
>> +
>> +	for ((grpid = 1; ; grpid++)); do
>> +		if [[ ! -e "${port_cfs}/ana_groups/${grpid}" ]]; then
>> +			break
>> +		fi
>> +	done
>> +
>> +	mkdir "${port_cfs}/ana_groups/${grpid}"
>> +
>> +	echo "${grpid}"
>> +}
>> +
>> +_remove_nvmet_anagroup() {
>> +	local port="$1"
>> +	local grpid="$2"
>> +	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
>> +
>> +	rmdir "${ana_cfs}"
>> +}
>> +
>> +_set_nvmet_anagroup_state() {
>> +	local port="$1"
>> +	local grpid="$2"
>> +	local state="$3"
>> +	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
>> +
>> +	echo "${state}" > "${ana_cfs}/ana_state"
>> +}
>> +
>> +_display_ana_state() {
>> +	local nvmet_subsystem="$1"
>> +	local grpid state cntlid
>> +
>> +	for nvme in /sys/class/nvme/* ; do
>> +		[ -f ${nvme}/subsysnqn ] || continue
>> +		subsys="$(cat "${nvme}/subsysnqn")"
>> +		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
>> +			continue
>> +		fi
>> +		cntlid="$(cat "${nvme}/cntlid")"
>> +		for c in ${nvme}/nvme* ; do
>> +			if [ ! -d ${c} ] ; then
>> +				echo "${cntlid}: ANA disabled"
>> +				continue
>> +			fi
>> +			grpid="$(cat "${c}/ana_grpid")"
>> +			state="$(cat "${c}/ana_state")"
>> +			echo "${cntlid}: grpid ${grpid} state ${state}"
>> +		done
>> +	done
>> +}
>> +
>>   _create_nvmet_ns() {
>>   	local nvmet_subsystem="$1"
>>   	local nsid="$2"
>> @@ -52,14 +109,22 @@ _create_nvmet_ns() {
>>   	local uuid="00000000-0000-0000-0000-000000000000"
>>   	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>>   	local ns_path="${subsys_path}/namespaces/${nsid}"
>> +	local ana_grpid
>>   
>> -	if [[ $# -eq 4 ]]; then
>> +	if [[ $# -ge 4 ]]; then
>>   		uuid="$4"
>>   	fi
>>   
>> +	if [[ $# -eq 5 ]]; then
>> +		ana_grpid="$5"
>> +	fi
>> +
>>   	mkdir "${ns_path}"
>>   	printf "%s" "${blkdev}" > "${ns_path}/device_path"
>>   	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
>> +	if [ -n "${ana_grpid}" ] ; then
>> +		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
>> +	fi
>>   	printf 1 > "${ns_path}/enable"
>>   }
>>   
>> @@ -67,11 +132,12 @@ _create_nvmet_subsystem() {
>>   	local nvmet_subsystem="$1"
>>   	local blkdev="$2"
>>   	local uuid=$3
>> +	local ana_grpid=$4
>>   	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
>>   
>>   	mkdir -p "${cfs_path}"
>>   	echo 1 > "${cfs_path}/attr_allow_any_host"
>> -	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
>> +	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" "${ana_grpid}"
>>   }
>>   
>>   _remove_nvmet_ns() {
>> @@ -115,6 +181,23 @@ _find_nvme_loop_dev() {
>>   		transport="$(cat "/sys/class/nvme/${dev}/transport")"
>>   		if [[ "$transport" == "loop" ]]; then
>>   			echo "$dev"
>> +			break
>> +		fi
>> +	done
>> +}
>> +
>> +_find_nvme_ns() {
>> +	local uuid=$1
>> +	local dev
>> +	local hidden
>> +
>> +	for dev in /sys/block/nvme*; do
>> +		hidden="$(cat "${dev}/hidden")"
>> +		[[ "$hidden" -eq 1 ]] && continue
>> +		u="$(cat "${dev}/uuid")"
>> +		if [[ "$uuid" = "$u" ]] ; then
>> +			echo "$(basename "${dev}")"
>> +			break
> 
> This fails if I have another NVMe device:
> 
> cat: /sys/block/nvme0n1/uuid: No such file or directory
> 
> And there are a bunch of shellcheck errors. The following fixes all of
> that:
> 
> diff --git a/tests/nvme/rc b/tests/nvme/rc
> index 3b3d852..0c8fc62 100644
> --- a/tests/nvme/rc
> +++ b/tests/nvme/rc
> @@ -84,14 +84,14 @@ _display_ana_state() {
>   	local grpid state cntlid
>   
>   	for nvme in /sys/class/nvme/* ; do
> -		[ -f ${nvme}/subsysnqn ] || continue
> +		[[ -f ${nvme}/subsysnqn ]] || continue
>   		subsys="$(cat "${nvme}/subsysnqn")"
> -		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
> +		if [[ "${subsys}" != "${nvmet_subsystem}" ]] ; then
>   			continue
>   		fi
>   		cntlid="$(cat "${nvme}/cntlid")"
> -		for c in ${nvme}/nvme* ; do
> -			if [ ! -d ${c} ] ; then
> +		for c in "${nvme}/nvme"* ; do
> +			if [[ ! -d ${c} ]] ; then
>   				echo "${cntlid}: ANA disabled"
>   				continue
>   			fi
> @@ -122,7 +122,7 @@ _create_nvmet_ns() {
>   	mkdir "${ns_path}"
>   	printf "%s" "${blkdev}" > "${ns_path}/device_path"
>   	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
> -	if [ -n "${ana_grpid}" ] ; then
> +	if [[ -n ${ana_grpid} ]] ; then
>   		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
>   	fi
>   	printf 1 > "${ns_path}/enable"
> @@ -194,9 +194,8 @@ _find_nvme_ns() {
>   	for dev in /sys/block/nvme*; do
>   		hidden="$(cat "${dev}/hidden")"
>   		[[ "$hidden" -eq 1 ]] && continue
> -		u="$(cat "${dev}/uuid")"
> -		if [[ "$uuid" = "$u" ]] ; then
> -			echo "$(basename "${dev}")"
> +		if [[ -e "${dev}/uuid" && "$(cat "${dev}/uuid")" = "$uuid" ]] ; then
> +			basename "${dev}"
>   			break
>   		fi
>   	done
> 
Thanks.

Should I resend my patchset?

Cheers,

Hannes
-- 
Dr. Hannes Reinecke		   Teamlead Storage & Networking
hare at suse.de			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: F. Imend?rffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton
HRB 21284 (AG N?rnberg)

[PATCH 1/3] nvme: enable ANA support

[Omar Sandoval]

On Tue, Oct 16, 2018@07:49:20AM +0200, Hannes Reinecke wrote:
> On 10/15/18 11:13 PM, Omar Sandoval wrote:
> > On Mon, Oct 15, 2018@09:01:28AM +0200, Hannes Reinecke wrote:
> > > Update functions to support ANA.
> > > 
> > > Signed-off-by: Hannes Reinecke <hare at suse.com>
> > > ---
> > >   tests/nvme/rc | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
> > >   1 file changed, 86 insertions(+), 3 deletions(-)
> > > 
> > > diff --git a/tests/nvme/rc b/tests/nvme/rc
> > > index eff1dd9..aaf438b 100644
> > > --- a/tests/nvme/rc
> > > +++ b/tests/nvme/rc
> > > @@ -14,7 +14,7 @@ group_device_requires() {
> > >   	_test_dev_is_nvme
> > >   }
> > > -NVMET_CFS="/sys/kernel/config/nvmet/"
> > > +NVMET_CFS="/sys/kernel/config/nvmet"
> > >   _test_dev_is_nvme() {
> > >   	if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then
> > > @@ -36,6 +36,7 @@ _create_nvmet_port() {
> > >   	mkdir "${NVMET_CFS}/ports/${port}"
> > >   	echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
> > > +	echo "${port}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
> > >   	echo "${port}"
> > >   }
> > > @@ -45,6 +46,62 @@ _remove_nvmet_port() {
> > >   	rmdir "${NVMET_CFS}/ports/${port}"
> > >   }
> > > +_create_nvmet_anagroup() {
> > > +	local port="$1"
> > > +	local port_cfs="${NVMET_CFS}/ports/${port}"
> > > +	local grpid
> > > +
> > > +	for ((grpid = 1; ; grpid++)); do
> > > +		if [[ ! -e "${port_cfs}/ana_groups/${grpid}" ]]; then
> > > +			break
> > > +		fi
> > > +	done
> > > +
> > > +	mkdir "${port_cfs}/ana_groups/${grpid}"
> > > +
> > > +	echo "${grpid}"
> > > +}
> > > +
> > > +_remove_nvmet_anagroup() {
> > > +	local port="$1"
> > > +	local grpid="$2"
> > > +	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
> > > +
> > > +	rmdir "${ana_cfs}"
> > > +}
> > > +
> > > +_set_nvmet_anagroup_state() {
> > > +	local port="$1"
> > > +	local grpid="$2"
> > > +	local state="$3"
> > > +	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
> > > +
> > > +	echo "${state}" > "${ana_cfs}/ana_state"
> > > +}
> > > +
> > > +_display_ana_state() {
> > > +	local nvmet_subsystem="$1"
> > > +	local grpid state cntlid
> > > +
> > > +	for nvme in /sys/class/nvme/* ; do
> > > +		[ -f ${nvme}/subsysnqn ] || continue
> > > +		subsys="$(cat "${nvme}/subsysnqn")"
> > > +		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
> > > +			continue
> > > +		fi
> > > +		cntlid="$(cat "${nvme}/cntlid")"
> > > +		for c in ${nvme}/nvme* ; do
> > > +			if [ ! -d ${c} ] ; then
> > > +				echo "${cntlid}: ANA disabled"
> > > +				continue
> > > +			fi
> > > +			grpid="$(cat "${c}/ana_grpid")"
> > > +			state="$(cat "${c}/ana_state")"
> > > +			echo "${cntlid}: grpid ${grpid} state ${state}"
> > > +		done
> > > +	done
> > > +}
> > > +
> > >   _create_nvmet_ns() {
> > >   	local nvmet_subsystem="$1"
> > >   	local nsid="$2"
> > > @@ -52,14 +109,22 @@ _create_nvmet_ns() {
> > >   	local uuid="00000000-0000-0000-0000-000000000000"
> > >   	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> > >   	local ns_path="${subsys_path}/namespaces/${nsid}"
> > > +	local ana_grpid
> > > -	if [[ $# -eq 4 ]]; then
> > > +	if [[ $# -ge 4 ]]; then
> > >   		uuid="$4"
> > >   	fi
> > > +	if [[ $# -eq 5 ]]; then
> > > +		ana_grpid="$5"
> > > +	fi
> > > +
> > >   	mkdir "${ns_path}"
> > >   	printf "%s" "${blkdev}" > "${ns_path}/device_path"
> > >   	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
> > > +	if [ -n "${ana_grpid}" ] ; then
> > > +		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
> > > +	fi
> > >   	printf 1 > "${ns_path}/enable"
> > >   }
> > > @@ -67,11 +132,12 @@ _create_nvmet_subsystem() {
> > >   	local nvmet_subsystem="$1"
> > >   	local blkdev="$2"
> > >   	local uuid=$3
> > > +	local ana_grpid=$4
> > >   	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
> > >   	mkdir -p "${cfs_path}"
> > >   	echo 1 > "${cfs_path}/attr_allow_any_host"
> > > -	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
> > > +	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" "${ana_grpid}"
> > >   }
> > >   _remove_nvmet_ns() {
> > > @@ -115,6 +181,23 @@ _find_nvme_loop_dev() {
> > >   		transport="$(cat "/sys/class/nvme/${dev}/transport")"
> > >   		if [[ "$transport" == "loop" ]]; then
> > >   			echo "$dev"
> > > +			break
> > > +		fi
> > > +	done
> > > +}
> > > +
> > > +_find_nvme_ns() {
> > > +	local uuid=$1
> > > +	local dev
> > > +	local hidden
> > > +
> > > +	for dev in /sys/block/nvme*; do
> > > +		hidden="$(cat "${dev}/hidden")"
> > > +		[[ "$hidden" -eq 1 ]] && continue
> > > +		u="$(cat "${dev}/uuid")"
> > > +		if [[ "$uuid" = "$u" ]] ; then
> > > +			echo "$(basename "${dev}")"
> > > +			break
> > 
> > This fails if I have another NVMe device:
> > 
> > cat: /sys/block/nvme0n1/uuid: No such file or directory
> > 
> > And there are a bunch of shellcheck errors. The following fixes all of
> > that:
> > 
> > diff --git a/tests/nvme/rc b/tests/nvme/rc
> > index 3b3d852..0c8fc62 100644
> > --- a/tests/nvme/rc
> > +++ b/tests/nvme/rc
> > @@ -84,14 +84,14 @@ _display_ana_state() {
> >   	local grpid state cntlid
> >   	for nvme in /sys/class/nvme/* ; do
> > -		[ -f ${nvme}/subsysnqn ] || continue
> > +		[[ -f ${nvme}/subsysnqn ]] || continue
> >   		subsys="$(cat "${nvme}/subsysnqn")"
> > -		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
> > +		if [[ "${subsys}" != "${nvmet_subsystem}" ]] ; then
> >   			continue
> >   		fi
> >   		cntlid="$(cat "${nvme}/cntlid")"
> > -		for c in ${nvme}/nvme* ; do
> > -			if [ ! -d ${c} ] ; then
> > +		for c in "${nvme}/nvme"* ; do
> > +			if [[ ! -d ${c} ]] ; then
> >   				echo "${cntlid}: ANA disabled"
> >   				continue
> >   			fi
> > @@ -122,7 +122,7 @@ _create_nvmet_ns() {
> >   	mkdir "${ns_path}"
> >   	printf "%s" "${blkdev}" > "${ns_path}/device_path"
> >   	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
> > -	if [ -n "${ana_grpid}" ] ; then
> > +	if [[ -n ${ana_grpid} ]] ; then
> >   		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
> >   	fi
> >   	printf 1 > "${ns_path}/enable"
> > @@ -194,9 +194,8 @@ _find_nvme_ns() {
> >   	for dev in /sys/block/nvme*; do
> >   		hidden="$(cat "${dev}/hidden")"
> >   		[[ "$hidden" -eq 1 ]] && continue
> > -		u="$(cat "${dev}/uuid")"
> > -		if [[ "$uuid" = "$u" ]] ; then
> > -			echo "$(basename "${dev}")"
> > +		if [[ -e "${dev}/uuid" && "$(cat "${dev}/uuid")" = "$uuid" ]] ; then
> > +			basename "${dev}"
> >   			break
> >   		fi
> >   	done
> > 
> Thanks.
> 
> Should I resend my patchset?

Yes, please, with the other mentioned issues addressed.

[PATCH 2/3] nvme: add test for ANA state transition

[Hannes Reinecke]

Signed-off-by: Hannes Reinecke <hare at suse.com>
---
 tests/nvme/018     | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/018.out |  18 +++++++
 2 files changed, 159 insertions(+)
 create mode 100755 tests/nvme/018
 create mode 100644 tests/nvme/018.out

diff --git a/tests/nvme/018 b/tests/nvme/018
new file mode 100755
index 0000000..4d9b6c7
--- /dev/null
+++ b/tests/nvme/018
@@ -0,0 +1,141 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Regression test for ANA base support
+#
+# Copyright (C) 2018 Hannes Reinecke, SUSE Linux GmbH
+#
+
+. tests/nvme/rc
+
+DESCRIPTION="test ANA optimized/transitioning/inaccessible support"
+QUICK=1
+
+switch_nvmet_anagroup() {
+	local port1="$1"
+	local port2="$2"
+	local mode="$3"
+
+	echo "ANA state ${mode}"
+
+	if [ "${mode}" = "change" ] ; then
+		_set_nvmet_anagroup_state "${port1}" "1" "change"
+		_set_nvmet_anagroup_state "${port1}" "2" "change"
+		_set_nvmet_anagroup_state "${port2}" "1" "change"
+		_set_nvmet_anagroup_state "${port2}" "2" "change"
+	elif [ "${mode}" = "failover" ] ; then
+		_set_nvmet_anagroup_state "${port1}" "1" "inaccessible"
+		_set_nvmet_anagroup_state "${port1}" "2" "optimized"
+		_set_nvmet_anagroup_state "${port2}" "1" "optimized"
+		_set_nvmet_anagroup_state "${port2}" "2" "inaccessible"
+	else
+		_set_nvmet_anagroup_state "${port1}" "1" "optimized"
+		_set_nvmet_anagroup_state "${port1}" "2" "inaccessible"
+		_set_nvmet_anagroup_state "${port2}" "1" "inaccessible"
+		_set_nvmet_anagroup_state "${port2}" "2" "optimized"
+	fi
+}
+
+requires() {
+	_have_program nvme && _have_module nvme-loop && _have_module loop && \
+		_have_configfs && _have_fio
+}
+
+test() {
+	local mount_dir="/mnt/blktests"
+	local subsys="blktests-subsystem-1"
+	local port1
+	local port2
+	local loop_dev1
+	local loop_dev2
+	local nvmedev
+
+	echo "Running ${TEST_NAME}"
+
+	modprobe nvmet
+	modprobe nvme-loop
+
+	mkdir -p "${mount_dir}" > /dev/null 2>&1
+
+	port1="$(_create_nvmet_port "loop")"
+	ag1="$(_create_nvmet_anagroup "${port1}")"
+
+	port2="$(_create_nvmet_port "loop")"
+	ag2="$(_create_nvmet_anagroup "${port2}")"
+
+	truncate -s 1G "$TMPDIR/img1"
+
+	loop_dev1="$(losetup -f --show "$TMPDIR/img1")"
+
+	_create_nvmet_subsystem "${subsys}" "${loop_dev1}" \
+		"91fdba0d-f87b-4c25-b80f-db7be1418b9e" "1"
+
+	truncate -s 1G "$TMPDIR/img2"
+
+	loop_dev2="$(losetup -f --show "$TMPDIR/img2")"
+
+	_create_nvmet_ns "${subsys}" "2" "${loop_dev2}" \
+		"9aed0138-bfd9-46f5-92ac-24c70377fd49" "2"
+
+	_add_nvmet_subsys_to_port "${port1}" "${subsys}"
+	_add_nvmet_subsys_to_port "${port2}" "${subsys}"
+
+	switch_nvmet_anagroup "${port1}" "${port2}" failback
+
+	nvme connect -t loop -a "${port1}" -n "${subsys}"
+	nvme connect -t loop -a "${port2}" -n "${subsys}"
+
+	sleep 1
+
+	_display_ana_state "${subsys}"
+
+	nvmedev="$(_find_nvme_ns 91fdba0d-f87b-4c25-b80f-db7be1418b9e)"
+
+	mkfs.xfs -f /dev/"${nvmedev}" > /dev/null 2>&1
+
+	mount /dev/"${nvmedev}" "${mount_dir}"
+
+	_run_fio_rand_io --size=63m --directory="${mount_dir}" --runtime=20s --time_based &
+	trap "kill $!" EXIT
+
+	sleep 10
+
+	switch_nvmet_anagroup "${port1}" "${port2}" "change"
+
+	# Insert a delay to allow the AEN to be processed
+	sleep 1
+
+	_display_ana_state "${subsys}"
+
+	sleep 6
+
+	switch_nvmet_anagroup "${port1}" "${port2}" "failover"
+
+	# Insert a delay to allow the AEN to be processed
+	sleep 1
+
+	_display_ana_state "${subsys}"
+
+	wait
+	trap - EXIT
+
+	umount "${mount_dir}" > /dev/null 2>&1
+
+	nvme disconnect -n "${subsys}"
+
+	_remove_nvmet_subsystem_from_port "${port1}" "${subsys}"
+	_remove_nvmet_subsystem_from_port "${port2}" "${subsys}"
+	_remove_nvmet_ns "${subsys}" "2"
+	_remove_nvmet_subsystem "${subsys}"
+	_remove_nvmet_anagroup "${port1}" "${ag1}"
+	_remove_nvmet_port "${port1}"
+	_remove_nvmet_anagroup "${port2}" "${ag2}"
+	_remove_nvmet_port "${port2}"
+	losetup -d "${loop_dev2}"
+	rm "$TMPDIR/img2"
+	losetup -d "${loop_dev1}"
+	rm "$TMPDIR/img1"
+
+	modprobe -r nvme-loop
+	modprobe -r nvmet
+	echo "Test complete"
+}
diff --git a/tests/nvme/018.out b/tests/nvme/018.out
new file mode 100644
index 0000000..77d52d2
--- /dev/null
+++ b/tests/nvme/018.out
@@ -0,0 +1,18 @@
+Running nvme/018
+ANA state failback
+1: grpid 1 state optimized
+1: grpid 2 state inaccessible
+2: grpid 1 state inaccessible
+2: grpid 2 state optimized
+ANA state change
+1: grpid 1 state change
+1: grpid 2 state change
+2: grpid 1 state change
+2: grpid 2 state change
+ANA state failover
+1: grpid 1 state inaccessible
+1: grpid 2 state optimized
+2: grpid 1 state optimized
+2: grpid 2 state inaccessible
+NQN:blktests-subsystem-1 disconnected 2 controller(s)
+Test complete
-- 
2.13.7

[PATCH 2/3] nvme: add test for ANA state transition

[Omar Sandoval]

On Mon, Oct 15, 2018@09:01:29AM +0200, Hannes Reinecke wrote:
> Signed-off-by: Hannes Reinecke <hare at suse.com>
> ---
>  tests/nvme/018     | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/nvme/018.out |  18 +++++++
>  2 files changed, 159 insertions(+)
>  create mode 100755 tests/nvme/018
>  create mode 100644 tests/nvme/018.out
> 
> diff --git a/tests/nvme/018 b/tests/nvme/018
> new file mode 100755
> index 0000000..4d9b6c7
> --- /dev/null
> +++ b/tests/nvme/018
> @@ -0,0 +1,141 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-2.0+
> +# Regression test for ANA base support
> +#
> +# Copyright (C) 2018 Hannes Reinecke, SUSE Linux GmbH
> +#
> +
> +. tests/nvme/rc
> +
> +DESCRIPTION="test ANA optimized/transitioning/inaccessible support"
> +QUICK=1
> +
> +switch_nvmet_anagroup() {
> +	local port1="$1"
> +	local port2="$2"
> +	local mode="$3"
> +
> +	echo "ANA state ${mode}"
> +
> +	if [ "${mode}" = "change" ] ; then
> +		_set_nvmet_anagroup_state "${port1}" "1" "change"
> +		_set_nvmet_anagroup_state "${port1}" "2" "change"
> +		_set_nvmet_anagroup_state "${port2}" "1" "change"
> +		_set_nvmet_anagroup_state "${port2}" "2" "change"
> +	elif [ "${mode}" = "failover" ] ; then
> +		_set_nvmet_anagroup_state "${port1}" "1" "inaccessible"
> +		_set_nvmet_anagroup_state "${port1}" "2" "optimized"
> +		_set_nvmet_anagroup_state "${port2}" "1" "optimized"
> +		_set_nvmet_anagroup_state "${port2}" "2" "inaccessible"
> +	else
> +		_set_nvmet_anagroup_state "${port1}" "1" "optimized"
> +		_set_nvmet_anagroup_state "${port1}" "2" "inaccessible"
> +		_set_nvmet_anagroup_state "${port2}" "1" "inaccessible"
> +		_set_nvmet_anagroup_state "${port2}" "2" "optimized"
> +	fi
> +}
> +
> +requires() {
> +	_have_program nvme && _have_module nvme-loop && _have_module loop && \
> +		_have_configfs && _have_fio

You didn't address the comments from last time (this needs _have_program
mkfs.xfs and _have_module nvmet, IIRC). You also didn't respond to my
rqeuest that this detects if ANA support is available in the kernel. I
had to figure out that I needed CONFIG_NVME_MULTIPATH=y.

[PATCH 2/3] nvme: add test for ANA state transition

[Hannes Reinecke]

On 10/15/18 11:26 PM, Omar Sandoval wrote:
> On Mon, Oct 15, 2018@09:01:29AM +0200, Hannes Reinecke wrote:
>> Signed-off-by: Hannes Reinecke <hare at suse.com>
>> ---
>>   tests/nvme/018     | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>>   tests/nvme/018.out |  18 +++++++
>>   2 files changed, 159 insertions(+)
>>   create mode 100755 tests/nvme/018
>>   create mode 100644 tests/nvme/018.out
>>
>> diff --git a/tests/nvme/018 b/tests/nvme/018
>> new file mode 100755
>> index 0000000..4d9b6c7
>> --- /dev/null
>> +++ b/tests/nvme/018
>> @@ -0,0 +1,141 @@
>> +#!/bin/bash
>> +# SPDX-License-Identifier: GPL-2.0+
>> +# Regression test for ANA base support
>> +#
>> +# Copyright (C) 2018 Hannes Reinecke, SUSE Linux GmbH
>> +#
>> +
>> +. tests/nvme/rc
>> +
>> +DESCRIPTION="test ANA optimized/transitioning/inaccessible support"
>> +QUICK=1
>> +
>> +switch_nvmet_anagroup() {
>> +	local port1="$1"
>> +	local port2="$2"
>> +	local mode="$3"
>> +
>> +	echo "ANA state ${mode}"
>> +
>> +	if [ "${mode}" = "change" ] ; then
>> +		_set_nvmet_anagroup_state "${port1}" "1" "change"
>> +		_set_nvmet_anagroup_state "${port1}" "2" "change"
>> +		_set_nvmet_anagroup_state "${port2}" "1" "change"
>> +		_set_nvmet_anagroup_state "${port2}" "2" "change"
>> +	elif [ "${mode}" = "failover" ] ; then
>> +		_set_nvmet_anagroup_state "${port1}" "1" "inaccessible"
>> +		_set_nvmet_anagroup_state "${port1}" "2" "optimized"
>> +		_set_nvmet_anagroup_state "${port2}" "1" "optimized"
>> +		_set_nvmet_anagroup_state "${port2}" "2" "inaccessible"
>> +	else
>> +		_set_nvmet_anagroup_state "${port1}" "1" "optimized"
>> +		_set_nvmet_anagroup_state "${port1}" "2" "inaccessible"
>> +		_set_nvmet_anagroup_state "${port2}" "1" "inaccessible"
>> +		_set_nvmet_anagroup_state "${port2}" "2" "optimized"
>> +	fi
>> +}
>> +
>> +requires() {
>> +	_have_program nvme && _have_module nvme-loop && _have_module loop && \
>> +		_have_configfs && _have_fio
> 
> You didn't address the comments from last time (this needs _have_program
> mkfs.xfs and _have_module nvmet, IIRC). You also didn't respond to my
> rqeuest that this detects if ANA support is available in the kernel. I
> had to figure out that I needed CONFIG_NVME_MULTIPATH=y.
> 
And even that is not sufficient; the user might have disabled the use of 
native nvme multipathing on the kernel commandline.
I see what I can do to fix it up.

Cheers,

Hannes
-- 
Dr. Hannes Reinecke		   Teamlead Storage & Networking
hare at suse.de			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: F. Imend?rffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton
HRB 21284 (AG N?rnberg)

[PATCH 3/3] nvme: Add test for fcloop

[Hannes Reinecke]

Signed-off-by: Hannes Reinecke <hare at suse.com>
---
 tests/nvme/029     | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/029.out |   2 +
 tests/nvme/rc      |  52 ++++++++++++++++++++++
 3 files changed, 177 insertions(+)
 create mode 100755 tests/nvme/029
 create mode 100644 tests/nvme/029.out

diff --git a/tests/nvme/029 b/tests/nvme/029
new file mode 100755
index 0000000..f8ed9b5
--- /dev/null
+++ b/tests/nvme/029
@@ -0,0 +1,123 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright (C) 2018 Johannes Thumshirn, Hannes Reinecke, SUSE Linux GmbH
+#
+# Reproducer for nvme-fcloop
+#
+
+. tests/nvme/rc
+
+DESCRIPTION="create nvme-fcloop with two ports and connect/disconnect"
+
+requires() {
+	_have_module nvme-fcloop && _have_module nvmet && _have_module loop && \
+		_have_configfs && _have_module nvme-core && \
+		_have_module nvmet-fc
+}
+
+test() {
+	local subsys="blktests-subsystem-1"
+	local remote_wwnn1="0x100140111111dbcc"
+	local remote_wwpn1="0x200140111111dbcc"
+	local remote_wwnn2="0x100140111111dbcd"
+	local remote_wwpn2="0x200140111111dbcd"
+	local host_wwnn1="0x100140111111dac8"
+	local host_wwpn1="0x200140111111dac8"
+	local host_wwnn2="0x100140111111dac9"
+	local host_wwpn2="0x200140111111dac9"
+
+	echo "Running ${TEST_NAME}"
+
+	modprobe nvmet-fc
+	modprobe nvme-fcloop
+
+	_nvme_fcloop_add_tport ${remote_wwnn1} ${remote_wwpn1}
+	_nvme_fcloop_add_tport ${remote_wwnn2} ${remote_wwpn2}
+
+	_nvme_fcloop_add_lport ${host_wwnn1} ${host_wwpn1}
+	_nvme_fcloop_add_lport ${host_wwnn2} ${host_wwpn2}
+
+	_nvme_fcloop_add_rport ${host_wwnn1} ${host_wwpn1} \
+		${remote_wwnn1} ${remote_wwpn1}
+	_nvme_fcloop_add_rport ${host_wwnn2} ${host_wwpn2} \
+		${remote_wwnn2} ${remote_wwpn2}
+
+	local port1
+	port1=$(_create_nvmet_port "fc")
+	ag1="$(_create_nvmet_anagroup "${port1}")"
+
+	local port2
+	port2=$(_create_nvmet_port "fc")
+	ag2="$(_create_nvmet_anagroup "${port2}")"
+
+	local remote_traddr1
+	remote_traddr1=$(printf "nn-%s:pn-%s" "${remote_wwnn1}" "${remote_wwpn1}")
+	echo fc > /sys/kernel/config/nvmet/ports/${port1}/addr_adrfam
+	echo "${remote_traddr1}" > /sys/kernel/config/nvmet/ports/${port1}/addr_traddr
+
+	local remote_traddr2
+	remote_traddr2=$(printf "nn-%s:pn-%s" "${remote_wwnn2}" "${remote_wwpn2}")
+	echo fc > /sys/kernel/config/nvmet/ports/${port2}/addr_adrfam
+	echo "${remote_traddr2}" > /sys/kernel/config/nvmet/ports/${port2}/addr_traddr
+
+	truncate -s 1G "$TMPDIR/img"
+
+	local loop_dev
+	loop_dev="$(losetup -f --show "$TMPDIR/img")"
+
+	_create_nvmet_subsystem "${subsys}" "${loop_dev}" \
+		"91fdba0d-f87b-4c25-b80f-db7be1418b9e" "1"
+	_add_nvmet_subsys_to_port "${port1}" "${subsys}"
+	_add_nvmet_subsys_to_port "${port2}" "${subsys}"
+
+	_set_nvmet_anagroup_state "${port2}" "1" "inaccessible"
+	_set_nvmet_anagroup_state "${port2}" "2" "optimized"
+
+	local host_traddr1
+	host_traddr1=$(printf "nn-%s:pn-%s" "${host_wwnn1}" "${host_wwpn1}")
+
+	nvme connect -t fc -a "${remote_traddr1}"  -w "${host_traddr1}" \
+		-n "${subsys}"
+
+	local host_traddr2
+	host_traddr2=$(printf "nn-%s:pn-%s" "${host_wwnn2}" "${host_wwpn2}")
+
+	nvme connect -t fc -a "${remote_traddr2}"  -w "${host_traddr2}" \
+		-n "${subsys}"
+
+	nvmedev="$(_find_nvme_ns 91fdba0d-f87b-4c25-b80f-db7be1418b9e)"
+
+	# Allow I/O to settle
+	udevadm settle
+
+	nvme disconnect -n "${subsys}"
+
+	_remove_nvmet_subsystem_from_port "${port1}" "${subsys}"
+	_remove_nvmet_subsystem_from_port "${port2}" "${subsys}"
+	_remove_nvmet_subsystem "${subsys}"
+	_remove_nvmet_anagroup "${port1}" "${ag1}"
+	_remove_nvmet_port "${port1}"
+	_remove_nvmet_anagroup "${port2}" "${ag2}"
+	_remove_nvmet_port "${port2}"
+
+	losetup -d "$loop_dev"
+	rm "$TMPDIR/img"
+
+	_nvme_fcloop_del_rport "${host_wwnn1}" "${host_wwpn1}" \
+		"${remote_wwnn1}" "${remote_wwpn1}"
+	_nvme_fcloop_del_rport "${host_wwnn2}" "${host_wwpn2}" \
+		"${remote_wwnn2}" "${remote_wwpn2}"
+
+	_nvme_fcloop_del_tport "${remote_wwnn1}" "${remote_wwpn1}"
+	_nvme_fcloop_del_tport "${remote_wwnn2}" "${remote_wwpn2}"
+
+	_nvme_fcloop_del_lport "${host_wwnn1}" "${host_wwpn1}"
+	_nvme_fcloop_del_lport "${host_wwnn2}" "${host_wwpn2}"
+
+	modprobe -r nvme-fcloop
+	modprobe -r nvmet-fc
+	modprobe -r nvmet
+	modprobe -r loop
+
+	echo "Test complete"
+}
diff --git a/tests/nvme/029.out b/tests/nvme/029.out
new file mode 100644
index 0000000..745f142
--- /dev/null
+++ b/tests/nvme/029.out
@@ -0,0 +1,2 @@
+Running nvme/029
+Test complete
diff --git a/tests/nvme/rc b/tests/nvme/rc
index aaf438b..3b3d852 100644
--- a/tests/nvme/rc
+++ b/tests/nvme/rc
@@ -201,3 +201,55 @@ _find_nvme_ns() {
 		fi
 	done
 }
+
+_nvme_fcloop_add_rport() {
+	local local_wwnn="$1"
+	local local_wwpn="$2"
+	local remote_wwnn="$3"
+	local remote_wwpn="$4"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn},lpwwnn=${local_wwnn},lpwwpn=${local_wwpn},roles=0x60" > ${loopctl}/add_remote_port
+}
+
+_nvme_fcloop_del_rport() {
+	local local_wwnn="$1"
+	local local_wwpn="$2"
+	local remote_wwnn="$3"
+	local remote_wwpn="$4"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${remote_wwnn},wwpn=${remote_wwpn}" > ${loopctl}/del_remote_port
+}
+
+_nvme_fcloop_add_lport() {
+	local wwnn="$1"
+	local wwpn="$2"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_local_port
+}
+
+_nvme_fcloop_del_lport() {
+	local wwnn="$1"
+	local wwpn="$2"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/del_local_port
+}
+
+_nvme_fcloop_add_tport() {
+	local wwnn="$1"
+	local wwpn="$2"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/add_target_port
+}
+
+_nvme_fcloop_del_tport() {
+	local wwnn="$1"
+	local wwpn="$2"
+	local loopctl=/sys/class/fcloop/ctl
+
+	echo "wwnn=${wwnn},wwpn=${wwpn}" > ${loopctl}/del_target_port
+}
-- 
2.13.7

[PATCH 3/3] nvme: Add test for fcloop

[Omar Sandoval]

On Mon, Oct 15, 2018@09:01:30AM +0200, Hannes Reinecke wrote:
> Signed-off-by: Hannes Reinecke <hare at suse.com>
> ---
>  tests/nvme/029     | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>  tests/nvme/029.out |   2 +
>  tests/nvme/rc      |  52 ++++++++++++++++++++++
>  3 files changed, 177 insertions(+)
>  create mode 100755 tests/nvme/029
>  create mode 100644 tests/nvme/029.out
> 
> diff --git a/tests/nvme/029 b/tests/nvme/029
> new file mode 100755
> index 0000000..f8ed9b5
> --- /dev/null
> +++ b/tests/nvme/029
> @@ -0,0 +1,123 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-2.0+
> +# Copyright (C) 2018 Johannes Thumshirn, Hannes Reinecke, SUSE Linux GmbH
> +#
> +# Reproducer for nvme-fcloop
> +#
> +
> +. tests/nvme/rc
> +
> +DESCRIPTION="create nvme-fcloop with two ports and connect/disconnect"
> +
> +requires() {
> +	_have_module nvme-fcloop && _have_module nvmet && _have_module loop && \
> +		_have_configfs && _have_module nvme-core && \
> +		_have_module nvmet-fc
> +}
> +
> +test() {
> +	local subsys="blktests-subsystem-1"
> +	local remote_wwnn1="0x100140111111dbcc"
> +	local remote_wwpn1="0x200140111111dbcc"
> +	local remote_wwnn2="0x100140111111dbcd"
> +	local remote_wwpn2="0x200140111111dbcd"
> +	local host_wwnn1="0x100140111111dac8"
> +	local host_wwpn1="0x200140111111dac8"
> +	local host_wwnn2="0x100140111111dac9"
> +	local host_wwpn2="0x200140111111dac9"
> +
> +	echo "Running ${TEST_NAME}"
> +
> +	modprobe nvmet-fc
> +	modprobe nvme-fcloop
> +
> +	_nvme_fcloop_add_tport ${remote_wwnn1} ${remote_wwpn1}
> +	_nvme_fcloop_add_tport ${remote_wwnn2} ${remote_wwpn2}
> +
> +	_nvme_fcloop_add_lport ${host_wwnn1} ${host_wwpn1}
> +	_nvme_fcloop_add_lport ${host_wwnn2} ${host_wwpn2}
> +
> +	_nvme_fcloop_add_rport ${host_wwnn1} ${host_wwpn1} \
> +		${remote_wwnn1} ${remote_wwpn1}
> +	_nvme_fcloop_add_rport ${host_wwnn2} ${host_wwpn2} \
> +		${remote_wwnn2} ${remote_wwpn2}
> +
> +	local port1
> +	port1=$(_create_nvmet_port "fc")
> +	ag1="$(_create_nvmet_anagroup "${port1}")"
> +
> +	local port2
> +	port2=$(_create_nvmet_port "fc")
> +	ag2="$(_create_nvmet_anagroup "${port2}")"
> +
> +	local remote_traddr1
> +	remote_traddr1=$(printf "nn-%s:pn-%s" "${remote_wwnn1}" "${remote_wwpn1}")
> +	echo fc > /sys/kernel/config/nvmet/ports/${port1}/addr_adrfam
> +	echo "${remote_traddr1}" > /sys/kernel/config/nvmet/ports/${port1}/addr_traddr
> +
> +	local remote_traddr2
> +	remote_traddr2=$(printf "nn-%s:pn-%s" "${remote_wwnn2}" "${remote_wwpn2}")
> +	echo fc > /sys/kernel/config/nvmet/ports/${port2}/addr_adrfam
> +	echo "${remote_traddr2}" > /sys/kernel/config/nvmet/ports/${port2}/addr_traddr
> +
> +	truncate -s 1G "$TMPDIR/img"
> +
> +	local loop_dev
> +	loop_dev="$(losetup -f --show "$TMPDIR/img")"
> +
> +	_create_nvmet_subsystem "${subsys}" "${loop_dev}" \
> +		"91fdba0d-f87b-4c25-b80f-db7be1418b9e" "1"
> +	_add_nvmet_subsys_to_port "${port1}" "${subsys}"
> +	_add_nvmet_subsys_to_port "${port2}" "${subsys}"
> +
> +	_set_nvmet_anagroup_state "${port2}" "1" "inaccessible"
> +	_set_nvmet_anagroup_state "${port2}" "2" "optimized"
> +
> +	local host_traddr1
> +	host_traddr1=$(printf "nn-%s:pn-%s" "${host_wwnn1}" "${host_wwpn1}")
> +
> +	nvme connect -t fc -a "${remote_traddr1}"  -w "${host_traddr1}" \
> +		-n "${subsys}"
> +
> +	local host_traddr2
> +	host_traddr2=$(printf "nn-%s:pn-%s" "${host_wwnn2}" "${host_wwpn2}")
> +
> +	nvme connect -t fc -a "${remote_traddr2}"  -w "${host_traddr2}" \
> +		-n "${subsys}"
> +
> +	nvmedev="$(_find_nvme_ns 91fdba0d-f87b-4c25-b80f-db7be1418b9e)"

nvmedev doesn't seem to be used anywhere (shellcheck warns about this).

> +	# Allow I/O to settle
> +	udevadm settle
> +
> +	nvme disconnect -n "${subsys}"

This outputs the following with my version (1.6) of nvme:

NQN:blktests-subsystem-1 disconnected 2 controller(s)

> +	_remove_nvmet_subsystem_from_port "${port1}" "${subsys}"
> +	_remove_nvmet_subsystem_from_port "${port2}" "${subsys}"
> +	_remove_nvmet_subsystem "${subsys}"
> +	_remove_nvmet_anagroup "${port1}" "${ag1}"
> +	_remove_nvmet_port "${port1}"
> +	_remove_nvmet_anagroup "${port2}" "${ag2}"
> +	_remove_nvmet_port "${port2}"
> +
> +	losetup -d "$loop_dev"
> +	rm "$TMPDIR/img"
> +
> +	_nvme_fcloop_del_rport "${host_wwnn1}" "${host_wwpn1}" \
> +		"${remote_wwnn1}" "${remote_wwpn1}"
> +	_nvme_fcloop_del_rport "${host_wwnn2}" "${host_wwpn2}" \
> +		"${remote_wwnn2}" "${remote_wwpn2}"
> +
> +	_nvme_fcloop_del_tport "${remote_wwnn1}" "${remote_wwpn1}"
> +	_nvme_fcloop_del_tport "${remote_wwnn2}" "${remote_wwpn2}"
> +
> +	_nvme_fcloop_del_lport "${host_wwnn1}" "${host_wwpn1}"
> +	_nvme_fcloop_del_lport "${host_wwnn2}" "${host_wwpn2}"
> +
> +	modprobe -r nvme-fcloop
> +	modprobe -r nvmet-fc
> +	modprobe -r nvmet
> +	modprobe -r loop

We don't want to try to remove loop, that's a generic enough module that
it might be in use elsewhere on the system.

[PATCH 3/3] nvme: Add test for fcloop

[Hannes Reinecke]

On 10/15/18 11:17 PM, Omar Sandoval wrote:
> On Mon, Oct 15, 2018@09:01:30AM +0200, Hannes Reinecke wrote:
>> Signed-off-by: Hannes Reinecke <hare at suse.com>
>> ---
>>   tests/nvme/029     | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++
>>   tests/nvme/029.out |   2 +
>>   tests/nvme/rc      |  52 ++++++++++++++++++++++
>>   3 files changed, 177 insertions(+)
>>   create mode 100755 tests/nvme/029
>>   create mode 100644 tests/nvme/029.out
>>
>> diff --git a/tests/nvme/029 b/tests/nvme/029
>> new file mode 100755
>> index 0000000..f8ed9b5
>> --- /dev/null
>> +++ b/tests/nvme/029
>> @@ -0,0 +1,123 @@
>> +#!/bin/bash
>> +# SPDX-License-Identifier: GPL-2.0+
>> +# Copyright (C) 2018 Johannes Thumshirn, Hannes Reinecke, SUSE Linux GmbH
>> +#
>> +# Reproducer for nvme-fcloop
>> +#
>> +
>> +. tests/nvme/rc
>> +
>> +DESCRIPTION="create nvme-fcloop with two ports and connect/disconnect"
>> +
>> +requires() {
>> +	_have_module nvme-fcloop && _have_module nvmet && _have_module loop && \
>> +		_have_configfs && _have_module nvme-core && \
>> +		_have_module nvmet-fc
>> +}
>> +
>> +test() {
>> +	local subsys="blktests-subsystem-1"
>> +	local remote_wwnn1="0x100140111111dbcc"
>> +	local remote_wwpn1="0x200140111111dbcc"
>> +	local remote_wwnn2="0x100140111111dbcd"
>> +	local remote_wwpn2="0x200140111111dbcd"
>> +	local host_wwnn1="0x100140111111dac8"
>> +	local host_wwpn1="0x200140111111dac8"
>> +	local host_wwnn2="0x100140111111dac9"
>> +	local host_wwpn2="0x200140111111dac9"
>> +
>> +	echo "Running ${TEST_NAME}"
>> +
>> +	modprobe nvmet-fc
>> +	modprobe nvme-fcloop
>> +
>> +	_nvme_fcloop_add_tport ${remote_wwnn1} ${remote_wwpn1}
>> +	_nvme_fcloop_add_tport ${remote_wwnn2} ${remote_wwpn2}
>> +
>> +	_nvme_fcloop_add_lport ${host_wwnn1} ${host_wwpn1}
>> +	_nvme_fcloop_add_lport ${host_wwnn2} ${host_wwpn2}
>> +
>> +	_nvme_fcloop_add_rport ${host_wwnn1} ${host_wwpn1} \
>> +		${remote_wwnn1} ${remote_wwpn1}
>> +	_nvme_fcloop_add_rport ${host_wwnn2} ${host_wwpn2} \
>> +		${remote_wwnn2} ${remote_wwpn2}
>> +
>> +	local port1
>> +	port1=$(_create_nvmet_port "fc")
>> +	ag1="$(_create_nvmet_anagroup "${port1}")"
>> +
>> +	local port2
>> +	port2=$(_create_nvmet_port "fc")
>> +	ag2="$(_create_nvmet_anagroup "${port2}")"
>> +
>> +	local remote_traddr1
>> +	remote_traddr1=$(printf "nn-%s:pn-%s" "${remote_wwnn1}" "${remote_wwpn1}")
>> +	echo fc > /sys/kernel/config/nvmet/ports/${port1}/addr_adrfam
>> +	echo "${remote_traddr1}" > /sys/kernel/config/nvmet/ports/${port1}/addr_traddr
>> +
>> +	local remote_traddr2
>> +	remote_traddr2=$(printf "nn-%s:pn-%s" "${remote_wwnn2}" "${remote_wwpn2}")
>> +	echo fc > /sys/kernel/config/nvmet/ports/${port2}/addr_adrfam
>> +	echo "${remote_traddr2}" > /sys/kernel/config/nvmet/ports/${port2}/addr_traddr
>> +
>> +	truncate -s 1G "$TMPDIR/img"
>> +
>> +	local loop_dev
>> +	loop_dev="$(losetup -f --show "$TMPDIR/img")"
>> +
>> +	_create_nvmet_subsystem "${subsys}" "${loop_dev}" \
>> +		"91fdba0d-f87b-4c25-b80f-db7be1418b9e" "1"
>> +	_add_nvmet_subsys_to_port "${port1}" "${subsys}"
>> +	_add_nvmet_subsys_to_port "${port2}" "${subsys}"
>> +
>> +	_set_nvmet_anagroup_state "${port2}" "1" "inaccessible"
>> +	_set_nvmet_anagroup_state "${port2}" "2" "optimized"
>> +
>> +	local host_traddr1
>> +	host_traddr1=$(printf "nn-%s:pn-%s" "${host_wwnn1}" "${host_wwpn1}")
>> +
>> +	nvme connect -t fc -a "${remote_traddr1}"  -w "${host_traddr1}" \
>> +		-n "${subsys}"
>> +
>> +	local host_traddr2
>> +	host_traddr2=$(printf "nn-%s:pn-%s" "${host_wwnn2}" "${host_wwpn2}")
>> +
>> +	nvme connect -t fc -a "${remote_traddr2}"  -w "${host_traddr2}" \
>> +		-n "${subsys}"
>> +
>> +	nvmedev="$(_find_nvme_ns 91fdba0d-f87b-4c25-b80f-db7be1418b9e)"
> 
> nvmedev doesn't seem to be used anywhere (shellcheck warns about this).
> 
Okay, will be removing the variable

>> +	# Allow I/O to settle
>> +	udevadm settle
>> +
>> +	nvme disconnect -n "${subsys}"
> 
> This outputs the following with my version (1.6) of nvme:
> 
> NQN:blktests-subsystem-1 disconnected 2 controller(s)
> 
And? That's what I expected, so where's the rub?

>> +	_remove_nvmet_subsystem_from_port "${port1}" "${subsys}"
>> +	_remove_nvmet_subsystem_from_port "${port2}" "${subsys}"
>> +	_remove_nvmet_subsystem "${subsys}"
>> +	_remove_nvmet_anagroup "${port1}" "${ag1}"
>> +	_remove_nvmet_port "${port1}"
>> +	_remove_nvmet_anagroup "${port2}" "${ag2}"
>> +	_remove_nvmet_port "${port2}"
>> +
>> +	losetup -d "$loop_dev"
>> +	rm "$TMPDIR/img"
>> +
>> +	_nvme_fcloop_del_rport "${host_wwnn1}" "${host_wwpn1}" \
>> +		"${remote_wwnn1}" "${remote_wwpn1}"
>> +	_nvme_fcloop_del_rport "${host_wwnn2}" "${host_wwpn2}" \
>> +		"${remote_wwnn2}" "${remote_wwpn2}"
>> +
>> +	_nvme_fcloop_del_tport "${remote_wwnn1}" "${remote_wwpn1}"
>> +	_nvme_fcloop_del_tport "${remote_wwnn2}" "${remote_wwpn2}"
>> +
>> +	_nvme_fcloop_del_lport "${host_wwnn1}" "${host_wwpn1}"
>> +	_nvme_fcloop_del_lport "${host_wwnn2}" "${host_wwpn2}"
>> +
>> +	modprobe -r nvme-fcloop
>> +	modprobe -r nvmet-fc
>> +	modprobe -r nvmet
>> +	modprobe -r loop
> 
> We don't want to try to remove loop, that's a generic enough module that
> it might be in use elsewhere on the system.
> 
Good point.

Will be fixing it up.

Cheers,

Hannes
-- 
Dr. Hannes Reinecke		   Teamlead Storage & Networking
hare at suse.de			               +49 911 74053 688
SUSE LINUX GmbH, Maxfeldstr. 5, 90409 N?rnberg
GF: F. Imend?rffer, J. Smithard, J. Guild, D. Upmanyu, G. Norton
HRB 21284 (AG N?rnberg)

[PATCH 3/3] nvme: Add test for fcloop

[Omar Sandoval]

On Tue, Oct 16, 2018@07:50:37AM +0200, Hannes Reinecke wrote:
> On 10/15/18 11:17 PM, Omar Sandoval wrote:
> > On Mon, Oct 15, 2018@09:01:30AM +0200, Hannes Reinecke wrote:
> > > Signed-off-by: Hannes Reinecke <hare at suse.com>
> > > ---
> > >   tests/nvme/029     | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++
> > >   tests/nvme/029.out |   2 +
> > >   tests/nvme/rc      |  52 ++++++++++++++++++++++
> > >   3 files changed, 177 insertions(+)
> > >   create mode 100755 tests/nvme/029
> > >   create mode 100644 tests/nvme/029.out
> > > 
> > > diff --git a/tests/nvme/029 b/tests/nvme/029
> > > new file mode 100755
> > > index 0000000..f8ed9b5
> > > --- /dev/null
> > > +++ b/tests/nvme/029
> > > @@ -0,0 +1,123 @@
> > > +#!/bin/bash
> > > +# SPDX-License-Identifier: GPL-2.0+
> > > +# Copyright (C) 2018 Johannes Thumshirn, Hannes Reinecke, SUSE Linux GmbH
> > > +#
> > > +# Reproducer for nvme-fcloop
> > > +#
> > > +
> > > +. tests/nvme/rc
> > > +
> > > +DESCRIPTION="create nvme-fcloop with two ports and connect/disconnect"
> > > +
> > > +requires() {
> > > +	_have_module nvme-fcloop && _have_module nvmet && _have_module loop && \
> > > +		_have_configfs && _have_module nvme-core && \
> > > +		_have_module nvmet-fc
> > > +}
> > > +
> > > +test() {
> > > +	local subsys="blktests-subsystem-1"
> > > +	local remote_wwnn1="0x100140111111dbcc"
> > > +	local remote_wwpn1="0x200140111111dbcc"
> > > +	local remote_wwnn2="0x100140111111dbcd"
> > > +	local remote_wwpn2="0x200140111111dbcd"
> > > +	local host_wwnn1="0x100140111111dac8"
> > > +	local host_wwpn1="0x200140111111dac8"
> > > +	local host_wwnn2="0x100140111111dac9"
> > > +	local host_wwpn2="0x200140111111dac9"
> > > +
> > > +	echo "Running ${TEST_NAME}"
> > > +
> > > +	modprobe nvmet-fc
> > > +	modprobe nvme-fcloop
> > > +
> > > +	_nvme_fcloop_add_tport ${remote_wwnn1} ${remote_wwpn1}
> > > +	_nvme_fcloop_add_tport ${remote_wwnn2} ${remote_wwpn2}
> > > +
> > > +	_nvme_fcloop_add_lport ${host_wwnn1} ${host_wwpn1}
> > > +	_nvme_fcloop_add_lport ${host_wwnn2} ${host_wwpn2}
> > > +
> > > +	_nvme_fcloop_add_rport ${host_wwnn1} ${host_wwpn1} \
> > > +		${remote_wwnn1} ${remote_wwpn1}
> > > +	_nvme_fcloop_add_rport ${host_wwnn2} ${host_wwpn2} \
> > > +		${remote_wwnn2} ${remote_wwpn2}
> > > +
> > > +	local port1
> > > +	port1=$(_create_nvmet_port "fc")
> > > +	ag1="$(_create_nvmet_anagroup "${port1}")"
> > > +
> > > +	local port2
> > > +	port2=$(_create_nvmet_port "fc")
> > > +	ag2="$(_create_nvmet_anagroup "${port2}")"
> > > +
> > > +	local remote_traddr1
> > > +	remote_traddr1=$(printf "nn-%s:pn-%s" "${remote_wwnn1}" "${remote_wwpn1}")
> > > +	echo fc > /sys/kernel/config/nvmet/ports/${port1}/addr_adrfam
> > > +	echo "${remote_traddr1}" > /sys/kernel/config/nvmet/ports/${port1}/addr_traddr
> > > +
> > > +	local remote_traddr2
> > > +	remote_traddr2=$(printf "nn-%s:pn-%s" "${remote_wwnn2}" "${remote_wwpn2}")
> > > +	echo fc > /sys/kernel/config/nvmet/ports/${port2}/addr_adrfam
> > > +	echo "${remote_traddr2}" > /sys/kernel/config/nvmet/ports/${port2}/addr_traddr
> > > +
> > > +	truncate -s 1G "$TMPDIR/img"
> > > +
> > > +	local loop_dev
> > > +	loop_dev="$(losetup -f --show "$TMPDIR/img")"
> > > +
> > > +	_create_nvmet_subsystem "${subsys}" "${loop_dev}" \
> > > +		"91fdba0d-f87b-4c25-b80f-db7be1418b9e" "1"
> > > +	_add_nvmet_subsys_to_port "${port1}" "${subsys}"
> > > +	_add_nvmet_subsys_to_port "${port2}" "${subsys}"
> > > +
> > > +	_set_nvmet_anagroup_state "${port2}" "1" "inaccessible"
> > > +	_set_nvmet_anagroup_state "${port2}" "2" "optimized"
> > > +
> > > +	local host_traddr1
> > > +	host_traddr1=$(printf "nn-%s:pn-%s" "${host_wwnn1}" "${host_wwpn1}")
> > > +
> > > +	nvme connect -t fc -a "${remote_traddr1}"  -w "${host_traddr1}" \
> > > +		-n "${subsys}"
> > > +
> > > +	local host_traddr2
> > > +	host_traddr2=$(printf "nn-%s:pn-%s" "${host_wwnn2}" "${host_wwpn2}")
> > > +
> > > +	nvme connect -t fc -a "${remote_traddr2}"  -w "${host_traddr2}" \
> > > +		-n "${subsys}"
> > > +
> > > +	nvmedev="$(_find_nvme_ns 91fdba0d-f87b-4c25-b80f-db7be1418b9e)"
> > 
> > nvmedev doesn't seem to be used anywhere (shellcheck warns about this).
> > 
> Okay, will be removing the variable
> 
> > > +	# Allow I/O to settle
> > > +	udevadm settle
> > > +
> > > +	nvme disconnect -n "${subsys}"
> > 
> > This outputs the following with my version (1.6) of nvme:
> > 
> > NQN:blktests-subsystem-1 disconnected 2 controller(s)
> > 
> And? That's what I expected, so where's the rub?

The expected test output does not include it.

[PATCH blktests 0/3] ANA and fcloop tests

[Hannes Reinecke]

Hi Omar,

here are two small tests which have been lingering in my repository
for far too long, so I figured it's time to finally post them.
The first is a rather trivial one for testing ANA states, the second
finally adds a test case for the FC Loop driver.
It also serves as a template on how to actually _use_ the FC Loop
driver itself :-)

However, that one uncovered a use-after-free issue in the NVMe-FC
stack; check the 'Fixme' line in the test. As it _might_ be a generic
issue and not something specific to fcloop I'm posting it now to
allow other people (Hi James!) to reproduce and possibly fix it.

As usual, comments and reviews are welcome.

Hannes Reinecke (3):
  nvme: enable ANA support
  nvme/033: add test for ANA state transition
  tests/nvme/034: Add a test for the fcloop driver

 common/fcloop      |  58 ++++++++++++++++++++++
 tests/nvme/033     | 139 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/033.out |  18 +++++++
 tests/nvme/034     | 122 ++++++++++++++++++++++++++++++++++++++++++++++
 tests/nvme/034.out |   3 ++
 tests/nvme/rc      |  88 +++++++++++++++++++++++++++++++--
 6 files changed, 425 insertions(+), 3 deletions(-)
 create mode 100644 common/fcloop
 create mode 100644 tests/nvme/033
 create mode 100644 tests/nvme/033.out
 create mode 100644 tests/nvme/034
 create mode 100644 tests/nvme/034.out

-- 
2.13.7

[PATCH 1/3] nvme: enable ANA support

[Hannes Reinecke]

Add ANA support functions and update _create_nvmet_ns() to
accept an additional ANA grpid parameter.

Signed-off-by: Hannes Reinecke <hare@suse.com>
---
 tests/nvme/rc | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 69 insertions(+), 3 deletions(-)

diff --git a/tests/nvme/rc b/tests/nvme/rc
index 40f0413..72e33c1 100644
--- a/tests/nvme/rc
+++ b/tests/nvme/rc
@@ -14,7 +14,7 @@ group_device_requires() {
 	_test_dev_is_nvme
 }
 
-NVMET_CFS="/sys/kernel/config/nvmet/"
+NVMET_CFS="/sys/kernel/config/nvmet"
 
 _test_dev_is_nvme() {
 	if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then
@@ -90,6 +90,7 @@ _create_nvmet_port() {
 
 	mkdir "${NVMET_CFS}/ports/${port}"
 	echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
+	echo "${port}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
 
 	echo "${port}"
 }
@@ -99,6 +100,62 @@ _remove_nvmet_port() {
 	rmdir "${NVMET_CFS}/ports/${port}"
 }
 
+_create_nvmet_anagroup() {
+	local port="$1"
+	local port_cfs="${NVMET_CFS}/ports/${port}"
+	local grpid
+
+	for ((grpid = 1; ; grpid++)); do
+		if [[ ! -e "${port_cfs}/ana_groups/${grpid}" ]]; then
+			break
+		fi
+	done
+
+	mkdir "${port_cfs}/ana_groups/${grpid}"
+
+	echo "${grpid}"
+}
+
+_remove_nvmet_anagroup() {
+	local port="$1"
+	local grpid="$2"
+	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
+
+	rmdir "${ana_cfs}"
+}
+
+_set_nvmet_anagroup_state() {
+	local port="$1"
+	local grpid="$2"
+	local state="$3"
+	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
+
+	echo "${state}" > "${ana_cfs}/ana_state"
+}
+
+_display_ana_state() {
+	local nvmet_subsystem="$1"
+	local grpid state cntlid
+
+	for nvme in /sys/class/nvme/* ; do
+		[ -f ${nvme}/subsysnqn ] || continue
+		subsys="$(cat "${nvme}/subsysnqn")"
+		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
+			continue
+		fi
+		cntlid="$(cat "${nvme}/cntlid")"
+		for c in ${nvme}/nvme* ; do
+			if [ ! -d ${c} ] ; then
+				echo "${cntlid}: ANA disabled"
+				continue
+			fi
+			grpid="$(cat "${c}/ana_grpid")"
+			state="$(cat "${c}/ana_state")"
+			echo "${cntlid}: grpid ${grpid} state ${state}"
+		done
+	done
+}
+
 _create_nvmet_ns() {
 	local nvmet_subsystem="$1"
 	local nsid="$2"
@@ -106,14 +163,22 @@ _create_nvmet_ns() {
 	local uuid="00000000-0000-0000-0000-000000000000"
 	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
 	local ns_path="${subsys_path}/namespaces/${nsid}"
+	local ana_grpid
 
-	if [[ $# -eq 4 ]]; then
+	if [[ $# -ge 4 ]]; then
 		uuid="$4"
 	fi
 
+	if [[ $# -eq 5 ]]; then
+		ana_grpid="$5"
+	fi
+
 	mkdir "${ns_path}"
 	printf "%s" "${blkdev}" > "${ns_path}/device_path"
 	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
+	if [ -n "${ana_grpid}" ] ; then
+		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
+	fi
 	printf 1 > "${ns_path}/enable"
 }
 
@@ -121,11 +186,12 @@ _create_nvmet_subsystem() {
 	local nvmet_subsystem="$1"
 	local blkdev="$2"
 	local uuid=$3
+	local ana_grpid=$4
 	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
 
 	mkdir -p "${cfs_path}"
 	echo 1 > "${cfs_path}/attr_allow_any_host"
-	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
+	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" "${ana_grpid}"
 }
 
 _remove_nvmet_ns() {
-- 
2.13.7

[PATCH 1/3] nvme: enable ANA support

[Hannes Reinecke]

Add ANA support functions and update _create_nvmet_ns() to
accept an additional ANA grpid parameter.

Signed-off-by: Hannes Reinecke <hare@suse.com>
---
 tests/nvme/rc | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 69 insertions(+), 3 deletions(-)

diff --git a/tests/nvme/rc b/tests/nvme/rc
index 40f0413..72e33c1 100644
--- a/tests/nvme/rc
+++ b/tests/nvme/rc
@@ -14,7 +14,7 @@ group_device_requires() {
 	_test_dev_is_nvme
 }
 
-NVMET_CFS="/sys/kernel/config/nvmet/"
+NVMET_CFS="/sys/kernel/config/nvmet"
 
 _test_dev_is_nvme() {
 	if ! readlink -f "$TEST_DEV_SYSFS/device" | grep -q nvme; then
@@ -90,6 +90,7 @@ _create_nvmet_port() {
 
 	mkdir "${NVMET_CFS}/ports/${port}"
 	echo "${trtype}" > "${NVMET_CFS}/ports/${port}/addr_trtype"
+	echo "${port}" > "${NVMET_CFS}/ports/${port}/addr_traddr"
 
 	echo "${port}"
 }
@@ -99,6 +100,62 @@ _remove_nvmet_port() {
 	rmdir "${NVMET_CFS}/ports/${port}"
 }
 
+_create_nvmet_anagroup() {
+	local port="$1"
+	local port_cfs="${NVMET_CFS}/ports/${port}"
+	local grpid
+
+	for ((grpid = 1; ; grpid++)); do
+		if [[ ! -e "${port_cfs}/ana_groups/${grpid}" ]]; then
+			break
+		fi
+	done
+
+	mkdir "${port_cfs}/ana_groups/${grpid}"
+
+	echo "${grpid}"
+}
+
+_remove_nvmet_anagroup() {
+	local port="$1"
+	local grpid="$2"
+	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
+
+	rmdir "${ana_cfs}"
+}
+
+_set_nvmet_anagroup_state() {
+	local port="$1"
+	local grpid="$2"
+	local state="$3"
+	local ana_cfs="${NVMET_CFS}/ports/${port}/ana_groups/${grpid}"
+
+	echo "${state}" > "${ana_cfs}/ana_state"
+}
+
+_display_ana_state() {
+	local nvmet_subsystem="$1"
+	local grpid state cntlid
+
+	for nvme in /sys/class/nvme/* ; do
+		[ -f ${nvme}/subsysnqn ] || continue
+		subsys="$(cat "${nvme}/subsysnqn")"
+		if [ "${subsys}" != "${nvmet_subsystem}" ] ; then
+			continue
+		fi
+		cntlid="$(cat "${nvme}/cntlid")"
+		for c in ${nvme}/nvme* ; do
+			if [ ! -d ${c} ] ; then
+				echo "${cntlid}: ANA disabled"
+				continue
+			fi
+			grpid="$(cat "${c}/ana_grpid")"
+			state="$(cat "${c}/ana_state")"
+			echo "${cntlid}: grpid ${grpid} state ${state}"
+		done
+	done
+}
+
 _create_nvmet_ns() {
 	local nvmet_subsystem="$1"
 	local nsid="$2"
@@ -106,14 +163,22 @@ _create_nvmet_ns() {
 	local uuid="00000000-0000-0000-0000-000000000000"
 	local subsys_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
 	local ns_path="${subsys_path}/namespaces/${nsid}"
+	local ana_grpid
 
-	if [[ $# -eq 4 ]]; then
+	if [[ $# -ge 4 ]]; then
 		uuid="$4"
 	fi
 
+	if [[ $# -eq 5 ]]; then
+		ana_grpid="$5"
+	fi
+
 	mkdir "${ns_path}"
 	printf "%s" "${blkdev}" > "${ns_path}/device_path"
 	printf "%s" "${uuid}" > "${ns_path}/device_uuid"
+	if [ -n "${ana_grpid}" ] ; then
+		printf "%s" "${ana_grpid}" > "${ns_path}/ana_grpid"
+	fi
 	printf 1 > "${ns_path}/enable"
 }
 
@@ -121,11 +186,12 @@ _create_nvmet_subsystem() {
 	local nvmet_subsystem="$1"
 	local blkdev="$2"
 	local uuid=$3
+	local ana_grpid=$4
 	local cfs_path="${NVMET_CFS}/subsystems/${nvmet_subsystem}"
 
 	mkdir -p "${cfs_path}"
 	echo 1 > "${cfs_path}/attr_allow_any_host"
-	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}"
+	_create_nvmet_ns "${nvmet_subsystem}" "1" "${blkdev}" "${uuid}" "${ana_grpid}"
 }
 
 _remove_nvmet_ns() {
-- 
2.13.7


_______________________________________________
linux-nvme mailing list
linux-nvme@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-nvme