Re: [PATCH blktests 2/2] nvme/070: Test multipath and marginal ports

[John Meneghini <jmeneghi@redhat.com>]

Reviewed-by: John Meneghini <jmeneghi@redhat.com>
Tested-by: John Meneghini <jmeneghi@redhat.com>

John A. Meneghini
Senior Principal Platform Storage Engineer
RHEL SST - Platform Storage Group
jmeneghi@redhat.com

On 8/12/26 13:45, Jesse Taube wrote:
> Add tests/nvme/070 to test various multipath and marginal port
> scenarios, while confirming the port useage and state. This test is
> intended to emulate receiving an FPIN event in a multipath environment.
> 
> Suggested-by: John Meneghini <jmeneghi@redhat.com>
> Signed-off-by: Jesse Taube <jtaubepe@redhat.com>
> ---
>   tests/nvme/070     | 521 +++++++++++++++++++++++++++++++++++++++++++++
>   tests/nvme/070.out |  43 ++++
>   2 files changed, 564 insertions(+)
>   create mode 100755 tests/nvme/070
>   create mode 100644 tests/nvme/070.out
> 
> diff --git a/tests/nvme/070 b/tests/nvme/070
> new file mode 100755
> index 0000000..a8002f8
> --- /dev/null
> +++ b/tests/nvme/070
> @@ -0,0 +1,521 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-3.0+
> +# Copyright (C) 2024 John Meneghini <jmeneghi@redhat.com>
> +#
> +# Test nvme-fc marginal path handling with fcloop
> +
> +. tests/nvme/rc
> +
> +DESCRIPTION="test nvme-fc marginal path handling"
> +
> +requires() {
> +	_nvme_requires
> +	_have_loop
> +	_have_fio
> +	_require_nvme_trtype fc
> +}
> +
> +set_conditions() {
> +	_set_nvme_trtype "$@"
> +}
> +
> +_subsys_rport_addr() {
> +	local RPORT=$1 # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	cat "$RPORT"/address
> +	# traddr=nn-0x2047d039ea98949e:pn-0x2048d039ea98949e,host_traddr=nn-0x200000109b9b7e4e:pn-0x100000109b9b7e4e
> +}
> +
> +_subsys_get_port() {
> +	local RPORT=$1 # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +	local port
> +	local address
> +
> +	address=$(_subsys_rport_addr "$RPORT")
> +	port=$(echo "$address" | sed -n 's/.*pn-\(.*\),.*/\1/p')
> +	echo $(( port - $(_remote_wwpn 0) ))
> +	# a number representing the port number, e.g. 0, 1, 2, 3
> +}
> +
> +_rport_set_iopolicy() {
> +	local RPORT=$1 # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +	local POLICY=$2 # numa queue-depth round-robin
> +
> +	echo "$POLICY" | sudo tee "$RPORT"/iopolicy > /dev/null
> +	# Returns none
> +}
> +
> +_rport_set_marginal() {
> +	local RPORT=$1 # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	_setup_nvmet_port_marginal "$(_subsys_get_port "$RPORT")" "marginal"
> +}
> +
> +_rport_set_online() {
> +	local RPORT=$1 # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	_setup_nvmet_port_marginal "$(_subsys_get_port "$RPORT")" "live"
> +}
> +
> +_rport_is_online() {
> +	local RPORT=$1 # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	[[ "$(cat "$RPORT"/state)" == "live" ]]
> +	# Returns Success or Fail
> +}
> +
> +_rport_is_marginal() {
> +	local RPORT=$1 # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	[[ "$(cat "$RPORT"/state)" == "marginal" ]]
> +	# Returns Success or Fail
> +}
> +
> +_rport_in_use() {
> +	local SUBSYS_PATH=$1 # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	# there should only be one "nvme3c9n1" afaik
> +	[[ "$(cat "$SUBSYS_PATH"/nvme*/stat | awk '{print $9}')" != "0" ]]
> +	# Returns exit code 0 if in use, 1 if not in use
> +}
> +
> +_rport_optimized() {
> +	local SUBSYS_PATH=$1 # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	# there should only be one "nvme3c9n1" afaik
> +	[[ "$(cat "$SUBSYS_PATH"/nvme*/ana_state)" == "optimized" ]]
> +	# Returns exit code 0 if optimized, 1 if not optimized
> +}
> +
> +# check if link is marginal or not
> +_rport_check_online() {
> +	local RPORT=$1 # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +	local STATE=$2 # "Online"
> +
> +	if [[ "$STATE" == "Online" ]]; then
> +		if ! _rport_is_online "$RPORT"; then
> +			echo  FC port \("$RPORT"\) is not online, expteced online.
> +			return 1
> +		fi
> +	else
> +		if ! _rport_is_marginal "$RPORT"; then
> +			echo FC port \("$RPORT"\) is not marginal, expteced marginal.
> +			return 1
> +		fi
> +	fi
> +	# Returns Success or Fail
> +}
> +
> +# check if link is in use
> +_rport_check_use() {
> +	local RPORT=$1 # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +	local STATE=$2 # "Online"
> +
> +	if [[ "$STATE" == "Online" ]]; then
> +		if ! _rport_in_use "$RPORT" ; then
> +			echo FC port on \("$RPORT"\) is not being used, expected use.
> +			return 1
> +		fi
> +	else
> +		if _rport_in_use "$RPORT" ; then
> +			echo FC port on \("$RPORT"\) is being used, expected no use.
> +			return 1
> +		fi
> +	fi
> +	# Returns Success or Fail
> +}
> +
> +# check if all link is in state and use state if optimized
> +_rport_check_opt() {
> +	local RPORT=$1 # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +	local STATE=$2 # "Online"
> +
> +	_rport_check_online "$RPORT" "$STATE"
> +	# Only optimized paths will be in use
> +	if _rport_optimized "$RPORT"; then
> +		_rport_check_use "$RPORT" "$STATE"
> +	else
> +		_rport_check_use "$RPORT" Marginal
> +	fi
> +	# Returns Success or Fail
> +}
> +
> +# check if all link is in state and use state
> +_rport_check() {
> +	local RPORT=$1 # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +	local STATE=$2 # "Online"
> +
> +	_rport_check_online "$RPORT" "$STATE"
> +	_rport_check_use "$RPORT" "$STATE"
> +	# Returns Success or Fail
> +}
> +
> +# Check if one of the marginal paths is in use
> +_rport_check_one_use_online() {
> +	local RPORTS_PATHS=("$@") #  /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	# Check if one of the online paths is in use
> +	for subsys_path in "${RPORTS_PATHS[@]}"; do
> +		# If path is Marginal, continue to the next path
> +		_rport_check_online "$subsys_path" Online || continue
> +		# One of the online paths should be in use
> +		# "!" inverts the return code, so returns false if path is in use,
> +		# running the return statement
> +		! _rport_in_use "$subsys_path" || return 0
> +	done
> +
> +	echo No FC ports are being used, expected atleast one in use when all are online in numa mode.
> +	# None of the online paths were in use
> +	return 1
> +
> +	# Returns Success or Fail
> +}
> +
> +test_set_all_online() {
> +	local IOPOLICY=$1 # "numa" "queue-depth" "round-robin"
> +	shift
> +	local -a RPORTS_PATHS=("$@") # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	for subsys_path in "${RPORTS_PATHS[@]}"; do
> +		_rport_set_online "$subsys_path" || return 1
> +		_rport_check_online "$subsys_path" Online || return 1
> +	done
> +
> +	if [ "$IOPOLICY" == "numa" ]; then
> +		_rport_check_one_use_online "${RPORTS_PATHS[@]}" || return 1
> +	else
> +		for subsys_path in "${RPORTS_PATHS[@]}"; do
> +			# Only optimized paths will be in use
> +			_rport_check_opt "$subsys_path" Online || return 1
> +		done
> +	fi
> +}
> +
> +test_set_one_host_marginal() {
> +	local HOST=$1 # host1
> +	shift
> +	local ARGS=("$@")
> +	local RPORTS_CNT="$(( $# / 2 ))"
> +	local PATHS_POS # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +	local HOSTS_POS # host1 host2
> +
> +	for (( PATHS_POS=0; PATHS_POS < RPORTS_CNT; PATHS_POS++ )); do
> +		HOSTS_POS="$(( RPORTS_CNT + PATHS_POS ))"
> +		if [[ "${ARGS[$HOSTS_POS]}" == "$HOST" ]]; then
> +			_rport_set_marginal "${ARGS[$PATHS_POS]}" || return 1
> +		fi
> +	done
> +
> +	# check if all links are of expected state on hosts
> +	for (( PATHS_POS=0; PATHS_POS < RPORTS_CNT; PATHS_POS++ )); do
> +		HOSTS_POS="$(( RPORTS_CNT + PATHS_POS ))"
> +		if [[ "${ARGS[$HOSTS_POS]}" == "$HOST" ]]; then
> +			_rport_check_opt "${ARGS[$PATHS_POS]}" "Marginal" || return 1
> +		else
> +			_rport_check_opt "${ARGS[$PATHS_POS]}" "Online" || return 1
> +		fi
> +	done
> +}
> +
> +
> +# Check if one of the marginal paths is in use
> +_rport_check_one_use_marginal() {
> +	local -a RPORTS_PATHS=("$@") # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	# Check if one of the marginal paths is in use
> +	for subsys_path in "${RPORTS_PATHS[@]}"; do
> +		_rport_check_online "$subsys_path" Marginal
> +		# One of the marginal paths should be in use
> +		# "!" inverts the return code, so returns false if path is in use,
> +		# running the return statement
> +		! _rport_in_use "$subsys_path" || return 0
> +	done
> +
> +	echo No FC ports are being used, expected one in use when all are marginal.
> +	# None of the marginal paths were in use
> +	return 1
> +
> +	# Returns Success or Fail
> +}
> +
> +test_set_all_marginal() {
> +	local -a RPORTS_PATHS=("$@") # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	for subsys_path in "${RPORTS_PATHS[@]}"; do
> +		_rport_set_marginal "$subsys_path" || return 1
> +	done
> +
> +	_rport_check_one_use_marginal "${RPORTS_PATHS[@]}"
> +}
> +
> +test_set_one_non_optimized_online() {
> +	local -a RPORTS_PATHS=("$@") # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	# Set one non-optimized online
> +	# First set one non-optimized online to online then check optimized paths
> +	local next_marginal=0
> +	for subsys_path in "${RPORTS_PATHS[@]}"; do
> +		if ! _rport_optimized "$subsys_path"; then
> +			if [ $next_marginal == 1 ]; then
> +				_rport_check "$subsys_path" Marginal || return 1
> +				break
> +			fi
> +			_rport_set_online "$subsys_path" || return 1
> +			_rport_check "$subsys_path" Online || return 1
> +			next_marginal=1
> +		fi
> +	done
> +	# Check optimized paths
> +	for subsys_path in "${RPORTS_PATHS[@]}"; do
> +		if _rport_optimized "$subsys_path"; then
> +			# Optimized paths will not be in use
> +			_rport_check_opt "$subsys_path" Marginal || return 1
> +		fi
> +	done
> +}
> +
> +test_set_all_non_optimized_online() {
> +	local IOPOLICY=$1 # "numa" "queue-depth" "round-robin"
> +	shift
> +	local -a RPORTS_PATHS=("$@") # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	# Set all non-optimized online
> +	local numa_in_use=0
> +	for subsys_path in "${RPORTS_PATHS[@]}"; do
> +		if _rport_optimized "$subsys_path"; then
> +			_rport_check "$subsys_path" Marginal || return 1
> +		else
> +			_rport_set_online "$subsys_path" || return 1
> +			if [ "$IOPOLICY" == "numa" ] && [ $numa_in_use == 0 ]; then
> +				_rport_check "$subsys_path" Online || return 1
> +				numa_in_use=1
> +			elif [ "$IOPOLICY" == "numa" ] && [ $numa_in_use == 1 ]; then
> +				_rport_check_online "$subsys_path" Online || return 1
> +				# Only one of the online paths should be in use
> +				_rport_check_use "$subsys_path" Marginal || return 1
> +			else
> +				_rport_check "$subsys_path" Online || return 1
> +			fi
> +		fi
> +	done
> +}
> +
> +set_one_optimized_online() {
> +	local -a RPORTS_PATHS=("$@") # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	# Set one optimized online
> +	local next_marginal=0
> +	for subsys_path in "${RPORTS_PATHS[@]}"; do
> +		if _rport_optimized "$subsys_path"; then
> +			if [ $next_marginal == 1 ]; then
> +				_rport_check "$subsys_path" Marginal || return 1
> +				break
> +			fi
> +			_rport_set_online "$subsys_path" || return 1
> +			_rport_check "$subsys_path" Online || return 1
> +			next_marginal=1
> +		fi
> +	done
> +}
> +
> +test_set_all_non_one_optimized_online() {
> +	local -a RPORTS_PATHS=("$@") # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	# Set one optimized online
> +	# First set one optimized online to online then check non-optimized paths
> +	set_one_optimized_online "${RPORTS_PATHS[@]}" || return 1
> +
> +	# Check non-optimized paths
> +	for subsys_path in "${RPORTS_PATHS[@]}"; do
> +		if ! _rport_optimized "$subsys_path"; then
> +			# Only optimized paths will be in use
> +			_rport_check_opt "$subsys_path" Online || return 1
> +		fi
> +	done
> +}
> +
> +test_set_one_optimized_online() {
> +	local -a RPORTS_PATHS=("$@") # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +
> +	# Set one optimized online
> +	# First set one optimized online to online then check non-optimized paths
> +	set_one_optimized_online "${RPORTS_PATHS[@]}" || return 1
> +
> +	# Check non-optimized paths
> +	for subsys_path in "${RPORTS_PATHS[@]}"; do
> +		if ! _rport_optimized "$subsys_path"; then
> +			# Only optimized paths will be in use
> +			_rport_check_opt "$subsys_path" Marginal || return 1
> +		fi
> +	done
> +}
> +
> +test_set_two_optimized_online() {
> +	local IOPOLICY=$1 # "numa" "queue-depth" "round-robin"
> +	shift
> +	local -a RPORTS_PATHS=("$@")
> +
> +	# Set two optimized online
> +	local numa_in_use=0
> +	for subsys_path in "${RPORTS_PATHS[@]}"; do
> +		if _rport_optimized "$subsys_path"; then
> +			_rport_set_online "$subsys_path" || return 1
> +			if [ "$IOPOLICY" == "numa" ] && [ $numa_in_use == 0 ]; then
> +				_rport_check "$subsys_path" Online || return 1
> +				numa_in_use=1
> +			elif [ "$IOPOLICY" == "numa" ] && [ $numa_in_use == 1 ]; then
> +				_rport_check_online "$subsys_path" Online || return 1
> +				# Only one of the online paths should be in use
> +				_rport_check_use "$subsys_path" Marginal || return 1
> +			else
> +				_rport_check "$subsys_path" Online || return 1
> +			fi
> +		else
> +			_rport_check "$subsys_path" Marginal || return 1
> +		fi
> +	done
> +}
> +
> +run_test() {
> +	local IOPOLICY=$1 # "numa" "queue-depth" "round-robin"
> +	shift
> +	local ARGS=("$@")
> +	local RPORTS_CNT="$(( $# / 2 ))"
> +	local -a RPORTS_PATHS # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +	local -a RPORTS_HOSTS # host1 host2
> +	local PATHS_POS # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +	local HOSTS_POS # host1 host2
> +
> +	# recrate the arrays
> +	for (( PATHS_POS=0; PATHS_POS < RPORTS_CNT; PATHS_POS++ )); do
> +		HOSTS_POS+="$(( RPORTS_CNT + PATHS_POS ))"
> +		RPORTS_PATHS+=("${ARGS[$PATHS_POS]}")
> +		RPORTS_HOSTS+=("${ARGS[$HOSTS_POS]}")
> +	done
> +
> +
> +	echo Changing FC links to online
> +	# Initial check to see if FC is operational and set ports to online
> +	test_set_all_online "$IOPOLICY" "${RPORTS_PATHS[@]}" && echo "Set all online: pass" || echo "Set all online: fail"
> +
> +	test_set_one_host_marginal "${RPORTS_HOSTS[0]}" "${RPORTS_PATHS[@]}" "${RPORTS_HOSTS[@]}" && echo "One host marginal: pass" || echo "One host marginal: fail"
> +	test_set_all_marginal "${RPORTS_PATHS[@]}" && echo "All marginal: pass" || echo "All marginal: fail"
> +
> +	test_set_one_non_optimized_online "${RPORTS_PATHS[@]}" && echo "One remote non-optimized online: pass" || echo "One remote non-optimized online: fail"
> +	test_set_all_non_optimized_online "$IOPOLICY" "${RPORTS_PATHS[@]}" && echo "Two remote non-optimized online: pass" || echo "Two remote non-optimized online: fail"
> +
> +	test_set_all_non_one_optimized_online "${RPORTS_PATHS[@]}" && echo "Two remote non-optimized, One remote optimized: pass" || echo "Two remote non-optimized, One remote optimized: fail"
> +	test_set_all_online "$IOPOLICY" "${RPORTS_PATHS[@]}" && echo "Set all online: pass" || echo "Set all online: fail"
> +
> +	test_set_all_marginal "${RPORTS_PATHS[@]}" && echo "All marginal: pass" || echo "All marginal: fail"
> +
> +	test_set_one_optimized_online "${RPORTS_PATHS[@]}" && echo "One remote optimized online: pass" || echo "One remote optimized online: fail"
> +	test_set_two_optimized_online "$IOPOLICY" "${RPORTS_PATHS[@]}" && echo "Two remote optimized online: pass" || echo "Two remote optimized online: fail"
> +	test_set_all_online "$IOPOLICY" "${RPORTS_PATHS[@]}" && echo "All online: pass" || echo "All online: fail"
> +}
> +
> +_nvmet_get_rport() {
> +	local PORT="$1"
> +
> +	local dev
> +	for dev in /sys/class/nvme/nvme*; do
> +		grep -q "io" "$dev/cntrltype" || continue
> +		[ -e "$dev" ] || continue
> +		dev="$(basename "$dev")"
> +		grep -q traddr="$(_fc_traddr "$PORT")" "/sys/class/nvme/$dev/address" && echo "$dev" || true
> +	done
> +	# nvme9
> +}
> +
> +run_tests() {
> +	local SUBSYS_PATH="$1"
> +	shift
> +	local PORTS=("$@")
> +	local -a RPORTS_PATHS # /sys/devices/virtual/nvme-subsystem/nvme-subsys3/nvme9
> +	local -a RPORTS_HOSTS # host1 host2
> +	local rport
> +
> +	for port in "${PORTS[@]}"; do
> +		RPORTS_HOSTS+=("$(_get_fc_host_port "$port")")
> +		rport="$(_nvmet_get_rport "$port")"
> +		if [ -z "$rport" ]; then
> +			echo "Could not find rport for port $port"
> +			return 1
> +		fi
> +		if [[ "$( echo "$rport" | sed -n '$=' )" -gt 1 ]]; then
> +			# One traddr has multiple /sys/class/nvme/nvme devices
> +			echo "Port $port has multiple rports with address"
> +			grep  traddr="$(_fc_traddr "$port")" /sys/class/nvme/nvme*/address
> +			return 1
> +		fi
> +		RPORTS_PATHS+=( "${SUBSYS_PATH}/$rport")
> +	done
> +
> +	local IOPOLICYS="numa queue-depth round-robin"
> +	for IOPOLICY in $IOPOLICYS; do
> +		_rport_set_iopolicy "$SUBSYS_PATH" "$IOPOLICY"
> +		echo "Testing iopolicy: $IOPOLICY"
> +		run_test "$IOPOLICY" "${RPORTS_PATHS[@]}" "${RPORTS_HOSTS[@]}"
> +	done
> +}
> +
> +_find_nvme_subsys() {
> +	local subsys=$1
> +	local subsysnqn
> +	local subsys_path
> +	for subsys_path in /sys/class/nvme-subsystem/nvme-subsys*; do
> +		[ -e "$subsys_path" ] || continue
> +		subsysnqn="$(cat "${subsys_path}/subsysnqn" 2>/dev/null)"
> +		if [[ "$subsysnqn" == "$subsys" ]]; then
> +			echo "$subsys_path"
> +			return
> +		fi
> +	done
> +}
> +
> +test() {
> +	local -a ports
> +	local ns
> +	local fio_pid
> +
> +	echo "Running ${TEST_NAME}"
> +
> +	_setup_nvmet 2
> +
> +	_nvmet_target_setup --ports 2
> +	_nvmet_target_add_ports --host_port 1 --ports 2
> +
> +	_get_nvmet_ports "${def_subsysnqn}" ports
> +
> +	for port in "${ports[@]}"; do
> +		if [[ $(_get_fc_host_port "${port}") == 0 ]]; then
> +			_setup_nvmet_port_ana "${port}" 1 "optimized"
> +		else
> +			_setup_nvmet_port_ana "${port}" 1 "non-optimized"
> +		fi
> +	done
> +
> +	for port in "${ports[@]}"; do
> +		_nvme_connect_subsys --port "${port}" --no-wait-ns || return 1
> +	done
> +
> +	# start fio job
> +	ns=$(_find_nvme_ns "$def_subsys_uuid")
> +	echo "Starting background I/O"
> +	_run_fio_verify_io --filename="/dev/${ns}" \
> +			   --group_reporting --ramp_time=5 \
> +			   --time_based --runtime=1m &> "$FULL" &
> +	fio_pid=$!
> +	sleep 10
> +
> +	run_tests "$(_find_nvme_subsys "${def_subsysnqn}")" "${ports[@]}"
> +
> +	# Stop background I/O
> +	echo "Stopping background I/O"
> +	{ kill "$fio_pid"; wait; } &> /dev/null
> +
> +	_nvme_disconnect_subsys
> +	_nvmet_target_cleanup
> +
> +	echo "Test complete"
> +}
> diff --git a/tests/nvme/070.out b/tests/nvme/070.out
> new file mode 100644
> index 0000000..61d5173
> --- /dev/null
> +++ b/tests/nvme/070.out
> @@ -0,0 +1,43 @@
> +Running nvme/070
> +Starting background I/O
> +Testing iopolicy: numa
> +Changing FC links to online
> +Set all online: pass
> +One host marginal: pass
> +All marginal: pass
> +One remote non-optimized online: pass
> +Two remote non-optimized online: pass
> +Two remote non-optimized, One remote optimized: pass
> +Set all online: pass
> +All marginal: pass
> +One remote optimized online: pass
> +Two remote optimized online: pass
> +All online: pass
> +Testing iopolicy: queue-depth
> +Changing FC links to online
> +Set all online: pass
> +One host marginal: pass
> +All marginal: pass
> +One remote non-optimized online: pass
> +Two remote non-optimized online: pass
> +Two remote non-optimized, One remote optimized: pass
> +Set all online: pass
> +All marginal: pass
> +One remote optimized online: pass
> +Two remote optimized online: pass
> +All online: pass
> +Testing iopolicy: round-robin
> +Changing FC links to online
> +Set all online: pass
> +One host marginal: pass
> +All marginal: pass
> +One remote non-optimized online: pass
> +Two remote non-optimized online: pass
> +Two remote non-optimized, One remote optimized: pass
> +Set all online: pass
> +All marginal: pass
> +One remote optimized online: pass
> +Two remote optimized online: pass
> +All online: pass
> +Stopping background I/O
> +Test complete