[PATCH blktests 2/7] md/rc: add _md_atomics_test

[John Garry <john.g.garry@oracle.com>]

The stacked device atomic writes testing is currently limited.

md/002 currently only tests scsi_debug. SCSI does not support atomic
boundaries, so it would be nice to test NVMe (which does support them).

Furthermore, the testing in md/002 for chunk boundaries is very limited,
in that we test once one boundary value. Indeed, for RAID0 and RAID10, a
boundary should always be set for testing.

Finally, md/002 only tests md RAID0/1/10. In future we will also want to
test the following stacked device personalities which support atomic
writes:
- md-linear (being upstreamed)
- dm-linear
- dm-stripe
- dm-mirror

To solve all those problems, add a generic test handler,
_md_atomics_test(). This can be extended for more extensive testing.

This test handler will accept a group of devices and test as follows:
a. calculate expected atomic write limits based on device limits
b. Take results from a., and refine expected limits based on any chunk
   size
c. loop through creating a stacked device for different chunk size. We loop
   once for any personality which does not have a chunk size, e.g. RAID1
d. test sysfs and statx limits vs what is calculated in a. and b.
e. test RWF_ATOMIC is accepted or rejected as expected

Steps c, d, and e are really same as md/002.

Signed-off-by: John Garry <john.g.garry@oracle.com>
---
 tests/md/rc | 372 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 372 insertions(+)

diff --git a/tests/md/rc b/tests/md/rc
index 96bcd97..105d283 100644
--- a/tests/md/rc
+++ b/tests/md/rc
@@ -5,9 +5,381 @@
 # Tests for md raid
 
 . common/rc
+. common/xfs
 
 group_requires() {
+	_have_kver 6 14 0
 	_have_root
 	_have_program mdadm
+	_have_xfs_io_atomic_write
+	_have_driver raid0
+	_have_driver raid1
+	_have_driver raid10
 	_have_driver md-mod
 }
+
+declare -A MD_DEVICES
+
+_max_pow_of_two_factor() {
+	part1=$1
+	part2=-$1
+	retval=$(($part1 & $part2))
+	echo "$retval"
+}
+
+# Find max atomic size given a boundary and chunk size
+# @unit is set if we want atomic write "unit" size, i.e power-of-2
+# @chunk must be > 0
+_md_atomics_boundaries_max() {
+	boundary=$1
+	chunk=$2
+	unit=$3
+
+	if [ "$boundary" -eq 0 ]
+	then
+		if [ "$unit" -eq 1 ]
+		then
+			retval=$(_max_pow_of_two_factor $chunk)
+			echo "$retval"
+			return 1
+		fi
+
+		echo "$chunk"
+		return 1
+	fi
+
+	# boundary is always a power-of-2
+	if [ "$boundary" -eq "$chunk" ]
+	then
+		echo "$boundary"
+		return 1
+	fi
+
+	if [ "$boundary" -gt "$chunk" ]
+	then
+		if (( $boundary % $chunk == 0))
+		then
+			if [ "$unit" -eq 1 ]
+			then
+				retval=$(_max_pow_of_two_factor $chunk)
+				echo "$retval"
+				return 1
+			fi
+			echo "$chunk"
+			return 1
+		fi
+		echo "0"
+		return 1
+	fi
+
+	if (( $chunk % $boundary == 0))
+	then
+		echo "$boundary"
+		return 1
+	fi
+
+	echo "0"
+}
+
+_md_atomics_test() {
+	local md_atomic_unit_max
+	local md_atomic_unit_min
+	local md_sysfs_max_hw_sectors_kb
+	local md_sysfs_max_hw
+	local md_chunk_size
+	local sysfs_logical_block_size
+	local sysfs_atomic_write_max
+	local sysfs_atomic_write_unit_min
+	local sysfs_atomic_write_unit_max
+	local bytes_to_write
+	local bytes_written
+	local test_desc
+	local md_dev
+	local md_dev_sysfs
+	local raw_atomic_write_unit_min
+	local raw_atomic_write_unit_max
+	local raw_atomic_write_max
+	local raw_atomic_write_boundary
+	local raw_atomic_write_supported=1
+
+	dev0=$1
+	dev1=$2
+	dev2=$3
+	dev3=$4
+	unset MD_DEVICES
+	MD_DEVICES=($dev0 $dev1 $dev2 $dev3);
+
+	# Calculate what we expect the atomic write limits to be
+	# Don't consider any chunk size at this stage
+	# Use the limits from the first device and then loop again to find
+	# lowest common supported
+	raw_atomic_write_unit_min=$(< /sys/block/"$dev0"/queue/atomic_write_unit_min_bytes);
+	raw_atomic_write_unit_max=$(< /sys/block/"$dev0"/queue/atomic_write_unit_max_bytes);
+	raw_atomic_write_max=$(< /sys/block/"$dev0"/queue/atomic_write_max_bytes);
+	raw_atomic_write_boundary=$(< /sys/block/"$dev0"/queue/atomic_write_boundary_bytes);
+
+	for i in "${MD_DEVICES[@]}"; do
+		if [[ $(< /sys/block/"$i"/queue/atomic_write_unit_min_bytes) -gt raw_atomic_write_unit_min ]]; then
+			raw_atomic_write_unit_min=$(< /sys/block/"$i"/queue/atomic_write_unit_min_bytes)
+		fi
+		if [[ $(< /sys/block/"$i"/queue/atomic_write_unit_max_bytes) -lt raw_atomic_write_unit_max ]]; then
+			raw_atomic_write_unit_max=$(< /sys/block/"$i"/queue/atomic_write_unit_max_bytes)
+		fi
+		if [[ $(< /sys/block/"$i"/queue/atomic_write_max_bytes) -lt raw_atomic_write_max ]]; then
+			raw_atomic_write_max=$(< /sys/block/"$i"/queue/atomic_write_max_bytes)
+		fi
+		# The kernel only supports same boundary size for all devices in the array
+		if [[ $(< /sys/block/"$i"/queue/atomic_write_boundary_bytes) -ne raw_atomic_write_boundary ]]; then
+			let raw_atomic_write_supported=0;
+		fi
+	done
+
+	# Check if we can support atomic writes for the array of devices given.
+	# If we cannot, then it is still worth trying to test that atomic
+	# writes don't work (as we would expect).
+
+	if [[ raw_atomic_write_supported -eq 0 ]]; then
+		let raw_atomic_write_unit_min=0;
+		let raw_atomic_write_unit_max=0;
+		let raw_atomic_write_max=0;
+		let raw_atomic_write_boundary=0;
+	fi
+
+	for personality in raid0 raid1 raid10; do
+		if [ "$personality" = raid0 ] || [ "$personality" = raid10 ]
+		then
+			step_limit=4
+		else
+			step_limit=1
+		fi
+		chunk_gran=$(( "$raw_atomic_write_unit_max" / 2))
+		if [ "$chunk_gran" -lt 4096 ]
+		then
+			let chunk_gran=4096
+		fi
+
+		local chunk_multiple=1
+		for step in `seq 1 $step_limit`
+		do
+			local expected_atomic_write_unit_min
+			local expected_atomic_write_unit_max
+			local expected_atomic_write_max
+			local expected_atomic_write_boundary
+
+			# only raid0 does not require a power-of-2 chunk size
+			if [ "$personality" = raid0 ]
+			then
+				chunk_multiple=$step
+			else
+				chunk_multiple=$(( 2 * "$chunk_multiple"))
+			fi
+			md_chunk_size=$(( "$chunk_gran" * "$chunk_multiple"))
+			md_chunk_size_kb=$(( "$md_chunk_size" / 1024))
+
+			# We may reassign these for RAID0/10
+			let expected_atomic_write_unit_min=$raw_atomic_write_unit_min
+			let expected_atomic_write_unit_max=$raw_atomic_write_unit_max
+			let expected_atomic_write_max=$raw_atomic_write_max
+			let expected_atomic_write_boundary=$raw_atomic_write_boundary
+
+			if [ "$personality" = raid0 ] || [ "$personality" = raid10 ]
+			then
+				echo y | mdadm --create /dev/md/blktests_md --level=$personality \
+					 --chunk="${md_chunk_size_kb}"K \
+					--raid-devices=4 --force /dev/"${dev0}" /dev/"${dev1}" \
+					/dev/"${dev2}" /dev/"${dev3}" 2> /dev/null 1>&2
+
+				atomics_boundaries_unit_max=$(_md_atomics_boundaries_max $raw_atomic_write_boundary $md_chunk_size "1")
+				atomics_boundaries_max=$(_md_atomics_boundaries_max $raw_atomic_write_boundary $md_chunk_size "0")
+				expected_atomic_write_unit_min=$(_min $expected_atomic_write_unit_min $atomics_boundaries_unit_max)
+				expected_atomic_write_unit_max=$(_min $expected_atomic_write_unit_max $atomics_boundaries_unit_max)
+				expected_atomic_write_max=$(_min $expected_atomic_write_max $atomics_boundaries_max)
+				if [ "$atomics_boundaries_max" -eq 0 ]
+				then
+					expected_atomic_write_boundary=0
+				fi
+				md_dev=$(readlink /dev/md/blktests_md | sed 's|\.\./||')
+			fi
+
+			if [ "$personality" = raid1 ]
+			then
+				echo y | mdadm --create /dev/md/blktests_md --level=$personality \
+					--raid-devices=4 --force /dev/"${dev0}" /dev/"${dev1}" \
+					/dev/"${dev2}" /dev/"${dev3}" 2> /dev/null 1>&2
+
+				md_dev=$(readlink /dev/md/blktests_md | sed 's|\.\./||')
+			fi
+
+			md_dev_sysfs="/sys/devices/virtual/block/${md_dev}"
+
+			sysfs_logical_block_size=$(< "${md_dev_sysfs}"/queue/logical_block_size)
+			md_sysfs_max_hw_sectors_kb=$(< "${md_dev_sysfs}"/queue/max_hw_sectors_kb)
+			md_sysfs_max_hw=$(( "$md_sysfs_max_hw_sectors_kb" * 1024 ))
+			sysfs_atomic_write_max=$(< "${md_dev_sysfs}"/queue/atomic_write_max_bytes)
+			sysfs_atomic_write_unit_max=$(< "${md_dev_sysfs}"/queue/atomic_write_unit_max_bytes)
+			sysfs_atomic_write_unit_min=$(< "${md_dev_sysfs}"/queue/atomic_write_unit_min_bytes)
+			sysfs_atomic_write_boundary=$(< "${md_dev_sysfs}"/queue/atomic_write_boundary_bytes)
+
+			test_desc="TEST 1 $personality step $step - Verify md sysfs atomic attributes matches"
+			if [ "$sysfs_atomic_write_unit_min" = "$expected_atomic_write_unit_min" ] &&
+				[ "$sysfs_atomic_write_unit_max" = "$expected_atomic_write_unit_max" ]
+			then
+				echo "$test_desc - pass"
+			else
+				echo "$test_desc - fail sysfs_atomic_write_unit_min="$sysfs_atomic_write_unit_min \
+					"expected_atomic_write_unit_min="$expected_atomic_write_unit_min \
+					"sysfs_atomic_write_unit_max="$sysfs_atomic_write_unit_max \
+					"expected_atomic_write_unit_max="$expected_atomic_write_unit_max \
+					"md_chunk_size="$md_chunk_size
+			fi
+
+			test_desc="TEST 2 $personality step $step - Verify sysfs atomic attributes"
+			if [ "$md_sysfs_max_hw" -ge "$sysfs_atomic_write_max" ] &&
+				[ "$sysfs_atomic_write_unit_max" -ge "$sysfs_atomic_write_unit_min" ] &&
+				[ "$sysfs_atomic_write_max" -ge "$sysfs_atomic_write_unit_max" ]
+			then
+				echo "$test_desc - pass"
+			else
+				echo "$test_desc - fail $md_sysfs_max_hw="$md_sysfs_max_hw \
+					"sysfs_atomic_write_max="$sysfs_atomic_write_max \
+					"sysfs_atomic_write_unit_min="$sysfs_atomic_write_unit_min \
+					"sysfs_atomic_write_unit_max="$sysfs_atomic_write_unit_max \
+					"md_chunk_size="$md_chunk_size
+			fi
+
+			test_desc="TEST 3 $personality step $step - Verify md sysfs_atomic_write_max is equal to "
+			test_desc+="expected_atomic_write_max"
+			if [ "$sysfs_atomic_write_max" -eq "$expected_atomic_write_max" ]
+			then
+				echo "$test_desc - pass"
+			else
+				echo "$test_desc - fail sysfs_atomic_write_max="$sysfs_atomic_write_max \
+					"expected_atomic_write_max="$expected_atomic_write_max \
+					"md_chunk_size="$md_chunk_size
+			fi
+
+			test_desc="TEST 4 $personality step $step - Verify sysfs atomic_write_unit_max_bytes =  expected_atomic_write_unit_max"
+			if [ "$sysfs_atomic_write_unit_max" = "$expected_atomic_write_unit_max" ]
+			then
+				echo "$test_desc - pass"
+			else
+				echo "$test_desc - fail sysfs_atomic_write_unit_max="$sysfs_atomic_write_unit_max \
+					"expected_atomic_write_unit_max="$expected_atomic_write_unit_max \
+					"md_chunk_size="$md_chunk_size
+			fi
+
+			test_desc="TEST 5 $personality step $step - Verify sysfs atomic_write_unit_boundary_bytes = expected atomic_write_unit_boundary_bytes"
+			if [ "$sysfs_atomic_write_boundary" = "$expected_atomic_write_boundary" ]
+			then
+				echo "$test_desc - pass"
+			else
+				echo "$test_desc - fail sysfs_atomic_write_boundary="$sysfs_atomic_write_boundary \
+					"expected_atomic_write_boundary="$expected_atomic_write_boundary
+			fi
+
+			test_desc="TEST 6 $personality step $step - Verify statx stx_atomic_write_unit_min"
+			statx_atomic_write_unit_min=$(run_xfs_io_xstat /dev/"$md_dev" "stat.atomic_write_unit_min")
+			if [ "$statx_atomic_write_unit_min" = "$sysfs_atomic_write_unit_min" ]
+			then
+				echo "$test_desc - pass"
+			else
+				echo "$test_desc - fail statx_atomic_write_unit_min="$statx_atomic_write_unit_min \
+					"sysfs_atomic_write_unit_min="$sysfs_atomic_write_unit_min \
+					"md_chunk_size="$md_chunk_size
+			fi
+
+			test_desc="TEST 7 $personality step $step - Verify statx stx_atomic_write_unit_max"
+			statx_atomic_write_unit_max=$(run_xfs_io_xstat /dev/"$md_dev" "stat.atomic_write_unit_max")
+			if [ "$statx_atomic_write_unit_max" = "$sysfs_atomic_write_unit_max" ]
+			then
+				echo "$test_desc - pass"
+			else
+				echo "$test_desc - fail statx_atomic_write_unit_max="$statx_atomic_write_unit_max \
+					"sysfs_atomic_write_unit_max="$sysfs_atomic_write_unit_max \
+					"md_chunk_size="$md_chunk_size
+			fi
+
+			test_desc="TEST 8 $personality step $step - perform a pwritev2 with size of sysfs_atomic_unit_max_bytes with "
+			test_desc+="RWF_ATOMIC flag - pwritev2 should fail"
+			if [ "$sysfs_atomic_write_unit_max" = 0 ]
+			then
+				echo "$test_desc - pass"
+			else
+				bytes_written=$(run_xfs_io_pwritev2_atomic /dev/"$md_dev" "$sysfs_atomic_write_unit_max")
+				if [ "$bytes_written" = "$sysfs_atomic_write_unit_max" ]
+				then
+					echo "$test_desc - pass"
+				else
+					echo "$test_desc - fail bytes_written="$bytes_written \
+						"sysfs_atomic_write_unit_max="$sysfs_atomic_write_unit_max \
+						"md_chunk_size="$md_chunk_size
+				fi
+			fi
+
+			test_desc="TEST 9 $personality step $step - perform a pwritev2 with size of sysfs_atomic_unit_max_bytes + LBS "
+			test_desc+="bytes with RWF_ATOMIC flag - pwritev2 should not be succesful"
+			if [ "$sysfs_atomic_write_unit_max" = 0 ]
+			then
+				echo "pwrite: Invalid argument"
+				echo "$test_desc - pass"
+			else
+				bytes_to_write=$(( "${sysfs_atomic_write_unit_max}" + "${sysfs_logical_block_size}" ))
+				bytes_written=$(run_xfs_io_pwritev2_atomic /dev/"$md_dev" "$bytes_to_write")
+				if [ "$bytes_written" = "" ]
+				then
+					echo "$test_desc - pass"
+				else
+					echo "$test_desc - fail bytes_written="$bytes_written \
+						"bytes_to_write="$bytes_to_write \
+						"sysfs_atomic_write_unit_max="$sysfs_atomic_write_unit_max \
+						"md_chunk_size="$md_chunk_size
+				fi
+			fi
+
+			test_desc="TEST 10 $personality step $step - perform a pwritev2 with size of sysfs_atomic_unit_min_bytes "
+			test_desc+="with RWF_ATOMIC flag - pwritev2 should fail"
+			if [ "$sysfs_atomic_write_unit_min" = 0 ]
+			then
+				echo "$test_desc - pass"
+			else
+				bytes_written=$(run_xfs_io_pwritev2_atomic /dev/"$md_dev" "$sysfs_atomic_write_unit_min")
+				if [ "$bytes_written" = "$sysfs_atomic_write_unit_min" ]
+				then
+					echo "$test_desc - pass"
+				else
+					echo "$test_desc - fail bytes_written="$bytes_written \
+						"sysfs_atomic_write_unit_min="$sysfs_atomic_write_unit_min \
+						"md_chunk_size="$md_chunk_size
+				fi
+			fi
+
+			test_desc="TEST 11 $personality step $step - perform a pwritev2 with a size of sysfs_atomic_write_unit_max_bytes - LBS "
+			test_desc+="bytes with RWF_ATOMIC flag - pwritev2 should fail"
+			if [ "${sysfs_atomic_write_unit_max}" -le "${sysfs_logical_block_size}" ]
+			then
+				echo "pwrite: Invalid argument"
+				echo "$test_desc - pass"
+			else
+				bytes_to_write=$(( "${sysfs_atomic_write_unit_max}" - "${sysfs_logical_block_size}" ))
+				bytes_written=$(run_xfs_io_pwritev2_atomic /dev/"$md_dev" "$bytes_to_write")
+				if [ "$bytes_written" = "" ]
+				then
+					echo "$test_desc - pass"
+				else
+					echo "$test_desc - fail bytes_written="$bytes_written \
+						"bytes_to_write="$bytes_to_write \
+						"md_chunk_size="$md_chunk_size
+				fi
+			fi
+
+			if [ "$personality" = raid0 ] || [ "$personality" = raid1 ] || [ "$personality" = raid10 ]
+			then
+				mdadm --stop /dev/md/blktests_md  2> /dev/null 1>&2
+				mdadm --zero-superblock /dev/"${dev0}" 2> /dev/null 1>&2
+				mdadm --zero-superblock /dev/"${dev1}" 2> /dev/null 1>&2
+				mdadm --zero-superblock /dev/"${dev2}" 2> /dev/null 1>&2
+				mdadm --zero-superblock /dev/"${dev3}" 2> /dev/null 1>&2
+			fi
+		done
+	done
+}
-- 
2.43.5