Re: [PATCH v2] RDMA/CM: add RDMA CM observability regression scripts

[Zhu Yanjun <yanjun.zhu@linux.dev>]

在 2026/4/28 0:12, Chenguang Zhao 写道:
> This adds a minimal RDMA CM selftest suite that captures observability
> baselines and runs trace, counter-delta, and fault-injection-oriented
> checks, plus a review-loop helper for repeated validation rounds.
> 
> Signed-off-by: Chenguang Zhao <zhaochenguang@kylinos.cn>
> ---
>   v2:
>   - Move the test scripts to tools/testing/selftests/rdma
>     as suggested by Leon.
> 
>   - rdma_cm_baseline.sh:rdma_cm_baseline.sh: Captures baseline
>     observability data for RDMA CM, establishing a reference
>     point for testing.
> 
>   - rdma_cm_counter_delta.sh: Monitors and reports counter deltas
>     during RDMA operations to detect unexpected behavior
> 
>   - rdma_cm_fault_injection.sh: Performs fault injection to test
>     RDMA CM's handling of failures and error scenarios.
> 
>   - rdma_cm_review_loop.sh: Automates repeated testing for continuous
>     validation of RDMA CM functionality.
> 
>   - rdma_cm_trace_sequence.sh: Captures a trace sequence of RDMA
>     operations to ensure proper flow and aid in troubleshooting.
> 
>   - rdma_common.sh: Contains shared utility functions used across
>     the above scripts for consistency and reusability.
> ---
>   tools/testing/selftests/Makefile              |   1 +
>   tools/testing/selftests/rdma/Makefile         |  10 ++
>   tools/testing/selftests/rdma/config           |   6 +
>   .../selftests/rdma/rdma_cm_baseline.sh        |  58 ++++++++
>   .../selftests/rdma/rdma_cm_counter_delta.sh   |  72 ++++++++++
>   .../selftests/rdma/rdma_cm_fault_injection.sh |  95 +++++++++++++
>   .../selftests/rdma/rdma_cm_review_loop.sh     |  35 +++++
>   .../selftests/rdma/rdma_cm_trace_sequence.sh  |  83 ++++++++++++
>   tools/testing/selftests/rdma/rdma_common.sh   | 126 ++++++++++++++++++
>   9 files changed, 486 insertions(+)
>   create mode 100755 tools/testing/selftests/rdma/rdma_cm_baseline.sh
>   create mode 100755 tools/testing/selftests/rdma/rdma_cm_counter_delta.sh
>   create mode 100755 tools/testing/selftests/rdma/rdma_cm_fault_injection.sh
>   create mode 100755 tools/testing/selftests/rdma/rdma_cm_review_loop.sh
>   create mode 100755 tools/testing/selftests/rdma/rdma_cm_trace_sequence.sh
>   create mode 100755 tools/testing/selftests/rdma/rdma_common.sh
> 
> diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile
> index 6e59b8f63e41..5794d55a92b2 100644
> --- a/tools/testing/selftests/Makefile
> +++ b/tools/testing/selftests/Makefile
> @@ -22,6 +22,7 @@ TARGETS += drivers/ntsync
>   TARGETS += drivers/s390x/uvdevice
>   TARGETS += drivers/net
>   TARGETS += drivers/net/bonding
> +TARGETS += rdma

In the latest linux kernel, in the file tools/testing/selftests/Makefile
"
...
  99 TARGETS += ptrace
100 TARGETS += openat2
101 TARGETS += rdma
102 TARGETS += resctrl
103 TARGETS += riscv
104 TARGETS += rlimits
...
"
rdma has already been in this file.

I am wondering if this rdma still needs to be added into this file.

Zhu Yanjun

>   TARGETS += drivers/net/netconsole
>   TARGETS += drivers/net/team
>   TARGETS += drivers/net/virtio_net
> diff --git a/tools/testing/selftests/rdma/Makefile b/tools/testing/selftests/rdma/Makefile
> index 7dd7cba7a73c..04c52db4b9d9 100644
> --- a/tools/testing/selftests/rdma/Makefile
> +++ b/tools/testing/selftests/rdma/Makefile
> @@ -4,4 +4,14 @@ TEST_PROGS := rxe_rping_between_netns.sh \
>   		rxe_socket_with_netns.sh \
>   		rxe_test_NETDEV_UNREGISTER.sh
>   
> +TEST_PROGS += \
> +	rdma_cm_baseline.sh \
> +	rdma_cm_trace_sequence.sh \
> +	rdma_cm_counter_delta.sh \
> +	rdma_cm_fault_injection.sh
> +
> +TEST_FILES += \
> +	rdma_common.sh \
> +	rdma_cm_review_loop.sh
> +
>   include ../lib.mk
> diff --git a/tools/testing/selftests/rdma/config b/tools/testing/selftests/rdma/config
> index 4ffb814e253b..e22141838c19 100644
> --- a/tools/testing/selftests/rdma/config
> +++ b/tools/testing/selftests/rdma/config
> @@ -1,3 +1,9 @@
>   CONFIG_TUN
>   CONFIG_VETH
>   CONFIG_RDMA_RXE
> +CONFIG_DEBUG_KERNEL
> +CONFIG_FAULT_INJECTION
> +CONFIG_SYSFS
> +CONFIG_DEBUG_FS
> +CONFIG_FAULT_INJECTION_DEBUG_FS
> +CONFIG_FAILSLAB
> diff --git a/tools/testing/selftests/rdma/rdma_cm_baseline.sh b/tools/testing/selftests/rdma/rdma_cm_baseline.sh
> new file mode 100755
> index 000000000000..b0d8b3e46470
> --- /dev/null
> +++ b/tools/testing/selftests/rdma/rdma_cm_baseline.sh
> @@ -0,0 +1,58 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-2.0
> +
> +set -euo pipefail
> +
> +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
> +source "${SCRIPT_DIR}/rdma_common.sh"
> +
> +require_root
> +require_cmd date
> +require_cmd uname
> +
> +trace_dir="$(tracefs_dir || true)"
> +counter_root="$(find_cm_counter_root || true)"
> +out_dir="/tmp/rdma_cm_baseline.$(date +%s)"
> +dmesg_lines=400
> +dmesg_pattern="ib_cm|infiniband|rdma|roce|mlx|hns_roce|irdma|siw|rxe"
> +
> +mkdir -p "${out_dir}"
> +
> +log_info "writing baseline to ${out_dir}"
> +
> +{
> +	echo "timestamp=$(date -u +%FT%TZ)"
> +	echo "kernel=$(uname -r)"
> +	echo "hostname=$(uname -n)"
> +	echo "dmesg_lines=${dmesg_lines}"
> +	echo "dmesg_pattern=${dmesg_pattern}"
> +} >"${out_dir}/env.txt"
> +
> +if [[ -n "${trace_dir}" && -d "${trace_dir}/events/ib_cma" ]]; then
> +	find "${trace_dir}/events/ib_cma" -maxdepth 2 -name enable -print \
> +		>"${out_dir}/trace_events.list" 2>/dev/null || true
> +else
> +	log_warn "tracefs or ib_cma trace events are unavailable"
> +fi
> +
> +if [[ -n "${counter_root}" ]]; then
> +	{
> +		echo "counter_root=${counter_root}"
> +		for group in "${RDMA_COUNTER_GROUPS[@]}"; do
> +			for attr in "${RDMA_COUNTER_ATTRS[@]}"; do
> +				value="$(read_cm_counter "${counter_root}" "${group}" "${attr}")"
> +				echo "${group}.${attr}=${value}"
> +			done
> +		done
> +	} >"${out_dir}/cm_counters.before"
> +else
> +	log_warn "cm counters are unavailable under /sys/class/infiniband"
> +fi
> +
> +if command -v dmesg >/dev/null 2>&1; then
> +	dmesg | tail -n "${dmesg_lines}" | grep -E "${dmesg_pattern}" \
> +		>"${out_dir}/dmesg.rdma.tail" || true
> +fi
> +
> +log_info "baseline collection completed"
> +exit 0
> diff --git a/tools/testing/selftests/rdma/rdma_cm_counter_delta.sh b/tools/testing/selftests/rdma/rdma_cm_counter_delta.sh
> new file mode 100755
> index 000000000000..060adf9fe78a
> --- /dev/null
> +++ b/tools/testing/selftests/rdma/rdma_cm_counter_delta.sh
> @@ -0,0 +1,72 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-2.0
> +
> +set -euo pipefail
> +
> +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
> +source "${SCRIPT_DIR}/rdma_common.sh"
> +
> +require_root
> +counter_root="$(find_cm_counter_root || true)"
> +counter_wait_sec=2
> +
> +if [[ -z "${counter_root}" ]]; then
> +	log_warn "cm counters are unavailable under /sys/class/infiniband"
> +	exit "${ksft_skip}"
> +fi
> +
> +declare -A before after
> +
> +for group in "${RDMA_COUNTER_GROUPS[@]}"; do
> +	for attr in "${RDMA_COUNTER_ATTRS[@]}"; do
> +		key="${group}.${attr}"
> +		before["${key}"]="$(read_cm_counter "${counter_root}" "${group}" "${attr}")"
> +	done
> +done
> +
> +if [[ "${counter_wait_sec}" != "0" ]]; then
> +	log_info "waiting ${counter_wait_sec}s before workload"
> +	sleep "${counter_wait_sec}"
> +fi
> +
> +workload_rc=0
> +run_workload || workload_rc=$?
> +if [[ "${workload_rc}" -eq "${ksft_skip}" ]]; then
> +	exit "${ksft_skip}"
> +fi
> +if [[ "${workload_rc}" -ne 0 ]]; then
> +	log_err "workload failed with rc=${workload_rc}"
> +	exit "${workload_rc}"
> +fi
> +
> +for group in "${RDMA_COUNTER_GROUPS[@]}"; do
> +	for attr in "${RDMA_COUNTER_ATTRS[@]}"; do
> +		key="${group}.${attr}"
> +		after["${key}"]="$(read_cm_counter "${counter_root}" "${group}" "${attr}")"
> +		delta=$((after["${key}"] - before["${key}"]))
> +		echo "${key}.delta=${delta}"
> +		if ((delta < 0)); then
> +			log_err "counter regressed: ${key}"
> +			exit 1
> +		fi
> +	done
> +done
> +
> +dup_limit=10
> +retry_limit=10
> +
> +for attr in "${RDMA_COUNTER_ATTRS[@]}"; do
> +	dup_delta=$((after["cm_rx_duplicates.${attr}"] - before["cm_rx_duplicates.${attr}"]))
> +	retry_delta=$((after["cm_tx_retries.${attr}"] - before["cm_tx_retries.${attr}"]))
> +
> +	if ((dup_delta > dup_limit)); then
> +		log_err "duplicate counter exceeds limit: ${attr}=${dup_delta}"
> +		exit 1
> +	fi
> +	if ((retry_delta > retry_limit)); then
> +		log_err "retry counter exceeds limit: ${attr}=${retry_delta}"
> +		exit 1
> +	fi
> +done
> +
> +exit 0
> diff --git a/tools/testing/selftests/rdma/rdma_cm_fault_injection.sh b/tools/testing/selftests/rdma/rdma_cm_fault_injection.sh
> new file mode 100755
> index 000000000000..0202ee901386
> --- /dev/null
> +++ b/tools/testing/selftests/rdma/rdma_cm_fault_injection.sh
> @@ -0,0 +1,95 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-2.0
> +
> +set -euo pipefail
> +
> +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
> +source "${SCRIPT_DIR}/rdma_common.sh"
> +
> +require_root
> +
> +debugfs_fail="/sys/kernel/debug/failslab"
> +recovery_wait_sec=2
> +if [[ ! -d "${debugfs_fail}" ]]; then
> +	log_warn "failslab is unavailable: ${debugfs_fail}"
> +	exit "${ksft_skip}"
> +fi
> +
> +for knob in probability interval times task-filter; do
> +	if [[ ! -f "${debugfs_fail}/${knob}" ]]; then
> +		log_warn "failslab knob missing: ${knob}"
> +		exit "${ksft_skip}"
> +	fi
> +done
> +
> +orig_probability="$(cat "${debugfs_fail}/probability")"
> +orig_interval="$(cat "${debugfs_fail}/interval")"
> +orig_times="$(cat "${debugfs_fail}/times")"
> +orig_task_filter="$(cat "${debugfs_fail}/task-filter")"
> +
> +restore_knobs()
> +{
> +	echo "${orig_probability}" >"${debugfs_fail}/probability" || true
> +	echo "${orig_interval}" >"${debugfs_fail}/interval" || true
> +	echo "${orig_times}" >"${debugfs_fail}/times" || true
> +	echo "${orig_task_filter}" >"${debugfs_fail}/task-filter" || true
> +}
> +
> +trap restore_knobs EXIT
> +
> +log_failslab_state()
> +{
> +	local state="$1"
> +	local task_filter probability interval times
> +
> +	task_filter="$(cat "${debugfs_fail}/task-filter")"
> +	probability="$(cat "${debugfs_fail}/probability")"
> +	interval="$(cat "${debugfs_fail}/interval")"
> +	times="$(cat "${debugfs_fail}/times")"
> +
> +	log_info "failslab ${state}: task-filter=${task_filter} probability=${probability}"
> +	log_info "failslab ${state}: interval=${interval} times=${times}"
> +}
> +
> +echo 1 >"${debugfs_fail}/task-filter"
> +echo 1 >"${debugfs_fail}/probability"
> +echo 100 >"${debugfs_fail}/interval"
> +echo 1 >"${debugfs_fail}/times"
> +log_failslab_state "enabled"
> +
> +if [[ -z "${CM_WORKLOAD_CMD:-}" && -n "${CM_VALIDATE_RECOVERY_CMD:-}" ]]; then
> +	CM_WORKLOAD_CMD="${CM_VALIDATE_RECOVERY_CMD}"
> +	log_warn "CM_WORKLOAD_CMD is not set; fallback to CM_VALIDATE_RECOVERY_CMD"
> +fi
> +
> +injected_rc=0
> +run_workload || injected_rc=$?
> +if [[ "${injected_rc}" -eq "${ksft_skip}" ]]; then
> +	exit "${ksft_skip}"
> +fi
> +log_info "workload rc under injection=${injected_rc}"
> +
> +echo 0 >"${debugfs_fail}/probability"
> +echo 0 >"${debugfs_fail}/times"
> +echo 0 >"${debugfs_fail}/task-filter"
> +log_failslab_state "disabled"
> +
> +recovery_cmd="${CM_VALIDATE_RECOVERY_CMD:-${CM_WORKLOAD_CMD:-}}"
> +if [[ -z "${recovery_cmd}" ]]; then
> +	log_warn "CM_VALIDATE_RECOVERY_CMD and CM_WORKLOAD_CMD are both unset"
> +	exit "${ksft_skip}"
> +fi
> +
> +if [[ "${recovery_wait_sec}" != "0" ]]; then
> +	log_info "waiting ${recovery_wait_sec}s before recovery workload"
> +	sleep "${recovery_wait_sec}"
> +fi
> +
> +log_info "running recovery workload: ${recovery_cmd}"
> +if ! bash -c "${recovery_cmd}"; then
> +	log_err "recovery workload failed after disabling fault injection"
> +	log_err "hint: ensure remote server is restarted and listening for a second connection"
> +	exit 1
> +fi
> +
> +exit 0
> diff --git a/tools/testing/selftests/rdma/rdma_cm_review_loop.sh b/tools/testing/selftests/rdma/rdma_cm_review_loop.sh
> new file mode 100755
> index 000000000000..c156090b17e3
> --- /dev/null
> +++ b/tools/testing/selftests/rdma/rdma_cm_review_loop.sh
> @@ -0,0 +1,35 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-2.0
> +
> +set -euo pipefail
> +
> +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
> +cd "${SCRIPT_DIR}"
> +
> +declare -A rc
> +
> +run_step()
> +{
> +	local name="$1"
> +	local cmd="$2"
> +
> +	echo "==== ${name} ===="
> +	if bash -c "${cmd}"; then
> +		rc["${name}"]=0
> +	else
> +		rc["${name}"]=$?
> +	fi
> +	echo "==== ${name} rc=${rc["${name}"]} ===="
> +}
> +
> +run_step baseline "./rdma_cm_baseline.sh"
> +run_step trace "./rdma_cm_trace_sequence.sh"
> +run_step counters "./rdma_cm_counter_delta.sh"
> +run_step fault_injection "./rdma_cm_fault_injection.sh"
> +
> +echo "==== summary ===="
> +for name in baseline trace counters fault_injection; do
> +	echo "${name}=${rc["${name}"]}"
> +done
> +
> +exit 0
> diff --git a/tools/testing/selftests/rdma/rdma_cm_trace_sequence.sh b/tools/testing/selftests/rdma/rdma_cm_trace_sequence.sh
> new file mode 100755
> index 000000000000..7e68289345e8
> --- /dev/null
> +++ b/tools/testing/selftests/rdma/rdma_cm_trace_sequence.sh
> @@ -0,0 +1,83 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-2.0
> +
> +set -euo pipefail
> +
> +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
> +source "${SCRIPT_DIR}/rdma_common.sh"
> +
> +require_root
> +require_cmd bash
> +require_cmd grep
> +
> +trace_dir="$(tracefs_dir || true)"
> +if [[ -z "${trace_dir}" ]]; then
> +	log_warn "tracefs is unavailable"
> +	exit "${ksft_skip}"
> +fi
> +
> +if [[ ! -d "${trace_dir}/events/ib_cma" ]]; then
> +	log_warn "ib_cma trace events are unavailable"
> +	exit "${ksft_skip}"
> +fi
> +
> +workload_rc=0
> +
> +cleanup_trace()
> +{
> +	local event
> +
> +	for event in icm_send_req icm_send_rep icm_send_rtu icm_recv_unknown_attr; do
> +		[[ -f "${trace_dir}/events/ib_cma/${event}/enable" ]] && \
> +			echo 0 >"${trace_dir}/events/ib_cma/${event}/enable"
> +	done
> +	[[ -f "${trace_dir}/events/ib_cma/enable" ]] && echo 0 >"${trace_dir}/events/ib_cma/enable"
> +	echo 0 >"${trace_dir}/tracing_on"
> +}
> +
> +trap cleanup_trace EXIT
> +
> +echo 0 >"${trace_dir}/tracing_on"
> +echo >"${trace_dir}/trace"
> +echo 1 >"${trace_dir}/events/ib_cma/enable"
> +
> +for event in icm_send_req icm_send_rep icm_send_rtu; do
> +	if [[ -f "${trace_dir}/events/ib_cma/${event}/enable" ]]; then
> +		echo 1 >"${trace_dir}/events/ib_cma/${event}/enable"
> +	fi
> +done
> +
> +echo 1 >"${trace_dir}/tracing_on"
> +run_workload || workload_rc=$?
> +echo 0 >"${trace_dir}/tracing_on"
> +
> +if [[ "${workload_rc}" -eq "${ksft_skip}" ]]; then
> +	exit "${ksft_skip}"
> +fi
> +
> +trace_log="/tmp/rdma_cm_trace.$(date +%s).log"
> +cat "${trace_dir}/trace" >"${trace_log}"
> +log_info "captured trace at ${trace_log}"
> +
> +if ! grep -Eq "icm_send_(req|rep|rtu)" "${trace_log}"; then
> +	log_err "missing CM send trace events (req/rep/rtu)"
> +	exit 1
> +fi
> +
> +err_lines="$(grep "icm_.*_err" "${trace_log}" || true)"
> +if [[ -n "${err_lines}" ]]; then
> +	# DREP send failure while already in TIMEWAIT is a common teardown
> +	# race and is tolerated for this smoke-style validation script.
> +	untolerated_err_lines="$(
> +		printf '%s\n' "${err_lines}" | \
> +			grep -Ev "icm_send_drep_err: .*state=TIMEWAIT" || true
> +	)"
> +	if [[ -n "${untolerated_err_lines}" ]]; then
> +		log_err "error trace event detected in ib_cma path"
> +		printf '%s\n' "${untolerated_err_lines}" >&2
> +		exit 1
> +	fi
> +	log_warn "only tolerated TIMEWAIT drep errors observed"
> +fi
> +
> +exit 0
> diff --git a/tools/testing/selftests/rdma/rdma_common.sh b/tools/testing/selftests/rdma/rdma_common.sh
> new file mode 100755
> index 000000000000..ee3d8b0d86b2
> --- /dev/null
> +++ b/tools/testing/selftests/rdma/rdma_common.sh
> @@ -0,0 +1,126 @@
> +#!/bin/bash
> +# SPDX-License-Identifier: GPL-2.0
> +
> +ksft_skip=4
> +RET=0
> +
> +RDMA_COUNTER_GROUPS=(
> +	cm_tx_msgs
> +	cm_tx_retries
> +	cm_rx_msgs
> +	cm_rx_duplicates
> +)
> +
> +RDMA_COUNTER_ATTRS=(
> +	req
> +	mra
> +	rej
> +	rep
> +	rtu
> +	dreq
> +	drep
> +	sidr_req
> +	sidr_rep
> +	lap
> +	apr
> +)
> +
> +log_info()
> +{
> +	echo "INFO: $*"
> +}
> +
> +log_warn()
> +{
> +	echo "WARN: $*" >&2
> +}
> +
> +log_err()
> +{
> +	echo "ERROR: $*" >&2
> +}
> +
> +require_root()
> +{
> +	if [[ "$(id -u)" -ne 0 ]]; then
> +		log_warn "this test requires root privileges"
> +		exit "${ksft_skip}"
> +	fi
> +}
> +
> +require_cmd()
> +{
> +	local cmd="$1"
> +
> +	command -v "${cmd}" >/dev/null 2>&1 || {
> +		log_warn "missing required command: ${cmd}"
> +		exit "${ksft_skip}"
> +	}
> +}
> +
> +tracefs_dir()
> +{
> +	if [[ -d /sys/kernel/tracing ]]; then
> +		echo /sys/kernel/tracing
> +	elif [[ -d /sys/kernel/debug/tracing ]]; then
> +		echo /sys/kernel/debug/tracing
> +	else
> +		return 1
> +	fi
> +}
> +
> +find_cm_counter_root()
> +{
> +	local base
> +	local port
> +	local candidate
> +
> +	for base in /sys/class/infiniband/*; do
> +		[[ -d "${base}" ]] || continue
> +
> +		for port in "${base}"/ports/*; do
> +			[[ -d "${port}" ]] || continue
> +			# RoCE / newer sysfs: cm_* groups live directly under ports/<N>/
> +			if [[ -d "${port}/cm_tx_msgs" ]]; then
> +				echo "${port}"
> +				return 0
> +			fi
> +			# Legacy layout: under counters/ or hw_counters/
> +			for candidate in "${port}/counters" "${port}/hw_counters"; do
> +				[[ -d "${candidate}/cm_tx_msgs" ]] || continue
> +				echo "${candidate}"
> +				return 0
> +			done
> +		done
> +	done
> +
> +	return 1
> +}
> +
> +read_cm_counter()
> +{
> +	local root="$1"
> +	local group="$2"
> +	local attr="$3"
> +	local path="${root}/${group}/${attr}"
> +
> +	if [[ -f "${path}" ]]; then
> +		cat "${path}" 2>/dev/null
> +	else
> +		echo 0
> +	fi
> +}
> +
> +run_workload()
> +{
> +	local cmd="${CM_WORKLOAD_CMD:-}"
> +
> +	if [[ -z "${cmd}" ]]; then
> +		log_warn "CM_WORKLOAD_CMD is not set"
> +		return "${ksft_skip}"
> +	fi
> +
> +	log_info "running workload: ${cmd}"
> +	bash -c "${cmd}"
> +}
> +