[PATCH v3] qtest/migration/rdma: Add test for rdma migration with ipv6

[PATCH v3] qtest/migration/rdma: Add test for rdma migration with ipv6

[Li Zhijian via]

Recently, we removed ipv6 restriction[0] from RDMA migration, add a
test for it.

[0] https://lore.kernel.org/qemu-devel/20250326095224.9918-1-jinpu.wang@ionos.com/

Cc: Jack Wang <jinpu.wang@ionos.com>
Cc: Michael R. Galaxy <mrgalaxy@nvidia.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Yu Zhang <yu.zhang@ionos.com>
Reviewed-by: Jack Wang <jinpu.wang@ionos.com>
Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
---
V3:
  - skip unsupported rxe interfaces: lo, tun, tap
  - setup/reuse both ipv4 and ipv4 in one setup shot
V2:
  - Collect Reviewed-by
  - quoate the whole string to adapt to the newer bash # Fedora40+
Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
---
 scripts/rdma-migration-helper.sh      | 48 ++++++++++++++++++++++-----
 tests/qtest/migration/precopy-tests.c | 21 +++++++++---
 2 files changed, 57 insertions(+), 12 deletions(-)

diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh
index a39f2fb0e5..404ce63a90 100755
--- a/scripts/rdma-migration-helper.sh
+++ b/scripts/rdma-migration-helper.sh
@@ -8,23 +8,39 @@ get_ipv4_addr()
         head -1 | tr -d '\n'
 }
 
+get_ipv6_addr() {
+    ipv6=$(ip -6 -o addr show dev "$1" |
+        sed -n 's/.*[[:blank:]]inet6[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' |
+        head -1 | tr -d '\n')
+
+    [ $? -eq 0 ] || return
+    echo -n "[$ipv6%$1]"
+}
+
 # existing rdma interfaces
 rdma_interfaces()
 {
-    rdma link show | sed -nE 's/^link .* netdev ([^ ]+).*$/\1 /p'
+    rdma link show | sed -nE 's/^link .* netdev ([^ ]+).*$/\1 /p' |
+    grep -Ev '^(lo|tun|tap)'
 }
 
 # existing valid ipv4 interfaces
 ipv4_interfaces()
 {
-    ip -o addr show | awk '/inet / {print $2}' | grep -v -w lo
+    ip -o addr show | awk '/inet / {print $2}' | grep -Ev '^(lo|tun|tap)'
+}
+
+ipv6_interfaces()
+{
+    ip -o addr show | awk '/inet6 / {print $2}' | grep -Ev '^(lo|tun|tap)'
 }
 
 rdma_rxe_detect()
 {
+    family=$1
     for r in $(rdma_interfaces)
     do
-        ipv4_interfaces | grep -qw $r && get_ipv4_addr $r && return
+        "$family"_interfaces | grep -qw $r && get_"$family"_addr $r && return
     done
 
     return 1
@@ -32,11 +48,16 @@ rdma_rxe_detect()
 
 rdma_rxe_setup()
 {
-    for i in $(ipv4_interfaces)
+    family=$1
+    for i in $("$family"_interfaces)
     do
-        rdma_interfaces | grep -qw $i && continue
+        if rdma_interfaces | grep -qw $i; then
+            echo "$family: Reuse the existing rdma/rxe ${i}_rxe for $i with $(get_"$family"_addr $i)"
+            return
+        fi
+
         rdma link add "${i}_rxe" type rxe netdev "$i" && {
-            echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_ipv4_addr $i)"
+            echo "$family: Setup new rdma/rxe ${i}_rxe for $i with $(get_"$family"_addr $i)"
             return
         }
     done
@@ -50,6 +71,12 @@ rdma_rxe_clean()
     modprobe -r rdma_rxe
 }
 
+IP_FAMILY=${IP_FAMILY:-ipv4}
+if [ "$IP_FAMILY" != "ipv6" ] && [ "$IP_FAMILY" != "ipv4" ]; then
+    echo "Unknown ip family '$IP_FAMILY', only ipv4 or ipv6 is supported." >&2
+    exit 1
+fi
+
 operation=${1:-detect}
 
 command -v rdma >/dev/null || {
@@ -62,9 +89,14 @@ if [ "$operation" == "setup" ] || [ "$operation" == "clean" ]; then
         echo "Root privilege is required to setup/clean a rdma/rxe link" >&2
         exit 1
     }
-    rdma_rxe_"$operation"
+    if [ "$operation" == "setup" ]; then
+        rdma_rxe_setup ipv4
+        rdma_rxe_setup ipv6
+    else
+        rdma_rxe_clean
+    fi
 elif [ "$operation" == "detect" ]; then
-    rdma_rxe_detect
+    rdma_rxe_detect "$IP_FAMILY"
 else
     echo "Usage: $0 [setup | detect | clean]"
 fi
diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c
index 4e32e61053..fb80c83967 100644
--- a/tests/qtest/migration/precopy-tests.c
+++ b/tests/qtest/migration/precopy-tests.c
@@ -131,12 +131,13 @@ static bool mlock_check(void)
 }
 
 #define RDMA_MIGRATION_HELPER "scripts/rdma-migration-helper.sh"
-static int new_rdma_link(char *buffer)
+static int new_rdma_link(char *buffer, bool ipv6)
 {
     char cmd[256];
     bool verbose = g_getenv("QTEST_LOG");
 
-    snprintf(cmd, sizeof(cmd), "%s detect %s", RDMA_MIGRATION_HELPER,
+    snprintf(cmd, sizeof(cmd), "IP_FAMILY=%s %s detect %s",
+             ipv6 ? "ipv6" : "ipv4", RDMA_MIGRATION_HELPER,
              verbose ? "" : "2>/dev/null");
 
     FILE *pipe = popen(cmd, "r");
@@ -161,7 +162,7 @@ static int new_rdma_link(char *buffer)
     return -1;
 }
 
-static void test_precopy_rdma_plain(void)
+static void __test_precopy_rdma_plain(bool ipv6)
 {
     char buffer[128] = {};
 
@@ -170,7 +171,7 @@ static void test_precopy_rdma_plain(void)
         return;
     }
 
-    if (new_rdma_link(buffer)) {
+    if (new_rdma_link(buffer, ipv6)) {
         g_test_skip("No rdma link available\n"
                     "# To enable the test:\n"
                     "# Run \'" RDMA_MIGRATION_HELPER " setup\' with root to "
@@ -193,6 +194,16 @@ static void test_precopy_rdma_plain(void)
 
     test_precopy_common(&args);
 }
+
+static void test_precopy_rdma_plain(void)
+{
+    __test_precopy_rdma_plain(0);
+}
+
+static void test_precopy_rdma_plain_ipv6(void)
+{
+    __test_precopy_rdma_plain(1);
+}
 #endif
 
 static void test_precopy_tcp_plain(void)
@@ -1226,6 +1237,8 @@ static void migration_test_add_precopy_smoke(MigrationTestEnv *env)
 #ifdef CONFIG_RDMA
     migration_test_add("/migration/precopy/rdma/plain",
                        test_precopy_rdma_plain);
+    migration_test_add("/migration/precopy/rdma/plain/ipv6",
+                       test_precopy_rdma_plain_ipv6);
 #endif
 }
 
-- 
2.41.0

Re: [PATCH v3] qtest/migration/rdma: Add test for rdma migration with ipv6

[Peter Xu]

On Mon, May 12, 2025 at 02:01:35PM +0800, Li Zhijian wrote:
> Recently, we removed ipv6 restriction[0] from RDMA migration, add a
> test for it.
> 
> [0] https://lore.kernel.org/qemu-devel/20250326095224.9918-1-jinpu.wang@ionos.com/
> 
> Cc: Jack Wang <jinpu.wang@ionos.com>
> Cc: Michael R. Galaxy <mrgalaxy@nvidia.com>
> Cc: Peter Xu <peterx@redhat.com>
> Cc: Yu Zhang <yu.zhang@ionos.com>
> Reviewed-by: Jack Wang <jinpu.wang@ionos.com>
> Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
> ---
> V3:
>   - skip unsupported rxe interfaces: lo, tun, tap
>   - setup/reuse both ipv4 and ipv4 in one setup shot

It still fails the ipv6 test here...

# Running /x86_64/migration/precopy/rdma/plain/ipv6
# Using machine type: pc-q35-10.1
# starting QEMU: exec ./qemu-system-x86_64 -qtest unix:/tmp/qtest-334248.sock -qtest-log /dev/null -chardev socket,path=/tmp/qtest-334248.qmp,id=char0 -mon char
dev=char0,mode=control -display none -audio none -accel kvm -accel tcg -machine pc-q35-10.1, -name source,debug-threads=on -m 150M  -serial file:/tmp/migration-
test-2TUX62/src_serial -drive if=none,id=d0,file=/tmp/migration-test-2TUX62/bootsect,format=raw -device ide-hd,drive=d0,secs=1,cyls=1,heads=1    -accel qtest
# starting QEMU: exec ./qemu-system-x86_64 -qtest unix:/tmp/qtest-334248.sock -qtest-log /dev/null -chardev socket,path=/tmp/qtest-334248.qmp,id=char0 -mon char
dev=char0,mode=control -display none -audio none -accel kvm -accel tcg -machine pc-q35-10.1, -name target,debug-threads=on -m 150M  -serial file:/tmp/migration-
test-2TUX62/dest_serial -incoming rdma:[fdd3:4fdc:97c9:ca4e:2837:28dd:1ec4:6b5a%wlp0s20f3]:29200  -drive if=none,id=d0,file=/tmp/migration-test-2TUX62/bootsect,
format=raw -device ide-hd,drive=d0,secs=1,cyls=1,heads=1    -accel qtest
qemu-system-x86_64: -incoming rdma:[fdd3:4fdc:97c9:ca4e:2837:28dd:1ec4:6b5a%wlp0s20f3]:29200: RDMA ERROR: could not rdma_getaddrinfo address fdd3:4fdc:97c9:ca4e
:2837:28dd:1ec4:6b5a%wlp0s20f3
Broken pipe
../tests/qtest/libqtest.c:199: kill_qemu() tried to terminate QEMU process but encountered exit status 1 (expected 0)
Aborted

Would below look right to you as a fix (I also adjusted a bool input):

diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c
index 1a67a9e2e4..a62d3c5378 100644
--- a/tests/qtest/migration/precopy-tests.c
+++ b/tests/qtest/migration/precopy-tests.c
@@ -197,12 +197,12 @@ static void __test_precopy_rdma_plain(bool ipv6)
 
 static void test_precopy_rdma_plain(void)
 {
-    __test_precopy_rdma_plain(0);
+    __test_precopy_rdma_plain(false);
 }
 
 static void test_precopy_rdma_plain_ipv6(void)
 {
-    __test_precopy_rdma_plain(1);
+    __test_precopy_rdma_plain(true);
 }
 #endif
 
diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh
index 404ce63a90..6ef61b9917 100755
--- a/scripts/rdma-migration-helper.sh
+++ b/scripts/rdma-migration-helper.sh
@@ -14,7 +14,7 @@ get_ipv6_addr() {
         head -1 | tr -d '\n')
 
     [ $? -eq 0 ] || return
-    echo -n "[$ipv6%$1]"
+    echo -n "[$ipv6]"
 }
 
With that the test runs.  If it looks good to you I can queue it with that
squashed.

Thanks,

-- 
Peter Xu

Re: [PATCH v3] qtest/migration/rdma: Add test for rdma migration with ipv6

[Zhijian Li (Fujitsu) via]

On 12/05/2025 21:56, Peter Xu wrote:
> On Mon, May 12, 2025 at 02:01:35PM +0800, Li Zhijian wrote:
>> Recently, we removed ipv6 restriction[0] from RDMA migration, add a
>> test for it.
>>
>> [0] https://lore.kernel.org/qemu-devel/20250326095224.9918-1-jinpu.wang@ionos.com/
>>
>> Cc: Jack Wang <jinpu.wang@ionos.com>
>> Cc: Michael R. Galaxy <mrgalaxy@nvidia.com>
>> Cc: Peter Xu <peterx@redhat.com>
>> Cc: Yu Zhang <yu.zhang@ionos.com>
>> Reviewed-by: Jack Wang <jinpu.wang@ionos.com>
>> Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
>> ---
>> V3:
>>    - skip unsupported rxe interfaces: lo, tun, tap
>>    - setup/reuse both ipv4 and ipv4 in one setup shot
> 
> It still fails the ipv6 test here...
> 
> # Running /x86_64/migration/precopy/rdma/plain/ipv6
> # Using machine type: pc-q35-10.1
> # starting QEMU: exec ./qemu-system-x86_64 -qtest unix:/tmp/qtest-334248.sock -qtest-log /dev/null -chardev socket,path=/tmp/qtest-334248.qmp,id=char0 -mon char
> dev=char0,mode=control -display none -audio none -accel kvm -accel tcg -machine pc-q35-10.1, -name source,debug-threads=on -m 150M  -serial file:/tmp/migration-
> test-2TUX62/src_serial -drive if=none,id=d0,file=/tmp/migration-test-2TUX62/bootsect,format=raw -device ide-hd,drive=d0,secs=1,cyls=1,heads=1    -accel qtest
> # starting QEMU: exec ./qemu-system-x86_64 -qtest unix:/tmp/qtest-334248.sock -qtest-log /dev/null -chardev socket,path=/tmp/qtest-334248.qmp,id=char0 -mon char
> dev=char0,mode=control -display none -audio none -accel kvm -accel tcg -machine pc-q35-10.1, -name target,debug-threads=on -m 150M  -serial file:/tmp/migration-
> test-2TUX62/dest_serial -incoming rdma:[fdd3:4fdc:97c9:ca4e:2837:28dd:1ec4:6b5a%wlp0s20f3]:29200  -drive if=none,id=d0,file=/tmp/migration-test-2TUX62/bootsect,
> format=raw -device ide-hd,drive=d0,secs=1,cyls=1,heads=1    -accel qtest
> qemu-system-x86_64: -incoming rdma:[fdd3:4fdc:97c9:ca4e:2837:28dd:1ec4:6b5a%wlp0s20f3]:29200: RDMA ERROR: could not rdma_getaddrinfo address fdd3:4fdc:97c9:ca4e
> :2837:28dd:1ec4:6b5a%wlp0s20f3
> Broken pipe
> ../tests/qtest/libqtest.c:199: kill_qemu() tried to terminate QEMU process but encountered exit status 1 (expected 0)
> Aborted
> 
> Would below look right to you as a fix (I also adjusted a bool input):
> 
> diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c
> index 1a67a9e2e4..a62d3c5378 100644
> --- a/tests/qtest/migration/precopy-tests.c
> +++ b/tests/qtest/migration/precopy-tests.c
> @@ -197,12 +197,12 @@ static void __test_precopy_rdma_plain(bool ipv6)
>   
>   static void test_precopy_rdma_plain(void)
>   {
> -    __test_precopy_rdma_plain(0);
> +    __test_precopy_rdma_plain(false);
>   }
>   
>   static void test_precopy_rdma_plain_ipv6(void)
>   {
> -    __test_precopy_rdma_plain(1);
> +    __test_precopy_rdma_plain(true);
>   }
>   #endif

Above 2 line are good to me.


>   
> diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh
> index 404ce63a90..6ef61b9917 100755
> --- a/scripts/rdma-migration-helper.sh
> +++ b/scripts/rdma-migration-helper.sh
> @@ -14,7 +14,7 @@ get_ipv6_addr() {
>           head -1 | tr -d '\n')
>   
>       [ $? -eq 0 ] || return
> -    echo -n "[$ipv6%$1]"
> +    echo -n "[$ipv6]"

Noop,

Without your above changes, it works in my environment:

qemu$ QTEST_QEMU_BINARY=./build/qemu-system-x86_64 ./build/tests/qtest/migration-test -p /x86_64/migration/precopy/rdma/plain/ipv6
# random seed: R02S42e93aea2e5a5cfa2ace5dcfdec0b421
# starting QEMU: exec ./build/qemu-system-x86_64 -qtest unix:/tmp/qtest-2842155.sock -qtest-log /dev/null -chardev socket,path=/tmp/qtest-2842155.qmp,id=char0 -mon chardev=char0,mode=control -display none -audio none -machine none -accel qtest
# Skipping test: userfaultfd not available
# Start of x86_64 tests
# Start of migration tests
# Start of precopy tests
# Start of rdma tests
# Start of plain tests
# Running /x86_64/migration/precopy/rdma/plain/ipv6
# Using machine type: pc-q35-10.1
# starting QEMU: exec ./build/qemu-system-x86_64 -qtest unix:/tmp/qtest-2842155.sock -qtest-log /dev/null -chardev socket,path=/tmp/qtest-2842155.qmp,id=char0 -mon chardev=char0,mode=control -display none -audio none -accel kvm -accel tcg -machine pc-q35-10.1, -name source,debug-threads=on -m 150M  -serial file:/tmp/migration-test-0WNM62/src_serial -drive if=none,id=d0,file=/tmp/migration-test-0WNM62/bootsect,format=raw -device ide-hd,drive=d0,secs=1,cyls=1,heads=1    -accel qtest
# starting QEMU: exec ./build/qemu-system-x86_64 -qtest unix:/tmp/qtest-2842155.sock -qtest-log /dev/null -chardev socket,path=/tmp/qtest-2842155.qmp,id=char0 -mon chardev=char0,mode=control -display none -audio none -accel kvm -accel tcg -machine pc-q35-10.1, -name target,debug-threads=on -m 150M  -serial file:/tmp/migration-test-0WNM62/dest_serial -incoming rdma:[fe80::a44a:e098:6a25:fede%eth1]:29200  -drive if=none,id=d0,file=/tmp/migration-test-0WNM62/bootsect,format=raw -device ide-hd,drive=d0,secs=1,cyls=1,heads=1    -accel qtest
ok 1 /x86_64/migration/precopy/rdma/plain/ipv6
# End of plain tests
# End of rdma tests
# End of precopy tests
# End of migration tests
# End of x86_64 tests
1..1


After applied your changes, it fails:

qemu$ QTEST_QEMU_BINARY=./build/qemu-system-x86_64 ./build/tests/qtest/migration-test -p /x86_64/migration/precopy/rdma/plain/ipv6
# random seed: R02Sc0b0224c0f011caca094195a40d40e74
# starting QEMU: exec ./build/qemu-system-x86_64 -qtest unix:/tmp/qtest-2842492.sock -qtest-log /dev/null -chardev socket,path=/tmp/qtest-2842492.qmp,id=char0 -mon chardev=char0,mode=control -display none -audio none -machine none -accel qtest
# Skipping test: userfaultfd not available
# Start of x86_64 tests
# Start of migration tests
# Start of precopy tests
# Start of rdma tests
# Start of plain tests
# Running /x86_64/migration/precopy/rdma/plain/ipv6
# Using machine type: pc-q35-10.1
# starting QEMU: exec ./build/qemu-system-x86_64 -qtest unix:/tmp/qtest-2842492.sock -qtest-log /dev/null -chardev socket,path=/tmp/qtest-2842492.qmp,id=char0 -mon chardev=char0,mode=control -display none -audio none -accel kvm -accel tcg -machine pc-q35-10.1, -name source,debug-threads=on -m 150M  -serial file:/tmp/migration-test-6IAM62/src_serial -drive if=none,id=d0,file=/tmp/migration-test-6IAM62/bootsect,format=raw -device ide-hd,drive=d0,secs=1,cyls=1,heads=1    -accel qtest
# starting QEMU: exec ./build/qemu-system-x86_64 -qtest unix:/tmp/qtest-2842492.sock -qtest-log /dev/null -chardev socket,path=/tmp/qtest-2842492.qmp,id=char0 -mon chardev=char0,mode=control -display none -audio none -accel kvm -accel tcg -machine pc-q35-10.1, -name target,debug-threads=on -m 150M  -serial file:/tmp/migration-test-6IAM62/dest_serial -incoming rdma:[fe80::a44a:e098:6a25:fede]:29200  -drive if=none,id=d0,file=/tmp/migration-test-6IAM62/bootsect,format=raw -device ide-hd,drive=d0,secs=1,cyls=1,heads=1    -accel qtest
qemu-system-x86_64: -incoming rdma:[fe80::a44a:e098:6a25:fede]:29200: RDMA ERROR: Error: could not rdma_bind_addr!
Broken pipe
../tests/qtest/libqtest.c:199: kill_qemu() tried to terminate QEMU process but encountered exit status 1 (expected 0)
Aborted


I just asked the AI

> Address Type	                            Return Format	        Scope	         Require Zone ID (Interface)
> Link-Local (fe80::/10)	           [address%interface]	     Local Link	          Yes
> ULA (fd00::/8) / GUA (e.g., 2001::/3)	   [address]	            Global / Site-Local	  No

> Notes:
> Link-Local (fe80::/10):
> 
> Applies to direct physical link communication.
> Requires explicit zone identifier (%interface) to resolve ambiguity (per RFC 4007).
> ULA (Unique Local Address) and GUA (Global Unicast Address):
> 
> ULA (fd00::/8) is used for private networks.
> GUA (2000::/3) is a public routable address.
> Both share the global/site-local scope and do not need interface specification.
> This table reflects the core logic of differentiating IPv6 address scopes and their usage in scripts.

He suggests this change for different type of ipv6 addresses().

--- a/scripts/rdma-migration-helper.sh
+++ b/scripts/rdma-migration-helper.sh
@@ -14,7 +14,12 @@ get_ipv6_addr() {
          head -1 | tr -d '\n')
  
      [ $? -eq 0 ] || return
-    echo -n "[$ipv6%$1]"
+
+    if [[ "$ipv6" =~ ^fe80: ]]; then
+        echo -n "[$ipv6%$1]"
+    else
+        echo -n "[$ipv6]"
+    fi
  }


>   }
>   
> With that the test runs.  If it looks good to you I can queue it with that
> squashed.
> 
> Thanks,
>