[PATCH] qtest/migration/rdma: Add test for rdma migration with ipv6

[PATCH] qtest/migration/rdma: Add test for rdma migration with ipv6

[Li Zhijian via]

Recently, we removed ipv6 restriction[0] from RDMA migration, add a
test for it.

[0] https://lore.kernel.org/qemu-devel/20250326095224.9918-1-jinpu.wang@ionos.com/

Cc: Jack Wang <jinpu.wang@ionos.com>
Cc: Michael R. Galaxy <mrgalaxy@nvidia.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Yu Zhang <yu.zhang@ionos.com>
Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
---

This test is added based on [1]

Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
---
 scripts/rdma-migration-helper.sh      | 26 +++++++++++++++++++++++---
 tests/qtest/migration/precopy-tests.c | 21 +++++++++++++++++----
 2 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh
index 019b9ad932..c358392b72 100755
--- a/scripts/rdma-migration-helper.sh
+++ b/scripts/rdma-migration-helper.sh
@@ -8,6 +8,15 @@ get_ipv4_addr()
         head -1 | tr -d '\n'
 }
 
+get_ipv6_addr() {
+    ipv6=$(ip -6 -o addr show dev "$1" |
+        sed -n 's/.*[[:blank:]]inet6[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' |
+        head -1 | tr -d '\n')
+
+    [ $? -eq 0 ] || return
+    echo -n ["$ipv6"%$1]
+}
+
 # existing rdma interfaces
 rdma_interfaces()
 {
@@ -20,11 +29,16 @@ ipv4_interfaces()
     ip -o addr show | awk '/inet / {print $2}' | sort -u | grep -v -w lo
 }
 
+ipv6_interfaces()
+{
+    ip -o addr show | awk '/inet6 / {print $2}' | sort -u | grep -v -w lo
+}
+
 rdma_rxe_detect()
 {
     for r in $(rdma_interfaces)
     do
-        ipv4_interfaces | grep -qw $r && get_ipv4_addr $r && return
+        "$IP_FAMILY"_interfaces | grep -qw $r && get_"$IP_FAMILY"_addr $r && return
     done
 
     return 1
@@ -32,11 +46,11 @@ rdma_rxe_detect()
 
 rdma_rxe_setup()
 {
-    for i in $(ipv4_interfaces)
+    for i in $("$IP_FAMILY"_interfaces)
     do
         rdma_interfaces | grep -qw $i && continue
         rdma link add "${i}_rxe" type rxe netdev "$i" && {
-            echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_ipv4_addr $i)"
+            echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_"$IP_FAMILY"_addr $i)"
             return
         }
     done
@@ -50,6 +64,12 @@ rdma_rxe_clean()
     modprobe -r rdma_rxe
 }
 
+IP_FAMILY=${IP_FAMILY:-ipv4}
+if [ "$IP_FAMILY" != "ipv6" ] && [ "$IP_FAMILY" != "ipv4" ]; then
+    echo "Unknown ip family '$IP_FAMILY', only ipv4 or ipv6 is supported," >&2
+    exit 1
+fi
+
 operation=${1:-detect}
 
 command -v rdma >/dev/null || {
diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c
index a137ea9f2c..9f7236dc59 100644
--- a/tests/qtest/migration/precopy-tests.c
+++ b/tests/qtest/migration/precopy-tests.c
@@ -102,12 +102,13 @@ static void test_precopy_unix_dirty_ring(void)
 #ifdef CONFIG_RDMA
 
 #define RDMA_MIGRATION_HELPER "scripts/rdma-migration-helper.sh"
-static int new_rdma_link(char *buffer)
+static int new_rdma_link(char *buffer, bool ipv6)
 {
     char cmd[256];
     bool verbose = g_getenv("QTEST_LOG");
 
-    snprintf(cmd, sizeof(cmd), "%s detect %s", RDMA_MIGRATION_HELPER,
+    snprintf(cmd, sizeof(cmd), "IP_FAMILY=%s %s detect %s",
+             ipv6 ? "ipv6" : "ipv4", RDMA_MIGRATION_HELPER,
              verbose ? "" : "2>/dev/null");
 
     FILE *pipe = popen(cmd, "r");
@@ -132,11 +133,11 @@ static int new_rdma_link(char *buffer)
     return -1;
 }
 
-static void test_precopy_rdma_plain(void)
+static void test_precopy_rdma_plain_ip(bool ipv6)
 {
     char buffer[128] = {};
 
-    if (new_rdma_link(buffer)) {
+    if (new_rdma_link(buffer, ipv6)) {
         g_test_skip("No rdma link available\n"
                     "# To enable the test:\n"
                     "# Run \'" RDMA_MIGRATION_HELPER " setup\' with root to setup"
@@ -159,6 +160,16 @@ static void test_precopy_rdma_plain(void)
 
     test_precopy_common(&args);
 }
+
+static void test_precopy_rdma_plain(void)
+{
+    test_precopy_rdma_plain_ip(0);
+}
+
+static void test_precopy_rdma_plain_ipv6(void)
+{
+    test_precopy_rdma_plain_ip(1);
+}
 #endif
 
 static void test_precopy_tcp_plain(void)
@@ -1189,6 +1200,8 @@ static void migration_test_add_precopy_smoke(MigrationTestEnv *env)
 #ifdef CONFIG_RDMA
     migration_test_add("/migration/precopy/rdma/plain",
                        test_precopy_rdma_plain);
+    migration_test_add("/migration/precopy/rdma/plain/ipv6",
+                       test_precopy_rdma_plain_ipv6);
 #endif
 }
 
-- 
2.41.0

Re: [PATCH] qtest/migration/rdma: Add test for rdma migration with ipv6

[Jinpu Wang]

On Thu, Mar 27, 2025 at 3:13 AM Li Zhijian <lizhijian@fujitsu.com> wrote:
>
> Recently, we removed ipv6 restriction[0] from RDMA migration, add a
> test for it.
>
> [0] https://lore.kernel.org/qemu-devel/20250326095224.9918-1-jinpu.wang@ionos.com/
>
> Cc: Jack Wang <jinpu.wang@ionos.com>
> Cc: Michael R. Galaxy <mrgalaxy@nvidia.com>
> Cc: Peter Xu <peterx@redhat.com>
> Cc: Yu Zhang <yu.zhang@ionos.com>
> Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
Hi Zhijian,

Thx for adding the tests. lgtm.
Reviewed-by: Jack Wang <jinpu.wang@ionos.com>
> ---
>
> This test is added based on [1]
>
> Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
> ---
>  scripts/rdma-migration-helper.sh      | 26 +++++++++++++++++++++++---
>  tests/qtest/migration/precopy-tests.c | 21 +++++++++++++++++----
>  2 files changed, 40 insertions(+), 7 deletions(-)
>
> diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh
> index 019b9ad932..c358392b72 100755
> --- a/scripts/rdma-migration-helper.sh
> +++ b/scripts/rdma-migration-helper.sh
> @@ -8,6 +8,15 @@ get_ipv4_addr()
>          head -1 | tr -d '\n'
>  }
>
> +get_ipv6_addr() {
> +    ipv6=$(ip -6 -o addr show dev "$1" |
> +        sed -n 's/.*[[:blank:]]inet6[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' |
> +        head -1 | tr -d '\n')
> +
> +    [ $? -eq 0 ] || return
> +    echo -n ["$ipv6"%$1]
> +}
> +
>  # existing rdma interfaces
>  rdma_interfaces()
>  {
> @@ -20,11 +29,16 @@ ipv4_interfaces()
>      ip -o addr show | awk '/inet / {print $2}' | sort -u | grep -v -w lo
>  }
>
> +ipv6_interfaces()
> +{
> +    ip -o addr show | awk '/inet6 / {print $2}' | sort -u | grep -v -w lo
> +}
> +
>  rdma_rxe_detect()
>  {
>      for r in $(rdma_interfaces)
>      do
> -        ipv4_interfaces | grep -qw $r && get_ipv4_addr $r && return
> +        "$IP_FAMILY"_interfaces | grep -qw $r && get_"$IP_FAMILY"_addr $r && return
>      done
>
>      return 1
> @@ -32,11 +46,11 @@ rdma_rxe_detect()
>
>  rdma_rxe_setup()
>  {
> -    for i in $(ipv4_interfaces)
> +    for i in $("$IP_FAMILY"_interfaces)
>      do
>          rdma_interfaces | grep -qw $i && continue
>          rdma link add "${i}_rxe" type rxe netdev "$i" && {
> -            echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_ipv4_addr $i)"
> +            echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_"$IP_FAMILY"_addr $i)"
>              return
>          }
>      done
> @@ -50,6 +64,12 @@ rdma_rxe_clean()
>      modprobe -r rdma_rxe
>  }
>
> +IP_FAMILY=${IP_FAMILY:-ipv4}
> +if [ "$IP_FAMILY" != "ipv6" ] && [ "$IP_FAMILY" != "ipv4" ]; then
> +    echo "Unknown ip family '$IP_FAMILY', only ipv4 or ipv6 is supported," >&2
> +    exit 1
> +fi
> +
>  operation=${1:-detect}
>
>  command -v rdma >/dev/null || {
> diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c
> index a137ea9f2c..9f7236dc59 100644
> --- a/tests/qtest/migration/precopy-tests.c
> +++ b/tests/qtest/migration/precopy-tests.c
> @@ -102,12 +102,13 @@ static void test_precopy_unix_dirty_ring(void)
>  #ifdef CONFIG_RDMA
>
>  #define RDMA_MIGRATION_HELPER "scripts/rdma-migration-helper.sh"
> -static int new_rdma_link(char *buffer)
> +static int new_rdma_link(char *buffer, bool ipv6)
>  {
>      char cmd[256];
>      bool verbose = g_getenv("QTEST_LOG");
>
> -    snprintf(cmd, sizeof(cmd), "%s detect %s", RDMA_MIGRATION_HELPER,
> +    snprintf(cmd, sizeof(cmd), "IP_FAMILY=%s %s detect %s",
> +             ipv6 ? "ipv6" : "ipv4", RDMA_MIGRATION_HELPER,
>               verbose ? "" : "2>/dev/null");
>
>      FILE *pipe = popen(cmd, "r");
> @@ -132,11 +133,11 @@ static int new_rdma_link(char *buffer)
>      return -1;
>  }
>
> -static void test_precopy_rdma_plain(void)
> +static void test_precopy_rdma_plain_ip(bool ipv6)
>  {
>      char buffer[128] = {};
>
> -    if (new_rdma_link(buffer)) {
> +    if (new_rdma_link(buffer, ipv6)) {
>          g_test_skip("No rdma link available\n"
>                      "# To enable the test:\n"
>                      "# Run \'" RDMA_MIGRATION_HELPER " setup\' with root to setup"
> @@ -159,6 +160,16 @@ static void test_precopy_rdma_plain(void)
>
>      test_precopy_common(&args);
>  }
> +
> +static void test_precopy_rdma_plain(void)
> +{
> +    test_precopy_rdma_plain_ip(0);
> +}
> +
> +static void test_precopy_rdma_plain_ipv6(void)
> +{
> +    test_precopy_rdma_plain_ip(1);
> +}
>  #endif
>
>  static void test_precopy_tcp_plain(void)
> @@ -1189,6 +1200,8 @@ static void migration_test_add_precopy_smoke(MigrationTestEnv *env)
>  #ifdef CONFIG_RDMA
>      migration_test_add("/migration/precopy/rdma/plain",
>                         test_precopy_rdma_plain);
> +    migration_test_add("/migration/precopy/rdma/plain/ipv6",
> +                       test_precopy_rdma_plain_ipv6);
>  #endif
>  }
>
> --
> 2.41.0
>

Re: [PATCH] qtest/migration/rdma: Add test for rdma migration with ipv6

[Peter Xu]

On Thu, Mar 27, 2025 at 10:12:34AM +0800, Li Zhijian wrote:
> Recently, we removed ipv6 restriction[0] from RDMA migration, add a
> test for it.
> 
> [0] https://lore.kernel.org/qemu-devel/20250326095224.9918-1-jinpu.wang@ionos.com/
> 
> Cc: Jack Wang <jinpu.wang@ionos.com>
> Cc: Michael R. Galaxy <mrgalaxy@nvidia.com>
> Cc: Peter Xu <peterx@redhat.com>
> Cc: Yu Zhang <yu.zhang@ionos.com>
> Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
> ---
> 
> This test is added based on [1]

Hmm, this cannot apply cleanly on top of your rdma test + Jake's ipv6 fix.
Did I made it wrong somewhere?

At the meantime, I found that the current rdma unit test failing if I did
the "setup" phase then run this test without root:

stderr:
qemu-system-x86_64: cannot get rkey
qemu-system-x86_64: error while loading state section id 2(ram)
qemu-system-x86_64: load of migration failed: Operation not permitted
qemu-system-x86_64: rdma migration: recv polling control error!
qemu-system-x86_64: RDMA is in an error state waiting migration to abort!
qemu-system-x86_64: failed to save SaveStateEntry with id(name): 2(ram): -1
qemu-system-x86_64: Channel error: Operation not permitted
**
ERROR:../tests/qtest/migration/migration-qmp.c:200:check_migration_status: assertion failed (current_status != "failed"): ("failed" != "failed")
qemu-system-x86_64: warning: Early error. Sending error.
qemu-system-x86_64: warning: rdma migration: send polling control error
../tests/qtest/libqtest.c:199: kill_qemu() tried to terminate QEMU process but encountered exit status 1 (expected 0)

So running the test also needs root?  Is it possible we fix the test so it
can also be smart enough to skip if it knows it'll hit the "cannot get
rkey" error (even if it sees the rdma link setup)?  Not something urgent
but definitely good to have.

OTOH, running the test using root always works for me.

> 
> Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
> ---
>  scripts/rdma-migration-helper.sh      | 26 +++++++++++++++++++++++---
>  tests/qtest/migration/precopy-tests.c | 21 +++++++++++++++++----
>  2 files changed, 40 insertions(+), 7 deletions(-)
> 
> diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh
> index 019b9ad932..c358392b72 100755
> --- a/scripts/rdma-migration-helper.sh
> +++ b/scripts/rdma-migration-helper.sh
> @@ -8,6 +8,15 @@ get_ipv4_addr()
>          head -1 | tr -d '\n'
>  }
>  
> +get_ipv6_addr() {
> +    ipv6=$(ip -6 -o addr show dev "$1" |
> +        sed -n 's/.*[[:blank:]]inet6[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' |
> +        head -1 | tr -d '\n')
> +
> +    [ $? -eq 0 ] || return
> +    echo -n ["$ipv6"%$1]
> +}
> +
>  # existing rdma interfaces
>  rdma_interfaces()
>  {
> @@ -20,11 +29,16 @@ ipv4_interfaces()
>      ip -o addr show | awk '/inet / {print $2}' | sort -u | grep -v -w lo
>  }
>  
> +ipv6_interfaces()
> +{
> +    ip -o addr show | awk '/inet6 / {print $2}' | sort -u | grep -v -w lo
> +}
> +
>  rdma_rxe_detect()
>  {
>      for r in $(rdma_interfaces)
>      do
> -        ipv4_interfaces | grep -qw $r && get_ipv4_addr $r && return
> +        "$IP_FAMILY"_interfaces | grep -qw $r && get_"$IP_FAMILY"_addr $r && return
>      done
>  
>      return 1
> @@ -32,11 +46,11 @@ rdma_rxe_detect()
>  
>  rdma_rxe_setup()
>  {
> -    for i in $(ipv4_interfaces)
> +    for i in $("$IP_FAMILY"_interfaces)
>      do
>          rdma_interfaces | grep -qw $i && continue
>          rdma link add "${i}_rxe" type rxe netdev "$i" && {
> -            echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_ipv4_addr $i)"
> +            echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_"$IP_FAMILY"_addr $i)"
>              return
>          }
>      done
> @@ -50,6 +64,12 @@ rdma_rxe_clean()
>      modprobe -r rdma_rxe
>  }
>  
> +IP_FAMILY=${IP_FAMILY:-ipv4}
> +if [ "$IP_FAMILY" != "ipv6" ] && [ "$IP_FAMILY" != "ipv4" ]; then
> +    echo "Unknown ip family '$IP_FAMILY', only ipv4 or ipv6 is supported," >&2
> +    exit 1
> +fi
> +
>  operation=${1:-detect}
>  
>  command -v rdma >/dev/null || {
> diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c
> index a137ea9f2c..9f7236dc59 100644
> --- a/tests/qtest/migration/precopy-tests.c
> +++ b/tests/qtest/migration/precopy-tests.c
> @@ -102,12 +102,13 @@ static void test_precopy_unix_dirty_ring(void)
>  #ifdef CONFIG_RDMA
>  
>  #define RDMA_MIGRATION_HELPER "scripts/rdma-migration-helper.sh"
> -static int new_rdma_link(char *buffer)
> +static int new_rdma_link(char *buffer, bool ipv6)
>  {
>      char cmd[256];
>      bool verbose = g_getenv("QTEST_LOG");
>  
> -    snprintf(cmd, sizeof(cmd), "%s detect %s", RDMA_MIGRATION_HELPER,
> +    snprintf(cmd, sizeof(cmd), "IP_FAMILY=%s %s detect %s",
> +             ipv6 ? "ipv6" : "ipv4", RDMA_MIGRATION_HELPER,
>               verbose ? "" : "2>/dev/null");
>  
>      FILE *pipe = popen(cmd, "r");
> @@ -132,11 +133,11 @@ static int new_rdma_link(char *buffer)
>      return -1;
>  }
>  
> -static void test_precopy_rdma_plain(void)
> +static void test_precopy_rdma_plain_ip(bool ipv6)
>  {
>      char buffer[128] = {};
>  
> -    if (new_rdma_link(buffer)) {
> +    if (new_rdma_link(buffer, ipv6)) {
>          g_test_skip("No rdma link available\n"
>                      "# To enable the test:\n"
>                      "# Run \'" RDMA_MIGRATION_HELPER " setup\' with root to setup"
> @@ -159,6 +160,16 @@ static void test_precopy_rdma_plain(void)
>  
>      test_precopy_common(&args);
>  }
> +
> +static void test_precopy_rdma_plain(void)
> +{
> +    test_precopy_rdma_plain_ip(0);
> +}
> +
> +static void test_precopy_rdma_plain_ipv6(void)
> +{
> +    test_precopy_rdma_plain_ip(1);
> +}
>  #endif
>  
>  static void test_precopy_tcp_plain(void)
> @@ -1189,6 +1200,8 @@ static void migration_test_add_precopy_smoke(MigrationTestEnv *env)
>  #ifdef CONFIG_RDMA
>      migration_test_add("/migration/precopy/rdma/plain",
>                         test_precopy_rdma_plain);
> +    migration_test_add("/migration/precopy/rdma/plain/ipv6",
> +                       test_precopy_rdma_plain_ipv6);
>  #endif
>  }
>  
> -- 
> 2.41.0
> 

-- 
Peter Xu

Re: [PATCH] qtest/migration/rdma: Add test for rdma migration with ipv6

[Zhijian Li (Fujitsu) via]

On 30/04/2025 05:09, Peter Xu wrote:
> On Thu, Mar 27, 2025 at 10:12:34AM +0800, Li Zhijian wrote:
>> Recently, we removed ipv6 restriction[0] from RDMA migration, add a
>> test for it.
>>
>> [0] https://lore.kernel.org/qemu-devel/20250326095224.9918-1-jinpu.wang@ionos.com/
>>
>> Cc: Jack Wang <jinpu.wang@ionos.com>
>> Cc: Michael R. Galaxy <mrgalaxy@nvidia.com>
>> Cc: Peter Xu <peterx@redhat.com>
>> Cc: Yu Zhang <yu.zhang@ionos.com>
>> Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
>> ---
>>
>> This test is added based on [1]
> 
> Hmm, this cannot apply cleanly on top of your rdma test + Jake's ipv6 fix.
> Did I made it wrong somewhere?
> 
> At the meantime, I found that the current rdma unit test failing if I did
> the "setup" phase then run this test without root:
> 

Thanks for you report, I reproduced it on fedora40 while ubuntu2204 always works well.

I will look into it.


Thanks
Zhijian


> stderr:
> qemu-system-x86_64: cannot get rkey
> qemu-system-x86_64: error while loading state section id 2(ram)
> qemu-system-x86_64: load of migration failed: Operation not permitted
> qemu-system-x86_64: rdma migration: recv polling control error!
> qemu-system-x86_64: RDMA is in an error state waiting migration to abort!
> qemu-system-x86_64: failed to save SaveStateEntry with id(name): 2(ram): -1
> qemu-system-x86_64: Channel error: Operation not permitted
> **
> ERROR:../tests/qtest/migration/migration-qmp.c:200:check_migration_status: assertion failed (current_status != "failed"): ("failed" != "failed")
> qemu-system-x86_64: warning: Early error. Sending error.
> qemu-system-x86_64: warning: rdma migration: send polling control error
> ../tests/qtest/libqtest.c:199: kill_qemu() tried to terminate QEMU process but encountered exit status 1 (expected 0)
> 
> So running the test also needs root?  Is it possible we fix the test so it
> can also be smart enough to skip if it knows it'll hit the "cannot get
> rkey" error (even if it sees the rdma link setup)?  Not something urgent
> but definitely good to have.
> 
> OTOH, running the test using root always works for me.


> 
>>
>> Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
>> ---
>>   scripts/rdma-migration-helper.sh      | 26 +++++++++++++++++++++++---
>>   tests/qtest/migration/precopy-tests.c | 21 +++++++++++++++++----
>>   2 files changed, 40 insertions(+), 7 deletions(-)
>>
>> diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh
>> index 019b9ad932..c358392b72 100755
>> --- a/scripts/rdma-migration-helper.sh
>> +++ b/scripts/rdma-migration-helper.sh
>> @@ -8,6 +8,15 @@ get_ipv4_addr()
>>           head -1 | tr -d '\n'
>>   }
>>   
>> +get_ipv6_addr() {
>> +    ipv6=$(ip -6 -o addr show dev "$1" |
>> +        sed -n 's/.*[[:blank:]]inet6[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' |
>> +        head -1 | tr -d '\n')
>> +
>> +    [ $? -eq 0 ] || return
>> +    echo -n ["$ipv6"%$1]
>> +}
>> +
>>   # existing rdma interfaces
>>   rdma_interfaces()
>>   {
>> @@ -20,11 +29,16 @@ ipv4_interfaces()
>>       ip -o addr show | awk '/inet / {print $2}' | sort -u | grep -v -w lo
>>   }
>>   
>> +ipv6_interfaces()
>> +{
>> +    ip -o addr show | awk '/inet6 / {print $2}' | sort -u | grep -v -w lo
>> +}
>> +
>>   rdma_rxe_detect()
>>   {
>>       for r in $(rdma_interfaces)
>>       do
>> -        ipv4_interfaces | grep -qw $r && get_ipv4_addr $r && return
>> +        "$IP_FAMILY"_interfaces | grep -qw $r && get_"$IP_FAMILY"_addr $r && return
>>       done
>>   
>>       return 1
>> @@ -32,11 +46,11 @@ rdma_rxe_detect()
>>   
>>   rdma_rxe_setup()
>>   {
>> -    for i in $(ipv4_interfaces)
>> +    for i in $("$IP_FAMILY"_interfaces)
>>       do
>>           rdma_interfaces | grep -qw $i && continue
>>           rdma link add "${i}_rxe" type rxe netdev "$i" && {
>> -            echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_ipv4_addr $i)"
>> +            echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_"$IP_FAMILY"_addr $i)"
>>               return
>>           }
>>       done
>> @@ -50,6 +64,12 @@ rdma_rxe_clean()
>>       modprobe -r rdma_rxe
>>   }
>>   
>> +IP_FAMILY=${IP_FAMILY:-ipv4}
>> +if [ "$IP_FAMILY" != "ipv6" ] && [ "$IP_FAMILY" != "ipv4" ]; then
>> +    echo "Unknown ip family '$IP_FAMILY', only ipv4 or ipv6 is supported," >&2
>> +    exit 1
>> +fi
>> +
>>   operation=${1:-detect}
>>   
>>   command -v rdma >/dev/null || {
>> diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c
>> index a137ea9f2c..9f7236dc59 100644
>> --- a/tests/qtest/migration/precopy-tests.c
>> +++ b/tests/qtest/migration/precopy-tests.c
>> @@ -102,12 +102,13 @@ static void test_precopy_unix_dirty_ring(void)
>>   #ifdef CONFIG_RDMA
>>   
>>   #define RDMA_MIGRATION_HELPER "scripts/rdma-migration-helper.sh"
>> -static int new_rdma_link(char *buffer)
>> +static int new_rdma_link(char *buffer, bool ipv6)
>>   {
>>       char cmd[256];
>>       bool verbose = g_getenv("QTEST_LOG");
>>   
>> -    snprintf(cmd, sizeof(cmd), "%s detect %s", RDMA_MIGRATION_HELPER,
>> +    snprintf(cmd, sizeof(cmd), "IP_FAMILY=%s %s detect %s",
>> +             ipv6 ? "ipv6" : "ipv4", RDMA_MIGRATION_HELPER,
>>                verbose ? "" : "2>/dev/null");
>>   
>>       FILE *pipe = popen(cmd, "r");
>> @@ -132,11 +133,11 @@ static int new_rdma_link(char *buffer)
>>       return -1;
>>   }
>>   
>> -static void test_precopy_rdma_plain(void)
>> +static void test_precopy_rdma_plain_ip(bool ipv6)
>>   {
>>       char buffer[128] = {};
>>   
>> -    if (new_rdma_link(buffer)) {
>> +    if (new_rdma_link(buffer, ipv6)) {
>>           g_test_skip("No rdma link available\n"
>>                       "# To enable the test:\n"
>>                       "# Run \'" RDMA_MIGRATION_HELPER " setup\' with root to setup"
>> @@ -159,6 +160,16 @@ static void test_precopy_rdma_plain(void)
>>   
>>       test_precopy_common(&args);
>>   }
>> +
>> +static void test_precopy_rdma_plain(void)
>> +{
>> +    test_precopy_rdma_plain_ip(0);
>> +}
>> +
>> +static void test_precopy_rdma_plain_ipv6(void)
>> +{
>> +    test_precopy_rdma_plain_ip(1);
>> +}
>>   #endif
>>   
>>   static void test_precopy_tcp_plain(void)
>> @@ -1189,6 +1200,8 @@ static void migration_test_add_precopy_smoke(MigrationTestEnv *env)
>>   #ifdef CONFIG_RDMA
>>       migration_test_add("/migration/precopy/rdma/plain",
>>                          test_precopy_rdma_plain);
>> +    migration_test_add("/migration/precopy/rdma/plain/ipv6",
>> +                       test_precopy_rdma_plain_ipv6);
>>   #endif
>>   }
>>   
>> -- 
>> 2.41.0
>>
> 

Re: [PATCH] qtest/migration/rdma: Add test for rdma migration with ipv6

[Zhijian Li (Fujitsu) via]

On 30/04/2025 16:28, Li Zhijian wrote:
> 
> 
> On 30/04/2025 05:09, Peter Xu wrote:
>> On Thu, Mar 27, 2025 at 10:12:34AM +0800, Li Zhijian wrote:
>>> Recently, we removed ipv6 restriction[0] from RDMA migration, add a
>>> test for it.
>>>
>>> [0] https://lore.kernel.org/qemu-devel/20250326095224.9918-1-jinpu.wang@ionos.com/
>>>
>>> Cc: Jack Wang <jinpu.wang@ionos.com>
>>> Cc: Michael R. Galaxy <mrgalaxy@nvidia.com>
>>> Cc: Peter Xu <peterx@redhat.com>
>>> Cc: Yu Zhang <yu.zhang@ionos.com>
>>> Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
>>> ---
>>>
>>> This test is added based on [1]
>>
>> Hmm, this cannot apply cleanly on top of your rdma test + Jake's ipv6 fix.
>> Did I made it wrong somewhere?
>>
>> At the meantime, I found that the current rdma unit test failing if I did
>> the "setup" phase then run this test without root:
>>
> 
> Thanks for you report, I reproduced it on fedora40 while ubuntu2204 always works well.
> 
> I will look into it.

A work around in my fedora40 is:

$ sudo setcap 'cap_ipc_lock+ep' ./build/qemu-system-x86_64

Well, that's interesting, the ./build/qemu-system-x86_64 in my ubuntu2204 works
without cap_ipc_lock cap..


> 
> 
> Thanks
> Zhijian
> 
> 
>> stderr:
>> qemu-system-x86_64: cannot get rkey
>> qemu-system-x86_64: error while loading state section id 2(ram)
>> qemu-system-x86_64: load of migration failed: Operation not permitted
>> qemu-system-x86_64: rdma migration: recv polling control error!
>> qemu-system-x86_64: RDMA is in an error state waiting migration to abort!
>> qemu-system-x86_64: failed to save SaveStateEntry with id(name): 2(ram): -1
>> qemu-system-x86_64: Channel error: Operation not permitted
>> **
>> ERROR:../tests/qtest/migration/migration-qmp.c:200:check_migration_status: assertion failed (current_status != "failed"): ("failed" != "failed")
>> qemu-system-x86_64: warning: Early error. Sending error.
>> qemu-system-x86_64: warning: rdma migration: send polling control error
>> ../tests/qtest/libqtest.c:199: kill_qemu() tried to terminate QEMU process but encountered exit status 1 (expected 0)
>>
>> So running the test also needs root?  Is it possible we fix the test so it
>> can also be smart enough to skip if it knows it'll hit the "cannot get
>> rkey" error (even if it sees the rdma link setup)?  Not something urgent
>> but definitely good to have.

It seems it's a security problem, I have no a good idea yet.

Let me see see...




>>
>> OTOH, running the test using root always works for me.
> 
> 
>>
>>>
>>> Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
>>> ---
>>>   scripts/rdma-migration-helper.sh      | 26 +++++++++++++++++++++++---
>>>   tests/qtest/migration/precopy-tests.c | 21 +++++++++++++++++----
>>>   2 files changed, 40 insertions(+), 7 deletions(-)
>>>
>>> diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh
>>> index 019b9ad932..c358392b72 100755
>>> --- a/scripts/rdma-migration-helper.sh
>>> +++ b/scripts/rdma-migration-helper.sh
>>> @@ -8,6 +8,15 @@ get_ipv4_addr()
>>>           head -1 | tr -d '\n'
>>>   }
>>> +get_ipv6_addr() {
>>> +    ipv6=$(ip -6 -o addr show dev "$1" |
>>> +        sed -n 's/.*[[:blank:]]inet6[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' |
>>> +        head -1 | tr -d '\n')
>>> +
>>> +    [ $? -eq 0 ] || return
>>> +    echo -n ["$ipv6"%$1]
>>> +}
>>> +
>>>   # existing rdma interfaces
>>>   rdma_interfaces()
>>>   {
>>> @@ -20,11 +29,16 @@ ipv4_interfaces()
>>>       ip -o addr show | awk '/inet / {print $2}' | sort -u | grep -v -w lo
>>>   }
>>> +ipv6_interfaces()
>>> +{
>>> +    ip -o addr show | awk '/inet6 / {print $2}' | sort -u | grep -v -w lo
>>> +}
>>> +
>>>   rdma_rxe_detect()
>>>   {
>>>       for r in $(rdma_interfaces)
>>>       do
>>> -        ipv4_interfaces | grep -qw $r && get_ipv4_addr $r && return
>>> +        "$IP_FAMILY"_interfaces | grep -qw $r && get_"$IP_FAMILY"_addr $r && return
>>>       done
>>>       return 1
>>> @@ -32,11 +46,11 @@ rdma_rxe_detect()
>>>   rdma_rxe_setup()
>>>   {
>>> -    for i in $(ipv4_interfaces)
>>> +    for i in $("$IP_FAMILY"_interfaces)
>>>       do
>>>           rdma_interfaces | grep -qw $i && continue
>>>           rdma link add "${i}_rxe" type rxe netdev "$i" && {
>>> -            echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_ipv4_addr $i)"
>>> +            echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_"$IP_FAMILY"_addr $i)"
>>>               return
>>>           }
>>>       done
>>> @@ -50,6 +64,12 @@ rdma_rxe_clean()
>>>       modprobe -r rdma_rxe
>>>   }
>>> +IP_FAMILY=${IP_FAMILY:-ipv4}
>>> +if [ "$IP_FAMILY" != "ipv6" ] && [ "$IP_FAMILY" != "ipv4" ]; then
>>> +    echo "Unknown ip family '$IP_FAMILY', only ipv4 or ipv6 is supported," >&2
>>> +    exit 1
>>> +fi
>>> +
>>>   operation=${1:-detect}
>>>   command -v rdma >/dev/null || {
>>> diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c
>>> index a137ea9f2c..9f7236dc59 100644
>>> --- a/tests/qtest/migration/precopy-tests.c
>>> +++ b/tests/qtest/migration/precopy-tests.c
>>> @@ -102,12 +102,13 @@ static void test_precopy_unix_dirty_ring(void)
>>>   #ifdef CONFIG_RDMA
>>>   #define RDMA_MIGRATION_HELPER "scripts/rdma-migration-helper.sh"
>>> -static int new_rdma_link(char *buffer)
>>> +static int new_rdma_link(char *buffer, bool ipv6)
>>>   {
>>>       char cmd[256];
>>>       bool verbose = g_getenv("QTEST_LOG");
>>> -    snprintf(cmd, sizeof(cmd), "%s detect %s", RDMA_MIGRATION_HELPER,
>>> +    snprintf(cmd, sizeof(cmd), "IP_FAMILY=%s %s detect %s",
>>> +             ipv6 ? "ipv6" : "ipv4", RDMA_MIGRATION_HELPER,
>>>                verbose ? "" : "2>/dev/null");
>>>       FILE *pipe = popen(cmd, "r");
>>> @@ -132,11 +133,11 @@ static int new_rdma_link(char *buffer)
>>>       return -1;
>>>   }
>>> -static void test_precopy_rdma_plain(void)
>>> +static void test_precopy_rdma_plain_ip(bool ipv6)
>>>   {
>>>       char buffer[128] = {};
>>> -    if (new_rdma_link(buffer)) {
>>> +    if (new_rdma_link(buffer, ipv6)) {
>>>           g_test_skip("No rdma link available\n"
>>>                       "# To enable the test:\n"
>>>                       "# Run \'" RDMA_MIGRATION_HELPER " setup\' with root to setup"
>>> @@ -159,6 +160,16 @@ static void test_precopy_rdma_plain(void)
>>>       test_precopy_common(&args);
>>>   }
>>> +
>>> +static void test_precopy_rdma_plain(void)
>>> +{
>>> +    test_precopy_rdma_plain_ip(0);
>>> +}
>>> +
>>> +static void test_precopy_rdma_plain_ipv6(void)
>>> +{
>>> +    test_precopy_rdma_plain_ip(1);
>>> +}
>>>   #endif
>>>   static void test_precopy_tcp_plain(void)
>>> @@ -1189,6 +1200,8 @@ static void migration_test_add_precopy_smoke(MigrationTestEnv *env)
>>>   #ifdef CONFIG_RDMA
>>>       migration_test_add("/migration/precopy/rdma/plain",
>>>                          test_precopy_rdma_plain);
>>> +    migration_test_add("/migration/precopy/rdma/plain/ipv6",
>>> +                       test_precopy_rdma_plain_ipv6);
>>>   #endif
>>>   }
>>> -- 
>>> 2.41.0
>>>
>>

Re: [PATCH] qtest/migration/rdma: Add test for rdma migration with ipv6

[Zhijian Li (Fujitsu) via]

On 30/04/2025 16:48, Zhijian Li (Fujitsu) via wrote:
>>> stderr:
>>> qemu-system-x86_64: cannot get rkey
>>> qemu-system-x86_64: error while loading state section id 2(ram)
>>> qemu-system-x86_64: load of migration failed: Operation not permitted
>>> qemu-system-x86_64: rdma migration: recv polling control error!
>>> qemu-system-x86_64: RDMA is in an error state waiting migration to abort!
>>> qemu-system-x86_64: failed to save SaveStateEntry with id(name): 2(ram): -1
>>> qemu-system-x86_64: Channel error: Operation not permitted
>>> **
>>> ERROR:../tests/qtest/migration/migration-qmp.c:200:check_migration_status: assertion failed (current_status != "failed"): ("failed" != "failed")
>>> qemu-system-x86_64: warning: Early error. Sending error.
>>> qemu-system-x86_64: warning: rdma migration: send polling control error
>>> ../tests/qtest/libqtest.c:199: kill_qemu() tried to terminate QEMU process but encountered exit status 1 (expected 0)
>>>
>>> So running the test also needs root?  Is it possible we fix the test so it
>>> can also be smart enough to skip if it knows it'll hit the "cannot get
>>> rkey" error (even if it sees the rdma link setup)?  Not something urgent
>>> but definitely good to have.
> It seems it's a security problem, I have no a good idea yet.
> 
> Let me see see...

Another workaround is update the 'ulimit -l' to >=128M for a non-root user(in practice
this value works well on fedora40)

So we would have something like this:

diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c
index 9f7236dc59f..1f24753c5a5 100644
--- a/tests/qtest/migration/precopy-tests.c
+++ b/tests/qtest/migration/precopy-tests.c
@@ -101,6 +101,26 @@ static void test_precopy_unix_dirty_ring(void)
  
  #ifdef CONFIG_RDMA
  
+#include <sys/resource.h>
+#define REQUIRED_MEMLOCK (128 * 1024 * 1024) // 128MB
+
+static bool mlock_check(void)
+{
+    uid_t uid;
+    struct rlimit rlim;
+
+    uid = getuid();
+    if (uid == 0) {
+        return true;
+    }
+
+    if (getrlimit(RLIMIT_MEMLOCK, &rlim) != 0) {
+        return false;
+    }
+
+    return rlim.rlim_cur >= REQUIRED_MEMLOCK;
+}
+
  #define RDMA_MIGRATION_HELPER "scripts/rdma-migration-helper.sh"
  static int new_rdma_link(char *buffer, bool ipv6)
  {
@@ -137,6 +157,11 @@ static void test_precopy_rdma_plain_ip(bool ipv6)
  {
      char buffer[128] = {};
  
+    if (!mlock_check()) {
+        g_test_skip("'ulimit -l' is too small, require 128M");
+        return;
+    }
+
      if (new_rdma_link(buffer, ipv6)) {
          g_test_skip("No rdma link available\n"
                      "# To enable the test:\n"

Re: [PATCH] qtest/migration/rdma: Add test for rdma migration with ipv6

[Peter Xu]

On Wed, Apr 30, 2025 at 09:49:40AM +0000, Zhijian Li (Fujitsu) wrote:
> 
> 
> On 30/04/2025 16:48, Zhijian Li (Fujitsu) via wrote:
> >>> stderr:
> >>> qemu-system-x86_64: cannot get rkey
> >>> qemu-system-x86_64: error while loading state section id 2(ram)
> >>> qemu-system-x86_64: load of migration failed: Operation not permitted
> >>> qemu-system-x86_64: rdma migration: recv polling control error!
> >>> qemu-system-x86_64: RDMA is in an error state waiting migration to abort!
> >>> qemu-system-x86_64: failed to save SaveStateEntry with id(name): 2(ram): -1
> >>> qemu-system-x86_64: Channel error: Operation not permitted
> >>> **
> >>> ERROR:../tests/qtest/migration/migration-qmp.c:200:check_migration_status: assertion failed (current_status != "failed"): ("failed" != "failed")
> >>> qemu-system-x86_64: warning: Early error. Sending error.
> >>> qemu-system-x86_64: warning: rdma migration: send polling control error
> >>> ../tests/qtest/libqtest.c:199: kill_qemu() tried to terminate QEMU process but encountered exit status 1 (expected 0)
> >>>
> >>> So running the test also needs root?  Is it possible we fix the test so it
> >>> can also be smart enough to skip if it knows it'll hit the "cannot get
> >>> rkey" error (even if it sees the rdma link setup)?  Not something urgent
> >>> but definitely good to have.
> > It seems it's a security problem, I have no a good idea yet.
> > 
> > Let me see see...
> 
> Another workaround is update the 'ulimit -l' to >=128M for a non-root user(in practice
> this value works well on fedora40)

OK so it's about the locked mem.. thanks for looking.

> 
> So we would have something like this:
> 
> diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c
> index 9f7236dc59f..1f24753c5a5 100644
> --- a/tests/qtest/migration/precopy-tests.c
> +++ b/tests/qtest/migration/precopy-tests.c
> @@ -101,6 +101,26 @@ static void test_precopy_unix_dirty_ring(void)
>   
>   #ifdef CONFIG_RDMA
>   
> +#include <sys/resource.h>
> +#define REQUIRED_MEMLOCK (128 * 1024 * 1024) // 128MB

How does the 128M come from?  Is it correlated to the VM size somehow?
Btw, migrate_start() says for x86 we use 150MB VM.

When you feel confident, feel free to send a formal patch, it can also
include the reposted version of the current patch so that can be a series.

It'll also be great if you could make sure they apply on top of:

https://gitlab.com/peterx/qemu/-/tree/migration-staging

Thanks,

> +
> +static bool mlock_check(void)
> +{
> +    uid_t uid;
> +    struct rlimit rlim;
> +
> +    uid = getuid();
> +    if (uid == 0) {
> +        return true;
> +    }
> +
> +    if (getrlimit(RLIMIT_MEMLOCK, &rlim) != 0) {
> +        return false;
> +    }
> +
> +    return rlim.rlim_cur >= REQUIRED_MEMLOCK;
> +}
> +
>   #define RDMA_MIGRATION_HELPER "scripts/rdma-migration-helper.sh"
>   static int new_rdma_link(char *buffer, bool ipv6)
>   {
> @@ -137,6 +157,11 @@ static void test_precopy_rdma_plain_ip(bool ipv6)
>   {
>       char buffer[128] = {};
>   
> +    if (!mlock_check()) {
> +        g_test_skip("'ulimit -l' is too small, require 128M");
> +        return;
> +    }
> +
>       if (new_rdma_link(buffer, ipv6)) {
>           g_test_skip("No rdma link available\n"
>                       "# To enable the test:\n"

-- 
Peter Xu

Re: [PATCH] qtest/migration/rdma: Add test for rdma migration with ipv6

[Zhijian Li (Fujitsu) via]

On 01/05/2025 05:52, Peter Xu wrote:
>> +#include <sys/resource.h>
>> +#define REQUIRED_MEMLOCK (128 * 1024 * 1024) // 128MB
> How does the 128M come from?  Is it correlated to the VM size somehow?
> Btw, migrate_start() says for x86 we use 150MB VM.

Good question, After a brief debugging session, it was discovered that the size of mlock remains
relatively stable at around 100M in this test(even though enlarge the guest memory to 1024M),
while the remainder is transmitted to the destination using zero-page compression.



> > When you feel confident, feel free to send a formal patch, it can also
> include the reposted version of the current patch so that can be a series.
> 
> It'll also be great if you could make sure they apply on top of:
> 
> https://gitlab.com/peterx/qemu/-/tree/migration-staging

No problem.

Thanks
Zhijian

> 
> Thanks,