Re: [PATCH] qtest/migration/rdma: Add test for rdma migration with ipv6

[Peter Xu <peterx@redhat.com>]

On Thu, Mar 27, 2025 at 10:12:34AM +0800, Li Zhijian wrote:
> Recently, we removed ipv6 restriction[0] from RDMA migration, add a
> test for it.
> 
> [0] https://lore.kernel.org/qemu-devel/20250326095224.9918-1-jinpu.wang@ionos.com/
> 
> Cc: Jack Wang <jinpu.wang@ionos.com>
> Cc: Michael R. Galaxy <mrgalaxy@nvidia.com>
> Cc: Peter Xu <peterx@redhat.com>
> Cc: Yu Zhang <yu.zhang@ionos.com>
> Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
> ---
> 
> This test is added based on [1]

Hmm, this cannot apply cleanly on top of your rdma test + Jake's ipv6 fix.
Did I made it wrong somewhere?

At the meantime, I found that the current rdma unit test failing if I did
the "setup" phase then run this test without root:

stderr:
qemu-system-x86_64: cannot get rkey
qemu-system-x86_64: error while loading state section id 2(ram)
qemu-system-x86_64: load of migration failed: Operation not permitted
qemu-system-x86_64: rdma migration: recv polling control error!
qemu-system-x86_64: RDMA is in an error state waiting migration to abort!
qemu-system-x86_64: failed to save SaveStateEntry with id(name): 2(ram): -1
qemu-system-x86_64: Channel error: Operation not permitted
**
ERROR:../tests/qtest/migration/migration-qmp.c:200:check_migration_status: assertion failed (current_status != "failed"): ("failed" != "failed")
qemu-system-x86_64: warning: Early error. Sending error.
qemu-system-x86_64: warning: rdma migration: send polling control error
../tests/qtest/libqtest.c:199: kill_qemu() tried to terminate QEMU process but encountered exit status 1 (expected 0)

So running the test also needs root?  Is it possible we fix the test so it
can also be smart enough to skip if it knows it'll hit the "cannot get
rkey" error (even if it sees the rdma link setup)?  Not something urgent
but definitely good to have.

OTOH, running the test using root always works for me.

> 
> Signed-off-by: Li Zhijian <lizhijian@fujitsu.com>
> ---
>  scripts/rdma-migration-helper.sh      | 26 +++++++++++++++++++++++---
>  tests/qtest/migration/precopy-tests.c | 21 +++++++++++++++++----
>  2 files changed, 40 insertions(+), 7 deletions(-)
> 
> diff --git a/scripts/rdma-migration-helper.sh b/scripts/rdma-migration-helper.sh
> index 019b9ad932..c358392b72 100755
> --- a/scripts/rdma-migration-helper.sh
> +++ b/scripts/rdma-migration-helper.sh
> @@ -8,6 +8,15 @@ get_ipv4_addr()
>          head -1 | tr -d '\n'
>  }
>  
> +get_ipv6_addr() {
> +    ipv6=$(ip -6 -o addr show dev "$1" |
> +        sed -n 's/.*[[:blank:]]inet6[[:blank:]]*\([^[:blank:]/]*\).*/\1/p' |
> +        head -1 | tr -d '\n')
> +
> +    [ $? -eq 0 ] || return
> +    echo -n ["$ipv6"%$1]
> +}
> +
>  # existing rdma interfaces
>  rdma_interfaces()
>  {
> @@ -20,11 +29,16 @@ ipv4_interfaces()
>      ip -o addr show | awk '/inet / {print $2}' | sort -u | grep -v -w lo
>  }
>  
> +ipv6_interfaces()
> +{
> +    ip -o addr show | awk '/inet6 / {print $2}' | sort -u | grep -v -w lo
> +}
> +
>  rdma_rxe_detect()
>  {
>      for r in $(rdma_interfaces)
>      do
> -        ipv4_interfaces | grep -qw $r && get_ipv4_addr $r && return
> +        "$IP_FAMILY"_interfaces | grep -qw $r && get_"$IP_FAMILY"_addr $r && return
>      done
>  
>      return 1
> @@ -32,11 +46,11 @@ rdma_rxe_detect()
>  
>  rdma_rxe_setup()
>  {
> -    for i in $(ipv4_interfaces)
> +    for i in $("$IP_FAMILY"_interfaces)
>      do
>          rdma_interfaces | grep -qw $i && continue
>          rdma link add "${i}_rxe" type rxe netdev "$i" && {
> -            echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_ipv4_addr $i)"
> +            echo "Setup new rdma/rxe ${i}_rxe for $i with $(get_"$IP_FAMILY"_addr $i)"
>              return
>          }
>      done
> @@ -50,6 +64,12 @@ rdma_rxe_clean()
>      modprobe -r rdma_rxe
>  }
>  
> +IP_FAMILY=${IP_FAMILY:-ipv4}
> +if [ "$IP_FAMILY" != "ipv6" ] && [ "$IP_FAMILY" != "ipv4" ]; then
> +    echo "Unknown ip family '$IP_FAMILY', only ipv4 or ipv6 is supported," >&2
> +    exit 1
> +fi
> +
>  operation=${1:-detect}
>  
>  command -v rdma >/dev/null || {
> diff --git a/tests/qtest/migration/precopy-tests.c b/tests/qtest/migration/precopy-tests.c
> index a137ea9f2c..9f7236dc59 100644
> --- a/tests/qtest/migration/precopy-tests.c
> +++ b/tests/qtest/migration/precopy-tests.c
> @@ -102,12 +102,13 @@ static void test_precopy_unix_dirty_ring(void)
>  #ifdef CONFIG_RDMA
>  
>  #define RDMA_MIGRATION_HELPER "scripts/rdma-migration-helper.sh"
> -static int new_rdma_link(char *buffer)
> +static int new_rdma_link(char *buffer, bool ipv6)
>  {
>      char cmd[256];
>      bool verbose = g_getenv("QTEST_LOG");
>  
> -    snprintf(cmd, sizeof(cmd), "%s detect %s", RDMA_MIGRATION_HELPER,
> +    snprintf(cmd, sizeof(cmd), "IP_FAMILY=%s %s detect %s",
> +             ipv6 ? "ipv6" : "ipv4", RDMA_MIGRATION_HELPER,
>               verbose ? "" : "2>/dev/null");
>  
>      FILE *pipe = popen(cmd, "r");
> @@ -132,11 +133,11 @@ static int new_rdma_link(char *buffer)
>      return -1;
>  }
>  
> -static void test_precopy_rdma_plain(void)
> +static void test_precopy_rdma_plain_ip(bool ipv6)
>  {
>      char buffer[128] = {};
>  
> -    if (new_rdma_link(buffer)) {
> +    if (new_rdma_link(buffer, ipv6)) {
>          g_test_skip("No rdma link available\n"
>                      "# To enable the test:\n"
>                      "# Run \'" RDMA_MIGRATION_HELPER " setup\' with root to setup"
> @@ -159,6 +160,16 @@ static void test_precopy_rdma_plain(void)
>  
>      test_precopy_common(&args);
>  }
> +
> +static void test_precopy_rdma_plain(void)
> +{
> +    test_precopy_rdma_plain_ip(0);
> +}
> +
> +static void test_precopy_rdma_plain_ipv6(void)
> +{
> +    test_precopy_rdma_plain_ip(1);
> +}
>  #endif
>  
>  static void test_precopy_tcp_plain(void)
> @@ -1189,6 +1200,8 @@ static void migration_test_add_precopy_smoke(MigrationTestEnv *env)
>  #ifdef CONFIG_RDMA
>      migration_test_add("/migration/precopy/rdma/plain",
>                         test_precopy_rdma_plain);
> +    migration_test_add("/migration/precopy/rdma/plain/ipv6",
> +                       test_precopy_rdma_plain_ipv6);
>  #endif
>  }
>  
> -- 
> 2.41.0
> 

-- 
Peter Xu