[RFC PATCH 0/3] Vhost-vdpa Shadow Virtqueue RSS Support

[RFC PATCH 0/3] Vhost-vdpa Shadow Virtqueue RSS Support

[Hawkins Jiawei]

This series enables shadowed CVQ to intercept RSS command
through shadowed CVQ, update the virtio NIC device model
so qemu send it in a migration, and the restore of that
RSS state in the destination.

Note that this patch should be based on
patch "Vhost-vdpa Shadow Virtqueue Hash calculation Support" at [1].

[1]. https://lore.kernel.org/all/cover.1691762906.git.yin31149@gmail.com/

TestStep
========
1. test the migration using vp-vdpa device
  - For L0 guest, boot QEMU with two virtio-net-pci net device with
`in-qemu` RSS, command line like:
	-netdev tap,vhost=off...
	-device virtio-net-pci,disable-legacy=on,disable-modern=off,
iommu_platform=on,mq=on,ctrl_vq=on,hash=on,rss=on,guest_announce=off,
indirect_desc=off,queue_reset=off,...

  - For L1 guest, apply the relative patch series and compile the
source code, start QEMU with two vdpa device with svq mode on,
enable the `ctrl_vq`, `mq`, `rss` features on, command line like:
      -netdev type=vhost-vdpa,x-svq=true,...
      -device virtio-net-pci,mq=on,guest_announce=off,ctrl_vq=on,
rss=on,...

  - For L2 source guest, run the following bash command:
```bash
#!/bin/sh

ethtool -K eth0 rxhash on
```

  - Execute the live migration in L2 source monitor

  - Result
    * with this series, L2 QEMU can execute without
triggering any error or warning. L0 QEMU echo
"Can't load eBPF RSS - fallback to software RSS".

Hawkins Jiawei (3):
  vdpa: Add SetSteeringEBPF method for NetClientState
  vdpa: Restore receive-side scaling state
  vdpa: Allow VIRTIO_NET_F_RSS in SVQ

 net/vhost-vdpa.c | 62 ++++++++++++++++++++++++++++++++++--------------
 1 file changed, 44 insertions(+), 18 deletions(-)

-- 
2.25.1

[RFC PATCH 1/3] vdpa: Add SetSteeringEBPF method for NetClientState

[Hawkins Jiawei]

At present, to enable the VIRTIO_NET_F_RSS feature, eBPF must
be loaded for the vhost backend.

Given that vhost-vdpa is one of the vhost backend, we need to
implement the SetSteeringEBPF method to support RSS for vhost-vdpa,
even if vhost-vdpa calculates the rss hash in the hardware device
instead of in the kernel by eBPF.

Although this requires QEMU to be compiled with `--enable-bpf`
configuration even if the vdpa device does not use eBPF to
calculate the rss hash, this can avoid adding the specific
conditional statements for vDPA case to enable the VIRTIO_NET_F_RSS
feature, which reduces code maintainbility.

Suggested-by: Eugenio Pérez <eperezma@redhat.com>
Signed-off-by: Hawkins Jiawei <yin31149@gmail.com>
---
 net/vhost-vdpa.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/net/vhost-vdpa.c b/net/vhost-vdpa.c
index a13b267250..4c8e4b19f6 100644
--- a/net/vhost-vdpa.c
+++ b/net/vhost-vdpa.c
@@ -238,6 +238,12 @@ static void vhost_vdpa_cleanup(NetClientState *nc)
     }
 }
 
+/** Dummy SetSteeringEBPF to support RSS for vhost-vdpa backend  */
+static bool vhost_vdpa_set_steering_ebpf(NetClientState *nc, int prog_fd)
+{
+    return true;
+}
+
 static bool vhost_vdpa_has_vnet_hdr(NetClientState *nc)
 {
     assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA);
@@ -400,6 +406,7 @@ static NetClientInfo net_vhost_vdpa_info = {
         .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
         .has_ufo = vhost_vdpa_has_ufo,
         .check_peer_type = vhost_vdpa_check_peer_type,
+        .set_steering_ebpf = vhost_vdpa_set_steering_ebpf,
 };
 
 static int64_t vhost_vdpa_get_vring_group(int device_fd, unsigned vq_index,
@@ -1215,6 +1222,7 @@ static NetClientInfo net_vhost_vdpa_cvq_info = {
     .has_vnet_hdr = vhost_vdpa_has_vnet_hdr,
     .has_ufo = vhost_vdpa_has_ufo,
     .check_peer_type = vhost_vdpa_check_peer_type,
+    .set_steering_ebpf = vhost_vdpa_set_steering_ebpf,
 };
 
 /*
-- 
2.25.1

[RFC PATCH 2/3] vdpa: Restore receive-side scaling state

[Hawkins Jiawei]

This patch reuses vhost_vdpa_net_load_rss() with some
refactorings to restore the receive-side scaling state
at device's startup.

Signed-off-by: Hawkins Jiawei <yin31149@gmail.com>
---
 net/vhost-vdpa.c | 53 ++++++++++++++++++++++++++++++++----------------
 1 file changed, 35 insertions(+), 18 deletions(-)

diff --git a/net/vhost-vdpa.c b/net/vhost-vdpa.c
index 4c8e4b19f6..7870cbe142 100644
--- a/net/vhost-vdpa.c
+++ b/net/vhost-vdpa.c
@@ -820,17 +820,28 @@ static int vhost_vdpa_net_load_rss(VhostVDPAState *s, const VirtIONet *n,
     }
 
     cfg.hash_types = cpu_to_le32(n->rss_data.hash_types);
-    /*
-     * According to VirtIO standard, "Field reserved MUST contain zeroes.
-     * It is defined to make the structure to match the layout of
-     * virtio_net_rss_config structure, defined in 5.1.6.5.7.".
-     *
-     * Therefore, we need to zero the fields in struct virtio_net_rss_config,
-     * which corresponds the `reserved` field in
-     * struct virtio_net_hash_config.
-     */
-    memset(&cfg.indirection_table_mask, 0,
-           sizeof_field(struct virtio_net_hash_config, reserved));
+    if (do_rss) {
+        /*
+         * According to VirtIO standard, "Number of entries in indirection_table
+         * is (indirection_table_mask + 1)".
+         */
+        cfg.indirection_table_mask = cpu_to_le16(n->rss_data.indirections_len -
+                                                 1);
+        cfg.unclassified_queue = cpu_to_le16(n->rss_data.default_queue);
+        cfg.max_tx_vq = cpu_to_le16(n->curr_queue_pairs);
+    } else {
+        /*
+         * According to VirtIO standard, "Field reserved MUST contain zeroes.
+         * It is defined to make the structure to match the layout of
+         * virtio_net_rss_config structure, defined in 5.1.6.5.7.".
+         *
+         * Therefore, we need to zero the fields in
+         * struct virtio_net_rss_config, which corresponds the `reserved` field
+         * in struct virtio_net_hash_config.
+         */
+        memset(&cfg.indirection_table_mask, 0,
+               sizeof_field(struct virtio_net_hash_config, reserved));
+    }
     /*
      * Consider that virtio_net_handle_rss() currently does not restore the
      * hash key length parsed from the CVQ command sent from the guest into
@@ -866,6 +877,7 @@ static int vhost_vdpa_net_load_rss(VhostVDPAState *s, const VirtIONet *n,
 
     r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
                                 VIRTIO_NET_CTRL_MQ,
+                                do_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG :
                                 VIRTIO_NET_CTRL_MQ_HASH_CONFIG,
                                 data, ARRAY_SIZE(data));
     if (unlikely(r < 0)) {
@@ -899,13 +911,18 @@ static int vhost_vdpa_net_load_mq(VhostVDPAState *s,
         return r;
     }
 
-    if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_HASH_REPORT)) {
-        return 0;
-    }
-
-    r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, false);
-    if (unlikely(r < 0)) {
-        return r;
+    if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_RSS)) {
+        /* Load the receive-side scaling state */
+        r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, true);
+        if (unlikely(r < 0)) {
+            return r;
+        }
+    } else if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_RSS)) {
+        /* Load the hash calculation state */
+        r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, false);
+        if (unlikely(r < 0)) {
+            return r;
+        }
     }
 
     return 0;
-- 
2.25.1

Re: [RFC PATCH 2/3] vdpa: Restore receive-side scaling state

[Hawkins Jiawei]

On 2023/8/11 23:28, Hawkins Jiawei wrote:
> This patch reuses vhost_vdpa_net_load_rss() with some
> refactorings to restore the receive-side scaling state
> at device's startup.
>
> Signed-off-by: Hawkins Jiawei <yin31149@gmail.com>
> ---
>   net/vhost-vdpa.c | 53 ++++++++++++++++++++++++++++++++----------------
>   1 file changed, 35 insertions(+), 18 deletions(-)
>
> diff --git a/net/vhost-vdpa.c b/net/vhost-vdpa.c
> index 4c8e4b19f6..7870cbe142 100644
> --- a/net/vhost-vdpa.c
> +++ b/net/vhost-vdpa.c
> @@ -820,17 +820,28 @@ static int vhost_vdpa_net_load_rss(VhostVDPAState *s, const VirtIONet *n,
>       }
>
>       cfg.hash_types = cpu_to_le32(n->rss_data.hash_types);
> -    /*
> -     * According to VirtIO standard, "Field reserved MUST contain zeroes.
> -     * It is defined to make the structure to match the layout of
> -     * virtio_net_rss_config structure, defined in 5.1.6.5.7.".
> -     *
> -     * Therefore, we need to zero the fields in struct virtio_net_rss_config,
> -     * which corresponds the `reserved` field in
> -     * struct virtio_net_hash_config.
> -     */
> -    memset(&cfg.indirection_table_mask, 0,
> -           sizeof_field(struct virtio_net_hash_config, reserved));
> +    if (do_rss) {
> +        /*
> +         * According to VirtIO standard, "Number of entries in indirection_table
> +         * is (indirection_table_mask + 1)".
> +         */
> +        cfg.indirection_table_mask = cpu_to_le16(n->rss_data.indirections_len -
> +                                                 1);
> +        cfg.unclassified_queue = cpu_to_le16(n->rss_data.default_queue);
> +        cfg.max_tx_vq = cpu_to_le16(n->curr_queue_pairs);
> +    } else {
> +        /*
> +         * According to VirtIO standard, "Field reserved MUST contain zeroes.
> +         * It is defined to make the structure to match the layout of
> +         * virtio_net_rss_config structure, defined in 5.1.6.5.7.".
> +         *
> +         * Therefore, we need to zero the fields in
> +         * struct virtio_net_rss_config, which corresponds the `reserved` field
> +         * in struct virtio_net_hash_config.
> +         */
> +        memset(&cfg.indirection_table_mask, 0,
> +               sizeof_field(struct virtio_net_hash_config, reserved));
> +    }
>       /*
>        * Consider that virtio_net_handle_rss() currently does not restore the
>        * hash key length parsed from the CVQ command sent from the guest into
> @@ -866,6 +877,7 @@ static int vhost_vdpa_net_load_rss(VhostVDPAState *s, const VirtIONet *n,
>
>       r = vhost_vdpa_net_load_cmd(s, out_cursor, in_cursor,
>                                   VIRTIO_NET_CTRL_MQ,
> +                                do_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG :
>                                   VIRTIO_NET_CTRL_MQ_HASH_CONFIG,
>                                   data, ARRAY_SIZE(data));
>       if (unlikely(r < 0)) {
> @@ -899,13 +911,18 @@ static int vhost_vdpa_net_load_mq(VhostVDPAState *s,
>           return r;
>       }
>
> -    if (!virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_HASH_REPORT)) {
> -        return 0;
> -    }
> -
> -    r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, false);
> -    if (unlikely(r < 0)) {
> -        return r;
> +    if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_RSS)) {
> +        /* Load the receive-side scaling state */
> +        r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, true);
> +        if (unlikely(r < 0)) {
> +            return r;
> +        }
> +    } else if (virtio_vdev_has_feature(&n->parent_obj, VIRTIO_NET_F_RSS)) {

The correct feature to be used here is VIRTIO_NET_F_HASH_REPORT, rather
than VIRTIO_NET_F_RSS. I will correct this in the v2 patch.

Thanks!


> +        /* Load the hash calculation state */
> +        r = vhost_vdpa_net_load_rss(s, n, out_cursor, in_cursor, false);
> +        if (unlikely(r < 0)) {
> +            return r;
> +        }
>       }
>
>       return 0;

[RFC PATCH 3/3] vdpa: Allow VIRTIO_NET_F_RSS in SVQ

[Hawkins Jiawei]

Enable SVQ with VIRTIO_NET_F_RSS feature.

Signed-off-by: Hawkins Jiawei <yin31149@gmail.com>
---
 net/vhost-vdpa.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/net/vhost-vdpa.c b/net/vhost-vdpa.c
index 7870cbe142..eb08530396 100644
--- a/net/vhost-vdpa.c
+++ b/net/vhost-vdpa.c
@@ -119,6 +119,7 @@ static const uint64_t vdpa_svq_device_features =
     /* VHOST_F_LOG_ALL is exposed by SVQ */
     BIT_ULL(VHOST_F_LOG_ALL) |
     BIT_ULL(VIRTIO_NET_F_HASH_REPORT) |
+    BIT_ULL(VIRTIO_NET_F_RSS) |
     BIT_ULL(VIRTIO_NET_F_RSC_EXT) |
     BIT_ULL(VIRTIO_NET_F_STANDBY) |
     BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX);
-- 
2.25.1