* [PATCH net-next v2 01/21] virtio_net: rename free_old_xmit_skbs to free_old_xmit
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 02/21] virtio_net: unify the code for recycling the xmit ptr Xuan Zhuo
` (21 subsequent siblings)
22 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
Since free_old_xmit_skbs not only deals with skb, but also xdp frame and
subsequent added xsk, so change the name of this function to
free_old_xmit.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Jason Wang <jasowang@redhat.com>
---
drivers/net/virtio_net.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index cf67c618b07b..e7dd131234b5 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -744,7 +744,7 @@ static void virtnet_rq_set_premapped(struct virtnet_info *vi)
}
}
-static void free_old_xmit_skbs(struct send_queue *sq, bool in_napi)
+static void free_old_xmit(struct send_queue *sq, bool in_napi)
{
unsigned int len;
unsigned int packets = 0;
@@ -816,7 +816,7 @@ static void check_sq_full_and_disable(struct virtnet_info *vi,
virtqueue_napi_schedule(&sq->napi, sq->vq);
} else if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
/* More just got used, free them then recheck. */
- free_old_xmit_skbs(sq, false);
+ free_old_xmit(sq, false);
if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) {
netif_start_subqueue(dev, qnum);
virtqueue_disable_cb(sq->vq);
@@ -2127,7 +2127,7 @@ static void virtnet_poll_cleantx(struct receive_queue *rq)
do {
virtqueue_disable_cb(sq->vq);
- free_old_xmit_skbs(sq, true);
+ free_old_xmit(sq, true);
} while (unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
@@ -2249,7 +2249,7 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
txq = netdev_get_tx_queue(vi->dev, index);
__netif_tx_lock(txq, raw_smp_processor_id());
virtqueue_disable_cb(sq->vq);
- free_old_xmit_skbs(sq, true);
+ free_old_xmit(sq, true);
if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
netif_tx_wake_queue(txq);
@@ -2339,7 +2339,7 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
if (use_napi)
virtqueue_disable_cb(sq->vq);
- free_old_xmit_skbs(sq, false);
+ free_old_xmit(sq, false);
} while (use_napi && kick &&
unlikely(!virtqueue_enable_cb_delayed(sq->vq)));
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* [PATCH net-next v2 02/21] virtio_net: unify the code for recycling the xmit ptr
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 01/21] virtio_net: rename free_old_xmit_skbs to free_old_xmit Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 03/21] virtio_net: independent directory Xuan Zhuo
` (20 subsequent siblings)
22 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
There are two completely similar and independent implementations. This
is inconvenient for the subsequent addition of new types. So extract a
function from this piece of code and call this function uniformly to
recover old xmit ptr.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Jason Wang <jasowang@redhat.com>
---
drivers/net/virtio_net.c | 66 +++++++++++++++++-----------------------
1 file changed, 28 insertions(+), 38 deletions(-)
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index e7dd131234b5..3fe26f48025f 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -352,6 +352,30 @@ static struct xdp_frame *ptr_to_xdp(void *ptr)
return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
}
+static void __free_old_xmit(struct send_queue *sq, bool in_napi,
+ u64 *bytes, u64 *packets)
+{
+ unsigned int len;
+ void *ptr;
+
+ while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
+ if (!is_xdp_frame(ptr)) {
+ struct sk_buff *skb = ptr;
+
+ pr_debug("Sent skb %p\n", skb);
+
+ *bytes += skb->len;
+ napi_consume_skb(skb, in_napi);
+ } else {
+ struct xdp_frame *frame = ptr_to_xdp(ptr);
+
+ *bytes += xdp_get_frame_len(frame);
+ xdp_return_frame(frame);
+ }
+ (*packets)++;
+ }
+}
+
/* Converting between virtqueue no. and kernel tx/rx queue no.
* 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
*/
@@ -746,27 +770,9 @@ static void virtnet_rq_set_premapped(struct virtnet_info *vi)
static void free_old_xmit(struct send_queue *sq, bool in_napi)
{
- unsigned int len;
- unsigned int packets = 0;
- unsigned int bytes = 0;
- void *ptr;
-
- while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
- if (likely(!is_xdp_frame(ptr))) {
- struct sk_buff *skb = ptr;
-
- pr_debug("Sent skb %p\n", skb);
+ u64 bytes, packets = 0;
- bytes += skb->len;
- napi_consume_skb(skb, in_napi);
- } else {
- struct xdp_frame *frame = ptr_to_xdp(ptr);
-
- bytes += xdp_get_frame_len(frame);
- xdp_return_frame(frame);
- }
- packets++;
- }
+ __free_old_xmit(sq, in_napi, &bytes, &packets);
/* Avoid overhead when no packets have been processed
* happens when called speculatively from start_xmit.
@@ -916,14 +922,11 @@ static int virtnet_xdp_xmit(struct net_device *dev,
{
struct virtnet_info *vi = netdev_priv(dev);
struct receive_queue *rq = vi->rq;
+ u64 bytes = 0, packets = 0;
struct bpf_prog *xdp_prog;
struct send_queue *sq;
- unsigned int len;
- int packets = 0;
- int bytes = 0;
int nxmit = 0;
int kicks = 0;
- void *ptr;
int ret;
int i;
@@ -942,20 +945,7 @@ static int virtnet_xdp_xmit(struct net_device *dev,
}
/* Free up any pending old buffers before queueing new ones. */
- while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
- if (likely(is_xdp_frame(ptr))) {
- struct xdp_frame *frame = ptr_to_xdp(ptr);
-
- bytes += xdp_get_frame_len(frame);
- xdp_return_frame(frame);
- } else {
- struct sk_buff *skb = ptr;
-
- bytes += skb->len;
- napi_consume_skb(skb, false);
- }
- packets++;
- }
+ __free_old_xmit(sq, false, &bytes, &packets);
for (i = 0; i < n; i++) {
struct xdp_frame *xdpf = frames[i];
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* [PATCH net-next v2 03/21] virtio_net: independent directory
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 01/21] virtio_net: rename free_old_xmit_skbs to free_old_xmit Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 02/21] virtio_net: unify the code for recycling the xmit ptr Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 04/21] virtio_net: move core structures to virtio_net.h Xuan Zhuo
` (19 subsequent siblings)
22 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
Create a separate directory for virtio-net. AF_XDP support will be added
later, then a separate xsk.c file will be added, so we should create a
directory for virtio-net.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Jason Wang <jasowang@redhat.com>
---
MAINTAINERS | 2 +-
drivers/net/Kconfig | 8 +-------
drivers/net/Makefile | 2 +-
drivers/net/virtio/Kconfig | 13 +++++++++++++
drivers/net/virtio/Makefile | 8 ++++++++
drivers/net/{virtio_net.c => virtio/main.c} | 0
6 files changed, 24 insertions(+), 9 deletions(-)
create mode 100644 drivers/net/virtio/Kconfig
create mode 100644 drivers/net/virtio/Makefile
rename drivers/net/{virtio_net.c => virtio/main.c} (100%)
diff --git a/MAINTAINERS b/MAINTAINERS
index 14e1194faa4b..81e7d31f6cc9 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -22905,7 +22905,7 @@ F: Documentation/devicetree/bindings/virtio/
F: Documentation/driver-api/virtio/
F: drivers/block/virtio_blk.c
F: drivers/crypto/virtio/
-F: drivers/net/virtio_net.c
+F: drivers/net/virtio/
F: drivers/vdpa/
F: drivers/virtio/
F: include/linux/vdpa.h
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index af0da4bb429b..a14ef645aa01 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -430,13 +430,7 @@ config VETH
When one end receives the packet it appears on its pair and vice
versa.
-config VIRTIO_NET
- tristate "Virtio network driver"
- depends on VIRTIO
- select NET_FAILOVER
- help
- This is the virtual network driver for virtio. It can be used with
- QEMU based VMMs (like KVM or Xen). Say Y or M.
+source "drivers/net/virtio/Kconfig"
config NLMON
tristate "Virtual netlink monitoring device"
diff --git a/drivers/net/Makefile b/drivers/net/Makefile
index 7cab36f94782..a205dd2be77e 100644
--- a/drivers/net/Makefile
+++ b/drivers/net/Makefile
@@ -32,7 +32,7 @@ obj-$(CONFIG_NET_TEAM) += team/
obj-$(CONFIG_TUN) += tun.o
obj-$(CONFIG_TAP) += tap.o
obj-$(CONFIG_VETH) += veth.o
-obj-$(CONFIG_VIRTIO_NET) += virtio_net.o
+obj-$(CONFIG_VIRTIO_NET) += virtio/
obj-$(CONFIG_VXLAN) += vxlan/
obj-$(CONFIG_GENEVE) += geneve.o
obj-$(CONFIG_BAREUDP) += bareudp.o
diff --git a/drivers/net/virtio/Kconfig b/drivers/net/virtio/Kconfig
new file mode 100644
index 000000000000..d8ccb3ac49df
--- /dev/null
+++ b/drivers/net/virtio/Kconfig
@@ -0,0 +1,13 @@
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# virtio-net device configuration
+#
+config VIRTIO_NET
+ tristate "Virtio network driver"
+ depends on VIRTIO
+ select NET_FAILOVER
+ help
+ This is the virtual network driver for virtio. It can be used with
+ QEMU based VMMs (like KVM or Xen).
+
+ Say Y or M.
diff --git a/drivers/net/virtio/Makefile b/drivers/net/virtio/Makefile
new file mode 100644
index 000000000000..15ed7c97fd4f
--- /dev/null
+++ b/drivers/net/virtio/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for the virtio network device drivers.
+#
+
+obj-$(CONFIG_VIRTIO_NET) += virtio_net.o
+
+virtio_net-y := main.o
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio/main.c
similarity index 100%
rename from drivers/net/virtio_net.c
rename to drivers/net/virtio/main.c
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* [PATCH net-next v2 04/21] virtio_net: move core structures to virtio_net.h
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (2 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 03/21] virtio_net: independent directory Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-09 6:03 ` Jason Wang
2023-11-07 3:12 ` [PATCH net-next v2 05/21] virtio_net: add prefix virtnet to all struct inside virtio_net.h Xuan Zhuo
` (18 subsequent siblings)
22 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
Move some core structures (send_queue, receive_queue, virtnet_info)
definitions and the relative structures definitions into the
virtio_net.h file.
That will be used by the other c code files.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 189 +------------------------------
drivers/net/virtio/virtio_net.h | 193 ++++++++++++++++++++++++++++++++
2 files changed, 195 insertions(+), 187 deletions(-)
create mode 100644 drivers/net/virtio/virtio_net.h
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 3fe26f48025f..66060cff9387 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -6,7 +6,6 @@
//#define DEBUG
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
-#include <linux/ethtool.h>
#include <linux/module.h>
#include <linux/virtio.h>
#include <linux/virtio_net.h>
@@ -16,7 +15,6 @@
#include <linux/if_vlan.h>
#include <linux/slab.h>
#include <linux/cpu.h>
-#include <linux/average.h>
#include <linux/filter.h>
#include <linux/kernel.h>
#include <net/route.h>
@@ -24,6 +22,8 @@
#include <net/net_failover.h>
#include <net/netdev_rx_queue.h>
+#include "virtio_net.h"
+
static int napi_weight = NAPI_POLL_WEIGHT;
module_param(napi_weight, int, 0444);
@@ -47,13 +47,6 @@ module_param(napi_tx, bool, 0644);
#define VIRTIO_XDP_FLAG BIT(0)
-/* RX packet size EWMA. The average packet size is used to determine the packet
- * buffer size when refilling RX rings. As the entire RX ring may be refilled
- * at once, the weight is chosen so that the EWMA will be insensitive to short-
- * term, transient changes in packet size.
- */
-DECLARE_EWMA(pkt_len, 0, 64)
-
#define VIRTNET_DRIVER_VERSION "1.0.0"
static const unsigned long guest_offloads[] = {
@@ -79,28 +72,6 @@ struct virtnet_stat_desc {
size_t offset;
};
-struct virtnet_sq_stats {
- struct u64_stats_sync syncp;
- u64_stats_t packets;
- u64_stats_t bytes;
- u64_stats_t xdp_tx;
- u64_stats_t xdp_tx_drops;
- u64_stats_t kicks;
- u64_stats_t tx_timeouts;
-};
-
-struct virtnet_rq_stats {
- struct u64_stats_sync syncp;
- u64_stats_t packets;
- u64_stats_t bytes;
- u64_stats_t drops;
- u64_stats_t xdp_packets;
- u64_stats_t xdp_tx;
- u64_stats_t xdp_redirects;
- u64_stats_t xdp_drops;
- u64_stats_t kicks;
-};
-
#define VIRTNET_SQ_STAT(m) offsetof(struct virtnet_sq_stats, m)
#define VIRTNET_RQ_STAT(m) offsetof(struct virtnet_rq_stats, m)
@@ -127,80 +98,6 @@ static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = {
#define VIRTNET_SQ_STATS_LEN ARRAY_SIZE(virtnet_sq_stats_desc)
#define VIRTNET_RQ_STATS_LEN ARRAY_SIZE(virtnet_rq_stats_desc)
-struct virtnet_interrupt_coalesce {
- u32 max_packets;
- u32 max_usecs;
-};
-
-/* The dma information of pages allocated at a time. */
-struct virtnet_rq_dma {
- dma_addr_t addr;
- u32 ref;
- u16 len;
- u16 need_sync;
-};
-
-/* Internal representation of a send virtqueue */
-struct send_queue {
- /* Virtqueue associated with this send _queue */
- struct virtqueue *vq;
-
- /* TX: fragments + linear part + virtio header */
- struct scatterlist sg[MAX_SKB_FRAGS + 2];
-
- /* Name of the send queue: output.$index */
- char name[16];
-
- struct virtnet_sq_stats stats;
-
- struct virtnet_interrupt_coalesce intr_coal;
-
- struct napi_struct napi;
-
- /* Record whether sq is in reset state. */
- bool reset;
-};
-
-/* Internal representation of a receive virtqueue */
-struct receive_queue {
- /* Virtqueue associated with this receive_queue */
- struct virtqueue *vq;
-
- struct napi_struct napi;
-
- struct bpf_prog __rcu *xdp_prog;
-
- struct virtnet_rq_stats stats;
-
- struct virtnet_interrupt_coalesce intr_coal;
-
- /* Chain pages by the private ptr. */
- struct page *pages;
-
- /* Average packet length for mergeable receive buffers. */
- struct ewma_pkt_len mrg_avg_pkt_len;
-
- /* Page frag for packet buffer allocation. */
- struct page_frag alloc_frag;
-
- /* RX: fragments + linear part + virtio header */
- struct scatterlist sg[MAX_SKB_FRAGS + 2];
-
- /* Min single buffer size for mergeable buffers case. */
- unsigned int min_buf_len;
-
- /* Name of this receive queue: input.$index */
- char name[16];
-
- struct xdp_rxq_info xdp_rxq;
-
- /* Record the last dma info to free after new pages is allocated. */
- struct virtnet_rq_dma *last_dma;
-
- /* Do dma by self */
- bool do_dma;
-};
-
/* This structure can contain rss message with maximum settings for indirection table and keysize
* Note, that default structure that describes RSS configuration virtio_net_rss_config
* contains same info but can't handle table values.
@@ -234,88 +131,6 @@ struct control_buf {
struct virtio_net_ctrl_coal_vq coal_vq;
};
-struct virtnet_info {
- struct virtio_device *vdev;
- struct virtqueue *cvq;
- struct net_device *dev;
- struct send_queue *sq;
- struct receive_queue *rq;
- unsigned int status;
-
- /* Max # of queue pairs supported by the device */
- u16 max_queue_pairs;
-
- /* # of queue pairs currently used by the driver */
- u16 curr_queue_pairs;
-
- /* # of XDP queue pairs currently used by the driver */
- u16 xdp_queue_pairs;
-
- /* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */
- bool xdp_enabled;
-
- /* I like... big packets and I cannot lie! */
- bool big_packets;
-
- /* number of sg entries allocated for big packets */
- unsigned int big_packets_num_skbfrags;
-
- /* Host will merge rx buffers for big packets (shake it! shake it!) */
- bool mergeable_rx_bufs;
-
- /* Host supports rss and/or hash report */
- bool has_rss;
- bool has_rss_hash_report;
- u8 rss_key_size;
- u16 rss_indir_table_size;
- u32 rss_hash_types_supported;
- u32 rss_hash_types_saved;
-
- /* Has control virtqueue */
- bool has_cvq;
-
- /* Host can handle any s/g split between our header and packet data */
- bool any_header_sg;
-
- /* Packet virtio header size */
- u8 hdr_len;
-
- /* Work struct for delayed refilling if we run low on memory. */
- struct delayed_work refill;
-
- /* Is delayed refill enabled? */
- bool refill_enabled;
-
- /* The lock to synchronize the access to refill_enabled */
- spinlock_t refill_lock;
-
- /* Work struct for config space updates */
- struct work_struct config_work;
-
- /* Does the affinity hint is set for virtqueues? */
- bool affinity_hint_set;
-
- /* CPU hotplug instances for online & dead */
- struct hlist_node node;
- struct hlist_node node_dead;
-
- struct control_buf *ctrl;
-
- /* Ethtool settings */
- u8 duplex;
- u32 speed;
-
- /* Interrupt coalescing settings */
- struct virtnet_interrupt_coalesce intr_coal_tx;
- struct virtnet_interrupt_coalesce intr_coal_rx;
-
- unsigned long guest_offloads;
- unsigned long guest_offloads_capable;
-
- /* failover when STANDBY feature enabled */
- struct failover *failover;
-};
-
struct padded_vnet_hdr {
struct virtio_net_hdr_v1_hash hdr;
/*
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
new file mode 100644
index 000000000000..38061e15d494
--- /dev/null
+++ b/drivers/net/virtio/virtio_net.h
@@ -0,0 +1,193 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#ifndef __VIRTIO_NET_H__
+#define __VIRTIO_NET_H__
+
+#include <linux/ethtool.h>
+#include <linux/average.h>
+
+/* RX packet size EWMA. The average packet size is used to determine the packet
+ * buffer size when refilling RX rings. As the entire RX ring may be refilled
+ * at once, the weight is chosen so that the EWMA will be insensitive to short-
+ * term, transient changes in packet size.
+ */
+DECLARE_EWMA(pkt_len, 0, 64)
+
+struct virtnet_sq_stats {
+ struct u64_stats_sync syncp;
+ u64_stats_t packets;
+ u64_stats_t bytes;
+ u64_stats_t xdp_tx;
+ u64_stats_t xdp_tx_drops;
+ u64_stats_t kicks;
+ u64_stats_t tx_timeouts;
+};
+
+struct virtnet_rq_stats {
+ struct u64_stats_sync syncp;
+ u64_stats_t packets;
+ u64_stats_t bytes;
+ u64_stats_t drops;
+ u64_stats_t xdp_packets;
+ u64_stats_t xdp_tx;
+ u64_stats_t xdp_redirects;
+ u64_stats_t xdp_drops;
+ u64_stats_t kicks;
+};
+
+struct virtnet_interrupt_coalesce {
+ u32 max_packets;
+ u32 max_usecs;
+};
+
+/* The dma information of pages allocated at a time. */
+struct virtnet_rq_dma {
+ dma_addr_t addr;
+ u32 ref;
+ u16 len;
+ u16 need_sync;
+};
+
+/* Internal representation of a send virtqueue */
+struct send_queue {
+ /* Virtqueue associated with this send _queue */
+ struct virtqueue *vq;
+
+ /* TX: fragments + linear part + virtio header */
+ struct scatterlist sg[MAX_SKB_FRAGS + 2];
+
+ /* Name of the send queue: output.$index */
+ char name[16];
+
+ struct virtnet_sq_stats stats;
+
+ struct virtnet_interrupt_coalesce intr_coal;
+
+ struct napi_struct napi;
+
+ /* Record whether sq is in reset state. */
+ bool reset;
+};
+
+/* Internal representation of a receive virtqueue */
+struct receive_queue {
+ /* Virtqueue associated with this receive_queue */
+ struct virtqueue *vq;
+
+ struct napi_struct napi;
+
+ struct bpf_prog __rcu *xdp_prog;
+
+ struct virtnet_rq_stats stats;
+
+ struct virtnet_interrupt_coalesce intr_coal;
+
+ /* Chain pages by the private ptr. */
+ struct page *pages;
+
+ /* Average packet length for mergeable receive buffers. */
+ struct ewma_pkt_len mrg_avg_pkt_len;
+
+ /* Page frag for packet buffer allocation. */
+ struct page_frag alloc_frag;
+
+ /* RX: fragments + linear part + virtio header */
+ struct scatterlist sg[MAX_SKB_FRAGS + 2];
+
+ /* Min single buffer size for mergeable buffers case. */
+ unsigned int min_buf_len;
+
+ /* Name of this receive queue: input.$index */
+ char name[16];
+
+ struct xdp_rxq_info xdp_rxq;
+
+ /* Record the last dma info to free after new pages is allocated. */
+ struct virtnet_rq_dma *last_dma;
+
+ /* Do dma by self */
+ bool do_dma;
+};
+
+struct virtnet_info {
+ struct virtio_device *vdev;
+ struct virtqueue *cvq;
+ struct net_device *dev;
+ struct send_queue *sq;
+ struct receive_queue *rq;
+ unsigned int status;
+
+ /* Max # of queue pairs supported by the device */
+ u16 max_queue_pairs;
+
+ /* # of queue pairs currently used by the driver */
+ u16 curr_queue_pairs;
+
+ /* # of XDP queue pairs currently used by the driver */
+ u16 xdp_queue_pairs;
+
+ /* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */
+ bool xdp_enabled;
+
+ /* I like... big packets and I cannot lie! */
+ bool big_packets;
+
+ /* number of sg entries allocated for big packets */
+ unsigned int big_packets_num_skbfrags;
+
+ /* Host will merge rx buffers for big packets (shake it! shake it!) */
+ bool mergeable_rx_bufs;
+
+ /* Host supports rss and/or hash report */
+ bool has_rss;
+ bool has_rss_hash_report;
+ u8 rss_key_size;
+ u16 rss_indir_table_size;
+ u32 rss_hash_types_supported;
+ u32 rss_hash_types_saved;
+
+ /* Has control virtqueue */
+ bool has_cvq;
+
+ /* Host can handle any s/g split between our header and packet data */
+ bool any_header_sg;
+
+ /* Packet virtio header size */
+ u8 hdr_len;
+
+ /* Work struct for delayed refilling if we run low on memory. */
+ struct delayed_work refill;
+
+ /* Is delayed refill enabled? */
+ bool refill_enabled;
+
+ /* The lock to synchronize the access to refill_enabled */
+ spinlock_t refill_lock;
+
+ /* Work struct for config space updates */
+ struct work_struct config_work;
+
+ /* Does the affinity hint is set for virtqueues? */
+ bool affinity_hint_set;
+
+ /* CPU hotplug instances for online & dead */
+ struct hlist_node node;
+ struct hlist_node node_dead;
+
+ struct control_buf *ctrl;
+
+ /* Ethtool settings */
+ u8 duplex;
+ u32 speed;
+
+ /* Interrupt coalescing settings */
+ struct virtnet_interrupt_coalesce intr_coal_tx;
+ struct virtnet_interrupt_coalesce intr_coal_rx;
+
+ unsigned long guest_offloads;
+ unsigned long guest_offloads_capable;
+
+ /* failover when STANDBY feature enabled */
+ struct failover *failover;
+};
+#endif
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 04/21] virtio_net: move core structures to virtio_net.h
2023-11-07 3:12 ` [PATCH net-next v2 04/21] virtio_net: move core structures to virtio_net.h Xuan Zhuo
@ 2023-11-09 6:03 ` Jason Wang
0 siblings, 0 replies; 61+ messages in thread
From: Jason Wang @ 2023-11-09 6:03 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Michael S. Tsirkin, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Tue, Nov 7, 2023 at 11:13 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
>
> Move some core structures (send_queue, receive_queue, virtnet_info)
> definitions and the relative structures definitions into the
> virtio_net.h file.
>
> That will be used by the other c code files.
>
> Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> ---
Acked-by: Jason Wang <jasowang@redhat.com>
Thanks
^ permalink raw reply [flat|nested] 61+ messages in thread
* [PATCH net-next v2 05/21] virtio_net: add prefix virtnet to all struct inside virtio_net.h
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (3 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 04/21] virtio_net: move core structures to virtio_net.h Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-09 6:04 ` Jason Wang
2023-11-07 3:12 ` [PATCH net-next v2 06/21] virtio_net: separate virtnet_rx_resize() Xuan Zhuo
` (17 subsequent siblings)
22 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
We move some structures to the header file, but these structures do not
prefixed with virtnet. This patch adds virtnet for these.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 100 ++++++++++++++++----------------
drivers/net/virtio/virtio_net.h | 12 ++--
2 files changed, 56 insertions(+), 56 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 66060cff9387..46a8be0c609c 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -167,7 +167,7 @@ static struct xdp_frame *ptr_to_xdp(void *ptr)
return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
}
-static void __free_old_xmit(struct send_queue *sq, bool in_napi,
+static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
u64 *bytes, u64 *packets)
{
unsigned int len;
@@ -224,7 +224,7 @@ skb_vnet_common_hdr(struct sk_buff *skb)
* private is used to chain pages for big packets, put the whole
* most recent used list in the beginning for reuse
*/
-static void give_pages(struct receive_queue *rq, struct page *page)
+static void give_pages(struct virtnet_rq *rq, struct page *page)
{
struct page *end;
@@ -234,7 +234,7 @@ static void give_pages(struct receive_queue *rq, struct page *page)
rq->pages = page;
}
-static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
+static struct page *get_a_page(struct virtnet_rq *rq, gfp_t gfp_mask)
{
struct page *p = rq->pages;
@@ -248,7 +248,7 @@ static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask)
}
static void virtnet_rq_free_buf(struct virtnet_info *vi,
- struct receive_queue *rq, void *buf)
+ struct virtnet_rq *rq, void *buf)
{
if (vi->mergeable_rx_bufs)
put_page(virt_to_head_page(buf));
@@ -345,7 +345,7 @@ static struct sk_buff *virtnet_build_skb(void *buf, unsigned int buflen,
/* Called from bottom half context */
static struct sk_buff *page_to_skb(struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
struct page *page, unsigned int offset,
unsigned int len, unsigned int truesize,
unsigned int headroom)
@@ -444,7 +444,7 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
return skb;
}
-static void virtnet_rq_unmap(struct receive_queue *rq, void *buf, u32 len)
+static void virtnet_rq_unmap(struct virtnet_rq *rq, void *buf, u32 len)
{
struct page *page = virt_to_head_page(buf);
struct virtnet_rq_dma *dma;
@@ -473,7 +473,7 @@ static void virtnet_rq_unmap(struct receive_queue *rq, void *buf, u32 len)
put_page(page);
}
-static void *virtnet_rq_get_buf(struct receive_queue *rq, u32 *len, void **ctx)
+static void *virtnet_rq_get_buf(struct virtnet_rq *rq, u32 *len, void **ctx)
{
void *buf;
@@ -484,7 +484,7 @@ static void *virtnet_rq_get_buf(struct receive_queue *rq, u32 *len, void **ctx)
return buf;
}
-static void virtnet_rq_init_one_sg(struct receive_queue *rq, void *buf, u32 len)
+static void virtnet_rq_init_one_sg(struct virtnet_rq *rq, void *buf, u32 len)
{
struct virtnet_rq_dma *dma;
dma_addr_t addr;
@@ -509,7 +509,7 @@ static void virtnet_rq_init_one_sg(struct receive_queue *rq, void *buf, u32 len)
rq->sg[0].length = len;
}
-static void *virtnet_rq_alloc(struct receive_queue *rq, u32 size, gfp_t gfp)
+static void *virtnet_rq_alloc(struct virtnet_rq *rq, u32 size, gfp_t gfp)
{
struct page_frag *alloc_frag = &rq->alloc_frag;
struct virtnet_rq_dma *dma;
@@ -583,7 +583,7 @@ static void virtnet_rq_set_premapped(struct virtnet_info *vi)
}
}
-static void free_old_xmit(struct send_queue *sq, bool in_napi)
+static void free_old_xmit(struct virtnet_sq *sq, bool in_napi)
{
u64 bytes, packets = 0;
@@ -613,7 +613,7 @@ static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
static void check_sq_full_and_disable(struct virtnet_info *vi,
struct net_device *dev,
- struct send_queue *sq)
+ struct virtnet_sq *sq)
{
bool use_napi = sq->napi.weight;
int qnum;
@@ -647,7 +647,7 @@ static void check_sq_full_and_disable(struct virtnet_info *vi,
}
static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
- struct send_queue *sq,
+ struct virtnet_sq *sq,
struct xdp_frame *xdpf)
{
struct virtio_net_hdr_mrg_rxbuf *hdr;
@@ -736,10 +736,10 @@ static int virtnet_xdp_xmit(struct net_device *dev,
int n, struct xdp_frame **frames, u32 flags)
{
struct virtnet_info *vi = netdev_priv(dev);
- struct receive_queue *rq = vi->rq;
+ struct virtnet_rq *rq = vi->rq;
u64 bytes = 0, packets = 0;
struct bpf_prog *xdp_prog;
- struct send_queue *sq;
+ struct virtnet_sq *sq;
int nxmit = 0;
int kicks = 0;
int ret;
@@ -879,7 +879,7 @@ static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
* across multiple buffers (num_buf > 1), and we make sure buffers
* have enough headroom.
*/
-static struct page *xdp_linearize_page(struct receive_queue *rq,
+static struct page *xdp_linearize_page(struct virtnet_rq *rq,
int *num_buf,
struct page *p,
int offset,
@@ -960,7 +960,7 @@ static struct sk_buff *receive_small_build_skb(struct virtnet_info *vi,
static struct sk_buff *receive_small_xdp(struct net_device *dev,
struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
struct bpf_prog *xdp_prog,
void *buf,
unsigned int xdp_headroom,
@@ -1047,7 +1047,7 @@ static struct sk_buff *receive_small_xdp(struct net_device *dev,
static struct sk_buff *receive_small(struct net_device *dev,
struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
void *buf, void *ctx,
unsigned int len,
unsigned int *xdp_xmit,
@@ -1094,7 +1094,7 @@ static struct sk_buff *receive_small(struct net_device *dev,
static struct sk_buff *receive_big(struct net_device *dev,
struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
void *buf,
unsigned int len,
struct virtnet_rq_stats *stats)
@@ -1115,7 +1115,7 @@ static struct sk_buff *receive_big(struct net_device *dev,
return NULL;
}
-static void mergeable_buf_free(struct receive_queue *rq, int num_buf,
+static void mergeable_buf_free(struct virtnet_rq *rq, int num_buf,
struct net_device *dev,
struct virtnet_rq_stats *stats)
{
@@ -1189,7 +1189,7 @@ static struct sk_buff *build_skb_from_xdp_buff(struct net_device *dev,
/* TODO: build xdp in big mode */
static int virtnet_build_xdp_buff_mrg(struct net_device *dev,
struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
struct xdp_buff *xdp,
void *buf,
unsigned int len,
@@ -1277,7 +1277,7 @@ static int virtnet_build_xdp_buff_mrg(struct net_device *dev,
}
static void *mergeable_xdp_get_buf(struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
struct bpf_prog *xdp_prog,
void *ctx,
unsigned int *frame_sz,
@@ -1352,7 +1352,7 @@ static void *mergeable_xdp_get_buf(struct virtnet_info *vi,
static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
struct bpf_prog *xdp_prog,
void *buf,
void *ctx,
@@ -1412,7 +1412,7 @@ static struct sk_buff *receive_mergeable_xdp(struct net_device *dev,
static struct sk_buff *receive_mergeable(struct net_device *dev,
struct virtnet_info *vi,
- struct receive_queue *rq,
+ struct virtnet_rq *rq,
void *buf,
void *ctx,
unsigned int len,
@@ -1557,7 +1557,7 @@ static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash,
skb_set_hash(skb, __le32_to_cpu(hdr_hash->hash_value), rss_hash_type);
}
-static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
+static void receive_buf(struct virtnet_info *vi, struct virtnet_rq *rq,
void *buf, unsigned int len, void **ctx,
unsigned int *xdp_xmit,
struct virtnet_rq_stats *stats)
@@ -1617,7 +1617,7 @@ static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
* not need to use mergeable_len_to_ctx here - it is enough
* to store the headroom as the context ignoring the truesize.
*/
-static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
+static int add_recvbuf_small(struct virtnet_info *vi, struct virtnet_rq *rq,
gfp_t gfp)
{
char *buf;
@@ -1646,7 +1646,7 @@ static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq,
return err;
}
-static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
+static int add_recvbuf_big(struct virtnet_info *vi, struct virtnet_rq *rq,
gfp_t gfp)
{
struct page *first, *list = NULL;
@@ -1695,7 +1695,7 @@ static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq,
return err;
}
-static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
+static unsigned int get_mergeable_buf_len(struct virtnet_rq *rq,
struct ewma_pkt_len *avg_pkt_len,
unsigned int room)
{
@@ -1713,7 +1713,7 @@ static unsigned int get_mergeable_buf_len(struct receive_queue *rq,
}
static int add_recvbuf_mergeable(struct virtnet_info *vi,
- struct receive_queue *rq, gfp_t gfp)
+ struct virtnet_rq *rq, gfp_t gfp)
{
struct page_frag *alloc_frag = &rq->alloc_frag;
unsigned int headroom = virtnet_get_headroom(vi);
@@ -1768,7 +1768,7 @@ static int add_recvbuf_mergeable(struct virtnet_info *vi,
* before we're receiving packets, or from refill_work which is
* careful to disable receiving (using napi_disable).
*/
-static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
+static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
gfp_t gfp)
{
int err;
@@ -1800,7 +1800,7 @@ static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq,
static void skb_recv_done(struct virtqueue *rvq)
{
struct virtnet_info *vi = rvq->vdev->priv;
- struct receive_queue *rq = &vi->rq[vq2rxq(rvq)];
+ struct virtnet_rq *rq = &vi->rq[vq2rxq(rvq)];
virtqueue_napi_schedule(&rq->napi, rvq);
}
@@ -1850,7 +1850,7 @@ static void refill_work(struct work_struct *work)
int i;
for (i = 0; i < vi->curr_queue_pairs; i++) {
- struct receive_queue *rq = &vi->rq[i];
+ struct virtnet_rq *rq = &vi->rq[i];
napi_disable(&rq->napi);
still_empty = !try_fill_recv(vi, rq, GFP_KERNEL);
@@ -1864,7 +1864,7 @@ static void refill_work(struct work_struct *work)
}
}
-static int virtnet_receive(struct receive_queue *rq, int budget,
+static int virtnet_receive(struct virtnet_rq *rq, int budget,
unsigned int *xdp_xmit)
{
struct virtnet_info *vi = rq->vq->vdev->priv;
@@ -1914,11 +1914,11 @@ static int virtnet_receive(struct receive_queue *rq, int budget,
return packets;
}
-static void virtnet_poll_cleantx(struct receive_queue *rq)
+static void virtnet_poll_cleantx(struct virtnet_rq *rq)
{
struct virtnet_info *vi = rq->vq->vdev->priv;
unsigned int index = vq2rxq(rq->vq);
- struct send_queue *sq = &vi->sq[index];
+ struct virtnet_sq *sq = &vi->sq[index];
struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);
if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index))
@@ -1944,10 +1944,10 @@ static void virtnet_poll_cleantx(struct receive_queue *rq)
static int virtnet_poll(struct napi_struct *napi, int budget)
{
- struct receive_queue *rq =
- container_of(napi, struct receive_queue, napi);
+ struct virtnet_rq *rq =
+ container_of(napi, struct virtnet_rq, napi);
struct virtnet_info *vi = rq->vq->vdev->priv;
- struct send_queue *sq;
+ struct virtnet_sq *sq;
unsigned int received;
unsigned int xdp_xmit = 0;
@@ -2038,7 +2038,7 @@ static int virtnet_open(struct net_device *dev)
static int virtnet_poll_tx(struct napi_struct *napi, int budget)
{
- struct send_queue *sq = container_of(napi, struct send_queue, napi);
+ struct virtnet_sq *sq = container_of(napi, struct virtnet_sq, napi);
struct virtnet_info *vi = sq->vq->vdev->priv;
unsigned int index = vq2txq(sq->vq);
struct netdev_queue *txq;
@@ -2082,7 +2082,7 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
return 0;
}
-static int xmit_skb(struct send_queue *sq, struct sk_buff *skb)
+static int xmit_skb(struct virtnet_sq *sq, struct sk_buff *skb)
{
struct virtio_net_hdr_mrg_rxbuf *hdr;
const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest;
@@ -2133,7 +2133,7 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
{
struct virtnet_info *vi = netdev_priv(dev);
int qnum = skb_get_queue_mapping(skb);
- struct send_queue *sq = &vi->sq[qnum];
+ struct virtnet_sq *sq = &vi->sq[qnum];
int err;
struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum);
bool kick = !netdev_xmit_more();
@@ -2187,7 +2187,7 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
}
static int virtnet_rx_resize(struct virtnet_info *vi,
- struct receive_queue *rq, u32 ring_num)
+ struct virtnet_rq *rq, u32 ring_num)
{
bool running = netif_running(vi->dev);
int err, qindex;
@@ -2210,7 +2210,7 @@ static int virtnet_rx_resize(struct virtnet_info *vi,
}
static int virtnet_tx_resize(struct virtnet_info *vi,
- struct send_queue *sq, u32 ring_num)
+ struct virtnet_sq *sq, u32 ring_num)
{
bool running = netif_running(vi->dev);
struct netdev_queue *txq;
@@ -2356,8 +2356,8 @@ static void virtnet_stats(struct net_device *dev,
for (i = 0; i < vi->max_queue_pairs; i++) {
u64 tpackets, tbytes, terrors, rpackets, rbytes, rdrops;
- struct receive_queue *rq = &vi->rq[i];
- struct send_queue *sq = &vi->sq[i];
+ struct virtnet_rq *rq = &vi->rq[i];
+ struct virtnet_sq *sq = &vi->sq[i];
do {
start = u64_stats_fetch_begin(&sq->stats.syncp);
@@ -2673,8 +2673,8 @@ static int virtnet_set_ringparam(struct net_device *dev,
{
struct virtnet_info *vi = netdev_priv(dev);
u32 rx_pending, tx_pending;
- struct receive_queue *rq;
- struct send_queue *sq;
+ struct virtnet_rq *rq;
+ struct virtnet_sq *sq;
int i, err;
if (ring->rx_mini_pending || ring->rx_jumbo_pending)
@@ -3003,7 +3003,7 @@ static void virtnet_get_ethtool_stats(struct net_device *dev,
size_t offset;
for (i = 0; i < vi->curr_queue_pairs; i++) {
- struct receive_queue *rq = &vi->rq[i];
+ struct virtnet_rq *rq = &vi->rq[i];
stats_base = (const u8 *)&rq->stats;
do {
@@ -3018,7 +3018,7 @@ static void virtnet_get_ethtool_stats(struct net_device *dev,
}
for (i = 0; i < vi->curr_queue_pairs; i++) {
- struct send_queue *sq = &vi->sq[i];
+ struct virtnet_sq *sq = &vi->sq[i];
stats_base = (const u8 *)&sq->stats;
do {
@@ -3705,7 +3705,7 @@ static int virtnet_set_features(struct net_device *dev,
static void virtnet_tx_timeout(struct net_device *dev, unsigned int txqueue)
{
struct virtnet_info *priv = netdev_priv(dev);
- struct send_queue *sq = &priv->sq[txqueue];
+ struct virtnet_sq *sq = &priv->sq[txqueue];
struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue);
u64_stats_update_begin(&sq->stats.syncp);
@@ -3839,7 +3839,7 @@ static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
{
struct virtnet_info *vi = vq->vdev->priv;
- struct receive_queue *rq;
+ struct virtnet_rq *rq;
int i = vq2rxq(vq);
rq = &vi->rq[i];
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index 38061e15d494..ebf9f344648a 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -49,8 +49,8 @@ struct virtnet_rq_dma {
};
/* Internal representation of a send virtqueue */
-struct send_queue {
- /* Virtqueue associated with this send _queue */
+struct virtnet_sq {
+ /* Virtqueue associated with this virtnet_sq */
struct virtqueue *vq;
/* TX: fragments + linear part + virtio header */
@@ -70,8 +70,8 @@ struct send_queue {
};
/* Internal representation of a receive virtqueue */
-struct receive_queue {
- /* Virtqueue associated with this receive_queue */
+struct virtnet_rq {
+ /* Virtqueue associated with this virtnet_rq */
struct virtqueue *vq;
struct napi_struct napi;
@@ -113,8 +113,8 @@ struct virtnet_info {
struct virtio_device *vdev;
struct virtqueue *cvq;
struct net_device *dev;
- struct send_queue *sq;
- struct receive_queue *rq;
+ struct virtnet_sq *sq;
+ struct virtnet_rq *rq;
unsigned int status;
/* Max # of queue pairs supported by the device */
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 05/21] virtio_net: add prefix virtnet to all struct inside virtio_net.h
2023-11-07 3:12 ` [PATCH net-next v2 05/21] virtio_net: add prefix virtnet to all struct inside virtio_net.h Xuan Zhuo
@ 2023-11-09 6:04 ` Jason Wang
0 siblings, 0 replies; 61+ messages in thread
From: Jason Wang @ 2023-11-09 6:04 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Michael S. Tsirkin, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Tue, Nov 7, 2023 at 11:13 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
>
> We move some structures to the header file, but these structures do not
> prefixed with virtnet. This patch adds virtnet for these.
>
> Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> ---
Acked-by: Jason Wang <jasowang@redhat.com>
Thanks
^ permalink raw reply [flat|nested] 61+ messages in thread
* [PATCH net-next v2 06/21] virtio_net: separate virtnet_rx_resize()
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (4 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 05/21] virtio_net: add prefix virtnet to all struct inside virtio_net.h Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 07/21] virtio_net: separate virtnet_tx_resize() Xuan Zhuo
` (16 subsequent siblings)
22 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
This patch separates two sub-functions from virtnet_rx_resize():
* virtnet_rx_pause
* virtnet_rx_resume
Then the subsequent reset rx for xsk can share these two functions.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Jason Wang <jasowang@redhat.com>
---
drivers/net/virtio/main.c | 29 +++++++++++++++++++++--------
drivers/net/virtio/virtio_net.h | 3 +++
2 files changed, 24 insertions(+), 8 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 46a8be0c609c..c9f8294153e2 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -2186,26 +2186,39 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
return NETDEV_TX_OK;
}
-static int virtnet_rx_resize(struct virtnet_info *vi,
- struct virtnet_rq *rq, u32 ring_num)
+void virtnet_rx_pause(struct virtnet_info *vi, struct virtnet_rq *rq)
{
bool running = netif_running(vi->dev);
- int err, qindex;
-
- qindex = rq - vi->rq;
if (running)
napi_disable(&rq->napi);
+}
- err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_free_unused_buf);
- if (err)
- netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err);
+void virtnet_rx_resume(struct virtnet_info *vi, struct virtnet_rq *rq)
+{
+ bool running = netif_running(vi->dev);
if (!try_fill_recv(vi, rq, GFP_KERNEL))
schedule_delayed_work(&vi->refill, 0);
if (running)
virtnet_napi_enable(rq->vq, &rq->napi);
+}
+
+static int virtnet_rx_resize(struct virtnet_info *vi,
+ struct virtnet_rq *rq, u32 ring_num)
+{
+ int err, qindex;
+
+ qindex = rq - vi->rq;
+
+ virtnet_rx_pause(vi, rq);
+
+ err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_free_unused_buf);
+ if (err)
+ netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err);
+
+ virtnet_rx_resume(vi, rq);
return err;
}
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index ebf9f344648a..693ca166fc93 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -190,4 +190,7 @@ struct virtnet_info {
/* failover when STANDBY feature enabled */
struct failover *failover;
};
+
+void virtnet_rx_pause(struct virtnet_info *vi, struct virtnet_rq *rq);
+void virtnet_rx_resume(struct virtnet_info *vi, struct virtnet_rq *rq);
#endif
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* [PATCH net-next v2 07/21] virtio_net: separate virtnet_tx_resize()
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (5 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 06/21] virtio_net: separate virtnet_rx_resize() Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 08/21] virtio_net: sq support premapped mode Xuan Zhuo
` (15 subsequent siblings)
22 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
This patch separates two sub-functions from virtnet_tx_resize():
* virtnet_tx_pause
* virtnet_tx_resume
Then the subsequent virtnet_tx_reset() can share these two functions.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Jason Wang <jasowang@redhat.com>
---
drivers/net/virtio/main.c | 35 +++++++++++++++++++++++++++------
drivers/net/virtio/virtio_net.h | 2 ++
2 files changed, 31 insertions(+), 6 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index c9f8294153e2..16e75c08639e 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -2222,12 +2222,11 @@ static int virtnet_rx_resize(struct virtnet_info *vi,
return err;
}
-static int virtnet_tx_resize(struct virtnet_info *vi,
- struct virtnet_sq *sq, u32 ring_num)
+void virtnet_tx_pause(struct virtnet_info *vi, struct virtnet_sq *sq)
{
bool running = netif_running(vi->dev);
struct netdev_queue *txq;
- int err, qindex;
+ int qindex;
qindex = sq - vi->sq;
@@ -2248,10 +2247,17 @@ static int virtnet_tx_resize(struct virtnet_info *vi,
netif_stop_subqueue(vi->dev, qindex);
__netif_tx_unlock_bh(txq);
+}
- err = virtqueue_resize(sq->vq, ring_num, virtnet_sq_free_unused_buf);
- if (err)
- netdev_err(vi->dev, "resize tx fail: tx queue index: %d err: %d\n", qindex, err);
+void virtnet_tx_resume(struct virtnet_info *vi, struct virtnet_sq *sq)
+{
+ bool running = netif_running(vi->dev);
+ struct netdev_queue *txq;
+ int qindex;
+
+ qindex = sq - vi->sq;
+
+ txq = netdev_get_tx_queue(vi->dev, qindex);
__netif_tx_lock_bh(txq);
sq->reset = false;
@@ -2260,6 +2266,23 @@ static int virtnet_tx_resize(struct virtnet_info *vi,
if (running)
virtnet_napi_tx_enable(vi, sq->vq, &sq->napi);
+}
+
+static int virtnet_tx_resize(struct virtnet_info *vi, struct virtnet_sq *sq,
+ u32 ring_num)
+{
+ int qindex, err;
+
+ qindex = sq - vi->sq;
+
+ virtnet_tx_pause(vi, sq);
+
+ err = virtqueue_resize(sq->vq, ring_num, virtnet_sq_free_unused_buf);
+ if (err)
+ netdev_err(vi->dev, "resize tx fail: tx queue index: %d err: %d\n", qindex, err);
+
+ virtnet_tx_resume(vi, sq);
+
return err;
}
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index 693ca166fc93..d814341d9f97 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -193,4 +193,6 @@ struct virtnet_info {
void virtnet_rx_pause(struct virtnet_info *vi, struct virtnet_rq *rq);
void virtnet_rx_resume(struct virtnet_info *vi, struct virtnet_rq *rq);
+void virtnet_tx_pause(struct virtnet_info *vi, struct virtnet_sq *sq);
+void virtnet_tx_resume(struct virtnet_info *vi, struct virtnet_sq *sq);
#endif
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* [PATCH net-next v2 08/21] virtio_net: sq support premapped mode
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (6 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 07/21] virtio_net: separate virtnet_tx_resize() Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-09 6:37 ` Jason Wang
2023-11-07 3:12 ` [PATCH net-next v2 09/21] virtio_net: xsk: bind/unbind xsk Xuan Zhuo
` (14 subsequent siblings)
22 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
If the xsk is enabling, the xsk tx will share the send queue.
But the xsk requires that the send queue use the premapped mode.
So the send queue must support premapped mode.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 163 ++++++++++++++++++++++++++++----
drivers/net/virtio/virtio_net.h | 16 ++++
2 files changed, 163 insertions(+), 16 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 16e75c08639e..f052db459156 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -46,6 +46,7 @@ module_param(napi_tx, bool, 0644);
#define VIRTIO_XDP_REDIR BIT(1)
#define VIRTIO_XDP_FLAG BIT(0)
+#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
#define VIRTNET_DRIVER_VERSION "1.0.0"
@@ -167,6 +168,29 @@ static struct xdp_frame *ptr_to_xdp(void *ptr)
return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
}
+static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
+{
+ struct virtnet_sq_dma *next, *head;
+
+ head = (void *)((unsigned long)data & ~VIRTIO_XMIT_DATA_MASK);
+
+ data = head->data;
+
+ while (head) {
+ virtqueue_dma_unmap_single_attrs(sq->vq, head->addr, head->len,
+ DMA_TO_DEVICE, 0);
+
+ next = head->next;
+
+ head->next = sq->dmainfo.free;
+ sq->dmainfo.free = head;
+
+ head = next;
+ }
+
+ return data;
+}
+
static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
u64 *bytes, u64 *packets)
{
@@ -175,14 +199,24 @@ static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
if (!is_xdp_frame(ptr)) {
- struct sk_buff *skb = ptr;
+ struct sk_buff *skb;
+
+ if (sq->do_dma)
+ ptr = virtnet_sq_unmap(sq, ptr);
+
+ skb = ptr;
pr_debug("Sent skb %p\n", skb);
*bytes += skb->len;
napi_consume_skb(skb, in_napi);
} else {
- struct xdp_frame *frame = ptr_to_xdp(ptr);
+ struct xdp_frame *frame;
+
+ if (sq->do_dma)
+ ptr = virtnet_sq_unmap(sq, ptr);
+
+ frame = ptr_to_xdp(ptr);
*bytes += xdp_get_frame_len(frame);
xdp_return_frame(frame);
@@ -567,22 +601,104 @@ static void *virtnet_rq_alloc(struct virtnet_rq *rq, u32 size, gfp_t gfp)
return buf;
}
-static void virtnet_rq_set_premapped(struct virtnet_info *vi)
+static int virtnet_sq_set_premapped(struct virtnet_sq *sq)
{
- int i;
+ struct virtnet_sq_dma *d;
+ int err, size, i;
- /* disable for big mode */
- if (!vi->mergeable_rx_bufs && vi->big_packets)
- return;
+ size = virtqueue_get_vring_size(sq->vq);
+
+ size += MAX_SKB_FRAGS + 2;
+
+ sq->dmainfo.head = kcalloc(size, sizeof(*sq->dmainfo.head), GFP_KERNEL);
+ if (!sq->dmainfo.head)
+ return -ENOMEM;
+
+ err = virtqueue_set_dma_premapped(sq->vq);
+ if (err) {
+ kfree(sq->dmainfo.head);
+ return err;
+ }
+
+ sq->dmainfo.free = NULL;
+
+ sq->do_dma = true;
+
+ for (i = 0; i < size; ++i) {
+ d = &sq->dmainfo.head[i];
+
+ d->next = sq->dmainfo.free;
+ sq->dmainfo.free = d;
+ }
+
+ return 0;
+}
+
+static void virtnet_set_premapped(struct virtnet_info *vi)
+{
+ int i;
for (i = 0; i < vi->max_queue_pairs; i++) {
- if (virtqueue_set_dma_premapped(vi->rq[i].vq))
- continue;
+ virtnet_sq_set_premapped(&vi->sq[i]);
- vi->rq[i].do_dma = true;
+ /* disable for big mode */
+ if (vi->mergeable_rx_bufs || !vi->big_packets) {
+ if (!virtqueue_set_dma_premapped(vi->rq[i].vq))
+ vi->rq[i].do_dma = true;
+ }
}
}
+static struct virtnet_sq_dma *virtnet_sq_map_sg(struct virtnet_sq *sq, int nents, void *data)
+{
+ struct virtnet_sq_dma *d, *head;
+ struct scatterlist *sg;
+ int i;
+
+ head = NULL;
+
+ for_each_sg(sq->sg, sg, nents, i) {
+ sg->dma_address = virtqueue_dma_map_single_attrs(sq->vq, sg_virt(sg),
+ sg->length,
+ DMA_TO_DEVICE, 0);
+ if (virtqueue_dma_mapping_error(sq->vq, sg->dma_address))
+ goto err;
+
+ d = sq->dmainfo.free;
+ sq->dmainfo.free = d->next;
+
+ d->addr = sg->dma_address;
+ d->len = sg->length;
+
+ d->next = head;
+ head = d;
+ }
+
+ head->data = data;
+
+ return (void *)((unsigned long)head | ((unsigned long)data & VIRTIO_XMIT_DATA_MASK));
+err:
+ virtnet_sq_unmap(sq, head);
+ return NULL;
+}
+
+static int virtnet_add_outbuf(struct virtnet_sq *sq, u32 num, void *data)
+{
+ int ret;
+
+ if (sq->do_dma) {
+ data = virtnet_sq_map_sg(sq, num, data);
+ if (!data)
+ return -ENOMEM;
+ }
+
+ ret = virtqueue_add_outbuf(sq->vq, sq->sg, num, data, GFP_ATOMIC);
+ if (ret && sq->do_dma)
+ virtnet_sq_unmap(sq, data);
+
+ return ret;
+}
+
static void free_old_xmit(struct virtnet_sq *sq, bool in_napi)
{
u64 bytes, packets = 0;
@@ -686,8 +802,7 @@ static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
skb_frag_size(frag), skb_frag_off(frag));
}
- err = virtqueue_add_outbuf(sq->vq, sq->sg, nr_frags + 1,
- xdp_to_ptr(xdpf), GFP_ATOMIC);
+ err = virtnet_add_outbuf(sq, nr_frags + 1, xdp_to_ptr(xdpf));
if (unlikely(err))
return -ENOSPC; /* Caller handle free/refcnt */
@@ -2126,7 +2241,8 @@ static int xmit_skb(struct virtnet_sq *sq, struct sk_buff *skb)
return num_sg;
num_sg++;
}
- return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
+
+ return virtnet_add_outbuf(sq, num_sg, skb);
}
static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
@@ -3818,6 +3934,8 @@ static void virtnet_free_queues(struct virtnet_info *vi)
for (i = 0; i < vi->max_queue_pairs; i++) {
__netif_napi_del(&vi->rq[i].napi);
__netif_napi_del(&vi->sq[i].napi);
+
+ kfree(vi->sq[i].dmainfo.head);
}
/* We called __netif_napi_del(),
@@ -3866,10 +3984,23 @@ static void free_receive_page_frags(struct virtnet_info *vi)
static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
{
- if (!is_xdp_frame(buf))
+ struct virtnet_info *vi = vq->vdev->priv;
+ struct virtnet_sq *sq;
+ int i = vq2rxq(vq);
+
+ sq = &vi->sq[i];
+
+ if (!is_xdp_frame(buf)) {
+ if (sq->do_dma)
+ buf = virtnet_sq_unmap(sq, buf);
+
dev_kfree_skb(buf);
- else
+ } else {
+ if (sq->do_dma)
+ buf = virtnet_sq_unmap(sq, buf);
+
xdp_return_frame(ptr_to_xdp(buf));
+ }
}
static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
@@ -4075,7 +4206,7 @@ static int init_vqs(struct virtnet_info *vi)
if (ret)
goto err_free;
- virtnet_rq_set_premapped(vi);
+ virtnet_set_premapped(vi);
cpus_read_lock();
virtnet_set_affinity(vi);
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index d814341d9f97..ce806afb6d64 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -48,6 +48,18 @@ struct virtnet_rq_dma {
u16 need_sync;
};
+struct virtnet_sq_dma {
+ struct virtnet_sq_dma *next;
+ dma_addr_t addr;
+ u32 len;
+ void *data;
+};
+
+struct virtnet_sq_dma_head {
+ struct virtnet_sq_dma *free;
+ struct virtnet_sq_dma *head;
+};
+
/* Internal representation of a send virtqueue */
struct virtnet_sq {
/* Virtqueue associated with this virtnet_sq */
@@ -67,6 +79,10 @@ struct virtnet_sq {
/* Record whether sq is in reset state. */
bool reset;
+
+ bool do_dma;
+
+ struct virtnet_sq_dma_head dmainfo;
};
/* Internal representation of a receive virtqueue */
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 08/21] virtio_net: sq support premapped mode
2023-11-07 3:12 ` [PATCH net-next v2 08/21] virtio_net: sq support premapped mode Xuan Zhuo
@ 2023-11-09 6:37 ` Jason Wang
2023-11-09 10:58 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Jason Wang @ 2023-11-09 6:37 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Michael S. Tsirkin, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Tue, Nov 7, 2023 at 11:12 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
>
> If the xsk is enabling, the xsk tx will share the send queue.
> But the xsk requires that the send queue use the premapped mode.
> So the send queue must support premapped mode.
>
> Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> ---
> drivers/net/virtio/main.c | 163 ++++++++++++++++++++++++++++----
> drivers/net/virtio/virtio_net.h | 16 ++++
> 2 files changed, 163 insertions(+), 16 deletions(-)
>
> diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> index 16e75c08639e..f052db459156 100644
> --- a/drivers/net/virtio/main.c
> +++ b/drivers/net/virtio/main.c
> @@ -46,6 +46,7 @@ module_param(napi_tx, bool, 0644);
> #define VIRTIO_XDP_REDIR BIT(1)
>
> #define VIRTIO_XDP_FLAG BIT(0)
> +#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
>
> #define VIRTNET_DRIVER_VERSION "1.0.0"
>
> @@ -167,6 +168,29 @@ static struct xdp_frame *ptr_to_xdp(void *ptr)
> return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
> }
>
> +static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> +{
> + struct virtnet_sq_dma *next, *head;
> +
> + head = (void *)((unsigned long)data & ~VIRTIO_XMIT_DATA_MASK);
> +
> + data = head->data;
> +
> + while (head) {
> + virtqueue_dma_unmap_single_attrs(sq->vq, head->addr, head->len,
> + DMA_TO_DEVICE, 0);
> +
> + next = head->next;
> +
> + head->next = sq->dmainfo.free;
> + sq->dmainfo.free = head;
> +
> + head = next;
> + }
> +
> + return data;
> +}
> +
> static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> u64 *bytes, u64 *packets)
> {
> @@ -175,14 +199,24 @@ static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
>
> while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
> if (!is_xdp_frame(ptr)) {
> - struct sk_buff *skb = ptr;
> + struct sk_buff *skb;
> +
> + if (sq->do_dma)
> + ptr = virtnet_sq_unmap(sq, ptr);
> +
> + skb = ptr;
>
> pr_debug("Sent skb %p\n", skb);
>
> *bytes += skb->len;
> napi_consume_skb(skb, in_napi);
> } else {
> - struct xdp_frame *frame = ptr_to_xdp(ptr);
> + struct xdp_frame *frame;
> +
> + if (sq->do_dma)
> + ptr = virtnet_sq_unmap(sq, ptr);
> +
> + frame = ptr_to_xdp(ptr);
>
> *bytes += xdp_get_frame_len(frame);
> xdp_return_frame(frame);
> @@ -567,22 +601,104 @@ static void *virtnet_rq_alloc(struct virtnet_rq *rq, u32 size, gfp_t gfp)
> return buf;
> }
>
> -static void virtnet_rq_set_premapped(struct virtnet_info *vi)
> +static int virtnet_sq_set_premapped(struct virtnet_sq *sq)
> {
> - int i;
> + struct virtnet_sq_dma *d;
> + int err, size, i;
>
> - /* disable for big mode */
> - if (!vi->mergeable_rx_bufs && vi->big_packets)
> - return;
> + size = virtqueue_get_vring_size(sq->vq);
> +
> + size += MAX_SKB_FRAGS + 2;
Btw, the dmainfo seems per sg? If I'm correct, how can vq_size +
MAX_SKB_FRAGS + 2 work?
> +
> + sq->dmainfo.head = kcalloc(size, sizeof(*sq->dmainfo.head), GFP_KERNEL);
> + if (!sq->dmainfo.head)
> + return -ENOMEM;
> +
> + err = virtqueue_set_dma_premapped(sq->vq);
> + if (err) {
> + kfree(sq->dmainfo.head);
> + return err;
> + }
Allocating after set_dma_premapped() seems easier.
Btw, is there a benchmark of TX PPS just for this patch to demonstrate
the impact of the performance?
> +
> + sq->dmainfo.free = NULL;
> +
> + sq->do_dma = true;
> +
> + for (i = 0; i < size; ++i) {
> + d = &sq->dmainfo.head[i];
> +
> + d->next = sq->dmainfo.free;
> + sq->dmainfo.free = d;
> + }
> +
> + return 0;
> +}
> +
> +static void virtnet_set_premapped(struct virtnet_info *vi)
> +{
> + int i;
>
> for (i = 0; i < vi->max_queue_pairs; i++) {
> - if (virtqueue_set_dma_premapped(vi->rq[i].vq))
> - continue;
> + virtnet_sq_set_premapped(&vi->sq[i]);
>
> - vi->rq[i].do_dma = true;
> + /* disable for big mode */
> + if (vi->mergeable_rx_bufs || !vi->big_packets) {
> + if (!virtqueue_set_dma_premapped(vi->rq[i].vq))
> + vi->rq[i].do_dma = true;
How about sticking a virtnet_rq_set_premapped() and calling it here?
It seems more clean.
Btw, the big mode support for pre mapping is still worthwhile
regardless whether or not XDP is supported. It has a page pool so we
can avoid redundant DMA map/unmap there.
> + }
> }
> }
>
> +static struct virtnet_sq_dma *virtnet_sq_map_sg(struct virtnet_sq *sq, int nents, void *data)
> +{
> + struct virtnet_sq_dma *d, *head;
> + struct scatterlist *sg;
> + int i;
> +
> + head = NULL;
> +
> + for_each_sg(sq->sg, sg, nents, i) {
> + sg->dma_address = virtqueue_dma_map_single_attrs(sq->vq, sg_virt(sg),
> + sg->length,
> + DMA_TO_DEVICE, 0);
> + if (virtqueue_dma_mapping_error(sq->vq, sg->dma_address))
> + goto err;
> +
> + d = sq->dmainfo.free;
> + sq->dmainfo.free = d->next;
> +
> + d->addr = sg->dma_address;
> + d->len = sg->length;
> +
> + d->next = head;
> + head = d;
> + }
> +
> + head->data = data;
> +
> + return (void *)((unsigned long)head | ((unsigned long)data & VIRTIO_XMIT_DATA_MASK));
So head contains a pointer to data, any reason we still need to pack a
data pointer here?
> +err:
> + virtnet_sq_unmap(sq, head);
> + return NULL;
> +}
> +
> +static int virtnet_add_outbuf(struct virtnet_sq *sq, u32 num, void *data)
> +{
> + int ret;
> +
> + if (sq->do_dma) {
> + data = virtnet_sq_map_sg(sq, num, data);
> + if (!data)
> + return -ENOMEM;
> + }
> +
> + ret = virtqueue_add_outbuf(sq->vq, sq->sg, num, data, GFP_ATOMIC);
> + if (ret && sq->do_dma)
> + virtnet_sq_unmap(sq, data);
> +
> + return ret;
> +}
> +
> static void free_old_xmit(struct virtnet_sq *sq, bool in_napi)
> {
> u64 bytes, packets = 0;
> @@ -686,8 +802,7 @@ static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
> skb_frag_size(frag), skb_frag_off(frag));
> }
>
> - err = virtqueue_add_outbuf(sq->vq, sq->sg, nr_frags + 1,
> - xdp_to_ptr(xdpf), GFP_ATOMIC);
> + err = virtnet_add_outbuf(sq, nr_frags + 1, xdp_to_ptr(xdpf));
> if (unlikely(err))
> return -ENOSPC; /* Caller handle free/refcnt */
>
> @@ -2126,7 +2241,8 @@ static int xmit_skb(struct virtnet_sq *sq, struct sk_buff *skb)
> return num_sg;
> num_sg++;
> }
> - return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
> +
> + return virtnet_add_outbuf(sq, num_sg, skb);
> }
>
> static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
> @@ -3818,6 +3934,8 @@ static void virtnet_free_queues(struct virtnet_info *vi)
> for (i = 0; i < vi->max_queue_pairs; i++) {
> __netif_napi_del(&vi->rq[i].napi);
> __netif_napi_del(&vi->sq[i].napi);
> +
> + kfree(vi->sq[i].dmainfo.head);
> }
>
> /* We called __netif_napi_del(),
> @@ -3866,10 +3984,23 @@ static void free_receive_page_frags(struct virtnet_info *vi)
>
> static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
> {
> - if (!is_xdp_frame(buf))
> + struct virtnet_info *vi = vq->vdev->priv;
> + struct virtnet_sq *sq;
> + int i = vq2rxq(vq);
> +
> + sq = &vi->sq[i];
> +
> + if (!is_xdp_frame(buf)) {
> + if (sq->do_dma)
> + buf = virtnet_sq_unmap(sq, buf);
> +
> dev_kfree_skb(buf);
> - else
> + } else {
> + if (sq->do_dma)
> + buf = virtnet_sq_unmap(sq, buf);
> +
> xdp_return_frame(ptr_to_xdp(buf));
> + }
> }
>
> static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
> @@ -4075,7 +4206,7 @@ static int init_vqs(struct virtnet_info *vi)
> if (ret)
> goto err_free;
>
> - virtnet_rq_set_premapped(vi);
> + virtnet_set_premapped(vi);
>
> cpus_read_lock();
> virtnet_set_affinity(vi);
> diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> index d814341d9f97..ce806afb6d64 100644
> --- a/drivers/net/virtio/virtio_net.h
> +++ b/drivers/net/virtio/virtio_net.h
> @@ -48,6 +48,18 @@ struct virtnet_rq_dma {
> u16 need_sync;
> };
>
> +struct virtnet_sq_dma {
> + struct virtnet_sq_dma *next;
> + dma_addr_t addr;
> + u32 len;
> + void *data;
I think we need to seek a way to reuse what has been stored by virtio
core. It should be much more efficient.
Thanks
> +};
> +
> +struct virtnet_sq_dma_head {
> + struct virtnet_sq_dma *free;
> + struct virtnet_sq_dma *head;
> +};
> +
> /* Internal representation of a send virtqueue */
> struct virtnet_sq {
> /* Virtqueue associated with this virtnet_sq */
> @@ -67,6 +79,10 @@ struct virtnet_sq {
>
> /* Record whether sq is in reset state. */
> bool reset;
> +
> + bool do_dma;
> +
> + struct virtnet_sq_dma_head dmainfo;
> };
>
> /* Internal representation of a receive virtqueue */
> --
> 2.32.0.3.g01195cf9f
>
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 08/21] virtio_net: sq support premapped mode
2023-11-09 6:37 ` Jason Wang
@ 2023-11-09 10:58 ` Xuan Zhuo
2023-11-14 3:26 ` Jason Wang
0 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-09 10:58 UTC (permalink / raw)
To: Jason Wang
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Michael S. Tsirkin, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Thu, 9 Nov 2023 14:37:38 +0800, Jason Wang <jasowang@redhat.com> wrote:
> On Tue, Nov 7, 2023 at 11:12 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> >
> > If the xsk is enabling, the xsk tx will share the send queue.
> > But the xsk requires that the send queue use the premapped mode.
> > So the send queue must support premapped mode.
> >
> > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > ---
> > drivers/net/virtio/main.c | 163 ++++++++++++++++++++++++++++----
> > drivers/net/virtio/virtio_net.h | 16 ++++
> > 2 files changed, 163 insertions(+), 16 deletions(-)
> >
> > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > index 16e75c08639e..f052db459156 100644
> > --- a/drivers/net/virtio/main.c
> > +++ b/drivers/net/virtio/main.c
> > @@ -46,6 +46,7 @@ module_param(napi_tx, bool, 0644);
> > #define VIRTIO_XDP_REDIR BIT(1)
> >
> > #define VIRTIO_XDP_FLAG BIT(0)
> > +#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
> >
> > #define VIRTNET_DRIVER_VERSION "1.0.0"
> >
> > @@ -167,6 +168,29 @@ static struct xdp_frame *ptr_to_xdp(void *ptr)
> > return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
> > }
> >
> > +static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> > +{
> > + struct virtnet_sq_dma *next, *head;
> > +
> > + head = (void *)((unsigned long)data & ~VIRTIO_XMIT_DATA_MASK);
> > +
> > + data = head->data;
> > +
> > + while (head) {
> > + virtqueue_dma_unmap_single_attrs(sq->vq, head->addr, head->len,
> > + DMA_TO_DEVICE, 0);
> > +
> > + next = head->next;
> > +
> > + head->next = sq->dmainfo.free;
> > + sq->dmainfo.free = head;
> > +
> > + head = next;
> > + }
> > +
> > + return data;
> > +}
> > +
> > static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > u64 *bytes, u64 *packets)
> > {
> > @@ -175,14 +199,24 @@ static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> >
> > while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
> > if (!is_xdp_frame(ptr)) {
> > - struct sk_buff *skb = ptr;
> > + struct sk_buff *skb;
> > +
> > + if (sq->do_dma)
> > + ptr = virtnet_sq_unmap(sq, ptr);
> > +
> > + skb = ptr;
> >
> > pr_debug("Sent skb %p\n", skb);
> >
> > *bytes += skb->len;
> > napi_consume_skb(skb, in_napi);
> > } else {
> > - struct xdp_frame *frame = ptr_to_xdp(ptr);
> > + struct xdp_frame *frame;
> > +
> > + if (sq->do_dma)
> > + ptr = virtnet_sq_unmap(sq, ptr);
> > +
> > + frame = ptr_to_xdp(ptr);
> >
> > *bytes += xdp_get_frame_len(frame);
> > xdp_return_frame(frame);
> > @@ -567,22 +601,104 @@ static void *virtnet_rq_alloc(struct virtnet_rq *rq, u32 size, gfp_t gfp)
> > return buf;
> > }
> >
> > -static void virtnet_rq_set_premapped(struct virtnet_info *vi)
> > +static int virtnet_sq_set_premapped(struct virtnet_sq *sq)
> > {
> > - int i;
> > + struct virtnet_sq_dma *d;
> > + int err, size, i;
> >
> > - /* disable for big mode */
> > - if (!vi->mergeable_rx_bufs && vi->big_packets)
> > - return;
> > + size = virtqueue_get_vring_size(sq->vq);
> > +
> > + size += MAX_SKB_FRAGS + 2;
>
> Btw, the dmainfo seems per sg? If I'm correct, how can vq_size +
> MAX_SKB_FRAGS + 2 work?
We may alloc dmainfo items when the vq is full. So I prepare more dmainfo items.
>
> > +
> > + sq->dmainfo.head = kcalloc(size, sizeof(*sq->dmainfo.head), GFP_KERNEL);
> > + if (!sq->dmainfo.head)
> > + return -ENOMEM;
> > +
> > + err = virtqueue_set_dma_premapped(sq->vq);
> > + if (err) {
> > + kfree(sq->dmainfo.head);
> > + return err;
> > + }
>
> Allocating after set_dma_premapped() seems easier.
Yes. But, we donot has the way to disable premapped mode.
That is my mistake. :)
>
> Btw, is there a benchmark of TX PPS just for this patch to demonstrate
> the impact of the performance?
We will have that.
>
> > +
> > + sq->dmainfo.free = NULL;
> > +
> > + sq->do_dma = true;
> > +
> > + for (i = 0; i < size; ++i) {
> > + d = &sq->dmainfo.head[i];
> > +
> > + d->next = sq->dmainfo.free;
> > + sq->dmainfo.free = d;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > +static void virtnet_set_premapped(struct virtnet_info *vi)
> > +{
> > + int i;
> >
> > for (i = 0; i < vi->max_queue_pairs; i++) {
> > - if (virtqueue_set_dma_premapped(vi->rq[i].vq))
> > - continue;
> > + virtnet_sq_set_premapped(&vi->sq[i]);
> >
> > - vi->rq[i].do_dma = true;
> > + /* disable for big mode */
> > + if (vi->mergeable_rx_bufs || !vi->big_packets) {
> > + if (!virtqueue_set_dma_premapped(vi->rq[i].vq))
> > + vi->rq[i].do_dma = true;
>
> How about sticking a virtnet_rq_set_premapped() and calling it here?
>
> It seems more clean.
OK.
>
> Btw, the big mode support for pre mapping is still worthwhile
> regardless whether or not XDP is supported. It has a page pool so we
> can avoid redundant DMA map/unmap there.
Yes.
I post other patch set to do this.
>
> > + }
> > }
> > }
> >
> > +static struct virtnet_sq_dma *virtnet_sq_map_sg(struct virtnet_sq *sq, int nents, void *data)
> > +{
> > + struct virtnet_sq_dma *d, *head;
> > + struct scatterlist *sg;
> > + int i;
> > +
> > + head = NULL;
> > +
> > + for_each_sg(sq->sg, sg, nents, i) {
> > + sg->dma_address = virtqueue_dma_map_single_attrs(sq->vq, sg_virt(sg),
> > + sg->length,
> > + DMA_TO_DEVICE, 0);
> > + if (virtqueue_dma_mapping_error(sq->vq, sg->dma_address))
> > + goto err;
> > +
> > + d = sq->dmainfo.free;
> > + sq->dmainfo.free = d->next;
> > +
> > + d->addr = sg->dma_address;
> > + d->len = sg->length;
> > +
> > + d->next = head;
> > + head = d;
> > + }
> > +
> > + head->data = data;
> > +
> > + return (void *)((unsigned long)head | ((unsigned long)data & VIRTIO_XMIT_DATA_MASK));
>
> So head contains a pointer to data, any reason we still need to pack a
> data pointer here?
Maybe you are right. We can skip this.
>
>
> > +err:
> > + virtnet_sq_unmap(sq, head);
> > + return NULL;
> > +}
> > +
> > +static int virtnet_add_outbuf(struct virtnet_sq *sq, u32 num, void *data)
> > +{
> > + int ret;
> > +
> > + if (sq->do_dma) {
> > + data = virtnet_sq_map_sg(sq, num, data);
> > + if (!data)
> > + return -ENOMEM;
> > + }
> > +
> > + ret = virtqueue_add_outbuf(sq->vq, sq->sg, num, data, GFP_ATOMIC);
> > + if (ret && sq->do_dma)
> > + virtnet_sq_unmap(sq, data);
> > +
> > + return ret;
> > +}
> > +
> > static void free_old_xmit(struct virtnet_sq *sq, bool in_napi)
> > {
> > u64 bytes, packets = 0;
> > @@ -686,8 +802,7 @@ static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
> > skb_frag_size(frag), skb_frag_off(frag));
> > }
> >
> > - err = virtqueue_add_outbuf(sq->vq, sq->sg, nr_frags + 1,
> > - xdp_to_ptr(xdpf), GFP_ATOMIC);
> > + err = virtnet_add_outbuf(sq, nr_frags + 1, xdp_to_ptr(xdpf));
> > if (unlikely(err))
> > return -ENOSPC; /* Caller handle free/refcnt */
> >
> > @@ -2126,7 +2241,8 @@ static int xmit_skb(struct virtnet_sq *sq, struct sk_buff *skb)
> > return num_sg;
> > num_sg++;
> > }
> > - return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
> > +
> > + return virtnet_add_outbuf(sq, num_sg, skb);
> > }
> >
> > static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
> > @@ -3818,6 +3934,8 @@ static void virtnet_free_queues(struct virtnet_info *vi)
> > for (i = 0; i < vi->max_queue_pairs; i++) {
> > __netif_napi_del(&vi->rq[i].napi);
> > __netif_napi_del(&vi->sq[i].napi);
> > +
> > + kfree(vi->sq[i].dmainfo.head);
> > }
> >
> > /* We called __netif_napi_del(),
> > @@ -3866,10 +3984,23 @@ static void free_receive_page_frags(struct virtnet_info *vi)
> >
> > static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
> > {
> > - if (!is_xdp_frame(buf))
> > + struct virtnet_info *vi = vq->vdev->priv;
> > + struct virtnet_sq *sq;
> > + int i = vq2rxq(vq);
> > +
> > + sq = &vi->sq[i];
> > +
> > + if (!is_xdp_frame(buf)) {
> > + if (sq->do_dma)
> > + buf = virtnet_sq_unmap(sq, buf);
> > +
> > dev_kfree_skb(buf);
> > - else
> > + } else {
> > + if (sq->do_dma)
> > + buf = virtnet_sq_unmap(sq, buf);
> > +
> > xdp_return_frame(ptr_to_xdp(buf));
> > + }
> > }
> >
> > static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
> > @@ -4075,7 +4206,7 @@ static int init_vqs(struct virtnet_info *vi)
> > if (ret)
> > goto err_free;
> >
> > - virtnet_rq_set_premapped(vi);
> > + virtnet_set_premapped(vi);
> >
> > cpus_read_lock();
> > virtnet_set_affinity(vi);
> > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > index d814341d9f97..ce806afb6d64 100644
> > --- a/drivers/net/virtio/virtio_net.h
> > +++ b/drivers/net/virtio/virtio_net.h
> > @@ -48,6 +48,18 @@ struct virtnet_rq_dma {
> > u16 need_sync;
> > };
> >
> > +struct virtnet_sq_dma {
> > + struct virtnet_sq_dma *next;
> > + dma_addr_t addr;
> > + u32 len;
> > + void *data;
>
> I think we need to seek a way to reuse what has been stored by virtio
> core. It should be much more efficient.
Yes.
But that is for net-next branch.
Can we do that as a fix after that is merged to 6.8?
Thanks.
>
> Thanks
>
> > +};
> > +
> > +struct virtnet_sq_dma_head {
> > + struct virtnet_sq_dma *free;
> > + struct virtnet_sq_dma *head;
> > +};
> > +
> > /* Internal representation of a send virtqueue */
> > struct virtnet_sq {
> > /* Virtqueue associated with this virtnet_sq */
> > @@ -67,6 +79,10 @@ struct virtnet_sq {
> >
> > /* Record whether sq is in reset state. */
> > bool reset;
> > +
> > + bool do_dma;
> > +
> > + struct virtnet_sq_dma_head dmainfo;
> > };
> >
> > /* Internal representation of a receive virtqueue */
> > --
> > 2.32.0.3.g01195cf9f
> >
>
>
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 08/21] virtio_net: sq support premapped mode
2023-11-09 10:58 ` Xuan Zhuo
@ 2023-11-14 3:26 ` Jason Wang
2023-11-14 3:28 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Jason Wang @ 2023-11-14 3:26 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Michael S. Tsirkin, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Thu, Nov 9, 2023 at 7:06 PM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
>
> On Thu, 9 Nov 2023 14:37:38 +0800, Jason Wang <jasowang@redhat.com> wrote:
> > On Tue, Nov 7, 2023 at 11:12 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> > >
> > > If the xsk is enabling, the xsk tx will share the send queue.
> > > But the xsk requires that the send queue use the premapped mode.
> > > So the send queue must support premapped mode.
> > >
> > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > ---
> > > drivers/net/virtio/main.c | 163 ++++++++++++++++++++++++++++----
> > > drivers/net/virtio/virtio_net.h | 16 ++++
> > > 2 files changed, 163 insertions(+), 16 deletions(-)
> > >
> > > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > > index 16e75c08639e..f052db459156 100644
> > > --- a/drivers/net/virtio/main.c
> > > +++ b/drivers/net/virtio/main.c
> > > @@ -46,6 +46,7 @@ module_param(napi_tx, bool, 0644);
> > > #define VIRTIO_XDP_REDIR BIT(1)
> > >
> > > #define VIRTIO_XDP_FLAG BIT(0)
> > > +#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
> > >
> > > #define VIRTNET_DRIVER_VERSION "1.0.0"
> > >
> > > @@ -167,6 +168,29 @@ static struct xdp_frame *ptr_to_xdp(void *ptr)
> > > return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
> > > }
> > >
> > > +static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> > > +{
> > > + struct virtnet_sq_dma *next, *head;
> > > +
> > > + head = (void *)((unsigned long)data & ~VIRTIO_XMIT_DATA_MASK);
> > > +
> > > + data = head->data;
> > > +
> > > + while (head) {
> > > + virtqueue_dma_unmap_single_attrs(sq->vq, head->addr, head->len,
> > > + DMA_TO_DEVICE, 0);
> > > +
> > > + next = head->next;
> > > +
> > > + head->next = sq->dmainfo.free;
> > > + sq->dmainfo.free = head;
> > > +
> > > + head = next;
> > > + }
> > > +
> > > + return data;
> > > +}
> > > +
> > > static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > > u64 *bytes, u64 *packets)
> > > {
> > > @@ -175,14 +199,24 @@ static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > >
> > > while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
> > > if (!is_xdp_frame(ptr)) {
> > > - struct sk_buff *skb = ptr;
> > > + struct sk_buff *skb;
> > > +
> > > + if (sq->do_dma)
> > > + ptr = virtnet_sq_unmap(sq, ptr);
> > > +
> > > + skb = ptr;
> > >
> > > pr_debug("Sent skb %p\n", skb);
> > >
> > > *bytes += skb->len;
> > > napi_consume_skb(skb, in_napi);
> > > } else {
> > > - struct xdp_frame *frame = ptr_to_xdp(ptr);
> > > + struct xdp_frame *frame;
> > > +
> > > + if (sq->do_dma)
> > > + ptr = virtnet_sq_unmap(sq, ptr);
> > > +
> > > + frame = ptr_to_xdp(ptr);
> > >
> > > *bytes += xdp_get_frame_len(frame);
> > > xdp_return_frame(frame);
> > > @@ -567,22 +601,104 @@ static void *virtnet_rq_alloc(struct virtnet_rq *rq, u32 size, gfp_t gfp)
> > > return buf;
> > > }
> > >
> > > -static void virtnet_rq_set_premapped(struct virtnet_info *vi)
> > > +static int virtnet_sq_set_premapped(struct virtnet_sq *sq)
> > > {
> > > - int i;
> > > + struct virtnet_sq_dma *d;
> > > + int err, size, i;
> > >
> > > - /* disable for big mode */
> > > - if (!vi->mergeable_rx_bufs && vi->big_packets)
> > > - return;
> > > + size = virtqueue_get_vring_size(sq->vq);
> > > +
> > > + size += MAX_SKB_FRAGS + 2;
> >
> > Btw, the dmainfo seems per sg? If I'm correct, how can vq_size +
> > MAX_SKB_FRAGS + 2 work?
>
>
> We may alloc dmainfo items when the vq is full. So I prepare more dmainfo items.
>
>
> >
> > > +
> > > + sq->dmainfo.head = kcalloc(size, sizeof(*sq->dmainfo.head), GFP_KERNEL);
> > > + if (!sq->dmainfo.head)
> > > + return -ENOMEM;
> > > +
> > > + err = virtqueue_set_dma_premapped(sq->vq);
> > > + if (err) {
> > > + kfree(sq->dmainfo.head);
> > > + return err;
> > > + }
> >
> > Allocating after set_dma_premapped() seems easier.
>
> Yes. But, we donot has the way to disable premapped mode.
>
> That is my mistake. :)
>
>
> >
> > Btw, is there a benchmark of TX PPS just for this patch to demonstrate
> > the impact of the performance?
>
> We will have that.
>
>
> >
> > > +
> > > + sq->dmainfo.free = NULL;
> > > +
> > > + sq->do_dma = true;
> > > +
> > > + for (i = 0; i < size; ++i) {
> > > + d = &sq->dmainfo.head[i];
> > > +
> > > + d->next = sq->dmainfo.free;
> > > + sq->dmainfo.free = d;
> > > + }
> > > +
> > > + return 0;
> > > +}
> > > +
> > > +static void virtnet_set_premapped(struct virtnet_info *vi)
> > > +{
> > > + int i;
> > >
> > > for (i = 0; i < vi->max_queue_pairs; i++) {
> > > - if (virtqueue_set_dma_premapped(vi->rq[i].vq))
> > > - continue;
> > > + virtnet_sq_set_premapped(&vi->sq[i]);
> > >
> > > - vi->rq[i].do_dma = true;
> > > + /* disable for big mode */
> > > + if (vi->mergeable_rx_bufs || !vi->big_packets) {
> > > + if (!virtqueue_set_dma_premapped(vi->rq[i].vq))
> > > + vi->rq[i].do_dma = true;
> >
> > How about sticking a virtnet_rq_set_premapped() and calling it here?
> >
> > It seems more clean.
>
> OK.
>
>
> >
> > Btw, the big mode support for pre mapping is still worthwhile
> > regardless whether or not XDP is supported. It has a page pool so we
> > can avoid redundant DMA map/unmap there.
>
> Yes.
>
> I post other patch set to do this.
>
>
> >
> > > + }
> > > }
> > > }
> > >
> > > +static struct virtnet_sq_dma *virtnet_sq_map_sg(struct virtnet_sq *sq, int nents, void *data)
> > > +{
> > > + struct virtnet_sq_dma *d, *head;
> > > + struct scatterlist *sg;
> > > + int i;
> > > +
> > > + head = NULL;
> > > +
> > > + for_each_sg(sq->sg, sg, nents, i) {
> > > + sg->dma_address = virtqueue_dma_map_single_attrs(sq->vq, sg_virt(sg),
> > > + sg->length,
> > > + DMA_TO_DEVICE, 0);
> > > + if (virtqueue_dma_mapping_error(sq->vq, sg->dma_address))
> > > + goto err;
> > > +
> > > + d = sq->dmainfo.free;
> > > + sq->dmainfo.free = d->next;
> > > +
> > > + d->addr = sg->dma_address;
> > > + d->len = sg->length;
> > > +
> > > + d->next = head;
> > > + head = d;
> > > + }
> > > +
> > > + head->data = data;
> > > +
> > > + return (void *)((unsigned long)head | ((unsigned long)data & VIRTIO_XMIT_DATA_MASK));
> >
> > So head contains a pointer to data, any reason we still need to pack a
> > data pointer here?
>
> Maybe you are right. We can skip this.
>
>
> >
> >
> > > +err:
> > > + virtnet_sq_unmap(sq, head);
> > > + return NULL;
> > > +}
> > > +
> > > +static int virtnet_add_outbuf(struct virtnet_sq *sq, u32 num, void *data)
> > > +{
> > > + int ret;
> > > +
> > > + if (sq->do_dma) {
> > > + data = virtnet_sq_map_sg(sq, num, data);
> > > + if (!data)
> > > + return -ENOMEM;
> > > + }
> > > +
> > > + ret = virtqueue_add_outbuf(sq->vq, sq->sg, num, data, GFP_ATOMIC);
> > > + if (ret && sq->do_dma)
> > > + virtnet_sq_unmap(sq, data);
> > > +
> > > + return ret;
> > > +}
> > > +
> > > static void free_old_xmit(struct virtnet_sq *sq, bool in_napi)
> > > {
> > > u64 bytes, packets = 0;
> > > @@ -686,8 +802,7 @@ static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
> > > skb_frag_size(frag), skb_frag_off(frag));
> > > }
> > >
> > > - err = virtqueue_add_outbuf(sq->vq, sq->sg, nr_frags + 1,
> > > - xdp_to_ptr(xdpf), GFP_ATOMIC);
> > > + err = virtnet_add_outbuf(sq, nr_frags + 1, xdp_to_ptr(xdpf));
> > > if (unlikely(err))
> > > return -ENOSPC; /* Caller handle free/refcnt */
> > >
> > > @@ -2126,7 +2241,8 @@ static int xmit_skb(struct virtnet_sq *sq, struct sk_buff *skb)
> > > return num_sg;
> > > num_sg++;
> > > }
> > > - return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
> > > +
> > > + return virtnet_add_outbuf(sq, num_sg, skb);
> > > }
> > >
> > > static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
> > > @@ -3818,6 +3934,8 @@ static void virtnet_free_queues(struct virtnet_info *vi)
> > > for (i = 0; i < vi->max_queue_pairs; i++) {
> > > __netif_napi_del(&vi->rq[i].napi);
> > > __netif_napi_del(&vi->sq[i].napi);
> > > +
> > > + kfree(vi->sq[i].dmainfo.head);
> > > }
> > >
> > > /* We called __netif_napi_del(),
> > > @@ -3866,10 +3984,23 @@ static void free_receive_page_frags(struct virtnet_info *vi)
> > >
> > > static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
> > > {
> > > - if (!is_xdp_frame(buf))
> > > + struct virtnet_info *vi = vq->vdev->priv;
> > > + struct virtnet_sq *sq;
> > > + int i = vq2rxq(vq);
> > > +
> > > + sq = &vi->sq[i];
> > > +
> > > + if (!is_xdp_frame(buf)) {
> > > + if (sq->do_dma)
> > > + buf = virtnet_sq_unmap(sq, buf);
> > > +
> > > dev_kfree_skb(buf);
> > > - else
> > > + } else {
> > > + if (sq->do_dma)
> > > + buf = virtnet_sq_unmap(sq, buf);
> > > +
> > > xdp_return_frame(ptr_to_xdp(buf));
> > > + }
> > > }
> > >
> > > static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
> > > @@ -4075,7 +4206,7 @@ static int init_vqs(struct virtnet_info *vi)
> > > if (ret)
> > > goto err_free;
> > >
> > > - virtnet_rq_set_premapped(vi);
> > > + virtnet_set_premapped(vi);
> > >
> > > cpus_read_lock();
> > > virtnet_set_affinity(vi);
> > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > index d814341d9f97..ce806afb6d64 100644
> > > --- a/drivers/net/virtio/virtio_net.h
> > > +++ b/drivers/net/virtio/virtio_net.h
> > > @@ -48,6 +48,18 @@ struct virtnet_rq_dma {
> > > u16 need_sync;
> > > };
> > >
> > > +struct virtnet_sq_dma {
> > > + struct virtnet_sq_dma *next;
> > > + dma_addr_t addr;
> > > + u32 len;
> > > + void *data;
> >
> > I think we need to seek a way to reuse what has been stored by virtio
> > core. It should be much more efficient.
>
>
> Yes.
>
> But that is for net-next branch.
>
> Can we do that as a fix after that is merged to 6.8?
We still have time. I would like to do it from the start.
Thanks
>
> Thanks.
>
>
> >
> > Thanks
> >
> > > +};
> > > +
> > > +struct virtnet_sq_dma_head {
> > > + struct virtnet_sq_dma *free;
> > > + struct virtnet_sq_dma *head;
> > > +};
> > > +
> > > /* Internal representation of a send virtqueue */
> > > struct virtnet_sq {
> > > /* Virtqueue associated with this virtnet_sq */
> > > @@ -67,6 +79,10 @@ struct virtnet_sq {
> > >
> > > /* Record whether sq is in reset state. */
> > > bool reset;
> > > +
> > > + bool do_dma;
> > > +
> > > + struct virtnet_sq_dma_head dmainfo;
> > > };
> > >
> > > /* Internal representation of a receive virtqueue */
> > > --
> > > 2.32.0.3.g01195cf9f
> > >
> >
> >
>
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 08/21] virtio_net: sq support premapped mode
2023-11-14 3:26 ` Jason Wang
@ 2023-11-14 3:28 ` Xuan Zhuo
2023-11-14 3:55 ` Jason Wang
0 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-14 3:28 UTC (permalink / raw)
To: Jason Wang
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Michael S. Tsirkin, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Tue, 14 Nov 2023 11:26:42 +0800, Jason Wang <jasowang@redhat.com> wrote:
> On Thu, Nov 9, 2023 at 7:06 PM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> >
> > On Thu, 9 Nov 2023 14:37:38 +0800, Jason Wang <jasowang@redhat.com> wrote:
> > > On Tue, Nov 7, 2023 at 11:12 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> > > >
> > > > If the xsk is enabling, the xsk tx will share the send queue.
> > > > But the xsk requires that the send queue use the premapped mode.
> > > > So the send queue must support premapped mode.
> > > >
> > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > > ---
> > > > drivers/net/virtio/main.c | 163 ++++++++++++++++++++++++++++----
> > > > drivers/net/virtio/virtio_net.h | 16 ++++
> > > > 2 files changed, 163 insertions(+), 16 deletions(-)
> > > >
> > > > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > > > index 16e75c08639e..f052db459156 100644
> > > > --- a/drivers/net/virtio/main.c
> > > > +++ b/drivers/net/virtio/main.c
> > > > @@ -46,6 +46,7 @@ module_param(napi_tx, bool, 0644);
> > > > #define VIRTIO_XDP_REDIR BIT(1)
> > > >
> > > > #define VIRTIO_XDP_FLAG BIT(0)
> > > > +#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
> > > >
> > > > #define VIRTNET_DRIVER_VERSION "1.0.0"
> > > >
> > > > @@ -167,6 +168,29 @@ static struct xdp_frame *ptr_to_xdp(void *ptr)
> > > > return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
> > > > }
> > > >
> > > > +static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> > > > +{
> > > > + struct virtnet_sq_dma *next, *head;
> > > > +
> > > > + head = (void *)((unsigned long)data & ~VIRTIO_XMIT_DATA_MASK);
> > > > +
> > > > + data = head->data;
> > > > +
> > > > + while (head) {
> > > > + virtqueue_dma_unmap_single_attrs(sq->vq, head->addr, head->len,
> > > > + DMA_TO_DEVICE, 0);
> > > > +
> > > > + next = head->next;
> > > > +
> > > > + head->next = sq->dmainfo.free;
> > > > + sq->dmainfo.free = head;
> > > > +
> > > > + head = next;
> > > > + }
> > > > +
> > > > + return data;
> > > > +}
> > > > +
> > > > static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > > > u64 *bytes, u64 *packets)
> > > > {
> > > > @@ -175,14 +199,24 @@ static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > > >
> > > > while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
> > > > if (!is_xdp_frame(ptr)) {
> > > > - struct sk_buff *skb = ptr;
> > > > + struct sk_buff *skb;
> > > > +
> > > > + if (sq->do_dma)
> > > > + ptr = virtnet_sq_unmap(sq, ptr);
> > > > +
> > > > + skb = ptr;
> > > >
> > > > pr_debug("Sent skb %p\n", skb);
> > > >
> > > > *bytes += skb->len;
> > > > napi_consume_skb(skb, in_napi);
> > > > } else {
> > > > - struct xdp_frame *frame = ptr_to_xdp(ptr);
> > > > + struct xdp_frame *frame;
> > > > +
> > > > + if (sq->do_dma)
> > > > + ptr = virtnet_sq_unmap(sq, ptr);
> > > > +
> > > > + frame = ptr_to_xdp(ptr);
> > > >
> > > > *bytes += xdp_get_frame_len(frame);
> > > > xdp_return_frame(frame);
> > > > @@ -567,22 +601,104 @@ static void *virtnet_rq_alloc(struct virtnet_rq *rq, u32 size, gfp_t gfp)
> > > > return buf;
> > > > }
> > > >
> > > > -static void virtnet_rq_set_premapped(struct virtnet_info *vi)
> > > > +static int virtnet_sq_set_premapped(struct virtnet_sq *sq)
> > > > {
> > > > - int i;
> > > > + struct virtnet_sq_dma *d;
> > > > + int err, size, i;
> > > >
> > > > - /* disable for big mode */
> > > > - if (!vi->mergeable_rx_bufs && vi->big_packets)
> > > > - return;
> > > > + size = virtqueue_get_vring_size(sq->vq);
> > > > +
> > > > + size += MAX_SKB_FRAGS + 2;
> > >
> > > Btw, the dmainfo seems per sg? If I'm correct, how can vq_size +
> > > MAX_SKB_FRAGS + 2 work?
> >
> >
> > We may alloc dmainfo items when the vq is full. So I prepare more dmainfo items.
> >
> >
> > >
> > > > +
> > > > + sq->dmainfo.head = kcalloc(size, sizeof(*sq->dmainfo.head), GFP_KERNEL);
> > > > + if (!sq->dmainfo.head)
> > > > + return -ENOMEM;
> > > > +
> > > > + err = virtqueue_set_dma_premapped(sq->vq);
> > > > + if (err) {
> > > > + kfree(sq->dmainfo.head);
> > > > + return err;
> > > > + }
> > >
> > > Allocating after set_dma_premapped() seems easier.
> >
> > Yes. But, we donot has the way to disable premapped mode.
> >
> > That is my mistake. :)
> >
> >
> > >
> > > Btw, is there a benchmark of TX PPS just for this patch to demonstrate
> > > the impact of the performance?
> >
> > We will have that.
> >
> >
> > >
> > > > +
> > > > + sq->dmainfo.free = NULL;
> > > > +
> > > > + sq->do_dma = true;
> > > > +
> > > > + for (i = 0; i < size; ++i) {
> > > > + d = &sq->dmainfo.head[i];
> > > > +
> > > > + d->next = sq->dmainfo.free;
> > > > + sq->dmainfo.free = d;
> > > > + }
> > > > +
> > > > + return 0;
> > > > +}
> > > > +
> > > > +static void virtnet_set_premapped(struct virtnet_info *vi)
> > > > +{
> > > > + int i;
> > > >
> > > > for (i = 0; i < vi->max_queue_pairs; i++) {
> > > > - if (virtqueue_set_dma_premapped(vi->rq[i].vq))
> > > > - continue;
> > > > + virtnet_sq_set_premapped(&vi->sq[i]);
> > > >
> > > > - vi->rq[i].do_dma = true;
> > > > + /* disable for big mode */
> > > > + if (vi->mergeable_rx_bufs || !vi->big_packets) {
> > > > + if (!virtqueue_set_dma_premapped(vi->rq[i].vq))
> > > > + vi->rq[i].do_dma = true;
> > >
> > > How about sticking a virtnet_rq_set_premapped() and calling it here?
> > >
> > > It seems more clean.
> >
> > OK.
> >
> >
> > >
> > > Btw, the big mode support for pre mapping is still worthwhile
> > > regardless whether or not XDP is supported. It has a page pool so we
> > > can avoid redundant DMA map/unmap there.
> >
> > Yes.
> >
> > I post other patch set to do this.
> >
> >
> > >
> > > > + }
> > > > }
> > > > }
> > > >
> > > > +static struct virtnet_sq_dma *virtnet_sq_map_sg(struct virtnet_sq *sq, int nents, void *data)
> > > > +{
> > > > + struct virtnet_sq_dma *d, *head;
> > > > + struct scatterlist *sg;
> > > > + int i;
> > > > +
> > > > + head = NULL;
> > > > +
> > > > + for_each_sg(sq->sg, sg, nents, i) {
> > > > + sg->dma_address = virtqueue_dma_map_single_attrs(sq->vq, sg_virt(sg),
> > > > + sg->length,
> > > > + DMA_TO_DEVICE, 0);
> > > > + if (virtqueue_dma_mapping_error(sq->vq, sg->dma_address))
> > > > + goto err;
> > > > +
> > > > + d = sq->dmainfo.free;
> > > > + sq->dmainfo.free = d->next;
> > > > +
> > > > + d->addr = sg->dma_address;
> > > > + d->len = sg->length;
> > > > +
> > > > + d->next = head;
> > > > + head = d;
> > > > + }
> > > > +
> > > > + head->data = data;
> > > > +
> > > > + return (void *)((unsigned long)head | ((unsigned long)data & VIRTIO_XMIT_DATA_MASK));
> > >
> > > So head contains a pointer to data, any reason we still need to pack a
> > > data pointer here?
> >
> > Maybe you are right. We can skip this.
> >
> >
> > >
> > >
> > > > +err:
> > > > + virtnet_sq_unmap(sq, head);
> > > > + return NULL;
> > > > +}
> > > > +
> > > > +static int virtnet_add_outbuf(struct virtnet_sq *sq, u32 num, void *data)
> > > > +{
> > > > + int ret;
> > > > +
> > > > + if (sq->do_dma) {
> > > > + data = virtnet_sq_map_sg(sq, num, data);
> > > > + if (!data)
> > > > + return -ENOMEM;
> > > > + }
> > > > +
> > > > + ret = virtqueue_add_outbuf(sq->vq, sq->sg, num, data, GFP_ATOMIC);
> > > > + if (ret && sq->do_dma)
> > > > + virtnet_sq_unmap(sq, data);
> > > > +
> > > > + return ret;
> > > > +}
> > > > +
> > > > static void free_old_xmit(struct virtnet_sq *sq, bool in_napi)
> > > > {
> > > > u64 bytes, packets = 0;
> > > > @@ -686,8 +802,7 @@ static int __virtnet_xdp_xmit_one(struct virtnet_info *vi,
> > > > skb_frag_size(frag), skb_frag_off(frag));
> > > > }
> > > >
> > > > - err = virtqueue_add_outbuf(sq->vq, sq->sg, nr_frags + 1,
> > > > - xdp_to_ptr(xdpf), GFP_ATOMIC);
> > > > + err = virtnet_add_outbuf(sq, nr_frags + 1, xdp_to_ptr(xdpf));
> > > > if (unlikely(err))
> > > > return -ENOSPC; /* Caller handle free/refcnt */
> > > >
> > > > @@ -2126,7 +2241,8 @@ static int xmit_skb(struct virtnet_sq *sq, struct sk_buff *skb)
> > > > return num_sg;
> > > > num_sg++;
> > > > }
> > > > - return virtqueue_add_outbuf(sq->vq, sq->sg, num_sg, skb, GFP_ATOMIC);
> > > > +
> > > > + return virtnet_add_outbuf(sq, num_sg, skb);
> > > > }
> > > >
> > > > static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev)
> > > > @@ -3818,6 +3934,8 @@ static void virtnet_free_queues(struct virtnet_info *vi)
> > > > for (i = 0; i < vi->max_queue_pairs; i++) {
> > > > __netif_napi_del(&vi->rq[i].napi);
> > > > __netif_napi_del(&vi->sq[i].napi);
> > > > +
> > > > + kfree(vi->sq[i].dmainfo.head);
> > > > }
> > > >
> > > > /* We called __netif_napi_del(),
> > > > @@ -3866,10 +3984,23 @@ static void free_receive_page_frags(struct virtnet_info *vi)
> > > >
> > > > static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
> > > > {
> > > > - if (!is_xdp_frame(buf))
> > > > + struct virtnet_info *vi = vq->vdev->priv;
> > > > + struct virtnet_sq *sq;
> > > > + int i = vq2rxq(vq);
> > > > +
> > > > + sq = &vi->sq[i];
> > > > +
> > > > + if (!is_xdp_frame(buf)) {
> > > > + if (sq->do_dma)
> > > > + buf = virtnet_sq_unmap(sq, buf);
> > > > +
> > > > dev_kfree_skb(buf);
> > > > - else
> > > > + } else {
> > > > + if (sq->do_dma)
> > > > + buf = virtnet_sq_unmap(sq, buf);
> > > > +
> > > > xdp_return_frame(ptr_to_xdp(buf));
> > > > + }
> > > > }
> > > >
> > > > static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
> > > > @@ -4075,7 +4206,7 @@ static int init_vqs(struct virtnet_info *vi)
> > > > if (ret)
> > > > goto err_free;
> > > >
> > > > - virtnet_rq_set_premapped(vi);
> > > > + virtnet_set_premapped(vi);
> > > >
> > > > cpus_read_lock();
> > > > virtnet_set_affinity(vi);
> > > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > > index d814341d9f97..ce806afb6d64 100644
> > > > --- a/drivers/net/virtio/virtio_net.h
> > > > +++ b/drivers/net/virtio/virtio_net.h
> > > > @@ -48,6 +48,18 @@ struct virtnet_rq_dma {
> > > > u16 need_sync;
> > > > };
> > > >
> > > > +struct virtnet_sq_dma {
> > > > + struct virtnet_sq_dma *next;
> > > > + dma_addr_t addr;
> > > > + u32 len;
> > > > + void *data;
> > >
> > > I think we need to seek a way to reuse what has been stored by virtio
> > > core. It should be much more efficient.
> >
> >
> > Yes.
> >
> > But that is for net-next branch.
> >
> > Can we do that as a fix after that is merged to 6.8?
>
> We still have time. I would like to do it from the start.
I want to finish the job including new AF_XDP ZC feature.
Because that this must wait the merge window.
Base on that, the optimizing work can be done everytime.
If we work from the new virtio prepare, that can be merged to 6.8.
And the AF_XDP zc must wait 6.9. right?
Thanks
>
> Thanks
>
> >
> > Thanks.
> >
> >
> > >
> > > Thanks
> > >
> > > > +};
> > > > +
> > > > +struct virtnet_sq_dma_head {
> > > > + struct virtnet_sq_dma *free;
> > > > + struct virtnet_sq_dma *head;
> > > > +};
> > > > +
> > > > /* Internal representation of a send virtqueue */
> > > > struct virtnet_sq {
> > > > /* Virtqueue associated with this virtnet_sq */
> > > > @@ -67,6 +79,10 @@ struct virtnet_sq {
> > > >
> > > > /* Record whether sq is in reset state. */
> > > > bool reset;
> > > > +
> > > > + bool do_dma;
> > > > +
> > > > + struct virtnet_sq_dma_head dmainfo;
> > > > };
> > > >
> > > > /* Internal representation of a receive virtqueue */
> > > > --
> > > > 2.32.0.3.g01195cf9f
> > > >
> > >
> > >
> >
>
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 08/21] virtio_net: sq support premapped mode
2023-11-14 3:28 ` Xuan Zhuo
@ 2023-11-14 3:55 ` Jason Wang
2023-11-14 3:57 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Jason Wang @ 2023-11-14 3:55 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Michael S. Tsirkin, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Tue, Nov 14, 2023 at 11:42 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
>
> On Tue, 14 Nov 2023 11:26:42 +0800, Jason Wang <jasowang@redhat.com> wrote:
> > On Thu, Nov 9, 2023 at 7:06 PM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> > >
> > > On Thu, 9 Nov 2023 14:37:38 +0800, Jason Wang <jasowang@redhat.com> wrote:
> > > > On Tue, Nov 7, 2023 at 11:12 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> > > > >
> > > > > If the xsk is enabling, the xsk tx will share the send queue.
> > > > > But the xsk requires that the send queue use the premapped mode.
> > > > > So the send queue must support premapped mode.
> > > > >
> > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > > > ---
> > > > > drivers/net/virtio/main.c | 163 ++++++++++++++++++++++++++++----
> > > > > drivers/net/virtio/virtio_net.h | 16 ++++
> > > > > 2 files changed, 163 insertions(+), 16 deletions(-)
> > > > >
[...]
> > > >
> > > > I think we need to seek a way to reuse what has been stored by virtio
> > > > core. It should be much more efficient.
> > >
> > >
> > > Yes.
> > >
> > > But that is for net-next branch.
> > >
> > > Can we do that as a fix after that is merged to 6.8?
> >
> > We still have time. I would like to do it from the start.
>
>
> I want to finish the job including new AF_XDP ZC feature.
> Because that this must wait the merge window.
> Base on that, the optimizing work can be done everytime.
>
> If we work from the new virtio prepare, that can be merged to 6.8.
> And the AF_XDP zc must wait 6.9. right?
It can be part of this series. Or anything I missed?
My understanding is that, since the information is handy, it just
requires new helpers. So I don't expect it needs a large series.
Thanks
>
> Thanks
>
^ permalink raw reply [flat|nested] 61+ messages in thread
* Re: [PATCH net-next v2 08/21] virtio_net: sq support premapped mode
2023-11-14 3:55 ` Jason Wang
@ 2023-11-14 3:57 ` Xuan Zhuo
2023-11-14 4:27 ` Jason Wang
0 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-14 3:57 UTC (permalink / raw)
To: Jason Wang
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Michael S. Tsirkin, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Tue, 14 Nov 2023 11:55:52 +0800, Jason Wang <jasowang@redhat.com> wrote:
> On Tue, Nov 14, 2023 at 11:42 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> >
> > On Tue, 14 Nov 2023 11:26:42 +0800, Jason Wang <jasowang@redhat.com> wrote:
> > > On Thu, Nov 9, 2023 at 7:06 PM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> > > >
> > > > On Thu, 9 Nov 2023 14:37:38 +0800, Jason Wang <jasowang@redhat.com> wrote:
> > > > > On Tue, Nov 7, 2023 at 11:12 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> > > > > >
> > > > > > If the xsk is enabling, the xsk tx will share the send queue.
> > > > > > But the xsk requires that the send queue use the premapped mode.
> > > > > > So the send queue must support premapped mode.
> > > > > >
> > > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > > > > ---
> > > > > > drivers/net/virtio/main.c | 163 ++++++++++++++++++++++++++++----
> > > > > > drivers/net/virtio/virtio_net.h | 16 ++++
> > > > > > 2 files changed, 163 insertions(+), 16 deletions(-)
> > > > > >
>
> [...]
>
> > > > >
> > > > > I think we need to seek a way to reuse what has been stored by virtio
> > > > > core. It should be much more efficient.
> > > >
> > > >
> > > > Yes.
> > > >
> > > > But that is for net-next branch.
> > > >
> > > > Can we do that as a fix after that is merged to 6.8?
> > >
> > > We still have time. I would like to do it from the start.
> >
> >
> > I want to finish the job including new AF_XDP ZC feature.
> > Because that this must wait the merge window.
> > Base on that, the optimizing work can be done everytime.
> >
> > If we work from the new virtio prepare, that can be merged to 6.8.
> > And the AF_XDP zc must wait 6.9. right?
>
> It can be part of this series. Or anything I missed?
>
> My understanding is that, since the information is handy, it just
> requires new helpers. So I don't expect it needs a large series.
Now, this is pushing to net-next.
If we add an new virtio-core helper. That must be pushed to virtio branch.
And this patch set must wait that.
Thanks.
>
> Thanks
>
> >
> > Thanks
> >
>
^ permalink raw reply [flat|nested] 61+ messages in thread
* Re: [PATCH net-next v2 08/21] virtio_net: sq support premapped mode
2023-11-14 3:57 ` Xuan Zhuo
@ 2023-11-14 4:27 ` Jason Wang
2023-11-14 4:45 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Jason Wang @ 2023-11-14 4:27 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Michael S. Tsirkin, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Tue, Nov 14, 2023 at 11:59 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
>
> On Tue, 14 Nov 2023 11:55:52 +0800, Jason Wang <jasowang@redhat.com> wrote:
> > On Tue, Nov 14, 2023 at 11:42 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> > >
> > > On Tue, 14 Nov 2023 11:26:42 +0800, Jason Wang <jasowang@redhat.com> wrote:
> > > > On Thu, Nov 9, 2023 at 7:06 PM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> > > > >
> > > > > On Thu, 9 Nov 2023 14:37:38 +0800, Jason Wang <jasowang@redhat.com> wrote:
> > > > > > On Tue, Nov 7, 2023 at 11:12 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> > > > > > >
> > > > > > > If the xsk is enabling, the xsk tx will share the send queue.
> > > > > > > But the xsk requires that the send queue use the premapped mode.
> > > > > > > So the send queue must support premapped mode.
> > > > > > >
> > > > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > > > > > ---
> > > > > > > drivers/net/virtio/main.c | 163 ++++++++++++++++++++++++++++----
> > > > > > > drivers/net/virtio/virtio_net.h | 16 ++++
> > > > > > > 2 files changed, 163 insertions(+), 16 deletions(-)
> > > > > > >
> >
> > [...]
> >
> > > > > >
> > > > > > I think we need to seek a way to reuse what has been stored by virtio
> > > > > > core. It should be much more efficient.
> > > > >
> > > > >
> > > > > Yes.
> > > > >
> > > > > But that is for net-next branch.
> > > > >
> > > > > Can we do that as a fix after that is merged to 6.8?
> > > >
> > > > We still have time. I would like to do it from the start.
> > >
> > >
> > > I want to finish the job including new AF_XDP ZC feature.
> > > Because that this must wait the merge window.
> > > Base on that, the optimizing work can be done everytime.
> > >
> > > If we work from the new virtio prepare, that can be merged to 6.8.
> > > And the AF_XDP zc must wait 6.9. right?
> >
> > It can be part of this series. Or anything I missed?
> >
> > My understanding is that, since the information is handy, it just
> > requires new helpers. So I don't expect it needs a large series.
>
> Now, this is pushing to net-next.
>
> If we add an new virtio-core helper. That must be pushed to virtio branch.
> And this patch set must wait that.
I don't think so if it's just a matter of new helpers. The
acknowledgement from the virtio maintainer should be sufficient.
Let's just try and see?
THanks
>
> Thanks.
>
>
> >
> > Thanks
> >
> > >
> > > Thanks
> > >
> >
>
^ permalink raw reply [flat|nested] 61+ messages in thread
* Re: [PATCH net-next v2 08/21] virtio_net: sq support premapped mode
2023-11-14 4:27 ` Jason Wang
@ 2023-11-14 4:45 ` Xuan Zhuo
0 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-14 4:45 UTC (permalink / raw)
To: Jason Wang
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Michael S. Tsirkin, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Tue, 14 Nov 2023 12:27:20 +0800, Jason Wang <jasowang@redhat.com> wrote:
> On Tue, Nov 14, 2023 at 11:59 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> >
> > On Tue, 14 Nov 2023 11:55:52 +0800, Jason Wang <jasowang@redhat.com> wrote:
> > > On Tue, Nov 14, 2023 at 11:42 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> > > >
> > > > On Tue, 14 Nov 2023 11:26:42 +0800, Jason Wang <jasowang@redhat.com> wrote:
> > > > > On Thu, Nov 9, 2023 at 7:06 PM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> > > > > >
> > > > > > On Thu, 9 Nov 2023 14:37:38 +0800, Jason Wang <jasowang@redhat.com> wrote:
> > > > > > > On Tue, Nov 7, 2023 at 11:12 AM Xuan Zhuo <xuanzhuo@linux.alibaba.com> wrote:
> > > > > > > >
> > > > > > > > If the xsk is enabling, the xsk tx will share the send queue.
> > > > > > > > But the xsk requires that the send queue use the premapped mode.
> > > > > > > > So the send queue must support premapped mode.
> > > > > > > >
> > > > > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > > > > > > ---
> > > > > > > > drivers/net/virtio/main.c | 163 ++++++++++++++++++++++++++++----
> > > > > > > > drivers/net/virtio/virtio_net.h | 16 ++++
> > > > > > > > 2 files changed, 163 insertions(+), 16 deletions(-)
> > > > > > > >
> > >
> > > [...]
> > >
> > > > > > >
> > > > > > > I think we need to seek a way to reuse what has been stored by virtio
> > > > > > > core. It should be much more efficient.
> > > > > >
> > > > > >
> > > > > > Yes.
> > > > > >
> > > > > > But that is for net-next branch.
> > > > > >
> > > > > > Can we do that as a fix after that is merged to 6.8?
> > > > >
> > > > > We still have time. I would like to do it from the start.
> > > >
> > > >
> > > > I want to finish the job including new AF_XDP ZC feature.
> > > > Because that this must wait the merge window.
> > > > Base on that, the optimizing work can be done everytime.
> > > >
> > > > If we work from the new virtio prepare, that can be merged to 6.8.
> > > > And the AF_XDP zc must wait 6.9. right?
> > >
> > > It can be part of this series. Or anything I missed?
> > >
> > > My understanding is that, since the information is handy, it just
> > > requires new helpers. So I don't expect it needs a large series.
> >
> > Now, this is pushing to net-next.
> >
> > If we add an new virtio-core helper. That must be pushed to virtio branch.
> > And this patch set must wait that.
>
> I don't think so if it's just a matter of new helpers. The
> acknowledgement from the virtio maintainer should be sufficient.
>
> Let's just try and see?
OK.
Thanks.
>
> THanks
>
> >
> > Thanks.
> >
> >
> > >
> > > Thanks
> > >
> > > >
> > > > Thanks
> > > >
> > >
> >
>
^ permalink raw reply [flat|nested] 61+ messages in thread
* [PATCH net-next v2 09/21] virtio_net: xsk: bind/unbind xsk
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (7 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 08/21] virtio_net: sq support premapped mode Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 10/21] virtio_net: xsk: prevent disable tx napi Xuan Zhuo
` (13 subsequent siblings)
22 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
This patch implement the logic of bind/unbind xsk pool to sq and rq.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/Makefile | 2 +-
drivers/net/virtio/main.c | 11 +-
drivers/net/virtio/virtio_net.h | 16 +++
drivers/net/virtio/xsk.c | 187 ++++++++++++++++++++++++++++++++
drivers/net/virtio/xsk.h | 7 ++
5 files changed, 216 insertions(+), 7 deletions(-)
create mode 100644 drivers/net/virtio/xsk.c
create mode 100644 drivers/net/virtio/xsk.h
diff --git a/drivers/net/virtio/Makefile b/drivers/net/virtio/Makefile
index 15ed7c97fd4f..8c2a884d2dba 100644
--- a/drivers/net/virtio/Makefile
+++ b/drivers/net/virtio/Makefile
@@ -5,4 +5,4 @@
obj-$(CONFIG_VIRTIO_NET) += virtio_net.o
-virtio_net-y := main.o
+virtio_net-y := main.o xsk.o
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index f052db459156..c4601784b6d1 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -8,7 +8,6 @@
#include <linux/etherdevice.h>
#include <linux/module.h>
#include <linux/virtio.h>
-#include <linux/virtio_net.h>
#include <linux/bpf.h>
#include <linux/bpf_trace.h>
#include <linux/scatterlist.h>
@@ -23,6 +22,7 @@
#include <net/netdev_rx_queue.h>
#include "virtio_net.h"
+#include "xsk.h"
static int napi_weight = NAPI_POLL_WEIGHT;
module_param(napi_weight, int, 0444);
@@ -150,9 +150,6 @@ struct virtio_net_common_hdr {
};
};
-static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf);
-static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf);
-
static bool is_xdp_frame(void *ptr)
{
return (unsigned long)ptr & VIRTIO_XDP_FLAG;
@@ -3797,6 +3794,8 @@ static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp)
switch (xdp->command) {
case XDP_SETUP_PROG:
return virtnet_xdp_set(dev, xdp->prog, xdp->extack);
+ case XDP_SETUP_XSK_POOL:
+ return virtnet_xsk_pool_setup(dev, xdp);
default:
return -EINVAL;
}
@@ -3982,7 +3981,7 @@ static void free_receive_page_frags(struct virtnet_info *vi)
}
}
-static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
+void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
{
struct virtnet_info *vi = vq->vdev->priv;
struct virtnet_sq *sq;
@@ -4003,7 +4002,7 @@ static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
}
}
-static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
+void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
{
struct virtnet_info *vi = vq->vdev->priv;
struct virtnet_rq *rq;
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index ce806afb6d64..98ba23cfdb20 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -5,6 +5,8 @@
#include <linux/ethtool.h>
#include <linux/average.h>
+#include <linux/virtio_net.h>
+#include <net/xdp_sock_drv.h>
/* RX packet size EWMA. The average packet size is used to determine the packet
* buffer size when refilling RX rings. As the entire RX ring may be refilled
@@ -83,6 +85,11 @@ struct virtnet_sq {
bool do_dma;
struct virtnet_sq_dma_head dmainfo;
+ struct {
+ struct xsk_buff_pool *pool;
+
+ dma_addr_t hdr_dma_address;
+ } xsk;
};
/* Internal representation of a receive virtqueue */
@@ -123,6 +130,13 @@ struct virtnet_rq {
/* Do dma by self */
bool do_dma;
+
+ struct {
+ struct xsk_buff_pool *pool;
+
+ /* xdp rxq used by xsk */
+ struct xdp_rxq_info xdp_rxq;
+ } xsk;
};
struct virtnet_info {
@@ -211,4 +225,6 @@ void virtnet_rx_pause(struct virtnet_info *vi, struct virtnet_rq *rq);
void virtnet_rx_resume(struct virtnet_info *vi, struct virtnet_rq *rq);
void virtnet_tx_pause(struct virtnet_info *vi, struct virtnet_sq *sq);
void virtnet_tx_resume(struct virtnet_info *vi, struct virtnet_sq *sq);
+void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf);
+void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf);
#endif
diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
new file mode 100644
index 000000000000..8b397787603f
--- /dev/null
+++ b/drivers/net/virtio/xsk.c
@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * virtio-net xsk
+ */
+
+#include "virtio_net.h"
+
+static struct virtio_net_hdr_mrg_rxbuf xsk_hdr;
+
+static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *rq,
+ struct xsk_buff_pool *pool)
+{
+ int err, qindex;
+
+ qindex = rq - vi->rq;
+
+ if (pool) {
+ err = xdp_rxq_info_reg(&rq->xsk.xdp_rxq, vi->dev, qindex, rq->napi.napi_id);
+ if (err < 0)
+ return err;
+
+ err = xdp_rxq_info_reg_mem_model(&rq->xsk.xdp_rxq,
+ MEM_TYPE_XSK_BUFF_POOL, NULL);
+ if (err < 0) {
+ xdp_rxq_info_unreg(&rq->xsk.xdp_rxq);
+ return err;
+ }
+
+ xsk_pool_set_rxq_info(pool, &rq->xsk.xdp_rxq);
+ }
+
+ virtnet_rx_pause(vi, rq);
+
+ err = virtqueue_reset(rq->vq, virtnet_rq_free_unused_buf);
+ if (err) {
+ netdev_err(vi->dev, "reset rx fail: rx queue index: %d err: %d\n", qindex, err);
+
+ pool = NULL;
+ }
+
+ if (!pool)
+ xdp_rxq_info_unreg(&rq->xsk.xdp_rxq);
+
+ rq->xsk.pool = pool;
+
+ virtnet_rx_resume(vi, rq);
+
+ return err;
+}
+
+static int virtnet_sq_bind_xsk_pool(struct virtnet_info *vi,
+ struct virtnet_sq *sq,
+ struct xsk_buff_pool *pool)
+{
+ int err, qindex;
+
+ qindex = sq - vi->sq;
+
+ virtnet_tx_pause(vi, sq);
+
+ err = virtqueue_reset(sq->vq, virtnet_sq_free_unused_buf);
+ if (err) {
+ pool = NULL;
+ netdev_err(vi->dev, "reset tx fail: tx queue index: %d err: %d\n", qindex, err);
+ }
+
+ sq->xsk.pool = pool;
+
+ virtnet_tx_resume(vi, sq);
+
+ return err;
+}
+
+static int virtnet_xsk_pool_enable(struct net_device *dev,
+ struct xsk_buff_pool *pool,
+ u16 qid)
+{
+ struct virtnet_info *vi = netdev_priv(dev);
+ struct virtnet_rq *rq;
+ struct virtnet_sq *sq;
+ struct device *dma_dev;
+ dma_addr_t hdr_dma;
+ int err;
+
+ /* In big_packets mode, xdp cannot work, so there is no need to
+ * initialize xsk of rq.
+ */
+ if (vi->big_packets && !vi->mergeable_rx_bufs)
+ return -ENOENT;
+
+ if (qid >= vi->curr_queue_pairs)
+ return -EINVAL;
+
+ sq = &vi->sq[qid];
+ rq = &vi->rq[qid];
+
+ /* xsk tx zerocopy depend on the tx napi.
+ *
+ * All xsk packets are actually consumed and sent out from the xsk tx
+ * queue under the tx napi mechanism.
+ */
+ if (!sq->napi.weight)
+ return -EPERM;
+
+ if (!rq->do_dma || !sq->do_dma)
+ return -EPERM;
+
+ /* For the xsk, the tx and rx should have the same device. But
+ * vq->dma_dev allows every vq has the respective dma dev. So I check
+ * the dma dev of vq and sq is the same dev.
+ */
+ if (virtqueue_dma_dev(rq->vq) != virtqueue_dma_dev(sq->vq))
+ return -EPERM;
+
+ dma_dev = virtqueue_dma_dev(rq->vq);
+ if (!dma_dev)
+ return -EPERM;
+
+ hdr_dma = dma_map_single(dma_dev, &xsk_hdr, vi->hdr_len, DMA_TO_DEVICE);
+ if (dma_mapping_error(dma_dev, hdr_dma))
+ return -ENOMEM;
+
+ err = xsk_pool_dma_map(pool, dma_dev, 0);
+ if (err)
+ goto err_xsk_map;
+
+ err = virtnet_rq_bind_xsk_pool(vi, rq, pool);
+ if (err)
+ goto err_rq;
+
+ err = virtnet_sq_bind_xsk_pool(vi, sq, pool);
+ if (err)
+ goto err_sq;
+
+ /* Now, we do not support tx offset, so all the tx virtnet hdr is zero.
+ * So all the tx packets can share a single hdr.
+ */
+ sq->xsk.hdr_dma_address = hdr_dma;
+
+ return 0;
+
+err_sq:
+ virtnet_rq_bind_xsk_pool(vi, rq, NULL);
+err_rq:
+ xsk_pool_dma_unmap(pool, 0);
+err_xsk_map:
+ dma_unmap_single(dma_dev, hdr_dma, vi->hdr_len, DMA_TO_DEVICE);
+ return err;
+}
+
+static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid)
+{
+ struct virtnet_info *vi = netdev_priv(dev);
+ struct xsk_buff_pool *pool;
+ struct device *dma_dev;
+ struct virtnet_rq *rq;
+ struct virtnet_sq *sq;
+ int err1, err2;
+
+ if (qid >= vi->curr_queue_pairs)
+ return -EINVAL;
+
+ sq = &vi->sq[qid];
+ rq = &vi->rq[qid];
+
+ pool = sq->xsk.pool;
+
+ err1 = virtnet_sq_bind_xsk_pool(vi, sq, NULL);
+ err2 = virtnet_rq_bind_xsk_pool(vi, rq, NULL);
+
+ xsk_pool_dma_unmap(pool, 0);
+
+ dma_dev = virtqueue_dma_dev(rq->vq);
+
+ dma_unmap_single(dma_dev, sq->xsk.hdr_dma_address, vi->hdr_len, DMA_TO_DEVICE);
+
+ return err1 | err2;
+}
+
+int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp)
+{
+ if (xdp->xsk.pool)
+ return virtnet_xsk_pool_enable(dev, xdp->xsk.pool,
+ xdp->xsk.queue_id);
+ else
+ return virtnet_xsk_pool_disable(dev, xdp->xsk.queue_id);
+}
diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
new file mode 100644
index 000000000000..1918285c310c
--- /dev/null
+++ b/drivers/net/virtio/xsk.h
@@ -0,0 +1,7 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#ifndef __XSK_H__
+#define __XSK_H__
+
+int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
+#endif
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* [PATCH net-next v2 10/21] virtio_net: xsk: prevent disable tx napi
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (8 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 09/21] virtio_net: xsk: bind/unbind xsk Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 11/21] virtio_net: move some api to header Xuan Zhuo
` (12 subsequent siblings)
22 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
Since xsk's TX queue is consumed by TX NAPI, if sq is bound to xsk, then
we must stop tx napi from being disabled.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Jason Wang <jasowang@redhat.com>
---
drivers/net/virtio/main.c | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index c4601784b6d1..02e054fd217c 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -3337,7 +3337,7 @@ static int virtnet_set_coalesce(struct net_device *dev,
struct netlink_ext_ack *extack)
{
struct virtnet_info *vi = netdev_priv(dev);
- int ret, queue_number, napi_weight;
+ int ret, queue_number, napi_weight, i;
bool update_napi = false;
/* Can't change NAPI weight if the link is up */
@@ -3366,6 +3366,14 @@ static int virtnet_set_coalesce(struct net_device *dev,
return ret;
if (update_napi) {
+ /* xsk xmit depends on the tx napi. So if xsk is active,
+ * prevent modifications to tx napi.
+ */
+ for (i = queue_number; i < vi->max_queue_pairs; i++) {
+ if (vi->sq[i].xsk.pool)
+ return -EBUSY;
+ }
+
for (; queue_number < vi->max_queue_pairs; queue_number++)
vi->sq[queue_number].napi.weight = napi_weight;
}
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* [PATCH net-next v2 11/21] virtio_net: move some api to header
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (9 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 10/21] virtio_net: xsk: prevent disable tx napi Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 12/21] virtio_net: xsk: tx: support tx Xuan Zhuo
` (11 subsequent siblings)
22 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
__free_old_xmit
is_xdp_raw_buffer_queue
These two APIs are needed by the xsk part.
So this commit move theses to the header. And add prefix "virtnet_".
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 94 +++------------------------------
drivers/net/virtio/virtio_net.h | 80 ++++++++++++++++++++++++++++
2 files changed, 87 insertions(+), 87 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 02e054fd217c..6c608b3ce27d 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -45,9 +45,6 @@ module_param(napi_tx, bool, 0644);
#define VIRTIO_XDP_TX BIT(0)
#define VIRTIO_XDP_REDIR BIT(1)
-#define VIRTIO_XDP_FLAG BIT(0)
-#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
-
#define VIRTNET_DRIVER_VERSION "1.0.0"
static const unsigned long guest_offloads[] = {
@@ -150,78 +147,11 @@ struct virtio_net_common_hdr {
};
};
-static bool is_xdp_frame(void *ptr)
-{
- return (unsigned long)ptr & VIRTIO_XDP_FLAG;
-}
-
static void *xdp_to_ptr(struct xdp_frame *ptr)
{
return (void *)((unsigned long)ptr | VIRTIO_XDP_FLAG);
}
-static struct xdp_frame *ptr_to_xdp(void *ptr)
-{
- return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
-}
-
-static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
-{
- struct virtnet_sq_dma *next, *head;
-
- head = (void *)((unsigned long)data & ~VIRTIO_XMIT_DATA_MASK);
-
- data = head->data;
-
- while (head) {
- virtqueue_dma_unmap_single_attrs(sq->vq, head->addr, head->len,
- DMA_TO_DEVICE, 0);
-
- next = head->next;
-
- head->next = sq->dmainfo.free;
- sq->dmainfo.free = head;
-
- head = next;
- }
-
- return data;
-}
-
-static void __free_old_xmit(struct virtnet_sq *sq, bool in_napi,
- u64 *bytes, u64 *packets)
-{
- unsigned int len;
- void *ptr;
-
- while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
- if (!is_xdp_frame(ptr)) {
- struct sk_buff *skb;
-
- if (sq->do_dma)
- ptr = virtnet_sq_unmap(sq, ptr);
-
- skb = ptr;
-
- pr_debug("Sent skb %p\n", skb);
-
- *bytes += skb->len;
- napi_consume_skb(skb, in_napi);
- } else {
- struct xdp_frame *frame;
-
- if (sq->do_dma)
- ptr = virtnet_sq_unmap(sq, ptr);
-
- frame = ptr_to_xdp(ptr);
-
- *bytes += xdp_get_frame_len(frame);
- xdp_return_frame(frame);
- }
- (*packets)++;
- }
-}
-
/* Converting between virtqueue no. and kernel tx/rx queue no.
* 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq
*/
@@ -700,7 +630,7 @@ static void free_old_xmit(struct virtnet_sq *sq, bool in_napi)
{
u64 bytes, packets = 0;
- __free_old_xmit(sq, in_napi, &bytes, &packets);
+ virtnet_free_old_xmit(sq, in_napi, &bytes, &packets);
/* Avoid overhead when no packets have been processed
* happens when called speculatively from start_xmit.
@@ -714,16 +644,6 @@ static void free_old_xmit(struct virtnet_sq *sq, bool in_napi)
u64_stats_update_end(&sq->stats.syncp);
}
-static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
-{
- if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
- return false;
- else if (q < vi->curr_queue_pairs)
- return true;
- else
- return false;
-}
-
static void check_sq_full_and_disable(struct virtnet_info *vi,
struct net_device *dev,
struct virtnet_sq *sq)
@@ -872,7 +792,7 @@ static int virtnet_xdp_xmit(struct net_device *dev,
}
/* Free up any pending old buffers before queueing new ones. */
- __free_old_xmit(sq, false, &bytes, &packets);
+ virtnet_free_old_xmit(sq, false, &bytes, &packets);
for (i = 0; i < n; i++) {
struct xdp_frame *xdpf = frames[i];
@@ -883,7 +803,7 @@ static int virtnet_xdp_xmit(struct net_device *dev,
}
ret = nxmit;
- if (!is_xdp_raw_buffer_queue(vi, sq - vi->sq))
+ if (!virtnet_is_xdp_raw_buffer_queue(vi, sq - vi->sq))
check_sq_full_and_disable(vi, dev, sq);
if (flags & XDP_XMIT_FLUSH) {
@@ -2033,7 +1953,7 @@ static void virtnet_poll_cleantx(struct virtnet_rq *rq)
struct virtnet_sq *sq = &vi->sq[index];
struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index);
- if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index))
+ if (!sq->napi.weight || virtnet_is_xdp_raw_buffer_queue(vi, index))
return;
if (__netif_tx_trylock(txq)) {
@@ -2157,7 +2077,7 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
int opaque;
bool done;
- if (unlikely(is_xdp_raw_buffer_queue(vi, index))) {
+ if (unlikely(virtnet_is_xdp_raw_buffer_queue(vi, index))) {
/* We don't need to enable cb for XDP */
napi_complete_done(napi, 0);
return 0;
@@ -3997,7 +3917,7 @@ void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
sq = &vi->sq[i];
- if (!is_xdp_frame(buf)) {
+ if (!virtnet_is_xdp_frame(buf)) {
if (sq->do_dma)
buf = virtnet_sq_unmap(sq, buf);
@@ -4006,7 +3926,7 @@ void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
if (sq->do_dma)
buf = virtnet_sq_unmap(sq, buf);
- xdp_return_frame(ptr_to_xdp(buf));
+ xdp_return_frame(virtnet_ptr_to_xdp(buf));
}
}
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index 98ba23cfdb20..442af4673bf8 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -8,6 +8,9 @@
#include <linux/virtio_net.h>
#include <net/xdp_sock_drv.h>
+#define VIRTIO_XDP_FLAG BIT(0)
+#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
+
/* RX packet size EWMA. The average packet size is used to determine the packet
* buffer size when refilling RX rings. As the entire RX ring may be refilled
* at once, the weight is chosen so that the EWMA will be insensitive to short-
@@ -221,6 +224,83 @@ struct virtnet_info {
struct failover *failover;
};
+static inline bool virtnet_is_xdp_frame(void *ptr)
+{
+ return (unsigned long)ptr & VIRTIO_XDP_FLAG;
+}
+
+static inline struct xdp_frame *virtnet_ptr_to_xdp(void *ptr)
+{
+ return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
+}
+
+static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
+{
+ struct virtnet_sq_dma *next, *head;
+
+ head = (void *)((unsigned long)data & ~VIRTIO_XMIT_DATA_MASK);
+
+ data = head->data;
+
+ while (head) {
+ virtqueue_dma_unmap_single_attrs(sq->vq, head->addr, head->len,
+ DMA_TO_DEVICE, 0);
+
+ next = head->next;
+
+ head->next = sq->dmainfo.free;
+ sq->dmainfo.free = head;
+
+ head = next;
+ }
+
+ return data;
+}
+
+static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
+ u64 *bytes, u64 *packets)
+{
+ unsigned int len;
+ void *ptr;
+
+ while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
+ if (!virtnet_is_xdp_frame(ptr)) {
+ struct sk_buff *skb;
+
+ if (sq->do_dma)
+ ptr = virtnet_sq_unmap(sq, ptr);
+
+ skb = ptr;
+
+ pr_debug("Sent skb %p\n", skb);
+
+ *bytes += skb->len;
+ napi_consume_skb(skb, in_napi);
+ } else {
+ struct xdp_frame *frame;
+
+ if (sq->do_dma)
+ ptr = virtnet_sq_unmap(sq, ptr);
+
+ frame = virtnet_ptr_to_xdp(ptr);
+
+ *bytes += xdp_get_frame_len(frame);
+ xdp_return_frame(frame);
+ }
+ (*packets)++;
+ }
+}
+
+static inline bool virtnet_is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
+{
+ if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs))
+ return false;
+ else if (q < vi->curr_queue_pairs)
+ return true;
+ else
+ return false;
+}
+
void virtnet_rx_pause(struct virtnet_info *vi, struct virtnet_rq *rq);
void virtnet_rx_resume(struct virtnet_info *vi, struct virtnet_rq *rq);
void virtnet_tx_pause(struct virtnet_info *vi, struct virtnet_sq *sq);
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* [PATCH net-next v2 12/21] virtio_net: xsk: tx: support tx
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (10 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 11/21] virtio_net: move some api to header Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-09 8:09 ` Michael S. Tsirkin
2023-11-07 3:12 ` [PATCH net-next v2 13/21] virtio_net: xsk: tx: support wakeup Xuan Zhuo
` (10 subsequent siblings)
22 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
The driver's tx napi is very important for XSK. It is responsible for
obtaining data from the XSK queue and sending it out.
At the beginning, we need to trigger tx napi.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 12 +++-
drivers/net/virtio/virtio_net.h | 3 +-
drivers/net/virtio/xsk.c | 110 ++++++++++++++++++++++++++++++++
drivers/net/virtio/xsk.h | 13 ++++
4 files changed, 136 insertions(+), 2 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 6c608b3ce27d..ff6bc764089d 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -2074,6 +2074,7 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
struct virtnet_info *vi = sq->vq->vdev->priv;
unsigned int index = vq2txq(sq->vq);
struct netdev_queue *txq;
+ int busy = 0;
int opaque;
bool done;
@@ -2086,11 +2087,20 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
txq = netdev_get_tx_queue(vi->dev, index);
__netif_tx_lock(txq, raw_smp_processor_id());
virtqueue_disable_cb(sq->vq);
- free_old_xmit(sq, true);
+
+ if (sq->xsk.pool)
+ busy |= virtnet_xsk_xmit(sq, sq->xsk.pool, budget);
+ else
+ free_old_xmit(sq, true);
if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
netif_tx_wake_queue(txq);
+ if (busy) {
+ __netif_tx_unlock(txq);
+ return budget;
+ }
+
opaque = virtqueue_enable_cb_prepare(sq->vq);
done = napi_complete_done(napi, 0);
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index 442af4673bf8..1c21af47e13c 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -9,7 +9,8 @@
#include <net/xdp_sock_drv.h>
#define VIRTIO_XDP_FLAG BIT(0)
-#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
+#define VIRTIO_XSK_FLAG BIT(1)
+#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG | VIRTIO_XSK_FLAG)
/* RX packet size EWMA. The average packet size is used to determine the packet
* buffer size when refilling RX rings. As the entire RX ring may be refilled
diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
index 8b397787603f..caa448308232 100644
--- a/drivers/net/virtio/xsk.c
+++ b/drivers/net/virtio/xsk.c
@@ -4,9 +4,119 @@
*/
#include "virtio_net.h"
+#include "xsk.h"
static struct virtio_net_hdr_mrg_rxbuf xsk_hdr;
+static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len)
+{
+ sg->dma_address = addr;
+ sg->length = len;
+}
+
+static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
+{
+ struct virtnet_info *vi = sq->vq->vdev->priv;
+ struct net_device *dev = vi->dev;
+ int qnum = sq - vi->sq;
+
+ /* If it is a raw buffer queue, it does not check whether the status
+ * of the queue is stopped when sending. So there is no need to check
+ * the situation of the raw buffer queue.
+ */
+ if (virtnet_is_xdp_raw_buffer_queue(vi, qnum))
+ return;
+
+ /* If this sq is not the exclusive queue of the current cpu,
+ * then it may be called by start_xmit, so check it running out
+ * of space.
+ *
+ * Stop the queue to avoid getting packets that we are
+ * then unable to transmit. Then wait the tx interrupt.
+ */
+ if (sq->vq->num_free < 2 + MAX_SKB_FRAGS)
+ netif_stop_subqueue(dev, qnum);
+}
+
+static int virtnet_xsk_xmit_one(struct virtnet_sq *sq,
+ struct xsk_buff_pool *pool,
+ struct xdp_desc *desc)
+{
+ struct virtnet_info *vi;
+ dma_addr_t addr;
+
+ vi = sq->vq->vdev->priv;
+
+ addr = xsk_buff_raw_get_dma(pool, desc->addr);
+ xsk_buff_raw_dma_sync_for_device(pool, addr, desc->len);
+
+ sg_init_table(sq->sg, 2);
+
+ sg_fill_dma(sq->sg, sq->xsk.hdr_dma_address, vi->hdr_len);
+ sg_fill_dma(sq->sg + 1, addr, desc->len);
+
+ return virtqueue_add_outbuf(sq->vq, sq->sg, 2,
+ virtnet_xsk_to_ptr(desc->len), GFP_ATOMIC);
+}
+
+static int virtnet_xsk_xmit_batch(struct virtnet_sq *sq,
+ struct xsk_buff_pool *pool,
+ unsigned int budget,
+ u64 *kicks)
+{
+ struct xdp_desc *descs = pool->tx_descs;
+ u32 nb_pkts, max_pkts, i;
+ bool kick = false;
+ int err;
+
+ /* Every xsk tx packet needs two desc(virtnet header and packet). So we
+ * use sq->vq->num_free / 2 as the limitation.
+ */
+ max_pkts = min_t(u32, budget, sq->vq->num_free / 2);
+
+ nb_pkts = xsk_tx_peek_release_desc_batch(pool, max_pkts);
+ if (!nb_pkts)
+ return 0;
+
+ for (i = 0; i < nb_pkts; i++) {
+ err = virtnet_xsk_xmit_one(sq, pool, &descs[i]);
+ if (unlikely(err))
+ break;
+
+ kick = true;
+ }
+
+ if (kick && virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
+ (*kicks)++;
+
+ return i;
+}
+
+bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
+ int budget)
+{
+ u64 bytes = 0, packets = 0, kicks = 0;
+ int sent;
+
+ virtnet_free_old_xmit(sq, true, &bytes, &packets);
+
+ sent = virtnet_xsk_xmit_batch(sq, pool, budget, &kicks);
+
+ virtnet_xsk_check_queue(sq);
+
+ u64_stats_update_begin(&sq->stats.syncp);
+ u64_stats_add(&sq->stats.packets, packets);
+ u64_stats_add(&sq->stats.bytes, bytes);
+ u64_stats_add(&sq->stats.kicks, kicks);
+ u64_stats_add(&sq->stats.xdp_tx, sent);
+ u64_stats_update_end(&sq->stats.syncp);
+
+ if (xsk_uses_need_wakeup(pool))
+ xsk_set_tx_need_wakeup(pool);
+
+ return sent == budget;
+}
+
static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *rq,
struct xsk_buff_pool *pool)
{
diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
index 1918285c310c..73ca8cd5308b 100644
--- a/drivers/net/virtio/xsk.h
+++ b/drivers/net/virtio/xsk.h
@@ -3,5 +3,18 @@
#ifndef __XSK_H__
#define __XSK_H__
+#define VIRTIO_XSK_FLAG_OFFSET 4
+
+static inline void *virtnet_xsk_to_ptr(u32 len)
+{
+ unsigned long p;
+
+ p = len << VIRTIO_XSK_FLAG_OFFSET;
+
+ return (void *)(p | VIRTIO_XSK_FLAG);
+}
+
int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
+bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
+ int budget);
#endif
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 12/21] virtio_net: xsk: tx: support tx
2023-11-07 3:12 ` [PATCH net-next v2 12/21] virtio_net: xsk: tx: support tx Xuan Zhuo
@ 2023-11-09 8:09 ` Michael S. Tsirkin
2023-11-09 11:06 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Michael S. Tsirkin @ 2023-11-09 8:09 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Tue, Nov 07, 2023 at 11:12:18AM +0800, Xuan Zhuo wrote:
> The driver's tx napi is very important for XSK. It is responsible for
> obtaining data from the XSK queue and sending it out.
>
> At the beginning, we need to trigger tx napi.
>
> Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> ---
> drivers/net/virtio/main.c | 12 +++-
> drivers/net/virtio/virtio_net.h | 3 +-
> drivers/net/virtio/xsk.c | 110 ++++++++++++++++++++++++++++++++
> drivers/net/virtio/xsk.h | 13 ++++
> 4 files changed, 136 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> index 6c608b3ce27d..ff6bc764089d 100644
> --- a/drivers/net/virtio/main.c
> +++ b/drivers/net/virtio/main.c
> @@ -2074,6 +2074,7 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
> struct virtnet_info *vi = sq->vq->vdev->priv;
> unsigned int index = vq2txq(sq->vq);
> struct netdev_queue *txq;
> + int busy = 0;
> int opaque;
> bool done;
>
> @@ -2086,11 +2087,20 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
> txq = netdev_get_tx_queue(vi->dev, index);
> __netif_tx_lock(txq, raw_smp_processor_id());
> virtqueue_disable_cb(sq->vq);
> - free_old_xmit(sq, true);
> +
> + if (sq->xsk.pool)
> + busy |= virtnet_xsk_xmit(sq, sq->xsk.pool, budget);
You use bitwise or on errno values? What's going on here?
> + else
> + free_old_xmit(sq, true);
>
> if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
> netif_tx_wake_queue(txq);
>
> + if (busy) {
> + __netif_tx_unlock(txq);
> + return budget;
> + }
> +
> opaque = virtqueue_enable_cb_prepare(sq->vq);
>
> done = napi_complete_done(napi, 0);
> diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> index 442af4673bf8..1c21af47e13c 100644
> --- a/drivers/net/virtio/virtio_net.h
> +++ b/drivers/net/virtio/virtio_net.h
> @@ -9,7 +9,8 @@
> #include <net/xdp_sock_drv.h>
>
> #define VIRTIO_XDP_FLAG BIT(0)
> -#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
> +#define VIRTIO_XSK_FLAG BIT(1)
> +#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG | VIRTIO_XSK_FLAG)
>
> /* RX packet size EWMA. The average packet size is used to determine the packet
> * buffer size when refilling RX rings. As the entire RX ring may be refilled
> diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> index 8b397787603f..caa448308232 100644
> --- a/drivers/net/virtio/xsk.c
> +++ b/drivers/net/virtio/xsk.c
> @@ -4,9 +4,119 @@
> */
>
> #include "virtio_net.h"
> +#include "xsk.h"
>
> static struct virtio_net_hdr_mrg_rxbuf xsk_hdr;
>
> +static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len)
> +{
> + sg->dma_address = addr;
> + sg->length = len;
> +}
> +
> +static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> +{
> + struct virtnet_info *vi = sq->vq->vdev->priv;
> + struct net_device *dev = vi->dev;
> + int qnum = sq - vi->sq;
> +
> + /* If it is a raw buffer queue, it does not check whether the status
> + * of the queue is stopped when sending. So there is no need to check
> + * the situation of the raw buffer queue.
> + */
> + if (virtnet_is_xdp_raw_buffer_queue(vi, qnum))
> + return;
> +
> + /* If this sq is not the exclusive queue of the current cpu,
> + * then it may be called by start_xmit, so check it running out
> + * of space.
> + *
> + * Stop the queue to avoid getting packets that we are
> + * then unable to transmit. Then wait the tx interrupt.
> + */
> + if (sq->vq->num_free < 2 + MAX_SKB_FRAGS)
what does MAX_SKB_FRAGS have to do with it? And where's 2 coming from?
> + netif_stop_subqueue(dev, qnum);
> +}
> +
> +static int virtnet_xsk_xmit_one(struct virtnet_sq *sq,
> + struct xsk_buff_pool *pool,
> + struct xdp_desc *desc)
> +{
> + struct virtnet_info *vi;
> + dma_addr_t addr;
> +
> + vi = sq->vq->vdev->priv;
> +
> + addr = xsk_buff_raw_get_dma(pool, desc->addr);
> + xsk_buff_raw_dma_sync_for_device(pool, addr, desc->len);
> +
> + sg_init_table(sq->sg, 2);
> +
> + sg_fill_dma(sq->sg, sq->xsk.hdr_dma_address, vi->hdr_len);
> + sg_fill_dma(sq->sg + 1, addr, desc->len);
> +
> + return virtqueue_add_outbuf(sq->vq, sq->sg, 2,
> + virtnet_xsk_to_ptr(desc->len), GFP_ATOMIC);
> +}
> +
> +static int virtnet_xsk_xmit_batch(struct virtnet_sq *sq,
> + struct xsk_buff_pool *pool,
> + unsigned int budget,
> + u64 *kicks)
> +{
> + struct xdp_desc *descs = pool->tx_descs;
> + u32 nb_pkts, max_pkts, i;
> + bool kick = false;
> + int err;
> +
> + /* Every xsk tx packet needs two desc(virtnet header and packet). So we
> + * use sq->vq->num_free / 2 as the limitation.
> + */
> + max_pkts = min_t(u32, budget, sq->vq->num_free / 2);
> +
> + nb_pkts = xsk_tx_peek_release_desc_batch(pool, max_pkts);
> + if (!nb_pkts)
> + return 0;
> +
> + for (i = 0; i < nb_pkts; i++) {
> + err = virtnet_xsk_xmit_one(sq, pool, &descs[i]);
> + if (unlikely(err))
> + break;
> +
> + kick = true;
> + }
> +
> + if (kick && virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
> + (*kicks)++;
> +
> + return i;
> +}
> +
> +bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> + int budget)
> +{
> + u64 bytes = 0, packets = 0, kicks = 0;
> + int sent;
> +
> + virtnet_free_old_xmit(sq, true, &bytes, &packets);
> +
> + sent = virtnet_xsk_xmit_batch(sq, pool, budget, &kicks);
> +
> + virtnet_xsk_check_queue(sq);
> +
> + u64_stats_update_begin(&sq->stats.syncp);
> + u64_stats_add(&sq->stats.packets, packets);
> + u64_stats_add(&sq->stats.bytes, bytes);
> + u64_stats_add(&sq->stats.kicks, kicks);
> + u64_stats_add(&sq->stats.xdp_tx, sent);
> + u64_stats_update_end(&sq->stats.syncp);
> +
> + if (xsk_uses_need_wakeup(pool))
> + xsk_set_tx_need_wakeup(pool);
> +
> + return sent == budget;
> +}
> +
> static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *rq,
> struct xsk_buff_pool *pool)
> {
> diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> index 1918285c310c..73ca8cd5308b 100644
> --- a/drivers/net/virtio/xsk.h
> +++ b/drivers/net/virtio/xsk.h
> @@ -3,5 +3,18 @@
> #ifndef __XSK_H__
> #define __XSK_H__
>
> +#define VIRTIO_XSK_FLAG_OFFSET 4
> +
> +static inline void *virtnet_xsk_to_ptr(u32 len)
> +{
> + unsigned long p;
> +
> + p = len << VIRTIO_XSK_FLAG_OFFSET;
> +
> + return (void *)(p | VIRTIO_XSK_FLAG);
> +}
> +
> int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> +bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> + int budget);
> #endif
> --
> 2.32.0.3.g01195cf9f
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 12/21] virtio_net: xsk: tx: support tx
2023-11-09 8:09 ` Michael S. Tsirkin
@ 2023-11-09 11:06 ` Xuan Zhuo
2023-11-09 11:58 ` Michael S. Tsirkin
0 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-09 11:06 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Thu, 9 Nov 2023 03:09:00 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> On Tue, Nov 07, 2023 at 11:12:18AM +0800, Xuan Zhuo wrote:
> > The driver's tx napi is very important for XSK. It is responsible for
> > obtaining data from the XSK queue and sending it out.
> >
> > At the beginning, we need to trigger tx napi.
> >
> > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > ---
> > drivers/net/virtio/main.c | 12 +++-
> > drivers/net/virtio/virtio_net.h | 3 +-
> > drivers/net/virtio/xsk.c | 110 ++++++++++++++++++++++++++++++++
> > drivers/net/virtio/xsk.h | 13 ++++
> > 4 files changed, 136 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > index 6c608b3ce27d..ff6bc764089d 100644
> > --- a/drivers/net/virtio/main.c
> > +++ b/drivers/net/virtio/main.c
> > @@ -2074,6 +2074,7 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
> > struct virtnet_info *vi = sq->vq->vdev->priv;
> > unsigned int index = vq2txq(sq->vq);
> > struct netdev_queue *txq;
> > + int busy = 0;
> > int opaque;
> > bool done;
> >
> > @@ -2086,11 +2087,20 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
> > txq = netdev_get_tx_queue(vi->dev, index);
> > __netif_tx_lock(txq, raw_smp_processor_id());
> > virtqueue_disable_cb(sq->vq);
> > - free_old_xmit(sq, true);
> > +
> > + if (sq->xsk.pool)
> > + busy |= virtnet_xsk_xmit(sq, sq->xsk.pool, budget);
>
> You use bitwise or on errno values? What's going on here?
virtnet_xsk_xmit() return that it is busy or not. Not the errno.
Here just record whether this handler is busy or not.
>
>
> > + else
> > + free_old_xmit(sq, true);
> >
> > if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
> > netif_tx_wake_queue(txq);
> >
> > + if (busy) {
> > + __netif_tx_unlock(txq);
> > + return budget;
> > + }
> > +
> > opaque = virtqueue_enable_cb_prepare(sq->vq);
> >
> > done = napi_complete_done(napi, 0);
> > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > index 442af4673bf8..1c21af47e13c 100644
> > --- a/drivers/net/virtio/virtio_net.h
> > +++ b/drivers/net/virtio/virtio_net.h
> > @@ -9,7 +9,8 @@
> > #include <net/xdp_sock_drv.h>
> >
> > #define VIRTIO_XDP_FLAG BIT(0)
> > -#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
> > +#define VIRTIO_XSK_FLAG BIT(1)
> > +#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG | VIRTIO_XSK_FLAG)
> >
> > /* RX packet size EWMA. The average packet size is used to determine the packet
> > * buffer size when refilling RX rings. As the entire RX ring may be refilled
> > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > index 8b397787603f..caa448308232 100644
> > --- a/drivers/net/virtio/xsk.c
> > +++ b/drivers/net/virtio/xsk.c
> > @@ -4,9 +4,119 @@
> > */
> >
> > #include "virtio_net.h"
> > +#include "xsk.h"
> >
> > static struct virtio_net_hdr_mrg_rxbuf xsk_hdr;
> >
> > +static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len)
> > +{
> > + sg->dma_address = addr;
> > + sg->length = len;
> > +}
> > +
> > +static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> > +{
> > + struct virtnet_info *vi = sq->vq->vdev->priv;
> > + struct net_device *dev = vi->dev;
> > + int qnum = sq - vi->sq;
> > +
> > + /* If it is a raw buffer queue, it does not check whether the status
> > + * of the queue is stopped when sending. So there is no need to check
> > + * the situation of the raw buffer queue.
> > + */
> > + if (virtnet_is_xdp_raw_buffer_queue(vi, qnum))
> > + return;
> > +
> > + /* If this sq is not the exclusive queue of the current cpu,
> > + * then it may be called by start_xmit, so check it running out
> > + * of space.
> > + *
> > + * Stop the queue to avoid getting packets that we are
> > + * then unable to transmit. Then wait the tx interrupt.
> > + */
> > + if (sq->vq->num_free < 2 + MAX_SKB_FRAGS)
>
> what does MAX_SKB_FRAGS have to do with it? And where's 2 coming from?
check_sq_full_and_disable()
Thanks.
>
> > + netif_stop_subqueue(dev, qnum);
> > +}
> > +
> > +static int virtnet_xsk_xmit_one(struct virtnet_sq *sq,
> > + struct xsk_buff_pool *pool,
> > + struct xdp_desc *desc)
> > +{
> > + struct virtnet_info *vi;
> > + dma_addr_t addr;
> > +
> > + vi = sq->vq->vdev->priv;
> > +
> > + addr = xsk_buff_raw_get_dma(pool, desc->addr);
> > + xsk_buff_raw_dma_sync_for_device(pool, addr, desc->len);
> > +
> > + sg_init_table(sq->sg, 2);
> > +
> > + sg_fill_dma(sq->sg, sq->xsk.hdr_dma_address, vi->hdr_len);
> > + sg_fill_dma(sq->sg + 1, addr, desc->len);
> > +
> > + return virtqueue_add_outbuf(sq->vq, sq->sg, 2,
> > + virtnet_xsk_to_ptr(desc->len), GFP_ATOMIC);
> > +}
> > +
> > +static int virtnet_xsk_xmit_batch(struct virtnet_sq *sq,
> > + struct xsk_buff_pool *pool,
> > + unsigned int budget,
> > + u64 *kicks)
> > +{
> > + struct xdp_desc *descs = pool->tx_descs;
> > + u32 nb_pkts, max_pkts, i;
> > + bool kick = false;
> > + int err;
> > +
> > + /* Every xsk tx packet needs two desc(virtnet header and packet). So we
> > + * use sq->vq->num_free / 2 as the limitation.
> > + */
> > + max_pkts = min_t(u32, budget, sq->vq->num_free / 2);
> > +
> > + nb_pkts = xsk_tx_peek_release_desc_batch(pool, max_pkts);
> > + if (!nb_pkts)
> > + return 0;
> > +
> > + for (i = 0; i < nb_pkts; i++) {
> > + err = virtnet_xsk_xmit_one(sq, pool, &descs[i]);
> > + if (unlikely(err))
> > + break;
> > +
> > + kick = true;
> > + }
> > +
> > + if (kick && virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
> > + (*kicks)++;
> > +
> > + return i;
> > +}
> > +
> > +bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > + int budget)
> > +{
> > + u64 bytes = 0, packets = 0, kicks = 0;
> > + int sent;
> > +
> > + virtnet_free_old_xmit(sq, true, &bytes, &packets);
> > +
> > + sent = virtnet_xsk_xmit_batch(sq, pool, budget, &kicks);
> > +
> > + virtnet_xsk_check_queue(sq);
> > +
> > + u64_stats_update_begin(&sq->stats.syncp);
> > + u64_stats_add(&sq->stats.packets, packets);
> > + u64_stats_add(&sq->stats.bytes, bytes);
> > + u64_stats_add(&sq->stats.kicks, kicks);
> > + u64_stats_add(&sq->stats.xdp_tx, sent);
> > + u64_stats_update_end(&sq->stats.syncp);
> > +
> > + if (xsk_uses_need_wakeup(pool))
> > + xsk_set_tx_need_wakeup(pool);
> > +
> > + return sent == budget;
> > +}
> > +
> > static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *rq,
> > struct xsk_buff_pool *pool)
> > {
> > diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> > index 1918285c310c..73ca8cd5308b 100644
> > --- a/drivers/net/virtio/xsk.h
> > +++ b/drivers/net/virtio/xsk.h
> > @@ -3,5 +3,18 @@
> > #ifndef __XSK_H__
> > #define __XSK_H__
> >
> > +#define VIRTIO_XSK_FLAG_OFFSET 4
> > +
> > +static inline void *virtnet_xsk_to_ptr(u32 len)
> > +{
> > + unsigned long p;
> > +
> > + p = len << VIRTIO_XSK_FLAG_OFFSET;
> > +
> > + return (void *)(p | VIRTIO_XSK_FLAG);
> > +}
> > +
> > int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> > +bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > + int budget);
> > #endif
> > --
> > 2.32.0.3.g01195cf9f
>
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 12/21] virtio_net: xsk: tx: support tx
2023-11-09 11:06 ` Xuan Zhuo
@ 2023-11-09 11:58 ` Michael S. Tsirkin
2023-11-10 1:51 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Michael S. Tsirkin @ 2023-11-09 11:58 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Thu, Nov 09, 2023 at 07:06:23PM +0800, Xuan Zhuo wrote:
> On Thu, 9 Nov 2023 03:09:00 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > On Tue, Nov 07, 2023 at 11:12:18AM +0800, Xuan Zhuo wrote:
> > > The driver's tx napi is very important for XSK. It is responsible for
> > > obtaining data from the XSK queue and sending it out.
> > >
> > > At the beginning, we need to trigger tx napi.
> > >
> > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > ---
> > > drivers/net/virtio/main.c | 12 +++-
> > > drivers/net/virtio/virtio_net.h | 3 +-
> > > drivers/net/virtio/xsk.c | 110 ++++++++++++++++++++++++++++++++
> > > drivers/net/virtio/xsk.h | 13 ++++
> > > 4 files changed, 136 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > > index 6c608b3ce27d..ff6bc764089d 100644
> > > --- a/drivers/net/virtio/main.c
> > > +++ b/drivers/net/virtio/main.c
> > > @@ -2074,6 +2074,7 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
> > > struct virtnet_info *vi = sq->vq->vdev->priv;
> > > unsigned int index = vq2txq(sq->vq);
> > > struct netdev_queue *txq;
> > > + int busy = 0;
> > > int opaque;
> > > bool done;
> > >
> > > @@ -2086,11 +2087,20 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
> > > txq = netdev_get_tx_queue(vi->dev, index);
> > > __netif_tx_lock(txq, raw_smp_processor_id());
> > > virtqueue_disable_cb(sq->vq);
> > > - free_old_xmit(sq, true);
> > > +
> > > + if (sq->xsk.pool)
> > > + busy |= virtnet_xsk_xmit(sq, sq->xsk.pool, budget);
> >
> > You use bitwise or on errno values? What's going on here?
>
> virtnet_xsk_xmit() return that it is busy or not. Not the errno.
> Here just record whether this handler is busy or not.
Ah I see it's a bool. So make busy a bool too.
> >
> >
> > > + else
> > > + free_old_xmit(sq, true);
> > >
> > > if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
> > > netif_tx_wake_queue(txq);
> > >
> > > + if (busy) {
> > > + __netif_tx_unlock(txq);
> > > + return budget;
> > > + }
> > > +
> > > opaque = virtqueue_enable_cb_prepare(sq->vq);
> > >
> > > done = napi_complete_done(napi, 0);
> > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > index 442af4673bf8..1c21af47e13c 100644
> > > --- a/drivers/net/virtio/virtio_net.h
> > > +++ b/drivers/net/virtio/virtio_net.h
> > > @@ -9,7 +9,8 @@
> > > #include <net/xdp_sock_drv.h>
> > >
> > > #define VIRTIO_XDP_FLAG BIT(0)
> > > -#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
> > > +#define VIRTIO_XSK_FLAG BIT(1)
> > > +#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG | VIRTIO_XSK_FLAG)
> > >
> > > /* RX packet size EWMA. The average packet size is used to determine the packet
> > > * buffer size when refilling RX rings. As the entire RX ring may be refilled
> > > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > > index 8b397787603f..caa448308232 100644
> > > --- a/drivers/net/virtio/xsk.c
> > > +++ b/drivers/net/virtio/xsk.c
> > > @@ -4,9 +4,119 @@
> > > */
> > >
> > > #include "virtio_net.h"
> > > +#include "xsk.h"
> > >
> > > static struct virtio_net_hdr_mrg_rxbuf xsk_hdr;
> > >
> > > +static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len)
> > > +{
> > > + sg->dma_address = addr;
> > > + sg->length = len;
> > > +}
> > > +
> > > +static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> > > +{
> > > + struct virtnet_info *vi = sq->vq->vdev->priv;
> > > + struct net_device *dev = vi->dev;
> > > + int qnum = sq - vi->sq;
> > > +
> > > + /* If it is a raw buffer queue, it does not check whether the status
> > > + * of the queue is stopped when sending. So there is no need to check
> > > + * the situation of the raw buffer queue.
> > > + */
> > > + if (virtnet_is_xdp_raw_buffer_queue(vi, qnum))
> > > + return;
> > > +
> > > + /* If this sq is not the exclusive queue of the current cpu,
> > > + * then it may be called by start_xmit, so check it running out
> > > + * of space.
> > > + *
> > > + * Stop the queue to avoid getting packets that we are
> > > + * then unable to transmit. Then wait the tx interrupt.
> > > + */
> > > + if (sq->vq->num_free < 2 + MAX_SKB_FRAGS)
> >
> > what does MAX_SKB_FRAGS have to do with it? And where's 2 coming from?
>
> check_sq_full_and_disable()
>
> Thanks.
This is one example of duplication I was talking about earlier.
> >
> > > + netif_stop_subqueue(dev, qnum);
> > > +}
> > > +
> > > +static int virtnet_xsk_xmit_one(struct virtnet_sq *sq,
> > > + struct xsk_buff_pool *pool,
> > > + struct xdp_desc *desc)
> > > +{
> > > + struct virtnet_info *vi;
> > > + dma_addr_t addr;
> > > +
> > > + vi = sq->vq->vdev->priv;
> > > +
> > > + addr = xsk_buff_raw_get_dma(pool, desc->addr);
> > > + xsk_buff_raw_dma_sync_for_device(pool, addr, desc->len);
> > > +
> > > + sg_init_table(sq->sg, 2);
> > > +
> > > + sg_fill_dma(sq->sg, sq->xsk.hdr_dma_address, vi->hdr_len);
> > > + sg_fill_dma(sq->sg + 1, addr, desc->len);
> > > +
> > > + return virtqueue_add_outbuf(sq->vq, sq->sg, 2,
> > > + virtnet_xsk_to_ptr(desc->len), GFP_ATOMIC);
> > > +}
> > > +
> > > +static int virtnet_xsk_xmit_batch(struct virtnet_sq *sq,
> > > + struct xsk_buff_pool *pool,
> > > + unsigned int budget,
> > > + u64 *kicks)
> > > +{
> > > + struct xdp_desc *descs = pool->tx_descs;
> > > + u32 nb_pkts, max_pkts, i;
> > > + bool kick = false;
> > > + int err;
> > > +
> > > + /* Every xsk tx packet needs two desc(virtnet header and packet). So we
> > > + * use sq->vq->num_free / 2 as the limitation.
> > > + */
> > > + max_pkts = min_t(u32, budget, sq->vq->num_free / 2);
> > > +
> > > + nb_pkts = xsk_tx_peek_release_desc_batch(pool, max_pkts);
> > > + if (!nb_pkts)
> > > + return 0;
> > > +
> > > + for (i = 0; i < nb_pkts; i++) {
> > > + err = virtnet_xsk_xmit_one(sq, pool, &descs[i]);
> > > + if (unlikely(err))
> > > + break;
> > > +
> > > + kick = true;
> > > + }
> > > +
> > > + if (kick && virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
> > > + (*kicks)++;
> > > +
> > > + return i;
> > > +}
> > > +
> > > +bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > > + int budget)
> > > +{
> > > + u64 bytes = 0, packets = 0, kicks = 0;
> > > + int sent;
> > > +
> > > + virtnet_free_old_xmit(sq, true, &bytes, &packets);
> > > +
> > > + sent = virtnet_xsk_xmit_batch(sq, pool, budget, &kicks);
> > > +
> > > + virtnet_xsk_check_queue(sq);
> > > +
> > > + u64_stats_update_begin(&sq->stats.syncp);
> > > + u64_stats_add(&sq->stats.packets, packets);
> > > + u64_stats_add(&sq->stats.bytes, bytes);
> > > + u64_stats_add(&sq->stats.kicks, kicks);
> > > + u64_stats_add(&sq->stats.xdp_tx, sent);
> > > + u64_stats_update_end(&sq->stats.syncp);
> > > +
> > > + if (xsk_uses_need_wakeup(pool))
> > > + xsk_set_tx_need_wakeup(pool);
> > > +
> > > + return sent == budget;
> > > +}
> > > +
> > > static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *rq,
> > > struct xsk_buff_pool *pool)
> > > {
> > > diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> > > index 1918285c310c..73ca8cd5308b 100644
> > > --- a/drivers/net/virtio/xsk.h
> > > +++ b/drivers/net/virtio/xsk.h
> > > @@ -3,5 +3,18 @@
> > > #ifndef __XSK_H__
> > > #define __XSK_H__
> > >
> > > +#define VIRTIO_XSK_FLAG_OFFSET 4
> > > +
> > > +static inline void *virtnet_xsk_to_ptr(u32 len)
> > > +{
> > > + unsigned long p;
> > > +
> > > + p = len << VIRTIO_XSK_FLAG_OFFSET;
> > > +
> > > + return (void *)(p | VIRTIO_XSK_FLAG);
> > > +}
> > > +
> > > int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> > > +bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > > + int budget);
> > > #endif
> > > --
> > > 2.32.0.3.g01195cf9f
> >
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 12/21] virtio_net: xsk: tx: support tx
2023-11-09 11:58 ` Michael S. Tsirkin
@ 2023-11-10 1:51 ` Xuan Zhuo
0 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-10 1:51 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Thu, 9 Nov 2023 06:58:48 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> On Thu, Nov 09, 2023 at 07:06:23PM +0800, Xuan Zhuo wrote:
> > On Thu, 9 Nov 2023 03:09:00 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > On Tue, Nov 07, 2023 at 11:12:18AM +0800, Xuan Zhuo wrote:
> > > > The driver's tx napi is very important for XSK. It is responsible for
> > > > obtaining data from the XSK queue and sending it out.
> > > >
> > > > At the beginning, we need to trigger tx napi.
> > > >
> > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > > ---
> > > > drivers/net/virtio/main.c | 12 +++-
> > > > drivers/net/virtio/virtio_net.h | 3 +-
> > > > drivers/net/virtio/xsk.c | 110 ++++++++++++++++++++++++++++++++
> > > > drivers/net/virtio/xsk.h | 13 ++++
> > > > 4 files changed, 136 insertions(+), 2 deletions(-)
> > > >
> > > > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > > > index 6c608b3ce27d..ff6bc764089d 100644
> > > > --- a/drivers/net/virtio/main.c
> > > > +++ b/drivers/net/virtio/main.c
> > > > @@ -2074,6 +2074,7 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
> > > > struct virtnet_info *vi = sq->vq->vdev->priv;
> > > > unsigned int index = vq2txq(sq->vq);
> > > > struct netdev_queue *txq;
> > > > + int busy = 0;
> > > > int opaque;
> > > > bool done;
> > > >
> > > > @@ -2086,11 +2087,20 @@ static int virtnet_poll_tx(struct napi_struct *napi, int budget)
> > > > txq = netdev_get_tx_queue(vi->dev, index);
> > > > __netif_tx_lock(txq, raw_smp_processor_id());
> > > > virtqueue_disable_cb(sq->vq);
> > > > - free_old_xmit(sq, true);
> > > > +
> > > > + if (sq->xsk.pool)
> > > > + busy |= virtnet_xsk_xmit(sq, sq->xsk.pool, budget);
> > >
> > > You use bitwise or on errno values? What's going on here?
> >
> > virtnet_xsk_xmit() return that it is busy or not. Not the errno.
> > Here just record whether this handler is busy or not.
>
>
> Ah I see it's a bool. So make busy a bool too.
>
>
> > >
> > >
> > > > + else
> > > > + free_old_xmit(sq, true);
> > > >
> > > > if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS)
> > > > netif_tx_wake_queue(txq);
> > > >
> > > > + if (busy) {
> > > > + __netif_tx_unlock(txq);
> > > > + return budget;
> > > > + }
> > > > +
> > > > opaque = virtqueue_enable_cb_prepare(sq->vq);
> > > >
> > > > done = napi_complete_done(napi, 0);
> > > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > > index 442af4673bf8..1c21af47e13c 100644
> > > > --- a/drivers/net/virtio/virtio_net.h
> > > > +++ b/drivers/net/virtio/virtio_net.h
> > > > @@ -9,7 +9,8 @@
> > > > #include <net/xdp_sock_drv.h>
> > > >
> > > > #define VIRTIO_XDP_FLAG BIT(0)
> > > > -#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG)
> > > > +#define VIRTIO_XSK_FLAG BIT(1)
> > > > +#define VIRTIO_XMIT_DATA_MASK (VIRTIO_XDP_FLAG | VIRTIO_XSK_FLAG)
> > > >
> > > > /* RX packet size EWMA. The average packet size is used to determine the packet
> > > > * buffer size when refilling RX rings. As the entire RX ring may be refilled
> > > > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > > > index 8b397787603f..caa448308232 100644
> > > > --- a/drivers/net/virtio/xsk.c
> > > > +++ b/drivers/net/virtio/xsk.c
> > > > @@ -4,9 +4,119 @@
> > > > */
> > > >
> > > > #include "virtio_net.h"
> > > > +#include "xsk.h"
> > > >
> > > > static struct virtio_net_hdr_mrg_rxbuf xsk_hdr;
> > > >
> > > > +static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len)
> > > > +{
> > > > + sg->dma_address = addr;
> > > > + sg->length = len;
> > > > +}
> > > > +
> > > > +static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> > > > +{
> > > > + struct virtnet_info *vi = sq->vq->vdev->priv;
> > > > + struct net_device *dev = vi->dev;
> > > > + int qnum = sq - vi->sq;
> > > > +
> > > > + /* If it is a raw buffer queue, it does not check whether the status
> > > > + * of the queue is stopped when sending. So there is no need to check
> > > > + * the situation of the raw buffer queue.
> > > > + */
> > > > + if (virtnet_is_xdp_raw_buffer_queue(vi, qnum))
> > > > + return;
> > > > +
> > > > + /* If this sq is not the exclusive queue of the current cpu,
> > > > + * then it may be called by start_xmit, so check it running out
> > > > + * of space.
> > > > + *
> > > > + * Stop the queue to avoid getting packets that we are
> > > > + * then unable to transmit. Then wait the tx interrupt.
> > > > + */
> > > > + if (sq->vq->num_free < 2 + MAX_SKB_FRAGS)
> > >
> > > what does MAX_SKB_FRAGS have to do with it? And where's 2 coming from?
> >
> > check_sq_full_and_disable()
> >
> > Thanks.
>
>
> This is one example of duplication I was talking about earlier.
OK. I write this function about two years ago. Let me rethink about this.
Thanks.
>
> > >
> > > > + netif_stop_subqueue(dev, qnum);
> > > > +}
> > > > +
> > > > +static int virtnet_xsk_xmit_one(struct virtnet_sq *sq,
> > > > + struct xsk_buff_pool *pool,
> > > > + struct xdp_desc *desc)
> > > > +{
> > > > + struct virtnet_info *vi;
> > > > + dma_addr_t addr;
> > > > +
> > > > + vi = sq->vq->vdev->priv;
> > > > +
> > > > + addr = xsk_buff_raw_get_dma(pool, desc->addr);
> > > > + xsk_buff_raw_dma_sync_for_device(pool, addr, desc->len);
> > > > +
> > > > + sg_init_table(sq->sg, 2);
> > > > +
> > > > + sg_fill_dma(sq->sg, sq->xsk.hdr_dma_address, vi->hdr_len);
> > > > + sg_fill_dma(sq->sg + 1, addr, desc->len);
> > > > +
> > > > + return virtqueue_add_outbuf(sq->vq, sq->sg, 2,
> > > > + virtnet_xsk_to_ptr(desc->len), GFP_ATOMIC);
> > > > +}
> > > > +
> > > > +static int virtnet_xsk_xmit_batch(struct virtnet_sq *sq,
> > > > + struct xsk_buff_pool *pool,
> > > > + unsigned int budget,
> > > > + u64 *kicks)
> > > > +{
> > > > + struct xdp_desc *descs = pool->tx_descs;
> > > > + u32 nb_pkts, max_pkts, i;
> > > > + bool kick = false;
> > > > + int err;
> > > > +
> > > > + /* Every xsk tx packet needs two desc(virtnet header and packet). So we
> > > > + * use sq->vq->num_free / 2 as the limitation.
> > > > + */
> > > > + max_pkts = min_t(u32, budget, sq->vq->num_free / 2);
> > > > +
> > > > + nb_pkts = xsk_tx_peek_release_desc_batch(pool, max_pkts);
> > > > + if (!nb_pkts)
> > > > + return 0;
> > > > +
> > > > + for (i = 0; i < nb_pkts; i++) {
> > > > + err = virtnet_xsk_xmit_one(sq, pool, &descs[i]);
> > > > + if (unlikely(err))
> > > > + break;
> > > > +
> > > > + kick = true;
> > > > + }
> > > > +
> > > > + if (kick && virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq))
> > > > + (*kicks)++;
> > > > +
> > > > + return i;
> > > > +}
> > > > +
> > > > +bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > > > + int budget)
> > > > +{
> > > > + u64 bytes = 0, packets = 0, kicks = 0;
> > > > + int sent;
> > > > +
> > > > + virtnet_free_old_xmit(sq, true, &bytes, &packets);
> > > > +
> > > > + sent = virtnet_xsk_xmit_batch(sq, pool, budget, &kicks);
> > > > +
> > > > + virtnet_xsk_check_queue(sq);
> > > > +
> > > > + u64_stats_update_begin(&sq->stats.syncp);
> > > > + u64_stats_add(&sq->stats.packets, packets);
> > > > + u64_stats_add(&sq->stats.bytes, bytes);
> > > > + u64_stats_add(&sq->stats.kicks, kicks);
> > > > + u64_stats_add(&sq->stats.xdp_tx, sent);
> > > > + u64_stats_update_end(&sq->stats.syncp);
> > > > +
> > > > + if (xsk_uses_need_wakeup(pool))
> > > > + xsk_set_tx_need_wakeup(pool);
> > > > +
> > > > + return sent == budget;
> > > > +}
> > > > +
> > > > static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *rq,
> > > > struct xsk_buff_pool *pool)
> > > > {
> > > > diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> > > > index 1918285c310c..73ca8cd5308b 100644
> > > > --- a/drivers/net/virtio/xsk.h
> > > > +++ b/drivers/net/virtio/xsk.h
> > > > @@ -3,5 +3,18 @@
> > > > #ifndef __XSK_H__
> > > > #define __XSK_H__
> > > >
> > > > +#define VIRTIO_XSK_FLAG_OFFSET 4
> > > > +
> > > > +static inline void *virtnet_xsk_to_ptr(u32 len)
> > > > +{
> > > > + unsigned long p;
> > > > +
> > > > + p = len << VIRTIO_XSK_FLAG_OFFSET;
> > > > +
> > > > + return (void *)(p | VIRTIO_XSK_FLAG);
> > > > +}
> > > > +
> > > > int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> > > > +bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > > > + int budget);
> > > > #endif
> > > > --
> > > > 2.32.0.3.g01195cf9f
> > >
>
>
^ permalink raw reply [flat|nested] 61+ messages in thread
* [PATCH net-next v2 13/21] virtio_net: xsk: tx: support wakeup
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (11 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 12/21] virtio_net: xsk: tx: support tx Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 14/21] virtio_net: xsk: tx: virtnet_free_old_xmit() distinguishes xsk buffer Xuan Zhuo
` (9 subsequent siblings)
22 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
xsk wakeup is used to trigger the logic for xsk xmit by xsk framework or
user.
Virtio-net does not support to actively generate an interruption, so it
tries to trigger tx NAPI on the local cpu.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 20 ++++++--------------
drivers/net/virtio/virtio_net.h | 9 +++++++++
drivers/net/virtio/xsk.c | 23 +++++++++++++++++++++++
drivers/net/virtio/xsk.h | 1 +
4 files changed, 39 insertions(+), 14 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index ff6bc764089d..6a5e74c482f3 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -233,15 +233,6 @@ static void disable_delayed_refill(struct virtnet_info *vi)
spin_unlock_bh(&vi->refill_lock);
}
-static void virtqueue_napi_schedule(struct napi_struct *napi,
- struct virtqueue *vq)
-{
- if (napi_schedule_prep(napi)) {
- virtqueue_disable_cb(vq);
- __napi_schedule(napi);
- }
-}
-
static void virtqueue_napi_complete(struct napi_struct *napi,
struct virtqueue *vq, int processed)
{
@@ -250,7 +241,7 @@ static void virtqueue_napi_complete(struct napi_struct *napi,
opaque = virtqueue_enable_cb_prepare(vq);
if (napi_complete_done(napi, processed)) {
if (unlikely(virtqueue_poll(vq, opaque)))
- virtqueue_napi_schedule(napi, vq);
+ virtnet_vq_napi_schedule(napi, vq);
} else {
virtqueue_disable_cb(vq);
}
@@ -265,7 +256,7 @@ static void skb_xmit_done(struct virtqueue *vq)
virtqueue_disable_cb(vq);
if (napi->weight)
- virtqueue_napi_schedule(napi, vq);
+ virtnet_vq_napi_schedule(napi, vq);
else
/* We were probably waiting for more output buffers. */
netif_wake_subqueue(vi->dev, vq2txq(vq));
@@ -667,7 +658,7 @@ static void check_sq_full_and_disable(struct virtnet_info *vi,
netif_stop_subqueue(dev, qnum);
if (use_napi) {
if (unlikely(!virtqueue_enable_cb_delayed(sq->vq)))
- virtqueue_napi_schedule(&sq->napi, sq->vq);
+ virtnet_vq_napi_schedule(&sq->napi, sq->vq);
} else if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) {
/* More just got used, free them then recheck. */
free_old_xmit(sq, false);
@@ -1834,7 +1825,7 @@ static void skb_recv_done(struct virtqueue *rvq)
struct virtnet_info *vi = rvq->vdev->priv;
struct virtnet_rq *rq = &vi->rq[vq2rxq(rvq)];
- virtqueue_napi_schedule(&rq->napi, rvq);
+ virtnet_vq_napi_schedule(&rq->napi, rvq);
}
static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi)
@@ -1846,7 +1837,7 @@ static void virtnet_napi_enable(struct virtqueue *vq, struct napi_struct *napi)
* Call local_bh_enable after to trigger softIRQ processing.
*/
local_bh_disable();
- virtqueue_napi_schedule(napi, vq);
+ virtnet_vq_napi_schedule(napi, vq);
local_bh_enable();
}
@@ -3818,6 +3809,7 @@ static const struct net_device_ops virtnet_netdev = {
.ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid,
.ndo_bpf = virtnet_xdp,
.ndo_xdp_xmit = virtnet_xdp_xmit,
+ .ndo_xsk_wakeup = virtnet_xsk_wakeup,
.ndo_features_check = passthru_features_check,
.ndo_get_phys_port_name = virtnet_get_phys_port_name,
.ndo_set_features = virtnet_set_features,
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index 1c21af47e13c..a431a2c1ee47 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -302,6 +302,15 @@ static inline bool virtnet_is_xdp_raw_buffer_queue(struct virtnet_info *vi, int
return false;
}
+static inline void virtnet_vq_napi_schedule(struct napi_struct *napi,
+ struct virtqueue *vq)
+{
+ if (napi_schedule_prep(napi)) {
+ virtqueue_disable_cb(vq);
+ __napi_schedule(napi);
+ }
+}
+
void virtnet_rx_pause(struct virtnet_info *vi, struct virtnet_rq *rq);
void virtnet_rx_resume(struct virtnet_info *vi, struct virtnet_rq *rq);
void virtnet_tx_pause(struct virtnet_info *vi, struct virtnet_sq *sq);
diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
index caa448308232..ea5804ddd44e 100644
--- a/drivers/net/virtio/xsk.c
+++ b/drivers/net/virtio/xsk.c
@@ -117,6 +117,29 @@ bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
return sent == budget;
}
+int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag)
+{
+ struct virtnet_info *vi = netdev_priv(dev);
+ struct virtnet_sq *sq;
+
+ if (!netif_running(dev))
+ return -ENETDOWN;
+
+ if (qid >= vi->curr_queue_pairs)
+ return -EINVAL;
+
+ sq = &vi->sq[qid];
+
+ if (napi_if_scheduled_mark_missed(&sq->napi))
+ return 0;
+
+ local_bh_disable();
+ virtnet_vq_napi_schedule(&sq->napi, sq->vq);
+ local_bh_enable();
+
+ return 0;
+}
+
static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *rq,
struct xsk_buff_pool *pool)
{
diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
index 73ca8cd5308b..1bd19dcda649 100644
--- a/drivers/net/virtio/xsk.h
+++ b/drivers/net/virtio/xsk.h
@@ -17,4 +17,5 @@ static inline void *virtnet_xsk_to_ptr(u32 len)
int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
int budget);
+int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag);
#endif
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* [PATCH net-next v2 14/21] virtio_net: xsk: tx: virtnet_free_old_xmit() distinguishes xsk buffer
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (12 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 13/21] virtio_net: xsk: tx: support wakeup Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-09 11:11 ` Michael S. Tsirkin
2023-11-07 3:12 ` [PATCH net-next v2 15/21] virtio_net: xsk: tx: virtnet_sq_free_unused_buf() check " Xuan Zhuo
` (8 subsequent siblings)
22 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
virtnet_free_old_xmit distinguishes three type ptr(skb, xdp frame, xsk
buffer) by the last bits of the pointer.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/virtio_net.h | 18 ++++++++++++++++--
drivers/net/virtio/xsk.h | 5 +++++
2 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index a431a2c1ee47..a13d6d301fdb 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -225,6 +225,11 @@ struct virtnet_info {
struct failover *failover;
};
+static inline bool virtnet_is_skb_ptr(void *ptr)
+{
+ return !((unsigned long)ptr & VIRTIO_XMIT_DATA_MASK);
+}
+
static inline bool virtnet_is_xdp_frame(void *ptr)
{
return (unsigned long)ptr & VIRTIO_XDP_FLAG;
@@ -235,6 +240,8 @@ static inline struct xdp_frame *virtnet_ptr_to_xdp(void *ptr)
return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
}
+static inline u32 virtnet_ptr_to_xsk(void *ptr);
+
static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
{
struct virtnet_sq_dma *next, *head;
@@ -261,11 +268,12 @@ static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
u64 *bytes, u64 *packets)
{
+ unsigned int xsknum = 0;
unsigned int len;
void *ptr;
while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
- if (!virtnet_is_xdp_frame(ptr)) {
+ if (virtnet_is_skb_ptr(ptr)) {
struct sk_buff *skb;
if (sq->do_dma)
@@ -277,7 +285,7 @@ static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
*bytes += skb->len;
napi_consume_skb(skb, in_napi);
- } else {
+ } else if (virtnet_is_xdp_frame(ptr)) {
struct xdp_frame *frame;
if (sq->do_dma)
@@ -287,9 +295,15 @@ static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
*bytes += xdp_get_frame_len(frame);
xdp_return_frame(frame);
+ } else {
+ *bytes += virtnet_ptr_to_xsk(ptr);
+ ++xsknum;
}
(*packets)++;
}
+
+ if (xsknum)
+ xsk_tx_completed(sq->xsk.pool, xsknum);
}
static inline bool virtnet_is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
index 1bd19dcda649..7ebc9bda7aee 100644
--- a/drivers/net/virtio/xsk.h
+++ b/drivers/net/virtio/xsk.h
@@ -14,6 +14,11 @@ static inline void *virtnet_xsk_to_ptr(u32 len)
return (void *)(p | VIRTIO_XSK_FLAG);
}
+static inline u32 virtnet_ptr_to_xsk(void *ptr)
+{
+ return ((unsigned long)ptr) >> VIRTIO_XSK_FLAG_OFFSET;
+}
+
int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
int budget);
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 14/21] virtio_net: xsk: tx: virtnet_free_old_xmit() distinguishes xsk buffer
2023-11-07 3:12 ` [PATCH net-next v2 14/21] virtio_net: xsk: tx: virtnet_free_old_xmit() distinguishes xsk buffer Xuan Zhuo
@ 2023-11-09 11:11 ` Michael S. Tsirkin
2023-11-09 11:16 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Michael S. Tsirkin @ 2023-11-09 11:11 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Tue, Nov 07, 2023 at 11:12:20AM +0800, Xuan Zhuo wrote:
> virtnet_free_old_xmit distinguishes three type ptr(skb, xdp frame, xsk
> buffer) by the last bits of the pointer.
>
> Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> ---
> drivers/net/virtio/virtio_net.h | 18 ++++++++++++++++--
> drivers/net/virtio/xsk.h | 5 +++++
> 2 files changed, 21 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> index a431a2c1ee47..a13d6d301fdb 100644
> --- a/drivers/net/virtio/virtio_net.h
> +++ b/drivers/net/virtio/virtio_net.h
> @@ -225,6 +225,11 @@ struct virtnet_info {
> struct failover *failover;
> };
>
> +static inline bool virtnet_is_skb_ptr(void *ptr)
> +{
> + return !((unsigned long)ptr & VIRTIO_XMIT_DATA_MASK);
> +}
> +
> static inline bool virtnet_is_xdp_frame(void *ptr)
> {
> return (unsigned long)ptr & VIRTIO_XDP_FLAG;
> @@ -235,6 +240,8 @@ static inline struct xdp_frame *virtnet_ptr_to_xdp(void *ptr)
> return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
> }
>
> +static inline u32 virtnet_ptr_to_xsk(void *ptr);
> +
I don't understand why you need this here.
> static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> {
> struct virtnet_sq_dma *next, *head;
> @@ -261,11 +268,12 @@ static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> u64 *bytes, u64 *packets)
> {
> + unsigned int xsknum = 0;
> unsigned int len;
> void *ptr;
>
> while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
> - if (!virtnet_is_xdp_frame(ptr)) {
> + if (virtnet_is_skb_ptr(ptr)) {
> struct sk_buff *skb;
>
> if (sq->do_dma)
> @@ -277,7 +285,7 @@ static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
>
> *bytes += skb->len;
> napi_consume_skb(skb, in_napi);
> - } else {
> + } else if (virtnet_is_xdp_frame(ptr)) {
> struct xdp_frame *frame;
>
> if (sq->do_dma)
> @@ -287,9 +295,15 @@ static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
>
> *bytes += xdp_get_frame_len(frame);
> xdp_return_frame(frame);
> + } else {
> + *bytes += virtnet_ptr_to_xsk(ptr);
> + ++xsknum;
> }
> (*packets)++;
> }
> +
> + if (xsknum)
> + xsk_tx_completed(sq->xsk.pool, xsknum);
> }
>
> static inline bool virtnet_is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
> diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> index 1bd19dcda649..7ebc9bda7aee 100644
> --- a/drivers/net/virtio/xsk.h
> +++ b/drivers/net/virtio/xsk.h
> @@ -14,6 +14,11 @@ static inline void *virtnet_xsk_to_ptr(u32 len)
> return (void *)(p | VIRTIO_XSK_FLAG);
> }
>
> +static inline u32 virtnet_ptr_to_xsk(void *ptr)
> +{
> + return ((unsigned long)ptr) >> VIRTIO_XSK_FLAG_OFFSET;
> +}
> +
> int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> int budget);
> --
> 2.32.0.3.g01195cf9f
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 14/21] virtio_net: xsk: tx: virtnet_free_old_xmit() distinguishes xsk buffer
2023-11-09 11:11 ` Michael S. Tsirkin
@ 2023-11-09 11:16 ` Xuan Zhuo
2023-11-09 11:59 ` Michael S. Tsirkin
0 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-09 11:16 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Thu, 9 Nov 2023 06:11:49 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> On Tue, Nov 07, 2023 at 11:12:20AM +0800, Xuan Zhuo wrote:
> > virtnet_free_old_xmit distinguishes three type ptr(skb, xdp frame, xsk
> > buffer) by the last bits of the pointer.
> >
> > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > ---
> > drivers/net/virtio/virtio_net.h | 18 ++++++++++++++++--
> > drivers/net/virtio/xsk.h | 5 +++++
> > 2 files changed, 21 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > index a431a2c1ee47..a13d6d301fdb 100644
> > --- a/drivers/net/virtio/virtio_net.h
> > +++ b/drivers/net/virtio/virtio_net.h
> > @@ -225,6 +225,11 @@ struct virtnet_info {
> > struct failover *failover;
> > };
> >
> > +static inline bool virtnet_is_skb_ptr(void *ptr)
> > +{
> > + return !((unsigned long)ptr & VIRTIO_XMIT_DATA_MASK);
> > +}
> > +
> > static inline bool virtnet_is_xdp_frame(void *ptr)
> > {
> > return (unsigned long)ptr & VIRTIO_XDP_FLAG;
> > @@ -235,6 +240,8 @@ static inline struct xdp_frame *virtnet_ptr_to_xdp(void *ptr)
> > return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
> > }
> >
> > +static inline u32 virtnet_ptr_to_xsk(void *ptr);
> > +
>
> I don't understand why you need this here.
The below function virtnet_free_old_xmit needs this.
Thanks.
>
>
> > static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> > {
> > struct virtnet_sq_dma *next, *head;
> > @@ -261,11 +268,12 @@ static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> > static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > u64 *bytes, u64 *packets)
> > {
> > + unsigned int xsknum = 0;
> > unsigned int len;
> > void *ptr;
> >
> > while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
> > - if (!virtnet_is_xdp_frame(ptr)) {
> > + if (virtnet_is_skb_ptr(ptr)) {
> > struct sk_buff *skb;
> >
> > if (sq->do_dma)
> > @@ -277,7 +285,7 @@ static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> >
> > *bytes += skb->len;
> > napi_consume_skb(skb, in_napi);
> > - } else {
> > + } else if (virtnet_is_xdp_frame(ptr)) {
> > struct xdp_frame *frame;
> >
> > if (sq->do_dma)
> > @@ -287,9 +295,15 @@ static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> >
> > *bytes += xdp_get_frame_len(frame);
> > xdp_return_frame(frame);
> > + } else {
> > + *bytes += virtnet_ptr_to_xsk(ptr);
> > + ++xsknum;
> > }
> > (*packets)++;
> > }
> > +
> > + if (xsknum)
> > + xsk_tx_completed(sq->xsk.pool, xsknum);
> > }
> >
> > static inline bool virtnet_is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
> > diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> > index 1bd19dcda649..7ebc9bda7aee 100644
> > --- a/drivers/net/virtio/xsk.h
> > +++ b/drivers/net/virtio/xsk.h
> > @@ -14,6 +14,11 @@ static inline void *virtnet_xsk_to_ptr(u32 len)
> > return (void *)(p | VIRTIO_XSK_FLAG);
> > }
> >
> > +static inline u32 virtnet_ptr_to_xsk(void *ptr)
> > +{
> > + return ((unsigned long)ptr) >> VIRTIO_XSK_FLAG_OFFSET;
> > +}
> > +
> > int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> > bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > int budget);
> > --
> > 2.32.0.3.g01195cf9f
>
>
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 14/21] virtio_net: xsk: tx: virtnet_free_old_xmit() distinguishes xsk buffer
2023-11-09 11:16 ` Xuan Zhuo
@ 2023-11-09 11:59 ` Michael S. Tsirkin
2023-11-10 1:44 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Michael S. Tsirkin @ 2023-11-09 11:59 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Thu, Nov 09, 2023 at 07:16:08PM +0800, Xuan Zhuo wrote:
> On Thu, 9 Nov 2023 06:11:49 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > On Tue, Nov 07, 2023 at 11:12:20AM +0800, Xuan Zhuo wrote:
> > > virtnet_free_old_xmit distinguishes three type ptr(skb, xdp frame, xsk
> > > buffer) by the last bits of the pointer.
> > >
> > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > ---
> > > drivers/net/virtio/virtio_net.h | 18 ++++++++++++++++--
> > > drivers/net/virtio/xsk.h | 5 +++++
> > > 2 files changed, 21 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > index a431a2c1ee47..a13d6d301fdb 100644
> > > --- a/drivers/net/virtio/virtio_net.h
> > > +++ b/drivers/net/virtio/virtio_net.h
> > > @@ -225,6 +225,11 @@ struct virtnet_info {
> > > struct failover *failover;
> > > };
> > >
> > > +static inline bool virtnet_is_skb_ptr(void *ptr)
> > > +{
> > > + return !((unsigned long)ptr & VIRTIO_XMIT_DATA_MASK);
> > > +}
> > > +
> > > static inline bool virtnet_is_xdp_frame(void *ptr)
> > > {
> > > return (unsigned long)ptr & VIRTIO_XDP_FLAG;
> > > @@ -235,6 +240,8 @@ static inline struct xdp_frame *virtnet_ptr_to_xdp(void *ptr)
> > > return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
> > > }
> > >
> > > +static inline u32 virtnet_ptr_to_xsk(void *ptr);
> > > +
> >
> > I don't understand why you need this here.
>
> The below function virtnet_free_old_xmit needs this.
>
> Thanks.
I don't understand why is virtnet_free_old_xmit inline, either.
> >
> >
> > > static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> > > {
> > > struct virtnet_sq_dma *next, *head;
> > > @@ -261,11 +268,12 @@ static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> > > static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > > u64 *bytes, u64 *packets)
> > > {
> > > + unsigned int xsknum = 0;
> > > unsigned int len;
> > > void *ptr;
> > >
> > > while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
> > > - if (!virtnet_is_xdp_frame(ptr)) {
> > > + if (virtnet_is_skb_ptr(ptr)) {
> > > struct sk_buff *skb;
> > >
> > > if (sq->do_dma)
> > > @@ -277,7 +285,7 @@ static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > >
> > > *bytes += skb->len;
> > > napi_consume_skb(skb, in_napi);
> > > - } else {
> > > + } else if (virtnet_is_xdp_frame(ptr)) {
> > > struct xdp_frame *frame;
> > >
> > > if (sq->do_dma)
> > > @@ -287,9 +295,15 @@ static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > >
> > > *bytes += xdp_get_frame_len(frame);
> > > xdp_return_frame(frame);
> > > + } else {
> > > + *bytes += virtnet_ptr_to_xsk(ptr);
> > > + ++xsknum;
> > > }
> > > (*packets)++;
> > > }
> > > +
> > > + if (xsknum)
> > > + xsk_tx_completed(sq->xsk.pool, xsknum);
> > > }
> > >
> > > static inline bool virtnet_is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
> > > diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> > > index 1bd19dcda649..7ebc9bda7aee 100644
> > > --- a/drivers/net/virtio/xsk.h
> > > +++ b/drivers/net/virtio/xsk.h
> > > @@ -14,6 +14,11 @@ static inline void *virtnet_xsk_to_ptr(u32 len)
> > > return (void *)(p | VIRTIO_XSK_FLAG);
> > > }
> > >
> > > +static inline u32 virtnet_ptr_to_xsk(void *ptr)
> > > +{
> > > + return ((unsigned long)ptr) >> VIRTIO_XSK_FLAG_OFFSET;
> > > +}
> > > +
> > > int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> > > bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > > int budget);
> > > --
> > > 2.32.0.3.g01195cf9f
> >
> >
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 14/21] virtio_net: xsk: tx: virtnet_free_old_xmit() distinguishes xsk buffer
2023-11-09 11:59 ` Michael S. Tsirkin
@ 2023-11-10 1:44 ` Xuan Zhuo
2023-11-10 5:32 ` Michael S. Tsirkin
0 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-10 1:44 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Thu, 9 Nov 2023 06:59:48 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> On Thu, Nov 09, 2023 at 07:16:08PM +0800, Xuan Zhuo wrote:
> > On Thu, 9 Nov 2023 06:11:49 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > On Tue, Nov 07, 2023 at 11:12:20AM +0800, Xuan Zhuo wrote:
> > > > virtnet_free_old_xmit distinguishes three type ptr(skb, xdp frame, xsk
> > > > buffer) by the last bits of the pointer.
> > > >
> > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > > ---
> > > > drivers/net/virtio/virtio_net.h | 18 ++++++++++++++++--
> > > > drivers/net/virtio/xsk.h | 5 +++++
> > > > 2 files changed, 21 insertions(+), 2 deletions(-)
> > > >
> > > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > > index a431a2c1ee47..a13d6d301fdb 100644
> > > > --- a/drivers/net/virtio/virtio_net.h
> > > > +++ b/drivers/net/virtio/virtio_net.h
> > > > @@ -225,6 +225,11 @@ struct virtnet_info {
> > > > struct failover *failover;
> > > > };
> > > >
> > > > +static inline bool virtnet_is_skb_ptr(void *ptr)
> > > > +{
> > > > + return !((unsigned long)ptr & VIRTIO_XMIT_DATA_MASK);
> > > > +}
> > > > +
> > > > static inline bool virtnet_is_xdp_frame(void *ptr)
> > > > {
> > > > return (unsigned long)ptr & VIRTIO_XDP_FLAG;
> > > > @@ -235,6 +240,8 @@ static inline struct xdp_frame *virtnet_ptr_to_xdp(void *ptr)
> > > > return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
> > > > }
> > > >
> > > > +static inline u32 virtnet_ptr_to_xsk(void *ptr);
> > > > +
> > >
> > > I don't understand why you need this here.
> >
> > The below function virtnet_free_old_xmit needs this.
> >
> > Thanks.
>
> I don't understand why is virtnet_free_old_xmit inline, either.
That is in the header file.
>
> > >
> > >
> > > > static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> > > > {
> > > > struct virtnet_sq_dma *next, *head;
> > > > @@ -261,11 +268,12 @@ static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> > > > static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > > > u64 *bytes, u64 *packets)
> > > > {
> > > > + unsigned int xsknum = 0;
> > > > unsigned int len;
> > > > void *ptr;
> > > >
> > > > while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
> > > > - if (!virtnet_is_xdp_frame(ptr)) {
> > > > + if (virtnet_is_skb_ptr(ptr)) {
> > > > struct sk_buff *skb;
> > > >
> > > > if (sq->do_dma)
> > > > @@ -277,7 +285,7 @@ static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > > >
> > > > *bytes += skb->len;
> > > > napi_consume_skb(skb, in_napi);
> > > > - } else {
> > > > + } else if (virtnet_is_xdp_frame(ptr)) {
> > > > struct xdp_frame *frame;
> > > >
> > > > if (sq->do_dma)
> > > > @@ -287,9 +295,15 @@ static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > > >
> > > > *bytes += xdp_get_frame_len(frame);
> > > > xdp_return_frame(frame);
> > > > + } else {
> > > > + *bytes += virtnet_ptr_to_xsk(ptr);
> > > > + ++xsknum;
> > > > }
> > > > (*packets)++;
> > > > }
> > > > +
> > > > + if (xsknum)
> > > > + xsk_tx_completed(sq->xsk.pool, xsknum);
> > > > }
> > > >
> > > > static inline bool virtnet_is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
> > > > diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> > > > index 1bd19dcda649..7ebc9bda7aee 100644
> > > > --- a/drivers/net/virtio/xsk.h
> > > > +++ b/drivers/net/virtio/xsk.h
> > > > @@ -14,6 +14,11 @@ static inline void *virtnet_xsk_to_ptr(u32 len)
> > > > return (void *)(p | VIRTIO_XSK_FLAG);
> > > > }
> > > >
> > > > +static inline u32 virtnet_ptr_to_xsk(void *ptr)
> > > > +{
> > > > + return ((unsigned long)ptr) >> VIRTIO_XSK_FLAG_OFFSET;
> > > > +}
> > > > +
> > > > int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> > > > bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > > > int budget);
> > > > --
> > > > 2.32.0.3.g01195cf9f
> > >
> > >
>
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 14/21] virtio_net: xsk: tx: virtnet_free_old_xmit() distinguishes xsk buffer
2023-11-10 1:44 ` Xuan Zhuo
@ 2023-11-10 5:32 ` Michael S. Tsirkin
2023-11-10 5:50 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Michael S. Tsirkin @ 2023-11-10 5:32 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Fri, Nov 10, 2023 at 09:44:32AM +0800, Xuan Zhuo wrote:
> On Thu, 9 Nov 2023 06:59:48 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > On Thu, Nov 09, 2023 at 07:16:08PM +0800, Xuan Zhuo wrote:
> > > On Thu, 9 Nov 2023 06:11:49 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > > On Tue, Nov 07, 2023 at 11:12:20AM +0800, Xuan Zhuo wrote:
> > > > > virtnet_free_old_xmit distinguishes three type ptr(skb, xdp frame, xsk
> > > > > buffer) by the last bits of the pointer.
> > > > >
> > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > > > ---
> > > > > drivers/net/virtio/virtio_net.h | 18 ++++++++++++++++--
> > > > > drivers/net/virtio/xsk.h | 5 +++++
> > > > > 2 files changed, 21 insertions(+), 2 deletions(-)
> > > > >
> > > > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > > > index a431a2c1ee47..a13d6d301fdb 100644
> > > > > --- a/drivers/net/virtio/virtio_net.h
> > > > > +++ b/drivers/net/virtio/virtio_net.h
> > > > > @@ -225,6 +225,11 @@ struct virtnet_info {
> > > > > struct failover *failover;
> > > > > };
> > > > >
> > > > > +static inline bool virtnet_is_skb_ptr(void *ptr)
> > > > > +{
> > > > > + return !((unsigned long)ptr & VIRTIO_XMIT_DATA_MASK);
> > > > > +}
> > > > > +
> > > > > static inline bool virtnet_is_xdp_frame(void *ptr)
> > > > > {
> > > > > return (unsigned long)ptr & VIRTIO_XDP_FLAG;
> > > > > @@ -235,6 +240,8 @@ static inline struct xdp_frame *virtnet_ptr_to_xdp(void *ptr)
> > > > > return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
> > > > > }
> > > > >
> > > > > +static inline u32 virtnet_ptr_to_xsk(void *ptr);
> > > > > +
> > > >
> > > > I don't understand why you need this here.
> > >
> > > The below function virtnet_free_old_xmit needs this.
> > >
> > > Thanks.
> >
> > I don't understand why is virtnet_free_old_xmit inline, either.
>
> That is in the header file.
>
It does not belong there.
> >
> > > >
> > > >
> > > > > static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> > > > > {
> > > > > struct virtnet_sq_dma *next, *head;
> > > > > @@ -261,11 +268,12 @@ static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> > > > > static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > > > > u64 *bytes, u64 *packets)
> > > > > {
> > > > > + unsigned int xsknum = 0;
> > > > > unsigned int len;
> > > > > void *ptr;
> > > > >
> > > > > while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
> > > > > - if (!virtnet_is_xdp_frame(ptr)) {
> > > > > + if (virtnet_is_skb_ptr(ptr)) {
> > > > > struct sk_buff *skb;
> > > > >
> > > > > if (sq->do_dma)
> > > > > @@ -277,7 +285,7 @@ static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > > > >
> > > > > *bytes += skb->len;
> > > > > napi_consume_skb(skb, in_napi);
> > > > > - } else {
> > > > > + } else if (virtnet_is_xdp_frame(ptr)) {
> > > > > struct xdp_frame *frame;
> > > > >
> > > > > if (sq->do_dma)
> > > > > @@ -287,9 +295,15 @@ static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > > > >
> > > > > *bytes += xdp_get_frame_len(frame);
> > > > > xdp_return_frame(frame);
> > > > > + } else {
> > > > > + *bytes += virtnet_ptr_to_xsk(ptr);
> > > > > + ++xsknum;
> > > > > }
> > > > > (*packets)++;
> > > > > }
> > > > > +
> > > > > + if (xsknum)
> > > > > + xsk_tx_completed(sq->xsk.pool, xsknum);
> > > > > }
> > > > >
> > > > > static inline bool virtnet_is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
> > > > > diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> > > > > index 1bd19dcda649..7ebc9bda7aee 100644
> > > > > --- a/drivers/net/virtio/xsk.h
> > > > > +++ b/drivers/net/virtio/xsk.h
> > > > > @@ -14,6 +14,11 @@ static inline void *virtnet_xsk_to_ptr(u32 len)
> > > > > return (void *)(p | VIRTIO_XSK_FLAG);
> > > > > }
> > > > >
> > > > > +static inline u32 virtnet_ptr_to_xsk(void *ptr)
> > > > > +{
> > > > > + return ((unsigned long)ptr) >> VIRTIO_XSK_FLAG_OFFSET;
> > > > > +}
> > > > > +
> > > > > int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> > > > > bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > > > > int budget);
> > > > > --
> > > > > 2.32.0.3.g01195cf9f
> > > >
> > > >
> >
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 14/21] virtio_net: xsk: tx: virtnet_free_old_xmit() distinguishes xsk buffer
2023-11-10 5:32 ` Michael S. Tsirkin
@ 2023-11-10 5:50 ` Xuan Zhuo
0 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-10 5:50 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Fri, 10 Nov 2023 00:32:50 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> On Fri, Nov 10, 2023 at 09:44:32AM +0800, Xuan Zhuo wrote:
> > On Thu, 9 Nov 2023 06:59:48 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > On Thu, Nov 09, 2023 at 07:16:08PM +0800, Xuan Zhuo wrote:
> > > > On Thu, 9 Nov 2023 06:11:49 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > > > On Tue, Nov 07, 2023 at 11:12:20AM +0800, Xuan Zhuo wrote:
> > > > > > virtnet_free_old_xmit distinguishes three type ptr(skb, xdp frame, xsk
> > > > > > buffer) by the last bits of the pointer.
> > > > > >
> > > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > > > > ---
> > > > > > drivers/net/virtio/virtio_net.h | 18 ++++++++++++++++--
> > > > > > drivers/net/virtio/xsk.h | 5 +++++
> > > > > > 2 files changed, 21 insertions(+), 2 deletions(-)
> > > > > >
> > > > > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > > > > index a431a2c1ee47..a13d6d301fdb 100644
> > > > > > --- a/drivers/net/virtio/virtio_net.h
> > > > > > +++ b/drivers/net/virtio/virtio_net.h
> > > > > > @@ -225,6 +225,11 @@ struct virtnet_info {
> > > > > > struct failover *failover;
> > > > > > };
> > > > > >
> > > > > > +static inline bool virtnet_is_skb_ptr(void *ptr)
> > > > > > +{
> > > > > > + return !((unsigned long)ptr & VIRTIO_XMIT_DATA_MASK);
> > > > > > +}
> > > > > > +
> > > > > > static inline bool virtnet_is_xdp_frame(void *ptr)
> > > > > > {
> > > > > > return (unsigned long)ptr & VIRTIO_XDP_FLAG;
> > > > > > @@ -235,6 +240,8 @@ static inline struct xdp_frame *virtnet_ptr_to_xdp(void *ptr)
> > > > > > return (struct xdp_frame *)((unsigned long)ptr & ~VIRTIO_XDP_FLAG);
> > > > > > }
> > > > > >
> > > > > > +static inline u32 virtnet_ptr_to_xsk(void *ptr);
> > > > > > +
> > > > >
> > > > > I don't understand why you need this here.
> > > >
> > > > The below function virtnet_free_old_xmit needs this.
> > > >
> > > > Thanks.
> > >
> > > I don't understand why is virtnet_free_old_xmit inline, either.
> >
> > That is in the header file.
> >
>
> It does not belong there.
Do you mean virtnet_free_old_xmit?
That will be called by xsk.c. So I move it to the header file.
Thanks.
>
>
> > >
> > > > >
> > > > >
> > > > > > static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> > > > > > {
> > > > > > struct virtnet_sq_dma *next, *head;
> > > > > > @@ -261,11 +268,12 @@ static inline void *virtnet_sq_unmap(struct virtnet_sq *sq, void *data)
> > > > > > static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > > > > > u64 *bytes, u64 *packets)
> > > > > > {
> > > > > > + unsigned int xsknum = 0;
> > > > > > unsigned int len;
> > > > > > void *ptr;
> > > > > >
> > > > > > while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) {
> > > > > > - if (!virtnet_is_xdp_frame(ptr)) {
> > > > > > + if (virtnet_is_skb_ptr(ptr)) {
> > > > > > struct sk_buff *skb;
> > > > > >
> > > > > > if (sq->do_dma)
> > > > > > @@ -277,7 +285,7 @@ static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > > > > >
> > > > > > *bytes += skb->len;
> > > > > > napi_consume_skb(skb, in_napi);
> > > > > > - } else {
> > > > > > + } else if (virtnet_is_xdp_frame(ptr)) {
> > > > > > struct xdp_frame *frame;
> > > > > >
> > > > > > if (sq->do_dma)
> > > > > > @@ -287,9 +295,15 @@ static inline void virtnet_free_old_xmit(struct virtnet_sq *sq, bool in_napi,
> > > > > >
> > > > > > *bytes += xdp_get_frame_len(frame);
> > > > > > xdp_return_frame(frame);
> > > > > > + } else {
> > > > > > + *bytes += virtnet_ptr_to_xsk(ptr);
> > > > > > + ++xsknum;
> > > > > > }
> > > > > > (*packets)++;
> > > > > > }
> > > > > > +
> > > > > > + if (xsknum)
> > > > > > + xsk_tx_completed(sq->xsk.pool, xsknum);
> > > > > > }
> > > > > >
> > > > > > static inline bool virtnet_is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q)
> > > > > > diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> > > > > > index 1bd19dcda649..7ebc9bda7aee 100644
> > > > > > --- a/drivers/net/virtio/xsk.h
> > > > > > +++ b/drivers/net/virtio/xsk.h
> > > > > > @@ -14,6 +14,11 @@ static inline void *virtnet_xsk_to_ptr(u32 len)
> > > > > > return (void *)(p | VIRTIO_XSK_FLAG);
> > > > > > }
> > > > > >
> > > > > > +static inline u32 virtnet_ptr_to_xsk(void *ptr)
> > > > > > +{
> > > > > > + return ((unsigned long)ptr) >> VIRTIO_XSK_FLAG_OFFSET;
> > > > > > +}
> > > > > > +
> > > > > > int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> > > > > > bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > > > > > int budget);
> > > > > > --
> > > > > > 2.32.0.3.g01195cf9f
> > > > >
> > > > >
> > >
>
^ permalink raw reply [flat|nested] 61+ messages in thread
* [PATCH net-next v2 15/21] virtio_net: xsk: tx: virtnet_sq_free_unused_buf() check xsk buffer
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (13 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 14/21] virtio_net: xsk: tx: virtnet_free_old_xmit() distinguishes xsk buffer Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 16/21] virtio_net: xsk: rx: introduce add_recvbuf_xsk() Xuan Zhuo
` (7 subsequent siblings)
22 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
virtnet_sq_free_unused_buf() check xsk buffer.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Jason Wang <jasowang@redhat.com>
---
drivers/net/virtio/main.c | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 6a5e74c482f3..6210a6e37396 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -3919,16 +3919,18 @@ void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf)
sq = &vi->sq[i];
- if (!virtnet_is_xdp_frame(buf)) {
+ if (virtnet_is_skb_ptr(buf)) {
if (sq->do_dma)
buf = virtnet_sq_unmap(sq, buf);
dev_kfree_skb(buf);
- } else {
+ } else if (virtnet_is_xdp_frame(buf)) {
if (sq->do_dma)
buf = virtnet_sq_unmap(sq, buf);
xdp_return_frame(virtnet_ptr_to_xdp(buf));
+ } else {
+ xsk_tx_completed(sq->xsk.pool, 1);
}
}
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* [PATCH net-next v2 16/21] virtio_net: xsk: rx: introduce add_recvbuf_xsk()
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (14 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 15/21] virtio_net: xsk: tx: virtnet_sq_free_unused_buf() check " Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-09 8:12 ` Michael S. Tsirkin
2023-11-07 3:12 ` [PATCH net-next v2 17/21] virtio_net: xsk: rx: skip dma unmap when rq is bind with AF_XDP Xuan Zhuo
` (6 subsequent siblings)
22 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
Implement the logic of filling rq with XSK buffers.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 4 ++-
drivers/net/virtio/virtio_net.h | 5 ++++
drivers/net/virtio/xsk.c | 49 ++++++++++++++++++++++++++++++++-
drivers/net/virtio/xsk.h | 2 ++
4 files changed, 58 insertions(+), 2 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 6210a6e37396..15943a22e17d 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -1798,7 +1798,9 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
bool oom;
do {
- if (vi->mergeable_rx_bufs)
+ if (rq->xsk.pool)
+ err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk.pool, gfp);
+ else if (vi->mergeable_rx_bufs)
err = add_recvbuf_mergeable(vi, rq, gfp);
else if (vi->big_packets)
err = add_recvbuf_big(vi, rq, gfp);
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index a13d6d301fdb..1242785e311e 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -140,6 +140,11 @@ struct virtnet_rq {
/* xdp rxq used by xsk */
struct xdp_rxq_info xdp_rxq;
+
+ struct xdp_buff **xsk_buffs;
+ u32 nxt_idx;
+ u32 num;
+ u32 size;
} xsk;
};
diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
index ea5804ddd44e..e737c3353212 100644
--- a/drivers/net/virtio/xsk.c
+++ b/drivers/net/virtio/xsk.c
@@ -38,6 +38,41 @@ static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
netif_stop_subqueue(dev, qnum);
}
+int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
+ struct xsk_buff_pool *pool, gfp_t gfp)
+{
+ struct xdp_buff **xsk_buffs;
+ dma_addr_t addr;
+ u32 len, i;
+ int err = 0;
+
+ xsk_buffs = rq->xsk.xsk_buffs;
+
+ if (rq->xsk.nxt_idx >= rq->xsk.num) {
+ rq->xsk.num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->xsk.size);
+ if (!rq->xsk.num)
+ return -ENOMEM;
+ rq->xsk.nxt_idx = 0;
+ }
+
+ i = rq->xsk.nxt_idx;
+
+ /* use the part of XDP_PACKET_HEADROOM as the virtnet hdr space */
+ addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len;
+ len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len;
+
+ sg_init_table(rq->sg, 1);
+ sg_fill_dma(rq->sg, addr, len);
+
+ err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, xsk_buffs[i], gfp);
+ if (err)
+ return err;
+
+ rq->xsk.nxt_idx++;
+
+ return 0;
+}
+
static int virtnet_xsk_xmit_one(struct virtnet_sq *sq,
struct xsk_buff_pool *pool,
struct xdp_desc *desc)
@@ -213,7 +248,7 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
struct virtnet_sq *sq;
struct device *dma_dev;
dma_addr_t hdr_dma;
- int err;
+ int err, size;
/* In big_packets mode, xdp cannot work, so there is no need to
* initialize xsk of rq.
@@ -249,6 +284,16 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
if (!dma_dev)
return -EPERM;
+ size = virtqueue_get_vring_size(rq->vq);
+
+ rq->xsk.xsk_buffs = kcalloc(size, sizeof(*rq->xsk.xsk_buffs), GFP_KERNEL);
+ if (!rq->xsk.xsk_buffs)
+ return -ENOMEM;
+
+ rq->xsk.size = size;
+ rq->xsk.nxt_idx = 0;
+ rq->xsk.num = 0;
+
hdr_dma = dma_map_single(dma_dev, &xsk_hdr, vi->hdr_len, DMA_TO_DEVICE);
if (dma_mapping_error(dma_dev, hdr_dma))
return -ENOMEM;
@@ -307,6 +352,8 @@ static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid)
dma_unmap_single(dma_dev, sq->xsk.hdr_dma_address, vi->hdr_len, DMA_TO_DEVICE);
+ kfree(rq->xsk.xsk_buffs);
+
return err1 | err2;
}
diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
index 7ebc9bda7aee..bef41a3f954e 100644
--- a/drivers/net/virtio/xsk.h
+++ b/drivers/net/virtio/xsk.h
@@ -23,4 +23,6 @@ int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
int budget);
int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag);
+int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
+ struct xsk_buff_pool *pool, gfp_t gfp);
#endif
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 16/21] virtio_net: xsk: rx: introduce add_recvbuf_xsk()
2023-11-07 3:12 ` [PATCH net-next v2 16/21] virtio_net: xsk: rx: introduce add_recvbuf_xsk() Xuan Zhuo
@ 2023-11-09 8:12 ` Michael S. Tsirkin
2023-11-09 11:11 ` Xuan Zhuo
2023-11-10 3:04 ` Xuan Zhuo
0 siblings, 2 replies; 61+ messages in thread
From: Michael S. Tsirkin @ 2023-11-09 8:12 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Tue, Nov 07, 2023 at 11:12:22AM +0800, Xuan Zhuo wrote:
> Implement the logic of filling rq with XSK buffers.
>
> Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> ---
> drivers/net/virtio/main.c | 4 ++-
> drivers/net/virtio/virtio_net.h | 5 ++++
> drivers/net/virtio/xsk.c | 49 ++++++++++++++++++++++++++++++++-
> drivers/net/virtio/xsk.h | 2 ++
> 4 files changed, 58 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> index 6210a6e37396..15943a22e17d 100644
> --- a/drivers/net/virtio/main.c
> +++ b/drivers/net/virtio/main.c
> @@ -1798,7 +1798,9 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
> bool oom;
>
> do {
> - if (vi->mergeable_rx_bufs)
> + if (rq->xsk.pool)
> + err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk.pool, gfp);
> + else if (vi->mergeable_rx_bufs)
> err = add_recvbuf_mergeable(vi, rq, gfp);
> else if (vi->big_packets)
> err = add_recvbuf_big(vi, rq, gfp);
I'm not sure I understand. How does this handle mergeable flag still being set?
> diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> index a13d6d301fdb..1242785e311e 100644
> --- a/drivers/net/virtio/virtio_net.h
> +++ b/drivers/net/virtio/virtio_net.h
> @@ -140,6 +140,11 @@ struct virtnet_rq {
>
> /* xdp rxq used by xsk */
> struct xdp_rxq_info xdp_rxq;
> +
> + struct xdp_buff **xsk_buffs;
> + u32 nxt_idx;
> + u32 num;
> + u32 size;
> } xsk;
> };
>
> diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> index ea5804ddd44e..e737c3353212 100644
> --- a/drivers/net/virtio/xsk.c
> +++ b/drivers/net/virtio/xsk.c
> @@ -38,6 +38,41 @@ static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> netif_stop_subqueue(dev, qnum);
> }
>
> +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> + struct xsk_buff_pool *pool, gfp_t gfp)
> +{
> + struct xdp_buff **xsk_buffs;
> + dma_addr_t addr;
> + u32 len, i;
> + int err = 0;
> +
> + xsk_buffs = rq->xsk.xsk_buffs;
> +
> + if (rq->xsk.nxt_idx >= rq->xsk.num) {
> + rq->xsk.num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->xsk.size);
> + if (!rq->xsk.num)
> + return -ENOMEM;
> + rq->xsk.nxt_idx = 0;
> + }
Another manually rolled linked list implementation.
Please, don't.
> +
> + i = rq->xsk.nxt_idx;
> +
> + /* use the part of XDP_PACKET_HEADROOM as the virtnet hdr space */
> + addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len;
> + len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len;
> +
> + sg_init_table(rq->sg, 1);
> + sg_fill_dma(rq->sg, addr, len);
> +
> + err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, xsk_buffs[i], gfp);
> + if (err)
> + return err;
> +
> + rq->xsk.nxt_idx++;
> +
> + return 0;
> +}
> +
> static int virtnet_xsk_xmit_one(struct virtnet_sq *sq,
> struct xsk_buff_pool *pool,
> struct xdp_desc *desc)
> @@ -213,7 +248,7 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
> struct virtnet_sq *sq;
> struct device *dma_dev;
> dma_addr_t hdr_dma;
> - int err;
> + int err, size;
>
> /* In big_packets mode, xdp cannot work, so there is no need to
> * initialize xsk of rq.
> @@ -249,6 +284,16 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
> if (!dma_dev)
> return -EPERM;
>
> + size = virtqueue_get_vring_size(rq->vq);
> +
> + rq->xsk.xsk_buffs = kcalloc(size, sizeof(*rq->xsk.xsk_buffs), GFP_KERNEL);
> + if (!rq->xsk.xsk_buffs)
> + return -ENOMEM;
> +
> + rq->xsk.size = size;
> + rq->xsk.nxt_idx = 0;
> + rq->xsk.num = 0;
> +
> hdr_dma = dma_map_single(dma_dev, &xsk_hdr, vi->hdr_len, DMA_TO_DEVICE);
> if (dma_mapping_error(dma_dev, hdr_dma))
> return -ENOMEM;
> @@ -307,6 +352,8 @@ static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid)
>
> dma_unmap_single(dma_dev, sq->xsk.hdr_dma_address, vi->hdr_len, DMA_TO_DEVICE);
>
> + kfree(rq->xsk.xsk_buffs);
> +
> return err1 | err2;
> }
>
> diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> index 7ebc9bda7aee..bef41a3f954e 100644
> --- a/drivers/net/virtio/xsk.h
> +++ b/drivers/net/virtio/xsk.h
> @@ -23,4 +23,6 @@ int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> int budget);
> int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag);
> +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> + struct xsk_buff_pool *pool, gfp_t gfp);
> #endif
> --
> 2.32.0.3.g01195cf9f
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 16/21] virtio_net: xsk: rx: introduce add_recvbuf_xsk()
2023-11-09 8:12 ` Michael S. Tsirkin
@ 2023-11-09 11:11 ` Xuan Zhuo
2023-11-09 16:26 ` Maciej Fijalkowski
2023-11-10 3:04 ` Xuan Zhuo
1 sibling, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-09 11:11 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Thu, 9 Nov 2023 03:12:27 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> On Tue, Nov 07, 2023 at 11:12:22AM +0800, Xuan Zhuo wrote:
> > Implement the logic of filling rq with XSK buffers.
> >
> > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > ---
> > drivers/net/virtio/main.c | 4 ++-
> > drivers/net/virtio/virtio_net.h | 5 ++++
> > drivers/net/virtio/xsk.c | 49 ++++++++++++++++++++++++++++++++-
> > drivers/net/virtio/xsk.h | 2 ++
> > 4 files changed, 58 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > index 6210a6e37396..15943a22e17d 100644
> > --- a/drivers/net/virtio/main.c
> > +++ b/drivers/net/virtio/main.c
> > @@ -1798,7 +1798,9 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
> > bool oom;
> >
> > do {
> > - if (vi->mergeable_rx_bufs)
> > + if (rq->xsk.pool)
> > + err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk.pool, gfp);
> > + else if (vi->mergeable_rx_bufs)
> > err = add_recvbuf_mergeable(vi, rq, gfp);
> > else if (vi->big_packets)
> > err = add_recvbuf_big(vi, rq, gfp);
>
> I'm not sure I understand. How does this handle mergeable flag still being set?
You has the same question as Jason.
So I think maybe I should put the handle into the
add_recvbuf_mergeable and add_recvbuf_small.
Let me think about this.
>
>
> > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > index a13d6d301fdb..1242785e311e 100644
> > --- a/drivers/net/virtio/virtio_net.h
> > +++ b/drivers/net/virtio/virtio_net.h
> > @@ -140,6 +140,11 @@ struct virtnet_rq {
> >
> > /* xdp rxq used by xsk */
> > struct xdp_rxq_info xdp_rxq;
> > +
> > + struct xdp_buff **xsk_buffs;
> > + u32 nxt_idx;
> > + u32 num;
> > + u32 size;
> > } xsk;
> > };
> >
> > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > index ea5804ddd44e..e737c3353212 100644
> > --- a/drivers/net/virtio/xsk.c
> > +++ b/drivers/net/virtio/xsk.c
> > @@ -38,6 +38,41 @@ static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> > netif_stop_subqueue(dev, qnum);
> > }
> >
> > +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> > + struct xsk_buff_pool *pool, gfp_t gfp)
> > +{
> > + struct xdp_buff **xsk_buffs;
> > + dma_addr_t addr;
> > + u32 len, i;
> > + int err = 0;
> > +
> > + xsk_buffs = rq->xsk.xsk_buffs;
> > +
> > + if (rq->xsk.nxt_idx >= rq->xsk.num) {
> > + rq->xsk.num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->xsk.size);
> > + if (!rq->xsk.num)
> > + return -ENOMEM;
> > + rq->xsk.nxt_idx = 0;
> > + }
>
> Another manually rolled linked list implementation.
> Please, don't.
The array is for speedup.
xsk_buff_alloc_batch will return many xsk_buff that will be more efficient than
the xsk_buff_alloc.
Thanks.
>
>
> > +
> > + i = rq->xsk.nxt_idx;
> > +
> > + /* use the part of XDP_PACKET_HEADROOM as the virtnet hdr space */
> > + addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len;
> > + len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len;
> > +
> > + sg_init_table(rq->sg, 1);
> > + sg_fill_dma(rq->sg, addr, len);
> > +
> > + err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, xsk_buffs[i], gfp);
> > + if (err)
> > + return err;
> > +
> > + rq->xsk.nxt_idx++;
> > +
> > + return 0;
> > +}
> > +
> > static int virtnet_xsk_xmit_one(struct virtnet_sq *sq,
> > struct xsk_buff_pool *pool,
> > struct xdp_desc *desc)
> > @@ -213,7 +248,7 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
> > struct virtnet_sq *sq;
> > struct device *dma_dev;
> > dma_addr_t hdr_dma;
> > - int err;
> > + int err, size;
> >
> > /* In big_packets mode, xdp cannot work, so there is no need to
> > * initialize xsk of rq.
> > @@ -249,6 +284,16 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
> > if (!dma_dev)
> > return -EPERM;
> >
> > + size = virtqueue_get_vring_size(rq->vq);
> > +
> > + rq->xsk.xsk_buffs = kcalloc(size, sizeof(*rq->xsk.xsk_buffs), GFP_KERNEL);
> > + if (!rq->xsk.xsk_buffs)
> > + return -ENOMEM;
> > +
> > + rq->xsk.size = size;
> > + rq->xsk.nxt_idx = 0;
> > + rq->xsk.num = 0;
> > +
> > hdr_dma = dma_map_single(dma_dev, &xsk_hdr, vi->hdr_len, DMA_TO_DEVICE);
> > if (dma_mapping_error(dma_dev, hdr_dma))
> > return -ENOMEM;
> > @@ -307,6 +352,8 @@ static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid)
> >
> > dma_unmap_single(dma_dev, sq->xsk.hdr_dma_address, vi->hdr_len, DMA_TO_DEVICE);
> >
> > + kfree(rq->xsk.xsk_buffs);
> > +
> > return err1 | err2;
> > }
> >
> > diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> > index 7ebc9bda7aee..bef41a3f954e 100644
> > --- a/drivers/net/virtio/xsk.h
> > +++ b/drivers/net/virtio/xsk.h
> > @@ -23,4 +23,6 @@ int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> > bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > int budget);
> > int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag);
> > +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> > + struct xsk_buff_pool *pool, gfp_t gfp);
> > #endif
> > --
> > 2.32.0.3.g01195cf9f
>
>
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 16/21] virtio_net: xsk: rx: introduce add_recvbuf_xsk()
2023-11-09 11:11 ` Xuan Zhuo
@ 2023-11-09 16:26 ` Maciej Fijalkowski
2023-11-10 2:38 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Maciej Fijalkowski @ 2023-11-09 16:26 UTC (permalink / raw)
To: Xuan Zhuo
Cc: Michael S. Tsirkin, netdev, David S. Miller, Eric Dumazet,
Jakub Kicinski, Paolo Abeni, Jason Wang, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Thu, Nov 09, 2023 at 07:11:46PM +0800, Xuan Zhuo wrote:
> On Thu, 9 Nov 2023 03:12:27 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > On Tue, Nov 07, 2023 at 11:12:22AM +0800, Xuan Zhuo wrote:
> > > Implement the logic of filling rq with XSK buffers.
> > >
> > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > ---
> > > drivers/net/virtio/main.c | 4 ++-
> > > drivers/net/virtio/virtio_net.h | 5 ++++
> > > drivers/net/virtio/xsk.c | 49 ++++++++++++++++++++++++++++++++-
> > > drivers/net/virtio/xsk.h | 2 ++
> > > 4 files changed, 58 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > > index 6210a6e37396..15943a22e17d 100644
> > > --- a/drivers/net/virtio/main.c
> > > +++ b/drivers/net/virtio/main.c
> > > @@ -1798,7 +1798,9 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
> > > bool oom;
> > >
> > > do {
> > > - if (vi->mergeable_rx_bufs)
> > > + if (rq->xsk.pool)
> > > + err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk.pool, gfp);
> > > + else if (vi->mergeable_rx_bufs)
> > > err = add_recvbuf_mergeable(vi, rq, gfp);
> > > else if (vi->big_packets)
> > > err = add_recvbuf_big(vi, rq, gfp);
> >
> > I'm not sure I understand. How does this handle mergeable flag still being set?
>
>
> You has the same question as Jason.
>
> So I think maybe I should put the handle into the
> add_recvbuf_mergeable and add_recvbuf_small.
>
> Let me think about this.
>
>
> >
> >
> > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > index a13d6d301fdb..1242785e311e 100644
> > > --- a/drivers/net/virtio/virtio_net.h
> > > +++ b/drivers/net/virtio/virtio_net.h
> > > @@ -140,6 +140,11 @@ struct virtnet_rq {
> > >
> > > /* xdp rxq used by xsk */
> > > struct xdp_rxq_info xdp_rxq;
> > > +
> > > + struct xdp_buff **xsk_buffs;
> > > + u32 nxt_idx;
> > > + u32 num;
> > > + u32 size;
> > > } xsk;
> > > };
> > >
> > > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > > index ea5804ddd44e..e737c3353212 100644
> > > --- a/drivers/net/virtio/xsk.c
> > > +++ b/drivers/net/virtio/xsk.c
> > > @@ -38,6 +38,41 @@ static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> > > netif_stop_subqueue(dev, qnum);
> > > }
> > >
> > > +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> > > + struct xsk_buff_pool *pool, gfp_t gfp)
> > > +{
> > > + struct xdp_buff **xsk_buffs;
> > > + dma_addr_t addr;
> > > + u32 len, i;
> > > + int err = 0;
> > > +
> > > + xsk_buffs = rq->xsk.xsk_buffs;
> > > +
> > > + if (rq->xsk.nxt_idx >= rq->xsk.num) {
> > > + rq->xsk.num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->xsk.size);
> > > + if (!rq->xsk.num)
> > > + return -ENOMEM;
> > > + rq->xsk.nxt_idx = 0;
> > > + }
> >
> > Another manually rolled linked list implementation.
> > Please, don't.
>
>
> The array is for speedup.
>
> xsk_buff_alloc_batch will return many xsk_buff that will be more efficient than
> the xsk_buff_alloc.
But your sg list just contains a single entry?
I think that you have to walk through the xsk_buffs array, retrieve dma
addrs from there and have sg list sized to the value
xsk_buff_alloc_batch() returned.
I don't think your logic based on nxt_idx is needed. Please take a look
how other drivers use xsk_buff_alloc_batch().
I don't see callsites of virtnet_add_recvbuf_xsk() though.
>
> Thanks.
>
> >
> >
> > > +
> > > + i = rq->xsk.nxt_idx;
> > > +
> > > + /* use the part of XDP_PACKET_HEADROOM as the virtnet hdr space */
> > > + addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len;
> > > + len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len;
> > > +
> > > + sg_init_table(rq->sg, 1);
> > > + sg_fill_dma(rq->sg, addr, len);
> > > +
> > > + err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, xsk_buffs[i], gfp);
> > > + if (err)
> > > + return err;
> > > +
> > > + rq->xsk.nxt_idx++;
> > > +
> > > + return 0;
> > > +}
> > > +
> > > static int virtnet_xsk_xmit_one(struct virtnet_sq *sq,
> > > struct xsk_buff_pool *pool,
> > > struct xdp_desc *desc)
> > > @@ -213,7 +248,7 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
> > > struct virtnet_sq *sq;
> > > struct device *dma_dev;
> > > dma_addr_t hdr_dma;
> > > - int err;
> > > + int err, size;
> > >
> > > /* In big_packets mode, xdp cannot work, so there is no need to
> > > * initialize xsk of rq.
> > > @@ -249,6 +284,16 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
> > > if (!dma_dev)
> > > return -EPERM;
> > >
> > > + size = virtqueue_get_vring_size(rq->vq);
> > > +
> > > + rq->xsk.xsk_buffs = kcalloc(size, sizeof(*rq->xsk.xsk_buffs), GFP_KERNEL);
> > > + if (!rq->xsk.xsk_buffs)
> > > + return -ENOMEM;
> > > +
> > > + rq->xsk.size = size;
> > > + rq->xsk.nxt_idx = 0;
> > > + rq->xsk.num = 0;
> > > +
> > > hdr_dma = dma_map_single(dma_dev, &xsk_hdr, vi->hdr_len, DMA_TO_DEVICE);
> > > if (dma_mapping_error(dma_dev, hdr_dma))
> > > return -ENOMEM;
> > > @@ -307,6 +352,8 @@ static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid)
> > >
> > > dma_unmap_single(dma_dev, sq->xsk.hdr_dma_address, vi->hdr_len, DMA_TO_DEVICE);
> > >
> > > + kfree(rq->xsk.xsk_buffs);
> > > +
> > > return err1 | err2;
> > > }
> > >
> > > diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> > > index 7ebc9bda7aee..bef41a3f954e 100644
> > > --- a/drivers/net/virtio/xsk.h
> > > +++ b/drivers/net/virtio/xsk.h
> > > @@ -23,4 +23,6 @@ int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> > > bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > > int budget);
> > > int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag);
> > > +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> > > + struct xsk_buff_pool *pool, gfp_t gfp);
> > > #endif
> > > --
> > > 2.32.0.3.g01195cf9f
> >
> >
>
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 16/21] virtio_net: xsk: rx: introduce add_recvbuf_xsk()
2023-11-09 16:26 ` Maciej Fijalkowski
@ 2023-11-10 2:38 ` Xuan Zhuo
2023-11-13 16:00 ` Maciej Fijalkowski
0 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-10 2:38 UTC (permalink / raw)
To: Maciej Fijalkowski
Cc: Michael S. Tsirkin, netdev, David S. Miller, Eric Dumazet,
Jakub Kicinski, Paolo Abeni, Jason Wang, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Thu, 9 Nov 2023 17:26:33 +0100, Maciej Fijalkowski <maciej.fijalkowski@intel.com> wrote:
> On Thu, Nov 09, 2023 at 07:11:46PM +0800, Xuan Zhuo wrote:
> > On Thu, 9 Nov 2023 03:12:27 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > On Tue, Nov 07, 2023 at 11:12:22AM +0800, Xuan Zhuo wrote:
> > > > Implement the logic of filling rq with XSK buffers.
> > > >
> > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > > ---
> > > > drivers/net/virtio/main.c | 4 ++-
> > > > drivers/net/virtio/virtio_net.h | 5 ++++
> > > > drivers/net/virtio/xsk.c | 49 ++++++++++++++++++++++++++++++++-
> > > > drivers/net/virtio/xsk.h | 2 ++
> > > > 4 files changed, 58 insertions(+), 2 deletions(-)
> > > >
> > > > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > > > index 6210a6e37396..15943a22e17d 100644
> > > > --- a/drivers/net/virtio/main.c
> > > > +++ b/drivers/net/virtio/main.c
> > > > @@ -1798,7 +1798,9 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
> > > > bool oom;
> > > >
> > > > do {
> > > > - if (vi->mergeable_rx_bufs)
> > > > + if (rq->xsk.pool)
> > > > + err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk.pool, gfp);
> > > > + else if (vi->mergeable_rx_bufs)
> > > > err = add_recvbuf_mergeable(vi, rq, gfp);
> > > > else if (vi->big_packets)
> > > > err = add_recvbuf_big(vi, rq, gfp);
> > >
> > > I'm not sure I understand. How does this handle mergeable flag still being set?
> >
> >
> > You has the same question as Jason.
> >
> > So I think maybe I should put the handle into the
> > add_recvbuf_mergeable and add_recvbuf_small.
> >
> > Let me think about this.
> >
> >
> > >
> > >
> > > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > > index a13d6d301fdb..1242785e311e 100644
> > > > --- a/drivers/net/virtio/virtio_net.h
> > > > +++ b/drivers/net/virtio/virtio_net.h
> > > > @@ -140,6 +140,11 @@ struct virtnet_rq {
> > > >
> > > > /* xdp rxq used by xsk */
> > > > struct xdp_rxq_info xdp_rxq;
> > > > +
> > > > + struct xdp_buff **xsk_buffs;
> > > > + u32 nxt_idx;
> > > > + u32 num;
> > > > + u32 size;
> > > > } xsk;
> > > > };
> > > >
> > > > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > > > index ea5804ddd44e..e737c3353212 100644
> > > > --- a/drivers/net/virtio/xsk.c
> > > > +++ b/drivers/net/virtio/xsk.c
> > > > @@ -38,6 +38,41 @@ static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> > > > netif_stop_subqueue(dev, qnum);
> > > > }
> > > >
> > > > +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> > > > + struct xsk_buff_pool *pool, gfp_t gfp)
> > > > +{
> > > > + struct xdp_buff **xsk_buffs;
> > > > + dma_addr_t addr;
> > > > + u32 len, i;
> > > > + int err = 0;
> > > > +
> > > > + xsk_buffs = rq->xsk.xsk_buffs;
> > > > +
> > > > + if (rq->xsk.nxt_idx >= rq->xsk.num) {
> > > > + rq->xsk.num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->xsk.size);
> > > > + if (!rq->xsk.num)
> > > > + return -ENOMEM;
> > > > + rq->xsk.nxt_idx = 0;
> > > > + }
> > >
> > > Another manually rolled linked list implementation.
> > > Please, don't.
> >
> >
> > The array is for speedup.
> >
> > xsk_buff_alloc_batch will return many xsk_buff that will be more efficient than
> > the xsk_buff_alloc.
>
> But your sg list just contains a single entry?
> I think that you have to walk through the xsk_buffs array, retrieve dma
> addrs from there and have sg list sized to the value
> xsk_buff_alloc_batch() returned.
>
> I don't think your logic based on nxt_idx is needed. Please take a look
> how other drivers use xsk_buff_alloc_batch().
>
> I don't see callsites of virtnet_add_recvbuf_xsk() though.
virtnet_add_recvbuf_xsk is called by the above try_fill_recv()
And the loop is in there.
Jason want to reuse the loop of the try_fill_recv().
So in this function I just consume one item.
The nxt_idx is used to cross the try_fill_recv.
If we drop the nxt_idx. This patch will like this:
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 6210a6e37396..88bff83ad0d8 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -1797,6 +1797,15 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
int err;
bool oom;
+ if (rq->xsk.pool) {
+ err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk.pool, gfp);
+ oom = err == -ENOMEM;
+ if (err > 0)
+ goto kick;
+
+ return err;
+ }
+
do {
if (vi->mergeable_rx_bufs)
err = add_recvbuf_mergeable(vi, rq, gfp);
@@ -1809,6 +1818,7 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
if (err)
break;
} while (rq->vq->num_free);
+kick:
if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) {
unsigned long flags;
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index a13d6d301fdb..184866014a19 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -140,6 +140,8 @@ struct virtnet_rq {
/* xdp rxq used by xsk */
struct xdp_rxq_info xdp_rxq;
+
+ struct xdp_buff **xsk_buffs;
} xsk;
};
diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
index ea5804ddd44e..73c9323bffd3 100644
--- a/drivers/net/virtio/xsk.c
+++ b/drivers/net/virtio/xsk.c
@@ -38,6 +38,46 @@ static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
netif_stop_subqueue(dev, qnum);
}
+int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
+ struct xsk_buff_pool *pool, gfp_t gfp)
+{
+ struct xdp_buff **xsk_buffs;
+ dma_addr_t addr;
+ u32 len, i;
+ int err = 0;
+ int num;
+
+ xsk_buffs = rq->xsk.xsk_buffs;
+
+ num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->vq->num_free);
+ if (!num)
+ return -ENOMEM;
+
+ for (i = 0; i < num; ++i) {
+ /* use the part of XDP_PACKET_HEADROOM as the virtnet hdr space */
+ addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len;
+ len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len;
+
+ sg_init_table(rq->sg, 1);
+ sg_fill_dma(rq->sg, addr, len);
+
+ err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, xsk_buffs[i], gfp);
+ if (err)
+ goto err;
+ }
+
+ return num;
+
+err:
+ if (i)
+ err = i;
+
+ for (; i < num; ++i)
+ xsk_buff_free(xsk_buffs[i]);
+
+ return err;
+}
+
static int virtnet_xsk_xmit_one(struct virtnet_sq *sq,
struct xsk_buff_pool *pool,
struct xdp_desc *desc)
@@ -213,7 +253,7 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
struct virtnet_sq *sq;
struct device *dma_dev;
dma_addr_t hdr_dma;
- int err;
+ int err, size;
/* In big_packets mode, xdp cannot work, so there is no need to
* initialize xsk of rq.
@@ -249,6 +289,12 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
if (!dma_dev)
return -EPERM;
+ size = virtqueue_get_vring_size(rq->vq);
+
+ rq->xsk.xsk_buffs = kcalloc(size, sizeof(*rq->xsk.xsk_buffs), GFP_KERNEL);
+ if (!rq->xsk.xsk_buffs)
+ return -ENOMEM;
+
hdr_dma = dma_map_single(dma_dev, &xsk_hdr, vi->hdr_len, DMA_TO_DEVICE);
if (dma_mapping_error(dma_dev, hdr_dma))
return -ENOMEM;
@@ -307,6 +353,8 @@ static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid)
dma_unmap_single(dma_dev, sq->xsk.hdr_dma_address, vi->hdr_len, DMA_TO_DEVICE);
+ kfree(rq->xsk.xsk_buffs);
+
return err1 | err2;
}
diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
index 7ebc9bda7aee..bef41a3f954e 100644
--- a/drivers/net/virtio/xsk.h
+++ b/drivers/net/virtio/xsk.h
@@ -23,4 +23,6 @@ int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
int budget);
int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag);
+int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
+ struct xsk_buff_pool *pool, gfp_t gfp);
#endif
^ permalink raw reply related [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 16/21] virtio_net: xsk: rx: introduce add_recvbuf_xsk()
2023-11-10 2:38 ` Xuan Zhuo
@ 2023-11-13 16:00 ` Maciej Fijalkowski
2023-11-14 3:16 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Maciej Fijalkowski @ 2023-11-13 16:00 UTC (permalink / raw)
To: Xuan Zhuo
Cc: Michael S. Tsirkin, netdev, David S. Miller, Eric Dumazet,
Jakub Kicinski, Paolo Abeni, Jason Wang, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Fri, Nov 10, 2023 at 10:38:04AM +0800, Xuan Zhuo wrote:
> On Thu, 9 Nov 2023 17:26:33 +0100, Maciej Fijalkowski <maciej.fijalkowski@intel.com> wrote:
> > On Thu, Nov 09, 2023 at 07:11:46PM +0800, Xuan Zhuo wrote:
> > > On Thu, 9 Nov 2023 03:12:27 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > > On Tue, Nov 07, 2023 at 11:12:22AM +0800, Xuan Zhuo wrote:
> > > > > Implement the logic of filling rq with XSK buffers.
> > > > >
> > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > > > ---
> > > > > drivers/net/virtio/main.c | 4 ++-
> > > > > drivers/net/virtio/virtio_net.h | 5 ++++
> > > > > drivers/net/virtio/xsk.c | 49 ++++++++++++++++++++++++++++++++-
> > > > > drivers/net/virtio/xsk.h | 2 ++
> > > > > 4 files changed, 58 insertions(+), 2 deletions(-)
> > > > >
> > > > > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > > > > index 6210a6e37396..15943a22e17d 100644
> > > > > --- a/drivers/net/virtio/main.c
> > > > > +++ b/drivers/net/virtio/main.c
> > > > > @@ -1798,7 +1798,9 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
> > > > > bool oom;
> > > > >
> > > > > do {
> > > > > - if (vi->mergeable_rx_bufs)
> > > > > + if (rq->xsk.pool)
> > > > > + err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk.pool, gfp);
> > > > > + else if (vi->mergeable_rx_bufs)
> > > > > err = add_recvbuf_mergeable(vi, rq, gfp);
> > > > > else if (vi->big_packets)
> > > > > err = add_recvbuf_big(vi, rq, gfp);
> > > >
> > > > I'm not sure I understand. How does this handle mergeable flag still being set?
> > >
> > >
> > > You has the same question as Jason.
> > >
> > > So I think maybe I should put the handle into the
> > > add_recvbuf_mergeable and add_recvbuf_small.
> > >
> > > Let me think about this.
> > >
> > >
> > > >
> > > >
> > > > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > > > index a13d6d301fdb..1242785e311e 100644
> > > > > --- a/drivers/net/virtio/virtio_net.h
> > > > > +++ b/drivers/net/virtio/virtio_net.h
> > > > > @@ -140,6 +140,11 @@ struct virtnet_rq {
> > > > >
> > > > > /* xdp rxq used by xsk */
> > > > > struct xdp_rxq_info xdp_rxq;
> > > > > +
> > > > > + struct xdp_buff **xsk_buffs;
> > > > > + u32 nxt_idx;
> > > > > + u32 num;
> > > > > + u32 size;
> > > > > } xsk;
> > > > > };
> > > > >
> > > > > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > > > > index ea5804ddd44e..e737c3353212 100644
> > > > > --- a/drivers/net/virtio/xsk.c
> > > > > +++ b/drivers/net/virtio/xsk.c
> > > > > @@ -38,6 +38,41 @@ static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> > > > > netif_stop_subqueue(dev, qnum);
> > > > > }
> > > > >
> > > > > +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> > > > > + struct xsk_buff_pool *pool, gfp_t gfp)
> > > > > +{
> > > > > + struct xdp_buff **xsk_buffs;
> > > > > + dma_addr_t addr;
> > > > > + u32 len, i;
> > > > > + int err = 0;
> > > > > +
> > > > > + xsk_buffs = rq->xsk.xsk_buffs;
> > > > > +
> > > > > + if (rq->xsk.nxt_idx >= rq->xsk.num) {
> > > > > + rq->xsk.num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->xsk.size);
> > > > > + if (!rq->xsk.num)
> > > > > + return -ENOMEM;
> > > > > + rq->xsk.nxt_idx = 0;
> > > > > + }
> > > >
> > > > Another manually rolled linked list implementation.
> > > > Please, don't.
> > >
> > >
> > > The array is for speedup.
> > >
> > > xsk_buff_alloc_batch will return many xsk_buff that will be more efficient than
> > > the xsk_buff_alloc.
> >
> > But your sg list just contains a single entry?
> > I think that you have to walk through the xsk_buffs array, retrieve dma
> > addrs from there and have sg list sized to the value
> > xsk_buff_alloc_batch() returned.
> >
> > I don't think your logic based on nxt_idx is needed. Please take a look
> > how other drivers use xsk_buff_alloc_batch().
> >
> > I don't see callsites of virtnet_add_recvbuf_xsk() though.
>
>
> virtnet_add_recvbuf_xsk is called by the above try_fill_recv()
> And the loop is in there.
Ah sorry I was looking for another patch to call it as it used to be in
v1.
>
> Jason want to reuse the loop of the try_fill_recv().
> So in this function I just consume one item.
>
> The nxt_idx is used to cross the try_fill_recv.
>
> If we drop the nxt_idx. This patch will like this:
>
> diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> index 6210a6e37396..88bff83ad0d8 100644
> --- a/drivers/net/virtio/main.c
> +++ b/drivers/net/virtio/main.c
> @@ -1797,6 +1797,15 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
> int err;
> bool oom;
>
> + if (rq->xsk.pool) {
> + err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk.pool, gfp);
> + oom = err == -ENOMEM;
> + if (err > 0)
> + goto kick;
> +
> + return err;
> + }
> +
> do {
> if (vi->mergeable_rx_bufs)
> err = add_recvbuf_mergeable(vi, rq, gfp);
> @@ -1809,6 +1818,7 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
> if (err)
> break;
> } while (rq->vq->num_free);
> +kick:
> if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) {
> unsigned long flags;
>
> diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> index a13d6d301fdb..184866014a19 100644
> --- a/drivers/net/virtio/virtio_net.h
> +++ b/drivers/net/virtio/virtio_net.h
> @@ -140,6 +140,8 @@ struct virtnet_rq {
>
> /* xdp rxq used by xsk */
> struct xdp_rxq_info xdp_rxq;
> +
> + struct xdp_buff **xsk_buffs;
> } xsk;
> };
>
> diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> index ea5804ddd44e..73c9323bffd3 100644
> --- a/drivers/net/virtio/xsk.c
> +++ b/drivers/net/virtio/xsk.c
> @@ -38,6 +38,46 @@ static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> netif_stop_subqueue(dev, qnum);
> }
>
> +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> + struct xsk_buff_pool *pool, gfp_t gfp)
> +{
> + struct xdp_buff **xsk_buffs;
> + dma_addr_t addr;
> + u32 len, i;
> + int err = 0;
> + int num;
> +
> + xsk_buffs = rq->xsk.xsk_buffs;
> +
> + num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->vq->num_free);
> + if (!num)
> + return -ENOMEM;
> +
> + for (i = 0; i < num; ++i) {
> + /* use the part of XDP_PACKET_HEADROOM as the virtnet hdr space */
> + addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len;
> + len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len;
len can be pulled out of loop...
> +
> + sg_init_table(rq->sg, 1);
> + sg_fill_dma(rq->sg, addr, len);
... but when I first commented I did not understand why you were not
passing dma from xsk_buff_pool like this:
sg_init_table(rq->sg, num);
len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len;
for (i = 0; i < num; ++i) {
/* use the part of XDP_PACKET_HEADROOM as the virtnet hdr space */
addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len;
/* TODO: extend scatterlist size in receive_queue */
sg_fill_dma(&rq->sg[i], addr, len);
}
err = virtqueue_add_inbuf(rq->vq, rq->sg, num, xsk_buffs, gfp);
and now I see that the problem is with 'data' argument above (or xsk_buffs
in this particular example).
Why do you need to pass xdp_buff to virtio_ring? You already have the
rq->xsk.xsk_buffs which you can use on rx side.
Can someone shed some light on it?
> +
> + err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, xsk_buffs[i], gfp);
> + if (err)
> + goto err;
> + }
> +
> + return num;
> +
> +err:
> + if (i)
> + err = i;
> +
> + for (; i < num; ++i)
> + xsk_buff_free(xsk_buffs[i]);
> +
> + return err;
> +}
> +
> static int virtnet_xsk_xmit_one(struct virtnet_sq *sq,
> struct xsk_buff_pool *pool,
> struct xdp_desc *desc)
> @@ -213,7 +253,7 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
> struct virtnet_sq *sq;
> struct device *dma_dev;
> dma_addr_t hdr_dma;
> - int err;
> + int err, size;
>
> /* In big_packets mode, xdp cannot work, so there is no need to
> * initialize xsk of rq.
> @@ -249,6 +289,12 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
> if (!dma_dev)
> return -EPERM;
>
> + size = virtqueue_get_vring_size(rq->vq);
> +
> + rq->xsk.xsk_buffs = kcalloc(size, sizeof(*rq->xsk.xsk_buffs), GFP_KERNEL);
> + if (!rq->xsk.xsk_buffs)
> + return -ENOMEM;
> +
> hdr_dma = dma_map_single(dma_dev, &xsk_hdr, vi->hdr_len, DMA_TO_DEVICE);
> if (dma_mapping_error(dma_dev, hdr_dma))
> return -ENOMEM;
> @@ -307,6 +353,8 @@ static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid)
>
> dma_unmap_single(dma_dev, sq->xsk.hdr_dma_address, vi->hdr_len, DMA_TO_DEVICE);
>
> + kfree(rq->xsk.xsk_buffs);
> +
> return err1 | err2;
> }
>
> diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> index 7ebc9bda7aee..bef41a3f954e 100644
> --- a/drivers/net/virtio/xsk.h
> +++ b/drivers/net/virtio/xsk.h
> @@ -23,4 +23,6 @@ int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> int budget);
> int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag);
> +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> + struct xsk_buff_pool *pool, gfp_t gfp);
> #endif
>
>
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 16/21] virtio_net: xsk: rx: introduce add_recvbuf_xsk()
2023-11-13 16:00 ` Maciej Fijalkowski
@ 2023-11-14 3:16 ` Xuan Zhuo
0 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-14 3:16 UTC (permalink / raw)
To: Maciej Fijalkowski
Cc: Michael S. Tsirkin, netdev, David S. Miller, Eric Dumazet,
Jakub Kicinski, Paolo Abeni, Jason Wang, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Mon, 13 Nov 2023 17:00:50 +0100, Maciej Fijalkowski <maciej.fijalkowski@intel.com> wrote:
> On Fri, Nov 10, 2023 at 10:38:04AM +0800, Xuan Zhuo wrote:
> > On Thu, 9 Nov 2023 17:26:33 +0100, Maciej Fijalkowski <maciej.fijalkowski@intel.com> wrote:
> > > On Thu, Nov 09, 2023 at 07:11:46PM +0800, Xuan Zhuo wrote:
> > > > On Thu, 9 Nov 2023 03:12:27 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > > > On Tue, Nov 07, 2023 at 11:12:22AM +0800, Xuan Zhuo wrote:
> > > > > > Implement the logic of filling rq with XSK buffers.
> > > > > >
> > > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > > > > ---
> > > > > > drivers/net/virtio/main.c | 4 ++-
> > > > > > drivers/net/virtio/virtio_net.h | 5 ++++
> > > > > > drivers/net/virtio/xsk.c | 49 ++++++++++++++++++++++++++++++++-
> > > > > > drivers/net/virtio/xsk.h | 2 ++
> > > > > > 4 files changed, 58 insertions(+), 2 deletions(-)
> > > > > >
> > > > > > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > > > > > index 6210a6e37396..15943a22e17d 100644
> > > > > > --- a/drivers/net/virtio/main.c
> > > > > > +++ b/drivers/net/virtio/main.c
> > > > > > @@ -1798,7 +1798,9 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
> > > > > > bool oom;
> > > > > >
> > > > > > do {
> > > > > > - if (vi->mergeable_rx_bufs)
> > > > > > + if (rq->xsk.pool)
> > > > > > + err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk.pool, gfp);
> > > > > > + else if (vi->mergeable_rx_bufs)
> > > > > > err = add_recvbuf_mergeable(vi, rq, gfp);
> > > > > > else if (vi->big_packets)
> > > > > > err = add_recvbuf_big(vi, rq, gfp);
> > > > >
> > > > > I'm not sure I understand. How does this handle mergeable flag still being set?
> > > >
> > > >
> > > > You has the same question as Jason.
> > > >
> > > > So I think maybe I should put the handle into the
> > > > add_recvbuf_mergeable and add_recvbuf_small.
> > > >
> > > > Let me think about this.
> > > >
> > > >
> > > > >
> > > > >
> > > > > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > > > > index a13d6d301fdb..1242785e311e 100644
> > > > > > --- a/drivers/net/virtio/virtio_net.h
> > > > > > +++ b/drivers/net/virtio/virtio_net.h
> > > > > > @@ -140,6 +140,11 @@ struct virtnet_rq {
> > > > > >
> > > > > > /* xdp rxq used by xsk */
> > > > > > struct xdp_rxq_info xdp_rxq;
> > > > > > +
> > > > > > + struct xdp_buff **xsk_buffs;
> > > > > > + u32 nxt_idx;
> > > > > > + u32 num;
> > > > > > + u32 size;
> > > > > > } xsk;
> > > > > > };
> > > > > >
> > > > > > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > > > > > index ea5804ddd44e..e737c3353212 100644
> > > > > > --- a/drivers/net/virtio/xsk.c
> > > > > > +++ b/drivers/net/virtio/xsk.c
> > > > > > @@ -38,6 +38,41 @@ static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> > > > > > netif_stop_subqueue(dev, qnum);
> > > > > > }
> > > > > >
> > > > > > +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> > > > > > + struct xsk_buff_pool *pool, gfp_t gfp)
> > > > > > +{
> > > > > > + struct xdp_buff **xsk_buffs;
> > > > > > + dma_addr_t addr;
> > > > > > + u32 len, i;
> > > > > > + int err = 0;
> > > > > > +
> > > > > > + xsk_buffs = rq->xsk.xsk_buffs;
> > > > > > +
> > > > > > + if (rq->xsk.nxt_idx >= rq->xsk.num) {
> > > > > > + rq->xsk.num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->xsk.size);
> > > > > > + if (!rq->xsk.num)
> > > > > > + return -ENOMEM;
> > > > > > + rq->xsk.nxt_idx = 0;
> > > > > > + }
> > > > >
> > > > > Another manually rolled linked list implementation.
> > > > > Please, don't.
> > > >
> > > >
> > > > The array is for speedup.
> > > >
> > > > xsk_buff_alloc_batch will return many xsk_buff that will be more efficient than
> > > > the xsk_buff_alloc.
> > >
> > > But your sg list just contains a single entry?
> > > I think that you have to walk through the xsk_buffs array, retrieve dma
> > > addrs from there and have sg list sized to the value
> > > xsk_buff_alloc_batch() returned.
> > >
> > > I don't think your logic based on nxt_idx is needed. Please take a look
> > > how other drivers use xsk_buff_alloc_batch().
> > >
> > > I don't see callsites of virtnet_add_recvbuf_xsk() though.
> >
> >
> > virtnet_add_recvbuf_xsk is called by the above try_fill_recv()
> > And the loop is in there.
>
> Ah sorry I was looking for another patch to call it as it used to be in
> v1.
>
> >
> > Jason want to reuse the loop of the try_fill_recv().
> > So in this function I just consume one item.
> >
> > The nxt_idx is used to cross the try_fill_recv.
> >
> > If we drop the nxt_idx. This patch will like this:
> >
> > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > index 6210a6e37396..88bff83ad0d8 100644
> > --- a/drivers/net/virtio/main.c
> > +++ b/drivers/net/virtio/main.c
> > @@ -1797,6 +1797,15 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
> > int err;
> > bool oom;
> >
> > + if (rq->xsk.pool) {
> > + err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk.pool, gfp);
> > + oom = err == -ENOMEM;
> > + if (err > 0)
> > + goto kick;
> > +
> > + return err;
> > + }
> > +
> > do {
> > if (vi->mergeable_rx_bufs)
> > err = add_recvbuf_mergeable(vi, rq, gfp);
> > @@ -1809,6 +1818,7 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
> > if (err)
> > break;
> > } while (rq->vq->num_free);
> > +kick:
> > if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) {
> > unsigned long flags;
> >
> > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > index a13d6d301fdb..184866014a19 100644
> > --- a/drivers/net/virtio/virtio_net.h
> > +++ b/drivers/net/virtio/virtio_net.h
> > @@ -140,6 +140,8 @@ struct virtnet_rq {
> >
> > /* xdp rxq used by xsk */
> > struct xdp_rxq_info xdp_rxq;
> > +
> > + struct xdp_buff **xsk_buffs;
> > } xsk;
> > };
> >
> > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > index ea5804ddd44e..73c9323bffd3 100644
> > --- a/drivers/net/virtio/xsk.c
> > +++ b/drivers/net/virtio/xsk.c
> > @@ -38,6 +38,46 @@ static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> > netif_stop_subqueue(dev, qnum);
> > }
> >
> > +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> > + struct xsk_buff_pool *pool, gfp_t gfp)
> > +{
> > + struct xdp_buff **xsk_buffs;
> > + dma_addr_t addr;
> > + u32 len, i;
> > + int err = 0;
> > + int num;
> > +
> > + xsk_buffs = rq->xsk.xsk_buffs;
> > +
> > + num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->vq->num_free);
> > + if (!num)
> > + return -ENOMEM;
> > +
> > + for (i = 0; i < num; ++i) {
> > + /* use the part of XDP_PACKET_HEADROOM as the virtnet hdr space */
> > + addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len;
> > + len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len;
>
> len can be pulled out of loop...
>
> > +
> > + sg_init_table(rq->sg, 1);
> > + sg_fill_dma(rq->sg, addr, len);
>
> ... but when I first commented I did not understand why you were not
> passing dma from xsk_buff_pool like this:
>
> sg_init_table(rq->sg, num);
> len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len;
>
> for (i = 0; i < num; ++i) {
> /* use the part of XDP_PACKET_HEADROOM as the virtnet hdr space */
> addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len;
> /* TODO: extend scatterlist size in receive_queue */
> sg_fill_dma(&rq->sg[i], addr, len);
> }
>
> err = virtqueue_add_inbuf(rq->vq, rq->sg, num, xsk_buffs, gfp);
Yes, you are right.
>
> and now I see that the problem is with 'data' argument above (or xsk_buffs
> in this particular example).
>
> Why do you need to pass xdp_buff to virtio_ring? You already have the
> rq->xsk.xsk_buffs which you can use on rx side.
>
> Can someone shed some light on it?
For virtio, we should not assume that the virtio ring is consumed
sequentially by default.
So we normally use the "data" to distinguish the received buffers.
Thanks.
>
> > +
> > + err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, xsk_buffs[i], gfp);
> > + if (err)
> > + goto err;
> > + }
> > +
> > + return num;
> > +
> > +err:
> > + if (i)
> > + err = i;
> > +
> > + for (; i < num; ++i)
> > + xsk_buff_free(xsk_buffs[i]);
> > +
> > + return err;
> > +}
> > +
> > static int virtnet_xsk_xmit_one(struct virtnet_sq *sq,
> > struct xsk_buff_pool *pool,
> > struct xdp_desc *desc)
> > @@ -213,7 +253,7 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
> > struct virtnet_sq *sq;
> > struct device *dma_dev;
> > dma_addr_t hdr_dma;
> > - int err;
> > + int err, size;
> >
> > /* In big_packets mode, xdp cannot work, so there is no need to
> > * initialize xsk of rq.
> > @@ -249,6 +289,12 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
> > if (!dma_dev)
> > return -EPERM;
> >
> > + size = virtqueue_get_vring_size(rq->vq);
> > +
> > + rq->xsk.xsk_buffs = kcalloc(size, sizeof(*rq->xsk.xsk_buffs), GFP_KERNEL);
> > + if (!rq->xsk.xsk_buffs)
> > + return -ENOMEM;
> > +
> > hdr_dma = dma_map_single(dma_dev, &xsk_hdr, vi->hdr_len, DMA_TO_DEVICE);
> > if (dma_mapping_error(dma_dev, hdr_dma))
> > return -ENOMEM;
> > @@ -307,6 +353,8 @@ static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid)
> >
> > dma_unmap_single(dma_dev, sq->xsk.hdr_dma_address, vi->hdr_len, DMA_TO_DEVICE);
> >
> > + kfree(rq->xsk.xsk_buffs);
> > +
> > return err1 | err2;
> > }
> >
> > diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> > index 7ebc9bda7aee..bef41a3f954e 100644
> > --- a/drivers/net/virtio/xsk.h
> > +++ b/drivers/net/virtio/xsk.h
> > @@ -23,4 +23,6 @@ int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> > bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > int budget);
> > int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag);
> > +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> > + struct xsk_buff_pool *pool, gfp_t gfp);
> > #endif
> >
> >
^ permalink raw reply [flat|nested] 61+ messages in thread
* Re: [PATCH net-next v2 16/21] virtio_net: xsk: rx: introduce add_recvbuf_xsk()
2023-11-09 8:12 ` Michael S. Tsirkin
2023-11-09 11:11 ` Xuan Zhuo
@ 2023-11-10 3:04 ` Xuan Zhuo
1 sibling, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-10 3:04 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Thu, 9 Nov 2023 03:12:27 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> On Tue, Nov 07, 2023 at 11:12:22AM +0800, Xuan Zhuo wrote:
> > Implement the logic of filling rq with XSK buffers.
> >
> > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > ---
> > drivers/net/virtio/main.c | 4 ++-
> > drivers/net/virtio/virtio_net.h | 5 ++++
> > drivers/net/virtio/xsk.c | 49 ++++++++++++++++++++++++++++++++-
> > drivers/net/virtio/xsk.h | 2 ++
> > 4 files changed, 58 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > index 6210a6e37396..15943a22e17d 100644
> > --- a/drivers/net/virtio/main.c
> > +++ b/drivers/net/virtio/main.c
> > @@ -1798,7 +1798,9 @@ static bool try_fill_recv(struct virtnet_info *vi, struct virtnet_rq *rq,
> > bool oom;
> >
> > do {
> > - if (vi->mergeable_rx_bufs)
> > + if (rq->xsk.pool)
> > + err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk.pool, gfp);
> > + else if (vi->mergeable_rx_bufs)
> > err = add_recvbuf_mergeable(vi, rq, gfp);
> > else if (vi->big_packets)
> > err = add_recvbuf_big(vi, rq, gfp);
>
> I'm not sure I understand. How does this handle mergeable flag still being set?
# xsk with merge
## fill ring
We fill the ring with the buffers from the xdp socket just like we alloc buffer
from the kernel.
## receive buffer
Now the xsk supported the multi-buffer recently. But I want to support that after
this packet set. So if the packet uses more buffers, I drop that.
If the xdp is not bound or the xdp prog does not redirect the packet to xsk socket.
all buffers are used to make skbs, whatever the packet uses one buffer or more.
But the buffers is from the xsk socket. So we must allocat new pages and copy
the data to the new pages. And free the buffers of xsk socket.
Thanks.
>
>
> > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > index a13d6d301fdb..1242785e311e 100644
> > --- a/drivers/net/virtio/virtio_net.h
> > +++ b/drivers/net/virtio/virtio_net.h
> > @@ -140,6 +140,11 @@ struct virtnet_rq {
> >
> > /* xdp rxq used by xsk */
> > struct xdp_rxq_info xdp_rxq;
> > +
> > + struct xdp_buff **xsk_buffs;
> > + u32 nxt_idx;
> > + u32 num;
> > + u32 size;
> > } xsk;
> > };
> >
> > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > index ea5804ddd44e..e737c3353212 100644
> > --- a/drivers/net/virtio/xsk.c
> > +++ b/drivers/net/virtio/xsk.c
> > @@ -38,6 +38,41 @@ static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> > netif_stop_subqueue(dev, qnum);
> > }
> >
> > +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> > + struct xsk_buff_pool *pool, gfp_t gfp)
> > +{
> > + struct xdp_buff **xsk_buffs;
> > + dma_addr_t addr;
> > + u32 len, i;
> > + int err = 0;
> > +
> > + xsk_buffs = rq->xsk.xsk_buffs;
> > +
> > + if (rq->xsk.nxt_idx >= rq->xsk.num) {
> > + rq->xsk.num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->xsk.size);
> > + if (!rq->xsk.num)
> > + return -ENOMEM;
> > + rq->xsk.nxt_idx = 0;
> > + }
>
> Another manually rolled linked list implementation.
> Please, don't.
>
>
> > +
> > + i = rq->xsk.nxt_idx;
> > +
> > + /* use the part of XDP_PACKET_HEADROOM as the virtnet hdr space */
> > + addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len;
> > + len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len;
> > +
> > + sg_init_table(rq->sg, 1);
> > + sg_fill_dma(rq->sg, addr, len);
> > +
> > + err = virtqueue_add_inbuf(rq->vq, rq->sg, 1, xsk_buffs[i], gfp);
> > + if (err)
> > + return err;
> > +
> > + rq->xsk.nxt_idx++;
> > +
> > + return 0;
> > +}
> > +
> > static int virtnet_xsk_xmit_one(struct virtnet_sq *sq,
> > struct xsk_buff_pool *pool,
> > struct xdp_desc *desc)
> > @@ -213,7 +248,7 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
> > struct virtnet_sq *sq;
> > struct device *dma_dev;
> > dma_addr_t hdr_dma;
> > - int err;
> > + int err, size;
> >
> > /* In big_packets mode, xdp cannot work, so there is no need to
> > * initialize xsk of rq.
> > @@ -249,6 +284,16 @@ static int virtnet_xsk_pool_enable(struct net_device *dev,
> > if (!dma_dev)
> > return -EPERM;
> >
> > + size = virtqueue_get_vring_size(rq->vq);
> > +
> > + rq->xsk.xsk_buffs = kcalloc(size, sizeof(*rq->xsk.xsk_buffs), GFP_KERNEL);
> > + if (!rq->xsk.xsk_buffs)
> > + return -ENOMEM;
> > +
> > + rq->xsk.size = size;
> > + rq->xsk.nxt_idx = 0;
> > + rq->xsk.num = 0;
> > +
> > hdr_dma = dma_map_single(dma_dev, &xsk_hdr, vi->hdr_len, DMA_TO_DEVICE);
> > if (dma_mapping_error(dma_dev, hdr_dma))
> > return -ENOMEM;
> > @@ -307,6 +352,8 @@ static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid)
> >
> > dma_unmap_single(dma_dev, sq->xsk.hdr_dma_address, vi->hdr_len, DMA_TO_DEVICE);
> >
> > + kfree(rq->xsk.xsk_buffs);
> > +
> > return err1 | err2;
> > }
> >
> > diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> > index 7ebc9bda7aee..bef41a3f954e 100644
> > --- a/drivers/net/virtio/xsk.h
> > +++ b/drivers/net/virtio/xsk.h
> > @@ -23,4 +23,6 @@ int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp);
> > bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > int budget);
> > int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag);
> > +int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> > + struct xsk_buff_pool *pool, gfp_t gfp);
> > #endif
> > --
> > 2.32.0.3.g01195cf9f
>
>
^ permalink raw reply [flat|nested] 61+ messages in thread
* [PATCH net-next v2 17/21] virtio_net: xsk: rx: skip dma unmap when rq is bind with AF_XDP
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (15 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 16/21] virtio_net: xsk: rx: introduce add_recvbuf_xsk() Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-09 8:15 ` Michael S. Tsirkin
2023-11-07 3:12 ` [PATCH net-next v2 18/21] virtio_net: xsk: rx: introduce receive_xsk() to recv xsk buffer Xuan Zhuo
` (5 subsequent siblings)
22 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
When rq is bound with AF_XDP, the buffer dma is managed
by the AF_XDP APIs. So the buffer got from the virtio core should
skip the dma unmap operation.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 8 +++++---
drivers/net/virtio/virtio_net.h | 3 +++
drivers/net/virtio/xsk.c | 1 +
3 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 15943a22e17d..a318b2533b94 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -430,7 +430,7 @@ static void *virtnet_rq_get_buf(struct virtnet_rq *rq, u32 *len, void **ctx)
void *buf;
buf = virtqueue_get_buf_ctx(rq->vq, len, ctx);
- if (buf && rq->do_dma)
+ if (buf && rq->do_dma_unmap)
virtnet_rq_unmap(rq, buf, *len);
return buf;
@@ -561,8 +561,10 @@ static void virtnet_set_premapped(struct virtnet_info *vi)
/* disable for big mode */
if (vi->mergeable_rx_bufs || !vi->big_packets) {
- if (!virtqueue_set_dma_premapped(vi->rq[i].vq))
+ if (!virtqueue_set_dma_premapped(vi->rq[i].vq)) {
vi->rq[i].do_dma = true;
+ vi->rq[i].do_dma_unmap = true;
+ }
}
}
}
@@ -3944,7 +3946,7 @@ void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
rq = &vi->rq[i];
- if (rq->do_dma)
+ if (rq->do_dma_unmap)
virtnet_rq_unmap(rq, buf, 0);
virtnet_rq_free_buf(vi, rq, buf);
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index 1242785e311e..2005d0cd22e2 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -135,6 +135,9 @@ struct virtnet_rq {
/* Do dma by self */
bool do_dma;
+ /* Do dma unmap after getting buf from virtio core. */
+ bool do_dma_unmap;
+
struct {
struct xsk_buff_pool *pool;
diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
index e737c3353212..b09c473c29fb 100644
--- a/drivers/net/virtio/xsk.c
+++ b/drivers/net/virtio/xsk.c
@@ -210,6 +210,7 @@ static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *
xdp_rxq_info_unreg(&rq->xsk.xdp_rxq);
rq->xsk.pool = pool;
+ rq->do_dma_unmap = !pool;
virtnet_rx_resume(vi, rq);
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 17/21] virtio_net: xsk: rx: skip dma unmap when rq is bind with AF_XDP
2023-11-07 3:12 ` [PATCH net-next v2 17/21] virtio_net: xsk: rx: skip dma unmap when rq is bind with AF_XDP Xuan Zhuo
@ 2023-11-09 8:15 ` Michael S. Tsirkin
2023-11-09 11:10 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Michael S. Tsirkin @ 2023-11-09 8:15 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Tue, Nov 07, 2023 at 11:12:23AM +0800, Xuan Zhuo wrote:
> When rq is bound with AF_XDP, the buffer dma is managed
> by the AF_XDP APIs. So the buffer got from the virtio core should
> skip the dma unmap operation.
>
> Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
I don't get it - is this like a bugfix?
And why do we need our own flag and checks?
Doesn't virtio core DTRT?
> ---
> drivers/net/virtio/main.c | 8 +++++---
> drivers/net/virtio/virtio_net.h | 3 +++
> drivers/net/virtio/xsk.c | 1 +
> 3 files changed, 9 insertions(+), 3 deletions(-)
>
> diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> index 15943a22e17d..a318b2533b94 100644
> --- a/drivers/net/virtio/main.c
> +++ b/drivers/net/virtio/main.c
> @@ -430,7 +430,7 @@ static void *virtnet_rq_get_buf(struct virtnet_rq *rq, u32 *len, void **ctx)
> void *buf;
>
> buf = virtqueue_get_buf_ctx(rq->vq, len, ctx);
> - if (buf && rq->do_dma)
> + if (buf && rq->do_dma_unmap)
> virtnet_rq_unmap(rq, buf, *len);
>
> return buf;
> @@ -561,8 +561,10 @@ static void virtnet_set_premapped(struct virtnet_info *vi)
>
> /* disable for big mode */
> if (vi->mergeable_rx_bufs || !vi->big_packets) {
> - if (!virtqueue_set_dma_premapped(vi->rq[i].vq))
> + if (!virtqueue_set_dma_premapped(vi->rq[i].vq)) {
> vi->rq[i].do_dma = true;
> + vi->rq[i].do_dma_unmap = true;
> + }
> }
> }
> }
> @@ -3944,7 +3946,7 @@ void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
>
> rq = &vi->rq[i];
>
> - if (rq->do_dma)
> + if (rq->do_dma_unmap)
> virtnet_rq_unmap(rq, buf, 0);
>
> virtnet_rq_free_buf(vi, rq, buf);
> diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> index 1242785e311e..2005d0cd22e2 100644
> --- a/drivers/net/virtio/virtio_net.h
> +++ b/drivers/net/virtio/virtio_net.h
> @@ -135,6 +135,9 @@ struct virtnet_rq {
> /* Do dma by self */
> bool do_dma;
>
> + /* Do dma unmap after getting buf from virtio core. */
> + bool do_dma_unmap;
> +
> struct {
> struct xsk_buff_pool *pool;
>
> diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> index e737c3353212..b09c473c29fb 100644
> --- a/drivers/net/virtio/xsk.c
> +++ b/drivers/net/virtio/xsk.c
> @@ -210,6 +210,7 @@ static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *
> xdp_rxq_info_unreg(&rq->xsk.xdp_rxq);
>
> rq->xsk.pool = pool;
> + rq->do_dma_unmap = !pool;
>
> virtnet_rx_resume(vi, rq);
>
> --
> 2.32.0.3.g01195cf9f
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 17/21] virtio_net: xsk: rx: skip dma unmap when rq is bind with AF_XDP
2023-11-09 8:15 ` Michael S. Tsirkin
@ 2023-11-09 11:10 ` Xuan Zhuo
2023-11-09 12:00 ` Michael S. Tsirkin
0 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-09 11:10 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Thu, 9 Nov 2023 03:15:03 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> On Tue, Nov 07, 2023 at 11:12:23AM +0800, Xuan Zhuo wrote:
> > When rq is bound with AF_XDP, the buffer dma is managed
> > by the AF_XDP APIs. So the buffer got from the virtio core should
> > skip the dma unmap operation.
> >
> > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
>
>
> I don't get it - is this like a bugfix?
I want focus on this. So let it as an independent commit.
> And why do we need our own flag and checks?
> Doesn't virtio core DTRT?
struct vring_virtqueue {
[....]
/* Do DMA mapping by driver */
bool premapped;
We can not.
So I add own flag.
Thanks.
>
> > ---
> > drivers/net/virtio/main.c | 8 +++++---
> > drivers/net/virtio/virtio_net.h | 3 +++
> > drivers/net/virtio/xsk.c | 1 +
> > 3 files changed, 9 insertions(+), 3 deletions(-)
> >
> > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > index 15943a22e17d..a318b2533b94 100644
> > --- a/drivers/net/virtio/main.c
> > +++ b/drivers/net/virtio/main.c
> > @@ -430,7 +430,7 @@ static void *virtnet_rq_get_buf(struct virtnet_rq *rq, u32 *len, void **ctx)
> > void *buf;
> >
> > buf = virtqueue_get_buf_ctx(rq->vq, len, ctx);
> > - if (buf && rq->do_dma)
> > + if (buf && rq->do_dma_unmap)
> > virtnet_rq_unmap(rq, buf, *len);
> >
> > return buf;
> > @@ -561,8 +561,10 @@ static void virtnet_set_premapped(struct virtnet_info *vi)
> >
> > /* disable for big mode */
> > if (vi->mergeable_rx_bufs || !vi->big_packets) {
> > - if (!virtqueue_set_dma_premapped(vi->rq[i].vq))
> > + if (!virtqueue_set_dma_premapped(vi->rq[i].vq)) {
> > vi->rq[i].do_dma = true;
> > + vi->rq[i].do_dma_unmap = true;
> > + }
> > }
> > }
> > }
> > @@ -3944,7 +3946,7 @@ void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
> >
> > rq = &vi->rq[i];
> >
> > - if (rq->do_dma)
> > + if (rq->do_dma_unmap)
> > virtnet_rq_unmap(rq, buf, 0);
> >
> > virtnet_rq_free_buf(vi, rq, buf);
> > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > index 1242785e311e..2005d0cd22e2 100644
> > --- a/drivers/net/virtio/virtio_net.h
> > +++ b/drivers/net/virtio/virtio_net.h
> > @@ -135,6 +135,9 @@ struct virtnet_rq {
> > /* Do dma by self */
> > bool do_dma;
> >
> > + /* Do dma unmap after getting buf from virtio core. */
> > + bool do_dma_unmap;
> > +
> > struct {
> > struct xsk_buff_pool *pool;
> >
> > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > index e737c3353212..b09c473c29fb 100644
> > --- a/drivers/net/virtio/xsk.c
> > +++ b/drivers/net/virtio/xsk.c
> > @@ -210,6 +210,7 @@ static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *
> > xdp_rxq_info_unreg(&rq->xsk.xdp_rxq);
> >
> > rq->xsk.pool = pool;
> > + rq->do_dma_unmap = !pool;
> >
> > virtnet_rx_resume(vi, rq);
> >
> > --
> > 2.32.0.3.g01195cf9f
>
>
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 17/21] virtio_net: xsk: rx: skip dma unmap when rq is bind with AF_XDP
2023-11-09 11:10 ` Xuan Zhuo
@ 2023-11-09 12:00 ` Michael S. Tsirkin
2023-11-10 1:47 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Michael S. Tsirkin @ 2023-11-09 12:00 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Thu, Nov 09, 2023 at 07:10:02PM +0800, Xuan Zhuo wrote:
> On Thu, 9 Nov 2023 03:15:03 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > On Tue, Nov 07, 2023 at 11:12:23AM +0800, Xuan Zhuo wrote:
> > > When rq is bound with AF_XDP, the buffer dma is managed
> > > by the AF_XDP APIs. So the buffer got from the virtio core should
> > > skip the dma unmap operation.
> > >
> > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> >
> >
> > I don't get it - is this like a bugfix?
>
> I want focus on this. So let it as an independent commit.
>
> > And why do we need our own flag and checks?
> > Doesn't virtio core DTRT?
>
>
> struct vring_virtqueue {
> [....]
>
> /* Do DMA mapping by driver */
> bool premapped;
>
> We can not.
>
> So I add own flag.
>
> Thanks.
Still don't get it. Why not check the premapped flag?
>
> >
> > > ---
> > > drivers/net/virtio/main.c | 8 +++++---
> > > drivers/net/virtio/virtio_net.h | 3 +++
> > > drivers/net/virtio/xsk.c | 1 +
> > > 3 files changed, 9 insertions(+), 3 deletions(-)
> > >
> > > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > > index 15943a22e17d..a318b2533b94 100644
> > > --- a/drivers/net/virtio/main.c
> > > +++ b/drivers/net/virtio/main.c
> > > @@ -430,7 +430,7 @@ static void *virtnet_rq_get_buf(struct virtnet_rq *rq, u32 *len, void **ctx)
> > > void *buf;
> > >
> > > buf = virtqueue_get_buf_ctx(rq->vq, len, ctx);
> > > - if (buf && rq->do_dma)
> > > + if (buf && rq->do_dma_unmap)
> > > virtnet_rq_unmap(rq, buf, *len);
> > >
> > > return buf;
> > > @@ -561,8 +561,10 @@ static void virtnet_set_premapped(struct virtnet_info *vi)
> > >
> > > /* disable for big mode */
> > > if (vi->mergeable_rx_bufs || !vi->big_packets) {
> > > - if (!virtqueue_set_dma_premapped(vi->rq[i].vq))
> > > + if (!virtqueue_set_dma_premapped(vi->rq[i].vq)) {
> > > vi->rq[i].do_dma = true;
> > > + vi->rq[i].do_dma_unmap = true;
> > > + }
> > > }
> > > }
> > > }
> > > @@ -3944,7 +3946,7 @@ void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
> > >
> > > rq = &vi->rq[i];
> > >
> > > - if (rq->do_dma)
> > > + if (rq->do_dma_unmap)
> > > virtnet_rq_unmap(rq, buf, 0);
> > >
> > > virtnet_rq_free_buf(vi, rq, buf);
> > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > index 1242785e311e..2005d0cd22e2 100644
> > > --- a/drivers/net/virtio/virtio_net.h
> > > +++ b/drivers/net/virtio/virtio_net.h
> > > @@ -135,6 +135,9 @@ struct virtnet_rq {
> > > /* Do dma by self */
> > > bool do_dma;
> > >
> > > + /* Do dma unmap after getting buf from virtio core. */
> > > + bool do_dma_unmap;
> > > +
> > > struct {
> > > struct xsk_buff_pool *pool;
> > >
> > > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > > index e737c3353212..b09c473c29fb 100644
> > > --- a/drivers/net/virtio/xsk.c
> > > +++ b/drivers/net/virtio/xsk.c
> > > @@ -210,6 +210,7 @@ static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *
> > > xdp_rxq_info_unreg(&rq->xsk.xdp_rxq);
> > >
> > > rq->xsk.pool = pool;
> > > + rq->do_dma_unmap = !pool;
> > >
> > > virtnet_rx_resume(vi, rq);
> > >
> > > --
> > > 2.32.0.3.g01195cf9f
> >
> >
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 17/21] virtio_net: xsk: rx: skip dma unmap when rq is bind with AF_XDP
2023-11-09 12:00 ` Michael S. Tsirkin
@ 2023-11-10 1:47 ` Xuan Zhuo
2023-11-10 5:33 ` Michael S. Tsirkin
0 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-10 1:47 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Thu, 9 Nov 2023 07:00:51 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> On Thu, Nov 09, 2023 at 07:10:02PM +0800, Xuan Zhuo wrote:
> > On Thu, 9 Nov 2023 03:15:03 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > On Tue, Nov 07, 2023 at 11:12:23AM +0800, Xuan Zhuo wrote:
> > > > When rq is bound with AF_XDP, the buffer dma is managed
> > > > by the AF_XDP APIs. So the buffer got from the virtio core should
> > > > skip the dma unmap operation.
> > > >
> > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > >
> > >
> > > I don't get it - is this like a bugfix?
> >
> > I want focus on this. So let it as an independent commit.
> >
> > > And why do we need our own flag and checks?
> > > Doesn't virtio core DTRT?
> >
> >
> > struct vring_virtqueue {
> > [....]
> >
> > /* Do DMA mapping by driver */
> > bool premapped;
> >
> > We can not.
> >
> > So I add own flag.
> >
> > Thanks.
>
> Still don't get it. Why not check the premapped flag?
premapped is in the struct vring_virtqueue.
We can not access it from the driver.
>
> >
> > >
> > > > ---
> > > > drivers/net/virtio/main.c | 8 +++++---
> > > > drivers/net/virtio/virtio_net.h | 3 +++
> > > > drivers/net/virtio/xsk.c | 1 +
> > > > 3 files changed, 9 insertions(+), 3 deletions(-)
> > > >
> > > > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > > > index 15943a22e17d..a318b2533b94 100644
> > > > --- a/drivers/net/virtio/main.c
> > > > +++ b/drivers/net/virtio/main.c
> > > > @@ -430,7 +430,7 @@ static void *virtnet_rq_get_buf(struct virtnet_rq *rq, u32 *len, void **ctx)
> > > > void *buf;
> > > >
> > > > buf = virtqueue_get_buf_ctx(rq->vq, len, ctx);
> > > > - if (buf && rq->do_dma)
> > > > + if (buf && rq->do_dma_unmap)
> > > > virtnet_rq_unmap(rq, buf, *len);
> > > >
> > > > return buf;
> > > > @@ -561,8 +561,10 @@ static void virtnet_set_premapped(struct virtnet_info *vi)
> > > >
> > > > /* disable for big mode */
> > > > if (vi->mergeable_rx_bufs || !vi->big_packets) {
> > > > - if (!virtqueue_set_dma_premapped(vi->rq[i].vq))
> > > > + if (!virtqueue_set_dma_premapped(vi->rq[i].vq)) {
> > > > vi->rq[i].do_dma = true;
> > > > + vi->rq[i].do_dma_unmap = true;
> > > > + }
> > > > }
> > > > }
> > > > }
> > > > @@ -3944,7 +3946,7 @@ void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
> > > >
> > > > rq = &vi->rq[i];
> > > >
> > > > - if (rq->do_dma)
> > > > + if (rq->do_dma_unmap)
> > > > virtnet_rq_unmap(rq, buf, 0);
> > > >
> > > > virtnet_rq_free_buf(vi, rq, buf);
> > > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > > index 1242785e311e..2005d0cd22e2 100644
> > > > --- a/drivers/net/virtio/virtio_net.h
> > > > +++ b/drivers/net/virtio/virtio_net.h
> > > > @@ -135,6 +135,9 @@ struct virtnet_rq {
> > > > /* Do dma by self */
> > > > bool do_dma;
> > > >
> > > > + /* Do dma unmap after getting buf from virtio core. */
> > > > + bool do_dma_unmap;
> > > > +
> > > > struct {
> > > > struct xsk_buff_pool *pool;
> > > >
> > > > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > > > index e737c3353212..b09c473c29fb 100644
> > > > --- a/drivers/net/virtio/xsk.c
> > > > +++ b/drivers/net/virtio/xsk.c
> > > > @@ -210,6 +210,7 @@ static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *
> > > > xdp_rxq_info_unreg(&rq->xsk.xdp_rxq);
> > > >
> > > > rq->xsk.pool = pool;
> > > > + rq->do_dma_unmap = !pool;
> > > >
> > > > virtnet_rx_resume(vi, rq);
> > > >
> > > > --
> > > > 2.32.0.3.g01195cf9f
> > >
> > >
>
>
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 17/21] virtio_net: xsk: rx: skip dma unmap when rq is bind with AF_XDP
2023-11-10 1:47 ` Xuan Zhuo
@ 2023-11-10 5:33 ` Michael S. Tsirkin
2023-11-10 5:51 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Michael S. Tsirkin @ 2023-11-10 5:33 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Fri, Nov 10, 2023 at 09:47:16AM +0800, Xuan Zhuo wrote:
> On Thu, 9 Nov 2023 07:00:51 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > On Thu, Nov 09, 2023 at 07:10:02PM +0800, Xuan Zhuo wrote:
> > > On Thu, 9 Nov 2023 03:15:03 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > > On Tue, Nov 07, 2023 at 11:12:23AM +0800, Xuan Zhuo wrote:
> > > > > When rq is bound with AF_XDP, the buffer dma is managed
> > > > > by the AF_XDP APIs. So the buffer got from the virtio core should
> > > > > skip the dma unmap operation.
> > > > >
> > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > >
> > > >
> > > > I don't get it - is this like a bugfix?
> > >
> > > I want focus on this. So let it as an independent commit.
> > >
> > > > And why do we need our own flag and checks?
> > > > Doesn't virtio core DTRT?
> > >
> > >
> > > struct vring_virtqueue {
> > > [....]
> > >
> > > /* Do DMA mapping by driver */
> > > bool premapped;
> > >
> > > We can not.
> > >
> > > So I add own flag.
> > >
> > > Thanks.
> >
> > Still don't get it. Why not check the premapped flag?
>
> premapped is in the struct vring_virtqueue.
>
> We can not access it from the driver.
If it's useful, move it.
>
> >
> > >
> > > >
> > > > > ---
> > > > > drivers/net/virtio/main.c | 8 +++++---
> > > > > drivers/net/virtio/virtio_net.h | 3 +++
> > > > > drivers/net/virtio/xsk.c | 1 +
> > > > > 3 files changed, 9 insertions(+), 3 deletions(-)
> > > > >
> > > > > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > > > > index 15943a22e17d..a318b2533b94 100644
> > > > > --- a/drivers/net/virtio/main.c
> > > > > +++ b/drivers/net/virtio/main.c
> > > > > @@ -430,7 +430,7 @@ static void *virtnet_rq_get_buf(struct virtnet_rq *rq, u32 *len, void **ctx)
> > > > > void *buf;
> > > > >
> > > > > buf = virtqueue_get_buf_ctx(rq->vq, len, ctx);
> > > > > - if (buf && rq->do_dma)
> > > > > + if (buf && rq->do_dma_unmap)
> > > > > virtnet_rq_unmap(rq, buf, *len);
> > > > >
> > > > > return buf;
> > > > > @@ -561,8 +561,10 @@ static void virtnet_set_premapped(struct virtnet_info *vi)
> > > > >
> > > > > /* disable for big mode */
> > > > > if (vi->mergeable_rx_bufs || !vi->big_packets) {
> > > > > - if (!virtqueue_set_dma_premapped(vi->rq[i].vq))
> > > > > + if (!virtqueue_set_dma_premapped(vi->rq[i].vq)) {
> > > > > vi->rq[i].do_dma = true;
> > > > > + vi->rq[i].do_dma_unmap = true;
> > > > > + }
> > > > > }
> > > > > }
> > > > > }
> > > > > @@ -3944,7 +3946,7 @@ void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
> > > > >
> > > > > rq = &vi->rq[i];
> > > > >
> > > > > - if (rq->do_dma)
> > > > > + if (rq->do_dma_unmap)
> > > > > virtnet_rq_unmap(rq, buf, 0);
> > > > >
> > > > > virtnet_rq_free_buf(vi, rq, buf);
> > > > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > > > index 1242785e311e..2005d0cd22e2 100644
> > > > > --- a/drivers/net/virtio/virtio_net.h
> > > > > +++ b/drivers/net/virtio/virtio_net.h
> > > > > @@ -135,6 +135,9 @@ struct virtnet_rq {
> > > > > /* Do dma by self */
> > > > > bool do_dma;
> > > > >
> > > > > + /* Do dma unmap after getting buf from virtio core. */
> > > > > + bool do_dma_unmap;
> > > > > +
> > > > > struct {
> > > > > struct xsk_buff_pool *pool;
> > > > >
> > > > > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > > > > index e737c3353212..b09c473c29fb 100644
> > > > > --- a/drivers/net/virtio/xsk.c
> > > > > +++ b/drivers/net/virtio/xsk.c
> > > > > @@ -210,6 +210,7 @@ static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *
> > > > > xdp_rxq_info_unreg(&rq->xsk.xdp_rxq);
> > > > >
> > > > > rq->xsk.pool = pool;
> > > > > + rq->do_dma_unmap = !pool;
> > > > >
> > > > > virtnet_rx_resume(vi, rq);
> > > > >
> > > > > --
> > > > > 2.32.0.3.g01195cf9f
> > > >
> > > >
> >
> >
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 17/21] virtio_net: xsk: rx: skip dma unmap when rq is bind with AF_XDP
2023-11-10 5:33 ` Michael S. Tsirkin
@ 2023-11-10 5:51 ` Xuan Zhuo
0 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-10 5:51 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Fri, 10 Nov 2023 00:33:27 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> On Fri, Nov 10, 2023 at 09:47:16AM +0800, Xuan Zhuo wrote:
> > On Thu, 9 Nov 2023 07:00:51 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > On Thu, Nov 09, 2023 at 07:10:02PM +0800, Xuan Zhuo wrote:
> > > > On Thu, 9 Nov 2023 03:15:03 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> > > > > On Tue, Nov 07, 2023 at 11:12:23AM +0800, Xuan Zhuo wrote:
> > > > > > When rq is bound with AF_XDP, the buffer dma is managed
> > > > > > by the AF_XDP APIs. So the buffer got from the virtio core should
> > > > > > skip the dma unmap operation.
> > > > > >
> > > > > > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > > > >
> > > > >
> > > > > I don't get it - is this like a bugfix?
> > > >
> > > > I want focus on this. So let it as an independent commit.
> > > >
> > > > > And why do we need our own flag and checks?
> > > > > Doesn't virtio core DTRT?
> > > >
> > > >
> > > > struct vring_virtqueue {
> > > > [....]
> > > >
> > > > /* Do DMA mapping by driver */
> > > > bool premapped;
> > > >
> > > > We can not.
> > > >
> > > > So I add own flag.
> > > >
> > > > Thanks.
> > >
> > > Still don't get it. Why not check the premapped flag?
> >
> > premapped is in the struct vring_virtqueue.
> >
> > We can not access it from the driver.
>
>
> If it's useful, move it.
We set that by API.
If we expose that to the driver. I worry some driver change it directly.
So I think it's better for the driver to add an own flag.
Thanks.
>
>
> >
> > >
> > > >
> > > > >
> > > > > > ---
> > > > > > drivers/net/virtio/main.c | 8 +++++---
> > > > > > drivers/net/virtio/virtio_net.h | 3 +++
> > > > > > drivers/net/virtio/xsk.c | 1 +
> > > > > > 3 files changed, 9 insertions(+), 3 deletions(-)
> > > > > >
> > > > > > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > > > > > index 15943a22e17d..a318b2533b94 100644
> > > > > > --- a/drivers/net/virtio/main.c
> > > > > > +++ b/drivers/net/virtio/main.c
> > > > > > @@ -430,7 +430,7 @@ static void *virtnet_rq_get_buf(struct virtnet_rq *rq, u32 *len, void **ctx)
> > > > > > void *buf;
> > > > > >
> > > > > > buf = virtqueue_get_buf_ctx(rq->vq, len, ctx);
> > > > > > - if (buf && rq->do_dma)
> > > > > > + if (buf && rq->do_dma_unmap)
> > > > > > virtnet_rq_unmap(rq, buf, *len);
> > > > > >
> > > > > > return buf;
> > > > > > @@ -561,8 +561,10 @@ static void virtnet_set_premapped(struct virtnet_info *vi)
> > > > > >
> > > > > > /* disable for big mode */
> > > > > > if (vi->mergeable_rx_bufs || !vi->big_packets) {
> > > > > > - if (!virtqueue_set_dma_premapped(vi->rq[i].vq))
> > > > > > + if (!virtqueue_set_dma_premapped(vi->rq[i].vq)) {
> > > > > > vi->rq[i].do_dma = true;
> > > > > > + vi->rq[i].do_dma_unmap = true;
> > > > > > + }
> > > > > > }
> > > > > > }
> > > > > > }
> > > > > > @@ -3944,7 +3946,7 @@ void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
> > > > > >
> > > > > > rq = &vi->rq[i];
> > > > > >
> > > > > > - if (rq->do_dma)
> > > > > > + if (rq->do_dma_unmap)
> > > > > > virtnet_rq_unmap(rq, buf, 0);
> > > > > >
> > > > > > virtnet_rq_free_buf(vi, rq, buf);
> > > > > > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > > > > > index 1242785e311e..2005d0cd22e2 100644
> > > > > > --- a/drivers/net/virtio/virtio_net.h
> > > > > > +++ b/drivers/net/virtio/virtio_net.h
> > > > > > @@ -135,6 +135,9 @@ struct virtnet_rq {
> > > > > > /* Do dma by self */
> > > > > > bool do_dma;
> > > > > >
> > > > > > + /* Do dma unmap after getting buf from virtio core. */
> > > > > > + bool do_dma_unmap;
> > > > > > +
> > > > > > struct {
> > > > > > struct xsk_buff_pool *pool;
> > > > > >
> > > > > > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > > > > > index e737c3353212..b09c473c29fb 100644
> > > > > > --- a/drivers/net/virtio/xsk.c
> > > > > > +++ b/drivers/net/virtio/xsk.c
> > > > > > @@ -210,6 +210,7 @@ static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct virtnet_rq *
> > > > > > xdp_rxq_info_unreg(&rq->xsk.xdp_rxq);
> > > > > >
> > > > > > rq->xsk.pool = pool;
> > > > > > + rq->do_dma_unmap = !pool;
> > > > > >
> > > > > > virtnet_rx_resume(vi, rq);
> > > > > >
> > > > > > --
> > > > > > 2.32.0.3.g01195cf9f
> > > > >
> > > > >
> > >
> > >
>
>
^ permalink raw reply [flat|nested] 61+ messages in thread
* [PATCH net-next v2 18/21] virtio_net: xsk: rx: introduce receive_xsk() to recv xsk buffer
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (16 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 17/21] virtio_net: xsk: rx: skip dma unmap when rq is bind with AF_XDP Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-13 16:11 ` Maciej Fijalkowski
2023-11-07 3:12 ` [PATCH net-next v2 19/21] virtio_net: xsk: rx: virtnet_rq_free_unused_buf() check " Xuan Zhuo
` (4 subsequent siblings)
22 siblings, 1 reply; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
The virtnet_xdp_handler() is re-used. But
1. We need to copy data to create skb for XDP_PASS.
2. We need to call xsk_buff_free() to release the buffer.
3. The handle for xdp_buff is difference.
If we pushed this logic into existing receive handle(merge and small),
we would have to maintain code scattered inside merge and small (and big).
So I think it is a good choice for us to put the xsk code into an
independent function.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 12 ++--
drivers/net/virtio/virtio_net.h | 4 ++
drivers/net/virtio/xsk.c | 120 ++++++++++++++++++++++++++++++++
drivers/net/virtio/xsk.h | 4 ++
4 files changed, 135 insertions(+), 5 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index a318b2533b94..095f4acb0577 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -831,10 +831,10 @@ static void put_xdp_frags(struct xdp_buff *xdp)
}
}
-static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
- struct net_device *dev,
- unsigned int *xdp_xmit,
- struct virtnet_rq_stats *stats)
+int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
+ struct net_device *dev,
+ unsigned int *xdp_xmit,
+ struct virtnet_rq_stats *stats)
{
struct xdp_frame *xdpf;
int err;
@@ -1598,7 +1598,9 @@ static void receive_buf(struct virtnet_info *vi, struct virtnet_rq *rq,
return;
}
- if (vi->mergeable_rx_bufs)
+ if (rq->xsk.pool)
+ skb = virtnet_receive_xsk(dev, vi, rq, buf, len, xdp_xmit, stats);
+ else if (vi->mergeable_rx_bufs)
skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit,
stats);
else if (vi->big_packets)
diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
index 2005d0cd22e2..f520fec06662 100644
--- a/drivers/net/virtio/virtio_net.h
+++ b/drivers/net/virtio/virtio_net.h
@@ -339,4 +339,8 @@ void virtnet_tx_pause(struct virtnet_info *vi, struct virtnet_sq *sq);
void virtnet_tx_resume(struct virtnet_info *vi, struct virtnet_sq *sq);
void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf);
void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf);
+int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
+ struct net_device *dev,
+ unsigned int *xdp_xmit,
+ struct virtnet_rq_stats *stats);
#endif
diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
index b09c473c29fb..5c7eb19ab04b 100644
--- a/drivers/net/virtio/xsk.c
+++ b/drivers/net/virtio/xsk.c
@@ -14,6 +14,18 @@ static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len)
sg->length = len;
}
+static unsigned int virtnet_receive_buf_num(struct virtnet_info *vi, char *buf)
+{
+ struct virtio_net_hdr_mrg_rxbuf *hdr;
+
+ if (vi->mergeable_rx_bufs) {
+ hdr = (struct virtio_net_hdr_mrg_rxbuf *)buf;
+ return virtio16_to_cpu(vi->vdev, hdr->num_buffers);
+ }
+
+ return 1;
+}
+
static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
{
struct virtnet_info *vi = sq->vq->vdev->priv;
@@ -38,6 +50,114 @@ static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
netif_stop_subqueue(dev, qnum);
}
+static void merge_drop_follow_xdp(struct net_device *dev,
+ struct virtnet_rq *rq,
+ u32 num_buf,
+ struct virtnet_rq_stats *stats)
+{
+ struct xdp_buff *xdp;
+ u32 len;
+
+ while (num_buf-- > 1) {
+ xdp = virtqueue_get_buf(rq->vq, &len);
+ if (unlikely(!xdp)) {
+ pr_debug("%s: rx error: %d buffers missing\n",
+ dev->name, num_buf);
+ dev->stats.rx_length_errors++;
+ break;
+ }
+ u64_stats_add(&stats->bytes, len);
+ xsk_buff_free(xdp);
+ }
+}
+
+static struct sk_buff *construct_skb(struct virtnet_rq *rq,
+ struct xdp_buff *xdp)
+{
+ unsigned int metasize = xdp->data - xdp->data_meta;
+ struct sk_buff *skb;
+ unsigned int size;
+
+ size = xdp->data_end - xdp->data_hard_start;
+ skb = napi_alloc_skb(&rq->napi, size);
+ if (unlikely(!skb))
+ return NULL;
+
+ skb_reserve(skb, xdp->data_meta - xdp->data_hard_start);
+
+ size = xdp->data_end - xdp->data_meta;
+ memcpy(__skb_put(skb, size), xdp->data_meta, size);
+
+ if (metasize) {
+ __skb_pull(skb, metasize);
+ skb_metadata_set(skb, metasize);
+ }
+
+ return skb;
+}
+
+struct sk_buff *virtnet_receive_xsk(struct net_device *dev, struct virtnet_info *vi,
+ struct virtnet_rq *rq, void *buf,
+ unsigned int len, unsigned int *xdp_xmit,
+ struct virtnet_rq_stats *stats)
+{
+ struct virtio_net_hdr_mrg_rxbuf *hdr;
+ struct sk_buff *skb = NULL;
+ u32 ret, headroom, num_buf;
+ struct bpf_prog *prog;
+ struct xdp_buff *xdp;
+
+ len -= vi->hdr_len;
+
+ xdp = (struct xdp_buff *)buf;
+
+ xsk_buff_set_size(xdp, len);
+
+ hdr = xdp->data - vi->hdr_len;
+
+ num_buf = virtnet_receive_buf_num(vi, (char *)hdr);
+ if (num_buf > 1)
+ goto drop;
+
+ headroom = xdp->data - xdp->data_hard_start;
+
+ xdp_prepare_buff(xdp, xdp->data_hard_start, headroom, len, true);
+ xsk_buff_dma_sync_for_cpu(xdp, rq->xsk.pool);
+
+ ret = XDP_PASS;
+ rcu_read_lock();
+ prog = rcu_dereference(rq->xdp_prog);
+ if (prog)
+ ret = virtnet_xdp_handler(prog, xdp, dev, xdp_xmit, stats);
+ rcu_read_unlock();
+
+ switch (ret) {
+ case XDP_PASS:
+ skb = construct_skb(rq, xdp);
+ xsk_buff_free(xdp);
+ break;
+
+ case XDP_TX:
+ case XDP_REDIRECT:
+ goto consumed;
+
+ default:
+ goto drop;
+ }
+
+ return skb;
+
+drop:
+ u64_stats_inc(&stats->drops);
+
+ xsk_buff_free(xdp);
+
+ if (num_buf > 1)
+ merge_drop_follow_xdp(dev, rq, num_buf, stats);
+consumed:
+ return NULL;
+}
+
int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
struct xsk_buff_pool *pool, gfp_t gfp)
{
diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
index bef41a3f954e..dbd2839a5f61 100644
--- a/drivers/net/virtio/xsk.h
+++ b/drivers/net/virtio/xsk.h
@@ -25,4 +25,8 @@ bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag);
int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
struct xsk_buff_pool *pool, gfp_t gfp);
+struct sk_buff *virtnet_receive_xsk(struct net_device *dev, struct virtnet_info *vi,
+ struct virtnet_rq *rq, void *buf,
+ unsigned int len, unsigned int *xdp_xmit,
+ struct virtnet_rq_stats *stats);
#endif
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 18/21] virtio_net: xsk: rx: introduce receive_xsk() to recv xsk buffer
2023-11-07 3:12 ` [PATCH net-next v2 18/21] virtio_net: xsk: rx: introduce receive_xsk() to recv xsk buffer Xuan Zhuo
@ 2023-11-13 16:11 ` Maciej Fijalkowski
2023-11-14 3:43 ` Xuan Zhuo
0 siblings, 1 reply; 61+ messages in thread
From: Maciej Fijalkowski @ 2023-11-13 16:11 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Michael S. Tsirkin, Jason Wang, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Tue, Nov 07, 2023 at 11:12:24AM +0800, Xuan Zhuo wrote:
> The virtnet_xdp_handler() is re-used. But
>
> 1. We need to copy data to create skb for XDP_PASS.
> 2. We need to call xsk_buff_free() to release the buffer.
> 3. The handle for xdp_buff is difference.
>
> If we pushed this logic into existing receive handle(merge and small),
> we would have to maintain code scattered inside merge and small (and big).
> So I think it is a good choice for us to put the xsk code into an
> independent function.
>
> Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> ---
> drivers/net/virtio/main.c | 12 ++--
> drivers/net/virtio/virtio_net.h | 4 ++
> drivers/net/virtio/xsk.c | 120 ++++++++++++++++++++++++++++++++
> drivers/net/virtio/xsk.h | 4 ++
> 4 files changed, 135 insertions(+), 5 deletions(-)
>
> diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> index a318b2533b94..095f4acb0577 100644
> --- a/drivers/net/virtio/main.c
> +++ b/drivers/net/virtio/main.c
> @@ -831,10 +831,10 @@ static void put_xdp_frags(struct xdp_buff *xdp)
> }
> }
>
> -static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
> - struct net_device *dev,
> - unsigned int *xdp_xmit,
> - struct virtnet_rq_stats *stats)
> +int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
> + struct net_device *dev,
> + unsigned int *xdp_xmit,
> + struct virtnet_rq_stats *stats)
> {
> struct xdp_frame *xdpf;
> int err;
> @@ -1598,7 +1598,9 @@ static void receive_buf(struct virtnet_info *vi, struct virtnet_rq *rq,
> return;
> }
>
> - if (vi->mergeable_rx_bufs)
> + if (rq->xsk.pool)
> + skb = virtnet_receive_xsk(dev, vi, rq, buf, len, xdp_xmit, stats);
> + else if (vi->mergeable_rx_bufs)
> skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit,
> stats);
> else if (vi->big_packets)
> diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> index 2005d0cd22e2..f520fec06662 100644
> --- a/drivers/net/virtio/virtio_net.h
> +++ b/drivers/net/virtio/virtio_net.h
> @@ -339,4 +339,8 @@ void virtnet_tx_pause(struct virtnet_info *vi, struct virtnet_sq *sq);
> void virtnet_tx_resume(struct virtnet_info *vi, struct virtnet_sq *sq);
> void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf);
> void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf);
> +int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
> + struct net_device *dev,
> + unsigned int *xdp_xmit,
> + struct virtnet_rq_stats *stats);
> #endif
> diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> index b09c473c29fb..5c7eb19ab04b 100644
> --- a/drivers/net/virtio/xsk.c
> +++ b/drivers/net/virtio/xsk.c
> @@ -14,6 +14,18 @@ static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len)
> sg->length = len;
> }
>
> +static unsigned int virtnet_receive_buf_num(struct virtnet_info *vi, char *buf)
> +{
> + struct virtio_net_hdr_mrg_rxbuf *hdr;
> +
> + if (vi->mergeable_rx_bufs) {
> + hdr = (struct virtio_net_hdr_mrg_rxbuf *)buf;
> + return virtio16_to_cpu(vi->vdev, hdr->num_buffers);
> + }
> +
> + return 1;
> +}
> +
> static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> {
> struct virtnet_info *vi = sq->vq->vdev->priv;
> @@ -38,6 +50,114 @@ static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> netif_stop_subqueue(dev, qnum);
> }
>
> +static void merge_drop_follow_xdp(struct net_device *dev,
> + struct virtnet_rq *rq,
> + u32 num_buf,
> + struct virtnet_rq_stats *stats)
> +{
> + struct xdp_buff *xdp;
> + u32 len;
> +
> + while (num_buf-- > 1) {
> + xdp = virtqueue_get_buf(rq->vq, &len);
> + if (unlikely(!xdp)) {
> + pr_debug("%s: rx error: %d buffers missing\n",
> + dev->name, num_buf);
> + dev->stats.rx_length_errors++;
> + break;
> + }
> + u64_stats_add(&stats->bytes, len);
> + xsk_buff_free(xdp);
> + }
> +}
> +
> +static struct sk_buff *construct_skb(struct virtnet_rq *rq,
could you name this to virtnet_construct_skb_zc
> + struct xdp_buff *xdp)
> +{
> + unsigned int metasize = xdp->data - xdp->data_meta;
> + struct sk_buff *skb;
> + unsigned int size;
> +
> + size = xdp->data_end - xdp->data_hard_start;
> + skb = napi_alloc_skb(&rq->napi, size);
> + if (unlikely(!skb))
> + return NULL;
> +
> + skb_reserve(skb, xdp->data_meta - xdp->data_hard_start);
> +
> + size = xdp->data_end - xdp->data_meta;
> + memcpy(__skb_put(skb, size), xdp->data_meta, size);
> +
> + if (metasize) {
> + __skb_pull(skb, metasize);
> + skb_metadata_set(skb, metasize);
> + }
> +
> + return skb;
> +}
> +
> +struct sk_buff *virtnet_receive_xsk(struct net_device *dev, struct virtnet_info *vi,
> + struct virtnet_rq *rq, void *buf,
> + unsigned int len, unsigned int *xdp_xmit,
> + struct virtnet_rq_stats *stats)
> +{
> + struct virtio_net_hdr_mrg_rxbuf *hdr;
> + struct sk_buff *skb = NULL;
> + u32 ret, headroom, num_buf;
> + struct bpf_prog *prog;
> + struct xdp_buff *xdp;
> +
> + len -= vi->hdr_len;
> +
> + xdp = (struct xdp_buff *)buf;
> +
> + xsk_buff_set_size(xdp, len);
> +
> + hdr = xdp->data - vi->hdr_len;
> +
> + num_buf = virtnet_receive_buf_num(vi, (char *)hdr);
> + if (num_buf > 1)
> + goto drop;
> +
> + headroom = xdp->data - xdp->data_hard_start;
> +
> + xdp_prepare_buff(xdp, xdp->data_hard_start, headroom, len, true);
Please don't.
xsk_buff_pool has ::data_hard_start initialized and you already
initialized ::data and ::data_end within xsk_buff_set_size().
> + xsk_buff_dma_sync_for_cpu(xdp, rq->xsk.pool);
> +
> + ret = XDP_PASS;
> + rcu_read_lock();
We don't need RCU sections for running XDP progs anymore.
> + prog = rcu_dereference(rq->xdp_prog);
> + if (prog)
Prog is always !NULL for ZC case. Just dereference it at the beginning of
rx processing instead of doing it for each buf.
> + ret = virtnet_xdp_handler(prog, xdp, dev, xdp_xmit, stats);
> + rcu_read_unlock();
> +
> + switch (ret) {
> + case XDP_PASS:
> + skb = construct_skb(rq, xdp);
> + xsk_buff_free(xdp);
> + break;
> +
> + case XDP_TX:
> + case XDP_REDIRECT:
> + goto consumed;
> +
> + default:
> + goto drop;
> + }
> +
> + return skb;
> +
> +drop:
> + u64_stats_inc(&stats->drops);
> +
> + xsk_buff_free(xdp);
> +
> + if (num_buf > 1)
> + merge_drop_follow_xdp(dev, rq, num_buf, stats);
> +consumed:
> + return NULL;
> +}
> +
> int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> struct xsk_buff_pool *pool, gfp_t gfp)
> {
> diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> index bef41a3f954e..dbd2839a5f61 100644
> --- a/drivers/net/virtio/xsk.h
> +++ b/drivers/net/virtio/xsk.h
> @@ -25,4 +25,8 @@ bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag);
> int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> struct xsk_buff_pool *pool, gfp_t gfp);
> +struct sk_buff *virtnet_receive_xsk(struct net_device *dev, struct virtnet_info *vi,
> + struct virtnet_rq *rq, void *buf,
> + unsigned int len, unsigned int *xdp_xmit,
> + struct virtnet_rq_stats *stats);
> #endif
> --
> 2.32.0.3.g01195cf9f
>
>
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 18/21] virtio_net: xsk: rx: introduce receive_xsk() to recv xsk buffer
2023-11-13 16:11 ` Maciej Fijalkowski
@ 2023-11-14 3:43 ` Xuan Zhuo
0 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-14 3:43 UTC (permalink / raw)
To: Maciej Fijalkowski
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Michael S. Tsirkin, Jason Wang, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Mon, 13 Nov 2023 17:11:04 +0100, Maciej Fijalkowski <maciej.fijalkowski@intel.com> wrote:
> On Tue, Nov 07, 2023 at 11:12:24AM +0800, Xuan Zhuo wrote:
> > The virtnet_xdp_handler() is re-used. But
> >
> > 1. We need to copy data to create skb for XDP_PASS.
> > 2. We need to call xsk_buff_free() to release the buffer.
> > 3. The handle for xdp_buff is difference.
> >
> > If we pushed this logic into existing receive handle(merge and small),
> > we would have to maintain code scattered inside merge and small (and big).
> > So I think it is a good choice for us to put the xsk code into an
> > independent function.
> >
> > Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
> > ---
> > drivers/net/virtio/main.c | 12 ++--
> > drivers/net/virtio/virtio_net.h | 4 ++
> > drivers/net/virtio/xsk.c | 120 ++++++++++++++++++++++++++++++++
> > drivers/net/virtio/xsk.h | 4 ++
> > 4 files changed, 135 insertions(+), 5 deletions(-)
> >
> > diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
> > index a318b2533b94..095f4acb0577 100644
> > --- a/drivers/net/virtio/main.c
> > +++ b/drivers/net/virtio/main.c
> > @@ -831,10 +831,10 @@ static void put_xdp_frags(struct xdp_buff *xdp)
> > }
> > }
> >
> > -static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
> > - struct net_device *dev,
> > - unsigned int *xdp_xmit,
> > - struct virtnet_rq_stats *stats)
> > +int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
> > + struct net_device *dev,
> > + unsigned int *xdp_xmit,
> > + struct virtnet_rq_stats *stats)
> > {
> > struct xdp_frame *xdpf;
> > int err;
> > @@ -1598,7 +1598,9 @@ static void receive_buf(struct virtnet_info *vi, struct virtnet_rq *rq,
> > return;
> > }
> >
> > - if (vi->mergeable_rx_bufs)
> > + if (rq->xsk.pool)
> > + skb = virtnet_receive_xsk(dev, vi, rq, buf, len, xdp_xmit, stats);
> > + else if (vi->mergeable_rx_bufs)
> > skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit,
> > stats);
> > else if (vi->big_packets)
> > diff --git a/drivers/net/virtio/virtio_net.h b/drivers/net/virtio/virtio_net.h
> > index 2005d0cd22e2..f520fec06662 100644
> > --- a/drivers/net/virtio/virtio_net.h
> > +++ b/drivers/net/virtio/virtio_net.h
> > @@ -339,4 +339,8 @@ void virtnet_tx_pause(struct virtnet_info *vi, struct virtnet_sq *sq);
> > void virtnet_tx_resume(struct virtnet_info *vi, struct virtnet_sq *sq);
> > void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf);
> > void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf);
> > +int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
> > + struct net_device *dev,
> > + unsigned int *xdp_xmit,
> > + struct virtnet_rq_stats *stats);
> > #endif
> > diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
> > index b09c473c29fb..5c7eb19ab04b 100644
> > --- a/drivers/net/virtio/xsk.c
> > +++ b/drivers/net/virtio/xsk.c
> > @@ -14,6 +14,18 @@ static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len)
> > sg->length = len;
> > }
> >
> > +static unsigned int virtnet_receive_buf_num(struct virtnet_info *vi, char *buf)
> > +{
> > + struct virtio_net_hdr_mrg_rxbuf *hdr;
> > +
> > + if (vi->mergeable_rx_bufs) {
> > + hdr = (struct virtio_net_hdr_mrg_rxbuf *)buf;
> > + return virtio16_to_cpu(vi->vdev, hdr->num_buffers);
> > + }
> > +
> > + return 1;
> > +}
> > +
> > static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> > {
> > struct virtnet_info *vi = sq->vq->vdev->priv;
> > @@ -38,6 +50,114 @@ static void virtnet_xsk_check_queue(struct virtnet_sq *sq)
> > netif_stop_subqueue(dev, qnum);
> > }
> >
> > +static void merge_drop_follow_xdp(struct net_device *dev,
> > + struct virtnet_rq *rq,
> > + u32 num_buf,
> > + struct virtnet_rq_stats *stats)
> > +{
> > + struct xdp_buff *xdp;
> > + u32 len;
> > +
> > + while (num_buf-- > 1) {
> > + xdp = virtqueue_get_buf(rq->vq, &len);
> > + if (unlikely(!xdp)) {
> > + pr_debug("%s: rx error: %d buffers missing\n",
> > + dev->name, num_buf);
> > + dev->stats.rx_length_errors++;
> > + break;
> > + }
> > + u64_stats_add(&stats->bytes, len);
> > + xsk_buff_free(xdp);
> > + }
> > +}
> > +
> > +static struct sk_buff *construct_skb(struct virtnet_rq *rq,
>
> could you name this to virtnet_construct_skb_zc
>
> > + struct xdp_buff *xdp)
> > +{
> > + unsigned int metasize = xdp->data - xdp->data_meta;
> > + struct sk_buff *skb;
> > + unsigned int size;
> > +
> > + size = xdp->data_end - xdp->data_hard_start;
> > + skb = napi_alloc_skb(&rq->napi, size);
> > + if (unlikely(!skb))
> > + return NULL;
> > +
> > + skb_reserve(skb, xdp->data_meta - xdp->data_hard_start);
> > +
> > + size = xdp->data_end - xdp->data_meta;
> > + memcpy(__skb_put(skb, size), xdp->data_meta, size);
> > +
> > + if (metasize) {
> > + __skb_pull(skb, metasize);
> > + skb_metadata_set(skb, metasize);
> > + }
> > +
> > + return skb;
> > +}
> > +
> > +struct sk_buff *virtnet_receive_xsk(struct net_device *dev, struct virtnet_info *vi,
> > + struct virtnet_rq *rq, void *buf,
> > + unsigned int len, unsigned int *xdp_xmit,
> > + struct virtnet_rq_stats *stats)
> > +{
> > + struct virtio_net_hdr_mrg_rxbuf *hdr;
> > + struct sk_buff *skb = NULL;
> > + u32 ret, headroom, num_buf;
> > + struct bpf_prog *prog;
> > + struct xdp_buff *xdp;
> > +
> > + len -= vi->hdr_len;
> > +
> > + xdp = (struct xdp_buff *)buf;
> > +
> > + xsk_buff_set_size(xdp, len);
> > +
> > + hdr = xdp->data - vi->hdr_len;
> > +
> > + num_buf = virtnet_receive_buf_num(vi, (char *)hdr);
> > + if (num_buf > 1)
> > + goto drop;
> > +
> > + headroom = xdp->data - xdp->data_hard_start;
> > +
> > + xdp_prepare_buff(xdp, xdp->data_hard_start, headroom, len, true);
>
> Please don't.
>
> xsk_buff_pool has ::data_hard_start initialized and you already
> initialized ::data and ::data_end within xsk_buff_set_size().
Yes, you are right.
>
> > + xsk_buff_dma_sync_for_cpu(xdp, rq->xsk.pool);
> > +
> > + ret = XDP_PASS;
> > + rcu_read_lock();
>
> We don't need RCU sections for running XDP progs anymore.
>
> > + prog = rcu_dereference(rq->xdp_prog);
> > + if (prog)
>
> Prog is always !NULL for ZC case. Just dereference it at the beginning of
> rx processing instead of doing it for each buf.
That is a problem of virtio-net. I notice that, I will post patch to fix
that.
Thanks.
>
> > + ret = virtnet_xdp_handler(prog, xdp, dev, xdp_xmit, stats);
> > + rcu_read_unlock();
> > +
> > + switch (ret) {
> > + case XDP_PASS:
> > + skb = construct_skb(rq, xdp);
> > + xsk_buff_free(xdp);
> > + break;
> > +
> > + case XDP_TX:
> > + case XDP_REDIRECT:
> > + goto consumed;
> > +
> > + default:
> > + goto drop;
> > + }
> > +
> > + return skb;
> > +
> > +drop:
> > + u64_stats_inc(&stats->drops);
> > +
> > + xsk_buff_free(xdp);
> > +
> > + if (num_buf > 1)
> > + merge_drop_follow_xdp(dev, rq, num_buf, stats);
> > +consumed:
> > + return NULL;
> > +}
> > +
> > int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> > struct xsk_buff_pool *pool, gfp_t gfp)
> > {
> > diff --git a/drivers/net/virtio/xsk.h b/drivers/net/virtio/xsk.h
> > index bef41a3f954e..dbd2839a5f61 100644
> > --- a/drivers/net/virtio/xsk.h
> > +++ b/drivers/net/virtio/xsk.h
> > @@ -25,4 +25,8 @@ bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
> > int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag);
> > int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct virtnet_rq *rq,
> > struct xsk_buff_pool *pool, gfp_t gfp);
> > +struct sk_buff *virtnet_receive_xsk(struct net_device *dev, struct virtnet_info *vi,
> > + struct virtnet_rq *rq, void *buf,
> > + unsigned int len, unsigned int *xdp_xmit,
> > + struct virtnet_rq_stats *stats);
> > #endif
> > --
> > 2.32.0.3.g01195cf9f
> >
> >
^ permalink raw reply [flat|nested] 61+ messages in thread
* [PATCH net-next v2 19/21] virtio_net: xsk: rx: virtnet_rq_free_unused_buf() check xsk buffer
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (17 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 18/21] virtio_net: xsk: rx: introduce receive_xsk() to recv xsk buffer Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 20/21] virtio_net: update tx timeout record Xuan Zhuo
` (3 subsequent siblings)
22 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
Since this will be called in other circumstances(freeze), we must check
whether it is xsk's buffer in this function. It cannot be judged outside
this function.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 095f4acb0577..42c7dbf53e63 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -3948,6 +3948,14 @@ void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf)
rq = &vi->rq[i];
+ if (rq->xsk.pool) {
+ struct xdp_buff *xdp;
+
+ xdp = (struct xdp_buff *)buf;
+ xsk_buff_free(xdp);
+ return;
+ }
+
if (rq->do_dma_unmap)
virtnet_rq_unmap(rq, buf, 0);
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* [PATCH net-next v2 20/21] virtio_net: update tx timeout record
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (18 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 19/21] virtio_net: xsk: rx: virtnet_rq_free_unused_buf() check " Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-07 3:12 ` [PATCH net-next v2 21/21] virtio_net: xdp_features add NETDEV_XDP_ACT_XSK_ZEROCOPY Xuan Zhuo
` (2 subsequent siblings)
22 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
If send queue sent some packets, we update the tx timeout
record to prevent the tx timeout.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
Acked-by: Jason Wang <jasowang@redhat.com>
---
drivers/net/virtio/xsk.c | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/drivers/net/virtio/xsk.c b/drivers/net/virtio/xsk.c
index 5c7eb19ab04b..96bf5d8260f8 100644
--- a/drivers/net/virtio/xsk.c
+++ b/drivers/net/virtio/xsk.c
@@ -259,6 +259,16 @@ bool virtnet_xsk_xmit(struct virtnet_sq *sq, struct xsk_buff_pool *pool,
virtnet_xsk_check_queue(sq);
+ if (packets) {
+ struct netdev_queue *txq;
+ struct virtnet_info *vi;
+
+ vi = sq->vq->vdev->priv;
+
+ txq = netdev_get_tx_queue(vi->dev, sq - vi->sq);
+ txq_trans_cond_update(txq);
+ }
+
u64_stats_update_begin(&sq->stats.syncp);
u64_stats_add(&sq->stats.packets, packets);
u64_stats_add(&sq->stats.bytes, bytes);
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* [PATCH net-next v2 21/21] virtio_net: xdp_features add NETDEV_XDP_ACT_XSK_ZEROCOPY
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (19 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 20/21] virtio_net: update tx timeout record Xuan Zhuo
@ 2023-11-07 3:12 ` Xuan Zhuo
2023-11-07 18:01 ` [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Jakub Kicinski
2023-11-09 8:19 ` Michael S. Tsirkin
22 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-07 3:12 UTC (permalink / raw)
To: netdev
Cc: David S. Miller, Eric Dumazet, Jakub Kicinski, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Xuan Zhuo, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
Now, we supported AF_XDP(xsk). Add NETDEV_XDP_ACT_XSK_ZEROCOPY to
xdp_features.
Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
drivers/net/virtio/main.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index 42c7dbf53e63..2c14f0c84e56 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -4363,7 +4363,8 @@ static int virtnet_probe(struct virtio_device *vdev)
dev->hw_features |= NETIF_F_GRO_HW;
dev->vlan_features = dev->features;
- dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT;
+ dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
+ NETDEV_XDP_ACT_XSK_ZEROCOPY;
/* MTU range: 68 - 65535 */
dev->min_mtu = MIN_MTU;
--
2.32.0.3.g01195cf9f
^ permalink raw reply related [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (20 preceding siblings ...)
2023-11-07 3:12 ` [PATCH net-next v2 21/21] virtio_net: xdp_features add NETDEV_XDP_ACT_XSK_ZEROCOPY Xuan Zhuo
@ 2023-11-07 18:01 ` Jakub Kicinski
2023-11-08 5:49 ` Xuan Zhuo
2023-11-09 8:19 ` Michael S. Tsirkin
22 siblings, 1 reply; 61+ messages in thread
From: Jakub Kicinski @ 2023-11-07 18:01 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Tue, 7 Nov 2023 11:12:06 +0800 Xuan Zhuo wrote:
> Please review.
## Form letter - net-next-closed
The merge window for v6.7 has begun and we have already posted our pull
request. Therefore net-next is closed for new drivers, features, code
refactoring and optimizations. We are currently accepting bug fixes only.
Please repost when net-next reopens after Nov 12th.
RFC patches sent for review only are obviously welcome at any time.
See: https://www.kernel.org/doc/html/next/process/maintainer-netdev.html#development-cycle
--
pw-bot: defer
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy
2023-11-07 18:01 ` [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Jakub Kicinski
@ 2023-11-08 5:49 ` Xuan Zhuo
0 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-08 5:49 UTC (permalink / raw)
To: Jakub Kicinski
Cc: netdev, David S. Miller, Eric Dumazet, Paolo Abeni,
Michael S. Tsirkin, Jason Wang, Alexei Starovoitov,
Daniel Borkmann, Jesper Dangaard Brouer, John Fastabend,
virtualization, bpf
On Tue, 7 Nov 2023 10:01:48 -0800, Jakub Kicinski <kuba@kernel.org> wrote:
> On Tue, 7 Nov 2023 11:12:06 +0800 Xuan Zhuo wrote:
> > Please review.
>
> ## Form letter - net-next-closed
>
> The merge window for v6.7 has begun and we have already posted our pull
> request. Therefore net-next is closed for new drivers, features, code
> refactoring and optimizations. We are currently accepting bug fixes only.
>
> Please repost when net-next reopens after Nov 12th.
>
> RFC patches sent for review only are obviously welcome at any time.
Yes. This is for review.
Thanks.
>
> See: https://www.kernel.org/doc/html/next/process/maintainer-netdev.html#development-cycle
> --
> pw-bot: defer
^ permalink raw reply [flat|nested] 61+ messages in thread
* Re: [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy
2023-11-07 3:12 [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Xuan Zhuo
` (21 preceding siblings ...)
2023-11-07 18:01 ` [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy Jakub Kicinski
@ 2023-11-09 8:19 ` Michael S. Tsirkin
2023-11-09 10:37 ` Xuan Zhuo
22 siblings, 1 reply; 61+ messages in thread
From: Michael S. Tsirkin @ 2023-11-09 8:19 UTC (permalink / raw)
To: Xuan Zhuo
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Tue, Nov 07, 2023 at 11:12:06AM +0800, Xuan Zhuo wrote:
> ## AF_XDP
>
> XDP socket(AF_XDP) is an excellent bypass kernel network framework. The zero
> copy feature of xsk (XDP socket) needs to be supported by the driver. The
> performance of zero copy is very good. mlx5 and intel ixgbe already support
> this feature, This patch set allows virtio-net to support xsk's zerocopy xmit
> feature.
>
> At present, we have completed some preparation:
>
> 1. vq-reset (virtio spec and kernel code)
> 2. virtio-core premapped dma
> 3. virtio-net xdp refactor
>
> So it is time for Virtio-Net to complete the support for the XDP Socket
> Zerocopy.
>
> Virtio-net can not increase the queue num at will, so xsk shares the queue with
> kernel.
>
> On the other hand, Virtio-Net does not support generate interrupt from driver
> manually, so when we wakeup tx xmit, we used some tips. If the CPU run by TX
> NAPI last time is other CPUs, use IPI to wake up NAPI on the remote CPU. If it
> is also the local CPU, then we wake up napi directly.
>
> This patch set includes some refactor to the virtio-net to let that to support
> AF_XDP.
>
> ## performance
>
> ENV: Qemu with vhost-user(polling mode).
> Host CPU: Intel(R) Xeon(R) Platinum 8163 CPU @ 2.50GHz
>
> ### virtio PMD in guest with testpmd
>
> testpmd> show port stats all
>
> ######################## NIC statistics for port 0 ########################
> RX-packets: 19531092064 RX-missed: 0 RX-bytes: 1093741155584
> RX-errors: 0
> RX-nombuf: 0
> TX-packets: 5959955552 TX-errors: 0 TX-bytes: 371030645664
>
>
> Throughput (since last show)
> Rx-pps: 8861574 Rx-bps: 3969985208
> Tx-pps: 8861493 Tx-bps: 3969962736
> ############################################################################
>
> ### AF_XDP PMD in guest with testpmd
>
> testpmd> show port stats all
>
> ######################## NIC statistics for port 0 ########################
> RX-packets: 68152727 RX-missed: 0 RX-bytes: 3816552712
> RX-errors: 0
> RX-nombuf: 0
> TX-packets: 68114967 TX-errors: 33216 TX-bytes: 3814438152
>
> Throughput (since last show)
> Rx-pps: 6333196 Rx-bps: 2837272088
> Tx-pps: 6333227 Tx-bps: 2837285936
> ############################################################################
>
> But AF_XDP consumes more CPU for tx and rx napi(100% and 86%).
>
> ## maintain
>
> I am currently a reviewer for virtio-net. I commit to maintain AF_XDP support in
> virtio-net.
>
> Please review.
>
> Thanks.
>
> v2
> 1. wakeup uses the way of GVE. No send ipi to wakeup napi on remote cpu.
> 2. remove rcu. Because we synchronize all operat, so the rcu is not needed.
> 3. split the commit "move to virtio_net.h" in last patch set. Just move the
> struct/api to header when we use them.
> 4. add comments for some code
>
> v1:
> 1. remove two virtio commits. Push this patchset to net-next
> 2. squash "virtio_net: virtnet_poll_tx support rescheduled" to xsk: support tx
> 3. fix some warnings
>
>
> Xuan Zhuo (21):
> virtio_net: rename free_old_xmit_skbs to free_old_xmit
> virtio_net: unify the code for recycling the xmit ptr
> virtio_net: independent directory
> virtio_net: move core structures to virtio_net.h
> virtio_net: add prefix virtnet to all struct inside virtio_net.h
> virtio_net: separate virtnet_rx_resize()
> virtio_net: separate virtnet_tx_resize()
> virtio_net: sq support premapped mode
> virtio_net: xsk: bind/unbind xsk
> virtio_net: xsk: prevent disable tx napi
> virtio_net: move some api to header
> virtio_net: xsk: tx: support tx
> virtio_net: xsk: tx: support wakeup
> virtio_net: xsk: tx: virtnet_free_old_xmit() distinguishes xsk buffer
> virtio_net: xsk: tx: virtnet_sq_free_unused_buf() check xsk buffer
> virtio_net: xsk: rx: introduce add_recvbuf_xsk()
> virtio_net: xsk: rx: skip dma unmap when rq is bind with AF_XDP
> virtio_net: xsk: rx: introduce receive_xsk() to recv xsk buffer
> virtio_net: xsk: rx: virtnet_rq_free_unused_buf() check xsk buffer
> virtio_net: update tx timeout record
> virtio_net: xdp_features add NETDEV_XDP_ACT_XSK_ZEROCOPY
>
> MAINTAINERS | 2 +-
> drivers/net/Kconfig | 8 +-
> drivers/net/Makefile | 2 +-
> drivers/net/virtio/Kconfig | 13 +
> drivers/net/virtio/Makefile | 8 +
> drivers/net/{virtio_net.c => virtio/main.c} | 630 +++++++++-----------
> drivers/net/virtio/virtio_net.h | 346 +++++++++++
> drivers/net/virtio/xsk.c | 498 ++++++++++++++++
> drivers/net/virtio/xsk.h | 32 +
> 9 files changed, 1174 insertions(+), 365 deletions(-)
> create mode 100644 drivers/net/virtio/Kconfig
> create mode 100644 drivers/net/virtio/Makefile
> rename drivers/net/{virtio_net.c => virtio/main.c} (92%)
> create mode 100644 drivers/net/virtio/virtio_net.h
> create mode 100644 drivers/net/virtio/xsk.c
> create mode 100644 drivers/net/virtio/xsk.h
Too much code duplications for my taste. Would there maybe be less if
everything was in a single file?
> --
> 2.32.0.3.g01195cf9f
^ permalink raw reply [flat|nested] 61+ messages in thread* Re: [PATCH net-next v2 00/21] virtio-net: support AF_XDP zero copy
2023-11-09 8:19 ` Michael S. Tsirkin
@ 2023-11-09 10:37 ` Xuan Zhuo
0 siblings, 0 replies; 61+ messages in thread
From: Xuan Zhuo @ 2023-11-09 10:37 UTC (permalink / raw)
To: Michael S. Tsirkin
Cc: netdev, David S. Miller, Eric Dumazet, Jakub Kicinski,
Paolo Abeni, Jason Wang, Alexei Starovoitov, Daniel Borkmann,
Jesper Dangaard Brouer, John Fastabend, virtualization, bpf
On Thu, 9 Nov 2023 03:19:04 -0500, "Michael S. Tsirkin" <mst@redhat.com> wrote:
> On Tue, Nov 07, 2023 at 11:12:06AM +0800, Xuan Zhuo wrote:
> > ## AF_XDP
> >
> > XDP socket(AF_XDP) is an excellent bypass kernel network framework. The zero
> > copy feature of xsk (XDP socket) needs to be supported by the driver. The
> > performance of zero copy is very good. mlx5 and intel ixgbe already support
> > this feature, This patch set allows virtio-net to support xsk's zerocopy xmit
> > feature.
> >
> > At present, we have completed some preparation:
> >
> > 1. vq-reset (virtio spec and kernel code)
> > 2. virtio-core premapped dma
> > 3. virtio-net xdp refactor
> >
> > So it is time for Virtio-Net to complete the support for the XDP Socket
> > Zerocopy.
> >
> > Virtio-net can not increase the queue num at will, so xsk shares the queue with
> > kernel.
> >
> > On the other hand, Virtio-Net does not support generate interrupt from driver
> > manually, so when we wakeup tx xmit, we used some tips. If the CPU run by TX
> > NAPI last time is other CPUs, use IPI to wake up NAPI on the remote CPU. If it
> > is also the local CPU, then we wake up napi directly.
> >
> > This patch set includes some refactor to the virtio-net to let that to support
> > AF_XDP.
> >
> > ## performance
> >
> > ENV: Qemu with vhost-user(polling mode).
> > Host CPU: Intel(R) Xeon(R) Platinum 8163 CPU @ 2.50GHz
> >
> > ### virtio PMD in guest with testpmd
> >
> > testpmd> show port stats all
> >
> > ######################## NIC statistics for port 0 ########################
> > RX-packets: 19531092064 RX-missed: 0 RX-bytes: 1093741155584
> > RX-errors: 0
> > RX-nombuf: 0
> > TX-packets: 5959955552 TX-errors: 0 TX-bytes: 371030645664
> >
> >
> > Throughput (since last show)
> > Rx-pps: 8861574 Rx-bps: 3969985208
> > Tx-pps: 8861493 Tx-bps: 3969962736
> > ############################################################################
> >
> > ### AF_XDP PMD in guest with testpmd
> >
> > testpmd> show port stats all
> >
> > ######################## NIC statistics for port 0 ########################
> > RX-packets: 68152727 RX-missed: 0 RX-bytes: 3816552712
> > RX-errors: 0
> > RX-nombuf: 0
> > TX-packets: 68114967 TX-errors: 33216 TX-bytes: 3814438152
> >
> > Throughput (since last show)
> > Rx-pps: 6333196 Rx-bps: 2837272088
> > Tx-pps: 6333227 Tx-bps: 2837285936
> > ############################################################################
> >
> > But AF_XDP consumes more CPU for tx and rx napi(100% and 86%).
> >
> > ## maintain
> >
> > I am currently a reviewer for virtio-net. I commit to maintain AF_XDP support in
> > virtio-net.
> >
> > Please review.
> >
> > Thanks.
> >
> > v2
> > 1. wakeup uses the way of GVE. No send ipi to wakeup napi on remote cpu.
> > 2. remove rcu. Because we synchronize all operat, so the rcu is not needed.
> > 3. split the commit "move to virtio_net.h" in last patch set. Just move the
> > struct/api to header when we use them.
> > 4. add comments for some code
> >
> > v1:
> > 1. remove two virtio commits. Push this patchset to net-next
> > 2. squash "virtio_net: virtnet_poll_tx support rescheduled" to xsk: support tx
> > 3. fix some warnings
> >
> >
> > Xuan Zhuo (21):
> > virtio_net: rename free_old_xmit_skbs to free_old_xmit
> > virtio_net: unify the code for recycling the xmit ptr
> > virtio_net: independent directory
> > virtio_net: move core structures to virtio_net.h
> > virtio_net: add prefix virtnet to all struct inside virtio_net.h
> > virtio_net: separate virtnet_rx_resize()
> > virtio_net: separate virtnet_tx_resize()
> > virtio_net: sq support premapped mode
> > virtio_net: xsk: bind/unbind xsk
> > virtio_net: xsk: prevent disable tx napi
> > virtio_net: move some api to header
> > virtio_net: xsk: tx: support tx
> > virtio_net: xsk: tx: support wakeup
> > virtio_net: xsk: tx: virtnet_free_old_xmit() distinguishes xsk buffer
> > virtio_net: xsk: tx: virtnet_sq_free_unused_buf() check xsk buffer
> > virtio_net: xsk: rx: introduce add_recvbuf_xsk()
> > virtio_net: xsk: rx: skip dma unmap when rq is bind with AF_XDP
> > virtio_net: xsk: rx: introduce receive_xsk() to recv xsk buffer
> > virtio_net: xsk: rx: virtnet_rq_free_unused_buf() check xsk buffer
> > virtio_net: update tx timeout record
> > virtio_net: xdp_features add NETDEV_XDP_ACT_XSK_ZEROCOPY
> >
> > MAINTAINERS | 2 +-
> > drivers/net/Kconfig | 8 +-
> > drivers/net/Makefile | 2 +-
> > drivers/net/virtio/Kconfig | 13 +
> > drivers/net/virtio/Makefile | 8 +
> > drivers/net/{virtio_net.c => virtio/main.c} | 630 +++++++++-----------
> > drivers/net/virtio/virtio_net.h | 346 +++++++++++
> > drivers/net/virtio/xsk.c | 498 ++++++++++++++++
> > drivers/net/virtio/xsk.h | 32 +
> > 9 files changed, 1174 insertions(+), 365 deletions(-)
> > create mode 100644 drivers/net/virtio/Kconfig
> > create mode 100644 drivers/net/virtio/Makefile
> > rename drivers/net/{virtio_net.c => virtio/main.c} (92%)
> > create mode 100644 drivers/net/virtio/virtio_net.h
> > create mode 100644 drivers/net/virtio/xsk.c
> > create mode 100644 drivers/net/virtio/xsk.h
>
>
> Too much code duplications for my taste. Would there maybe be less if
> everything was in a single file?
What duplication?
Yes. If everything is in a single file, something maybe simpler. But I
do not like that, that will be more trouble in the future. We want to
make the virtnet more like the modern network card, then we will
introduce more features to the virtio-net. I think it's time to
split the single file to modules. I think letting every module simple is
beneficial for the maintaining.
About the duplications, I do not understand. No duplication,
We just refer the functions from each other.
Do you mean the receive_xsk()?
Let we resolve this problem if you think it's a duplicate. I think this is the
right way.
Thanks.
>
>
> > --
> > 2.32.0.3.g01195cf9f
>
>
^ permalink raw reply [flat|nested] 61+ messages in thread