[PATCH v5 4/4] net/af_packet: add software checksum offload support

[scott.k.mitch1@gmail.com]

From: Scott Mitchell <scott.k.mitch1@gmail.com>

Add software checksum offload support and configurable TX poll
behavior to improve flexibility and performance.

Add rte_net_ip_udptcp_cksum_mbuf in rte_net.h which is shared
between rte_eth_tap and rte_eth_af_packet that supports
IPv4/UDP/TCP checksums in software due to hardware offload
and context propagation not being supported.

Signed-off-by: Scott Mitchell <scott.k.mitch1@gmail.com>
---
Depends-on: patch-160679 ("eal: add __rte_may_alias and __rte_aligned to unaligned typedefs")

 doc/guides/nics/features/afpacket.ini     |  2 +
 doc/guides/rel_notes/release_26_03.rst    |  2 +
 drivers/net/af_packet/rte_eth_af_packet.c | 42 ++++++++++----
 drivers/net/tap/rte_eth_tap.c             | 70 ++---------------------
 lib/net/rte_net.c                         | 68 ++++++++++++++++++++++
 lib/net/rte_net.h                         | 22 +++++++
 6 files changed, 130 insertions(+), 76 deletions(-)

diff --git a/doc/guides/nics/features/afpacket.ini b/doc/guides/nics/features/afpacket.ini
index 391f79b173..4bb81c84ff 100644
--- a/doc/guides/nics/features/afpacket.ini
+++ b/doc/guides/nics/features/afpacket.ini
@@ -7,5 +7,7 @@
 Link status          = Y
 Promiscuous mode     = Y
 MTU update           = Y
+L3 checksum offload  = Y
+L4 checksum offload  = Y
 Basic stats          = Y
 Stats per queue      = Y
diff --git a/doc/guides/rel_notes/release_26_03.rst b/doc/guides/rel_notes/release_26_03.rst
index 3b6be19645..2946acce99 100644
--- a/doc/guides/rel_notes/release_26_03.rst
+++ b/doc/guides/rel_notes/release_26_03.rst
@@ -60,6 +60,8 @@ New Features
   * Fixed kernel memory barrier protocol for memory availability
   * Fixed shared memory frame overhead offset calculation
   * Added ``txpollnotrdy`` devarg to avoid ``poll()`` blocking calls
+  * Added checksum offload support for ``IPV4_CKSUM``, ``UDP_CKSUM``,
+    and ``TCP_CKSUM``
 
 Removed Items
 -------------
diff --git a/drivers/net/af_packet/rte_eth_af_packet.c b/drivers/net/af_packet/rte_eth_af_packet.c
index 9df1b1fd4c..662341ffc7 100644
--- a/drivers/net/af_packet/rte_eth_af_packet.c
+++ b/drivers/net/af_packet/rte_eth_af_packet.c
@@ -10,6 +10,8 @@
 #include <rte_string_fns.h>
 #include <rte_mbuf.h>
 #include <rte_atomic.h>
+#include <rte_ip.h>
+#include <rte_net.h>
 #include <rte_bitops.h>
 #include <ethdev_driver.h>
 #include <ethdev_vdev.h>
@@ -101,6 +103,7 @@ struct pmd_internals {
 	struct pkt_tx_queue *tx_queue;
 	uint8_t vlan_strip;
 	uint8_t timestamp_offloading;
+	bool tx_sw_cksum;
 };
 
 static const char *valid_arguments[] = {
@@ -220,7 +223,7 @@ eth_af_packet_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		/* account for the receive frame */
 		bufs[i] = mbuf;
 		num_rx++;
-		num_rx_bytes += mbuf->pkt_len;
+		num_rx_bytes += rte_pktmbuf_pkt_len(mbuf);
 	}
 	pkt_q->framenum = framenum;
 	pkt_q->rx_pkts += num_rx;
@@ -256,6 +259,7 @@ eth_af_packet_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 {
 	struct tpacket2_hdr *ppd;
 	struct rte_mbuf *mbuf;
+	struct rte_mbuf *seg;
 	uint8_t *pbuf;
 	unsigned int framecount, framenum;
 	struct pollfd pfd;
@@ -277,7 +281,7 @@ eth_af_packet_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 		mbuf = bufs[i];
 
 		/* Drop oversized packets. Insert VLAN if necessary */
-		if (unlikely(mbuf->pkt_len > pkt_q->frame_data_size ||
+		if (unlikely(rte_pktmbuf_pkt_len(mbuf) > pkt_q->frame_data_size ||
 			    ((mbuf->ol_flags & RTE_MBUF_F_TX_VLAN) != 0 &&
 			     rte_vlan_insert(&mbuf) != 0))) {
 			continue;
@@ -308,23 +312,32 @@ eth_af_packet_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 
 		pbuf = (uint8_t *)ppd + ETH_AF_PACKET_FRAME_OVERHEAD;
 
-		ppd->tp_len = mbuf->pkt_len;
-		ppd->tp_snaplen = mbuf->pkt_len;
+		if (pkt_q->sw_cksum) {
+			seg = rte_net_ip_udptcp_cksum_mbuf(mbuf);
+			if (!seg)
+				continue;
 
-		struct rte_mbuf *tmp_mbuf = mbuf;
+			mbuf = seg;
+			bufs[i] = seg;
+		}
+
+		ppd->tp_len = rte_pktmbuf_pkt_len(mbuf);
+		ppd->tp_snaplen = rte_pktmbuf_pkt_len(mbuf);
+
+		seg = mbuf;
 		do {
-			uint16_t data_len = rte_pktmbuf_data_len(tmp_mbuf);
-			memcpy(pbuf, rte_pktmbuf_mtod(tmp_mbuf, void*), data_len);
+			uint16_t data_len = rte_pktmbuf_data_len(seg);
+			memcpy(pbuf, rte_pktmbuf_mtod(seg, void*), data_len);
 			pbuf += data_len;
-			tmp_mbuf = tmp_mbuf->next;
-		} while (tmp_mbuf);
+			seg = seg->next;
+		} while (seg);
 
 		/* release incoming frame and advance ring buffer */
 		tpacket_write_status(&ppd->tp_status, TP_STATUS_SEND_REQUEST);
 		if (++framenum >= framecount)
 			framenum = 0;
 		num_tx++;
-		num_tx_bytes += mbuf->pkt_len;
+		num_tx_bytes += rte_pktmbuf_pkt_len(mbuf);
 	}
 
 	rte_pktmbuf_free_bulk(&bufs[0], i);
@@ -396,10 +409,13 @@ eth_dev_configure(struct rte_eth_dev *dev __rte_unused)
 {
 	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
 	const struct rte_eth_rxmode *rxmode = &dev_conf->rxmode;
+	const struct rte_eth_txmode *txmode = &dev_conf->txmode;
 	struct pmd_internals *internals = dev->data->dev_private;
 
 	internals->vlan_strip = !!(rxmode->offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP);
 	internals->timestamp_offloading = !!(rxmode->offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP);
+	internals->tx_sw_cksum = !!(txmode->offloads & (RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
+			RTE_ETH_TX_OFFLOAD_UDP_CKSUM | RTE_ETH_TX_OFFLOAD_TCP_CKSUM));
 	return 0;
 }
 
@@ -417,7 +433,10 @@ eth_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 	dev_info->max_tx_queues = (uint16_t)internals->nb_queues;
 	dev_info->min_rx_bufsize = ETH_AF_PACKET_ETH_OVERHEAD;
 	dev_info->tx_offload_capa = RTE_ETH_TX_OFFLOAD_MULTI_SEGS |
-		RTE_ETH_TX_OFFLOAD_VLAN_INSERT;
+		RTE_ETH_TX_OFFLOAD_VLAN_INSERT |
+		RTE_ETH_TX_OFFLOAD_IPV4_CKSUM |
+		RTE_ETH_TX_OFFLOAD_UDP_CKSUM |
+		RTE_ETH_TX_OFFLOAD_TCP_CKSUM;
 	dev_info->rx_offload_capa = RTE_ETH_RX_OFFLOAD_VLAN_STRIP |
 		RTE_ETH_RX_OFFLOAD_TIMESTAMP;
 
@@ -618,6 +637,7 @@ eth_tx_queue_setup(struct rte_eth_dev *dev,
 {
 
 	struct pmd_internals *internals = dev->data->dev_private;
+	internals->tx_queue[tx_queue_id].sw_cksum = internals->tx_sw_cksum;
 
 	dev->data->tx_queues[tx_queue_id] = &internals->tx_queue[tx_queue_id];
 	return 0;
diff --git a/drivers/net/tap/rte_eth_tap.c b/drivers/net/tap/rte_eth_tap.c
index 730f1859bd..c7ed6dfb8b 100644
--- a/drivers/net/tap/rte_eth_tap.c
+++ b/drivers/net/tap/rte_eth_tap.c
@@ -525,7 +525,6 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
 		struct iovec iovecs[mbuf->nb_segs + 2];
 		struct tun_pi pi = { .flags = 0, .proto = 0x00 };
 		struct rte_mbuf *seg = mbuf;
-		uint64_t l4_ol_flags;
 		int proto;
 		int n;
 		int j;
@@ -556,74 +555,15 @@ tap_write_mbufs(struct tx_queue *txq, uint16_t num_mbufs,
 		iovecs[k].iov_len = sizeof(pi);
 		k++;
 
-		l4_ol_flags = mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK;
-		if (txq->csum && (mbuf->ol_flags & RTE_MBUF_F_TX_IP_CKSUM ||
-				l4_ol_flags == RTE_MBUF_F_TX_UDP_CKSUM ||
-				l4_ol_flags == RTE_MBUF_F_TX_TCP_CKSUM)) {
-			unsigned int hdrlens = mbuf->l2_len + mbuf->l3_len;
-			uint16_t *l4_cksum;
-			void *l3_hdr;
-
-			if (l4_ol_flags == RTE_MBUF_F_TX_UDP_CKSUM)
-				hdrlens += sizeof(struct rte_udp_hdr);
-			else if (l4_ol_flags == RTE_MBUF_F_TX_TCP_CKSUM)
-				hdrlens += sizeof(struct rte_tcp_hdr);
-			else if (l4_ol_flags != RTE_MBUF_F_TX_L4_NO_CKSUM)
+		if (txq->csum) {
+			seg = rte_net_ip_udptcp_cksum_mbuf(mbuf);
+			if (!seg)
 				return -1;
 
-			/* Support only packets with at least layer 4
-			 * header included in the first segment
-			 */
-			if (rte_pktmbuf_data_len(mbuf) < hdrlens)
-				return -1;
-
-			/* To change checksums (considering that a mbuf can be
-			 * indirect, for example), copy l2, l3 and l4 headers
-			 * in a new segment and chain it to existing data
-			 */
-			seg = rte_pktmbuf_copy(mbuf, mbuf->pool, 0, hdrlens);
-			if (seg == NULL)
-				return -1;
-			rte_pktmbuf_adj(mbuf, hdrlens);
-			rte_pktmbuf_chain(seg, mbuf);
-			pmbufs[i] = mbuf = seg;
-
-			l3_hdr = rte_pktmbuf_mtod_offset(mbuf, void *, mbuf->l2_len);
-			if (mbuf->ol_flags & RTE_MBUF_F_TX_IP_CKSUM) {
-				struct rte_ipv4_hdr *iph = l3_hdr;
-
-				iph->hdr_checksum = 0;
-				iph->hdr_checksum = rte_ipv4_cksum(iph);
-			}
-
-			if (l4_ol_flags == RTE_MBUF_F_TX_L4_NO_CKSUM)
-				goto skip_l4_cksum;
-
-			if (l4_ol_flags == RTE_MBUF_F_TX_UDP_CKSUM) {
-				struct rte_udp_hdr *udp_hdr;
-
-				udp_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_udp_hdr *,
-					mbuf->l2_len + mbuf->l3_len);
-				l4_cksum = &udp_hdr->dgram_cksum;
-			} else {
-				struct rte_tcp_hdr *tcp_hdr;
-
-				tcp_hdr = rte_pktmbuf_mtod_offset(mbuf, struct rte_tcp_hdr *,
-					mbuf->l2_len + mbuf->l3_len);
-				l4_cksum = &tcp_hdr->cksum;
-			}
-
-			*l4_cksum = 0;
-			if (mbuf->ol_flags & RTE_MBUF_F_TX_IPV4) {
-				*l4_cksum = rte_ipv4_udptcp_cksum_mbuf(mbuf, l3_hdr,
-					mbuf->l2_len + mbuf->l3_len);
-			} else {
-				*l4_cksum = rte_ipv6_udptcp_cksum_mbuf(mbuf, l3_hdr,
-					mbuf->l2_len + mbuf->l3_len);
-			}
+			mbuf = seg;
+			pmbufs[i] = seg;
 		}
 
-skip_l4_cksum:
 		for (j = 0; j < mbuf->nb_segs; j++) {
 			iovecs[k].iov_len = rte_pktmbuf_data_len(seg);
 			iovecs[k].iov_base = rte_pktmbuf_mtod(seg, void *);
diff --git a/lib/net/rte_net.c b/lib/net/rte_net.c
index 458b4814a9..1a0397bcd7 100644
--- a/lib/net/rte_net.c
+++ b/lib/net/rte_net.c
@@ -615,3 +615,71 @@ uint32_t rte_net_get_ptype(const struct rte_mbuf *m,
 
 	return pkt_type;
 }
+
+RTE_EXPORT_EXPERIMENTAL_SYMBOL(rte_net_ip_udptcp_cksum_mbuf, 26.03)
+struct rte_mbuf *
+rte_net_ip_udptcp_cksum_mbuf(struct rte_mbuf *mbuf)
+{
+	const uint64_t l4_ol_flags = mbuf->ol_flags & RTE_MBUF_F_TX_L4_MASK;
+	const uint32_t l4_offset = mbuf->l2_len + mbuf->l3_len;
+	uint32_t hdrlens = l4_offset;
+	unaligned_uint16_t *l4_cksum = NULL;
+	void *l3_hdr;
+
+	/* Quick check - nothing to do if no checksum offloads requested */
+	if (!(mbuf->ol_flags & (RTE_MBUF_F_TX_IP_CKSUM | RTE_MBUF_F_TX_L4_MASK)))
+		return mbuf;
+
+	/* Determine total header length needed */
+	if (l4_ol_flags == RTE_MBUF_F_TX_UDP_CKSUM)
+		hdrlens += sizeof(struct rte_udp_hdr);
+	else if (l4_ol_flags == RTE_MBUF_F_TX_TCP_CKSUM)
+		hdrlens += sizeof(struct rte_tcp_hdr);
+	else if (l4_ol_flags != RTE_MBUF_F_TX_L4_NO_CKSUM)
+		return NULL; /* Unsupported L4 checksum type */
+
+	/* Validate we at least have L2+L3 headers */
+	if (unlikely(rte_pktmbuf_data_len(mbuf) < l4_offset))
+		return NULL;
+
+	if (!RTE_MBUF_DIRECT(mbuf) || rte_mbuf_refcnt_read(mbuf) > 1) {
+		/* Indirect or shared - must copy, cannot modify in-place */
+		struct rte_mbuf *seg = rte_pktmbuf_copy(mbuf, mbuf->pool, 0, hdrlens);
+		if (!seg)
+			return NULL;
+
+		rte_pktmbuf_adj(mbuf, hdrlens);
+		rte_pktmbuf_chain(seg, mbuf);
+		mbuf = seg;
+	} else if (rte_pktmbuf_data_len(mbuf) < hdrlens &&
+		(rte_pktmbuf_linearize(mbuf) < 0 || rte_pktmbuf_data_len(mbuf) < hdrlens)) {
+		/* failed: direct, non-shared, but segmented headers linearize in-place */
+		return NULL;
+	}
+	/* else: Direct, non-shared, contiguous - can modify in-place, nothing to do */
+
+	l3_hdr = rte_pktmbuf_mtod_offset(mbuf, void *, mbuf->l2_len);
+
+	/* IPv4 header checksum */
+	if (mbuf->ol_flags & RTE_MBUF_F_TX_IP_CKSUM) {
+		struct rte_ipv4_hdr *iph = (struct rte_ipv4_hdr *)l3_hdr;
+		iph->hdr_checksum = 0;
+		iph->hdr_checksum = rte_ipv4_cksum(iph);
+	}
+
+	/* L4 checksum */
+	if (l4_ol_flags == RTE_MBUF_F_TX_UDP_CKSUM)
+		l4_cksum = &rte_pktmbuf_mtod_offset(mbuf, struct rte_udp_hdr *,
+				l4_offset)->dgram_cksum;
+	else if (l4_ol_flags == RTE_MBUF_F_TX_TCP_CKSUM)
+		l4_cksum = &rte_pktmbuf_mtod_offset(mbuf, struct rte_tcp_hdr *, l4_offset)->cksum;
+
+	if (l4_cksum) {
+		*l4_cksum = 0;
+		*l4_cksum = (mbuf->ol_flags & RTE_MBUF_F_TX_IPV4) ?
+				rte_ipv4_udptcp_cksum_mbuf(mbuf, l3_hdr, l4_offset) :
+				rte_ipv6_udptcp_cksum_mbuf(mbuf, l3_hdr, l4_offset);
+	}
+
+	return mbuf;
+}
diff --git a/lib/net/rte_net.h b/lib/net/rte_net.h
index 65d724b84b..b258a86928 100644
--- a/lib/net/rte_net.h
+++ b/lib/net/rte_net.h
@@ -246,6 +246,28 @@ rte_net_intel_cksum_prepare(struct rte_mbuf *m)
 	return rte_net_intel_cksum_flags_prepare(m, m->ol_flags);
 }
 
+/**
+ * Compute IP and L4 checksums in software for mbufs with
+ * RTE_MBUF_F_TX_IP_CKSUM, RTE_MBUF_F_TX_UDP_CKSUM, or
+ * RTE_MBUF_F_TX_TCP_CKSUM offload flags set.
+ *
+ * On success, this function takes ownership of the input mbuf. The mbuf may be
+ * modified in-place (for direct, non-shared mbufs) or a new mbuf chain may be
+ * created (for indirect/shared mbufs) with the original becoming part of the chain.
+ *
+ * @param mbuf
+ *   The packet mbuf to checksum.
+ * @return
+ *   - On success: pointer to mbuf with checksums computed (may be same as input
+ *     or a new mbuf chain). Caller must free only this returned pointer; the input
+ *     mbuf pointer should not be freed separately as it may be part of the returned
+ *     chain or may be the same as the returned pointer.
+ *   - On error: NULL. Original mbuf remains valid and owned by caller.
+ */
+__rte_experimental
+struct rte_mbuf *
+rte_net_ip_udptcp_cksum_mbuf(struct rte_mbuf *mbuf);
+
 #ifdef __cplusplus
 }
 #endif
-- 
2.39.5 (Apple Git-154)