[RFC: DMA_PMD 18/22] net/gve: Use DMA_PMD memory for tx header bounce buffers

[Luigi Rizzo <lrizzo@google.com>]

When dev->dma_pmd_tx_hdrs is enabled, allocate a MAX_TCP_HEADER
bounce buffer per TX slot/packet out of the DMA_PMD arena in raw-addressing
mode (both GQI and DQO). When skb_headlen(skb) <= MAX_TCP_HEADER, the
linear portion is copied into the pre-mapped bounce buffer instead of
calling dma_map_single()/dma_unmap_single() on every packet.

Signed-off-by: Luigi Rizzo <lrizzo@google.com>
---
 drivers/net/ethernet/google/gve/gve.h        |  3 ++
 drivers/net/ethernet/google/gve/gve_tx.c     | 38 ++++++++++++++---
 drivers/net/ethernet/google/gve/gve_tx_dqo.c | 45 +++++++++++++++-----
 3 files changed, 69 insertions(+), 17 deletions(-)

diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h
index 11330404f681b..00e438fe05a50 100644
--- a/drivers/net/ethernet/google/gve/gve.h
+++ b/drivers/net/ethernet/google/gve/gve.h
@@ -18,6 +18,7 @@
 #include <linux/ptp_clock_kernel.h>
 #include <linux/u64_stats_sync.h>
 #include <net/page_pool/helpers.h>
+#include <net/tcp.h>
 #include <net/xdp.h>
 
 #include "gve_desc.h"
@@ -642,6 +643,8 @@ struct gve_tx_ring {
 			};
 		} dqo;
 	} ____cacheline_aligned;
+	void *tx_hdr_bufs;
+	dma_addr_t tx_hdr_bufs_dma;
 	struct netdev_queue *netdev_txq;
 	struct gve_queue_resources *q_resources; /* head and tail pointer idx */
 	struct device *dev;
diff --git a/drivers/net/ethernet/google/gve/gve_tx.c b/drivers/net/ethernet/google/gve/gve_tx.c
index 79338fd9bf66e..9610b54ef4792 100644
--- a/drivers/net/ethernet/google/gve/gve_tx.c
+++ b/drivers/net/ethernet/google/gve/gve_tx.c
@@ -224,6 +224,11 @@ static void gve_tx_free_ring_gqi(struct gve_priv *priv, struct gve_tx_ring *tx,
 	u32 slots;
 
 	slots = tx->mask + 1;
+	if (tx->tx_hdr_bufs) {
+		dma_free_coherent(hdev, slots * MAX_TCP_HEADER,
+				  tx->tx_hdr_bufs, tx->tx_hdr_bufs_dma);
+		tx->tx_hdr_bufs = NULL;
+	}
 	dma_free_coherent(hdev, sizeof(*tx->q_resources),
 			  tx->q_resources, tx->q_resources_bus);
 	tx->q_resources = NULL;
@@ -298,6 +303,11 @@ static int gve_tx_alloc_ring_gqi(struct gve_priv *priv,
 		/* map Tx FIFO */
 		if (gve_tx_fifo_init(priv, &tx->tx_fifo))
 			goto abort_with_qpl;
+	} else if (hdev->dma_pmd_tx_hdrs) {
+		tx->tx_hdr_bufs = dma_alloc_coherent(hdev,
+						     cfg->ring_size * MAX_TCP_HEADER,
+						     &tx->tx_hdr_bufs_dma,
+						     GFP_KERNEL);
 	}
 
 	tx->q_resources =
@@ -311,6 +321,12 @@ static int gve_tx_alloc_ring_gqi(struct gve_priv *priv,
 	return 0;
 
 abort_with_fifo:
+	if (tx->tx_hdr_bufs) {
+		dma_free_coherent(hdev,
+				  cfg->ring_size * MAX_TCP_HEADER,
+				  tx->tx_hdr_bufs, tx->tx_hdr_bufs_dma);
+		tx->tx_hdr_bufs = NULL;
+	}
 	if (!tx->raw_addressing)
 		gve_tx_fifo_release(priv, &tx->tx_fifo);
 abort_with_qpl:
@@ -423,6 +439,8 @@ static inline int gve_skb_fifo_bytes_required(struct gve_tx_ring *tx,
 #define MAX_TX_DESC_NEEDED	(MAX_SKB_FRAGS + 4)
 static void gve_tx_unmap_buf(struct device *dev, struct gve_tx_buffer_state *info)
 {
+	if (!dma_unmap_len(info, len))
+		return;
 	if (info->skb) {
 		dma_unmap_single(dev, dma_unmap_addr(info, dma),
 				 dma_unmap_len(info, len),
@@ -657,13 +675,21 @@ static int gve_tx_add_skb_no_copy(struct gve_priv *priv, struct gve_tx_ring *tx,
 
 	info->skb =  skb;
 
-	addr = dma_map_single(tx->dev, skb->data, len, DMA_TO_DEVICE);
-	if (unlikely(dma_mapping_error(tx->dev, addr))) {
-		tx->dma_mapping_error++;
-		goto drop;
+	if (tx->tx_hdr_bufs && len <= MAX_TCP_HEADER) {
+		u32 hdr_off = idx * MAX_TCP_HEADER;
+
+		memcpy(tx->tx_hdr_bufs + hdr_off, skb->data, len);
+		addr = tx->tx_hdr_bufs_dma + hdr_off;
+		dma_unmap_len_set(info, len, 0);
+	} else {
+		addr = dma_map_single(tx->dev, skb->data, len, DMA_TO_DEVICE);
+		if (unlikely(dma_mapping_error(tx->dev, addr))) {
+			tx->dma_mapping_error++;
+			goto drop;
+		}
+		dma_unmap_len_set(info, len, len);
+		dma_unmap_addr_set(info, dma, addr);
 	}
-	dma_unmap_len_set(info, len, len);
-	dma_unmap_addr_set(info, dma, addr);
 
 	num_descriptors = 1 + shinfo->nr_frags;
 	if (hlen < len)
diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c
index ad99cbb77e291..ac69793492c9f 100644
--- a/drivers/net/ethernet/google/gve/gve_tx_dqo.c
+++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c
@@ -176,8 +176,9 @@ static void gve_unmap_packet(struct device *dev,
 		return;
 
 	/* SKB linear portion is guaranteed to be mapped */
-	dma_unmap_single(dev, dma_unmap_addr(pkt, dma[0]),
-			 dma_unmap_len(pkt, len[0]), DMA_TO_DEVICE);
+	if (dma_unmap_len(pkt, len[0]))
+		dma_unmap_single(dev, dma_unmap_addr(pkt, dma[0]),
+				 dma_unmap_len(pkt, len[0]), DMA_TO_DEVICE);
 	for (i = 1; i < pkt->num_bufs; i++) {
 		netmem_dma_unmap_page_attrs(dev, dma_unmap_addr(pkt, dma[i]),
 					    dma_unmap_len(pkt, len[i]),
@@ -232,6 +233,13 @@ static void gve_tx_free_ring_dqo(struct gve_priv *priv, struct gve_tx_ring *tx,
 	size_t bytes;
 	u32 qpl_id;
 
+	if (tx->tx_hdr_bufs) {
+		dma_free_coherent(hdev,
+				  tx->dqo.num_pending_packets * MAX_TCP_HEADER,
+				  tx->tx_hdr_bufs, tx->tx_hdr_bufs_dma);
+		tx->tx_hdr_bufs = NULL;
+	}
+
 	if (tx->q_resources) {
 		dma_free_coherent(hdev, sizeof(*tx->q_resources),
 				  tx->q_resources, tx->q_resources_bus);
@@ -399,6 +407,12 @@ static int gve_tx_alloc_ring_dqo(struct gve_priv *priv,
 
 		if (gve_tx_qpl_buf_init(tx))
 			goto err;
+	} else if (hdev->dma_pmd_tx_hdrs) {
+		tx->tx_hdr_bufs =
+			dma_alloc_coherent(hdev,
+					   tx->dqo.num_pending_packets * MAX_TCP_HEADER,
+					   &tx->tx_hdr_bufs_dma,
+					   GFP_KERNEL);
 	}
 
 	return 0;
@@ -712,12 +726,20 @@ static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx,
 		u32 len = skb_headlen(skb);
 		dma_addr_t addr;
 
-		addr = dma_map_single(tx->dev, skb->data, len, DMA_TO_DEVICE);
-		if (unlikely(dma_mapping_error(tx->dev, addr)))
-			goto err;
+		if (tx->tx_hdr_bufs && len <= MAX_TCP_HEADER) {
+			u32 hdr_off = (u32)completion_tag * MAX_TCP_HEADER;
 
-		dma_unmap_len_set(pkt, len[pkt->num_bufs], len);
-		dma_unmap_addr_set(pkt, dma[pkt->num_bufs], addr);
+			memcpy(tx->tx_hdr_bufs + hdr_off, skb->data, len);
+			addr = tx->tx_hdr_bufs_dma + hdr_off;
+			dma_unmap_len_set(pkt, len[pkt->num_bufs], 0);
+		} else {
+			addr = dma_map_single(tx->dev, skb->data, len,
+					      DMA_TO_DEVICE);
+			if (unlikely(dma_mapping_error(tx->dev, addr)))
+				goto err;
+			dma_unmap_len_set(pkt, len[pkt->num_bufs], len);
+			dma_unmap_addr_set(pkt, dma[pkt->num_bufs], addr);
+		}
 		++pkt->num_bufs;
 
 		gve_tx_fill_pkt_desc_dqo(tx, desc_idx, enable_csum, len, addr,
@@ -748,10 +770,11 @@ static int gve_tx_add_skb_no_copy_dqo(struct gve_tx_ring *tx,
 err:
 	for (i = 0; i < pkt->num_bufs; i++) {
 		if (i == 0) {
-			dma_unmap_single(tx->dev,
-					 dma_unmap_addr(pkt, dma[i]),
-					 dma_unmap_len(pkt, len[i]),
-					 DMA_TO_DEVICE);
+			if (dma_unmap_len(pkt, len[i]))
+				dma_unmap_single(tx->dev,
+						 dma_unmap_addr(pkt, dma[i]),
+						 dma_unmap_len(pkt, len[i]),
+						 DMA_TO_DEVICE);
 		} else {
 			dma_unmap_page(tx->dev,
 				       dma_unmap_addr(pkt, dma[i]),
-- 
2.56.0.rc1.315.gc6ed9934b7-goog