[PATCH net-next] net: mana: Extend RX CQE coalescing up to 8 packets

[Haiyang Zhang <haiyangz@linux.microsoft.com>]

From: Haiyang Zhang <haiyangz@microsoft.com>

To support up to 8 packets per CQE, put two packet lengths and
hash values into one PPI entry by using the reserved fields.
Update ethtool handlers to set this feature.
Update per queue stat to show the coalesced CQE counters.
This feature is supported on NIC hardware showing the relevant
PF flag.

Signed-off-by: Haiyang Zhang <haiyangz@microsoft.com>
---
 drivers/net/ethernet/microsoft/mana/mana_en.c | 137 ++++++++++++------
 .../ethernet/microsoft/mana/mana_ethtool.c    |  30 +++-
 include/net/mana/gdma.h                       |   4 +
 include/net/mana/mana.h                       |  43 ++++--
 4 files changed, 150 insertions(+), 64 deletions(-)

diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
index a8c329bdbacf..720d22e6aea9 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_en.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
@@ -1241,6 +1241,9 @@ int mana_gd_query_device_cfg(struct gdma_context *gc, u32 proto_major_ver,
 		return err;
 	}
 
+	if (resp.pf_cap_flags1 & MANA_PF_FLAG_1_CQE_8_COALESCING_SUPPORTED)
+		gc->cqe8_coalescing_sup = true;
+
 	*max_num_vports = resp.max_num_vports;
 
 	if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2) {
@@ -1426,7 +1429,11 @@ static int mana_cfg_vport_steering(struct mana_port_context *apc,
 	mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size,
 			     sizeof(resp));
 
-	req->hdr.req.msg_version = GDMA_MESSAGE_V2;
+	/* Request & response versions can be different.
+	 * HW can handle newer msg versions, by skipping
+	 * new fields.
+	 */
+	req->hdr.req.msg_version = GDMA_MESSAGE_V5;
 	req->hdr.resp.msg_version = GDMA_MESSAGE_V2;
 
 	req->vport = apc->port_handle;
@@ -1440,8 +1447,13 @@ static int mana_cfg_vport_steering(struct mana_port_context *apc,
 	req->update_indir_tab = update_tab;
 	req->default_rxobj = apc->default_rxobj;
 
-	if (rx != TRI_STATE_FALSE)
+	/* Request-msg v5 requires this field */
+	req->rss_hash_types = MANA_HASH_ENABLE_SUPPORTED;
+
+	if (rx != TRI_STATE_FALSE) {
 		req->cqe_coalescing_enable = apc->cqe_coalescing_enable;
+		req->cqe8_coalescing_enable = apc->cqe8_coalescing_enable;
+	}
 
 	if (update_key)
 		memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE);
@@ -2046,16 +2058,14 @@ static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va,
 
 static void mana_rx_skb(void *buf_va, bool from_pool,
 			struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq,
-			int i)
+			u32 pkt_len, u32 pkt_hash)
 {
 	struct mana_stats_rx *rx_stats = &rxq->stats;
 	struct net_device *ndev = rxq->ndev;
-	uint pkt_len = cqe->ppi[i].pkt_len;
 	u16 rxq_idx = rxq->rxq_idx;
 	struct napi_struct *napi;
 	struct xdp_buff xdp = {};
 	struct sk_buff *skb;
-	u32 hash_value;
 	u32 act;
 
 	rxq->rx_cq.work_done++;
@@ -2094,12 +2104,10 @@ static void mana_rx_skb(void *buf_va, bool from_pool,
 	}
 
 	if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) {
-		hash_value = cqe->ppi[i].pkt_hash;
-
 		if (cqe->rx_hashtype & MANA_HASH_L4)
-			skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4);
+			skb_set_hash(skb, pkt_hash, PKT_HASH_TYPE_L4);
 		else
-			skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L3);
+			skb_set_hash(skb, pkt_hash, PKT_HASH_TYPE_L3);
 	}
 
 	if (cqe->rx_vlantag_present) {
@@ -2240,6 +2248,43 @@ static void mana_refill_rx_oob(struct device *dev, struct mana_rxq *rxq,
 	rxoob->dma_sync_offset = dma_sync_offset;
 }
 
+static void mana_process_one_rx_pkt(struct device *dev, struct mana_rxq *rxq,
+				    struct mana_rxcomp_oob *oob,
+				    u32 pktlen, u32 pkt_hash)
+{
+	struct mana_recv_buf_oob *rxbuf_oob;
+	struct net_device *ndev = rxq->ndev;
+	void *old_buf = NULL;
+	bool old_fp;
+
+	rxbuf_oob = &rxq->rx_oobs[rxq->buf_index];
+	WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1);
+
+	if (unlikely(pktlen > rxq->datasize)) {
+		/* Increase it even if mana_rx_skb() isn't called. */
+		rxq->rx_cq.work_done++;
+
+		++ndev->stats.rx_dropped;
+		netdev_warn_once(ndev,
+				 "Dropped oversized RX packet: len=%u, datasize=%u\n",
+				 pktlen, rxq->datasize);
+
+		/* Reuse the RX buffer since rxbuf_oob is unchanged. */
+	} else {
+		mana_refill_rx_oob(dev, rxq, rxbuf_oob, pktlen,
+				   &old_buf, &old_fp);
+
+		/* Unsuccessful refill will have old_buf == NULL.
+		 * In this case, mana_rx_skb() will drop the packet.
+		 */
+		mana_rx_skb(old_buf, old_fp, oob, rxq, pktlen, pkt_hash);
+	}
+
+	mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu);
+
+	mana_post_pkt_rxq(rxq);
+}
+
 static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 				struct gdma_comp *cqe)
 {
@@ -2249,10 +2294,10 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 	struct mana_recv_buf_oob *rxbuf_oob;
 	struct mana_port_context *apc;
 	struct device *dev = gc->dev;
+	bool coalesced_8 = false;
 	bool coalesced = false;
-	void *old_buf = NULL;
-	u32 curr, pktlen;
-	bool old_fp;
+	u32 pktlen;
+	int pkt_i;
 	int i;
 
 	apc = netdev_priv(ndev);
@@ -2275,6 +2320,11 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 		coalesced = true;
 		break;
 
+	case CQE_RX_COALESCED_8:
+		coalesced = true;
+		coalesced_8 = true;
+		break;
+
 	case CQE_RX_OBJECT_FENCE:
 		complete(&rxq->fence_event);
 		return;
@@ -2286,54 +2336,48 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
 		return;
 	}
 
+	pkt_i = 0;
 	for (i = 0; i < MANA_RXCOMP_OOB_NUM_PPI; i++) {
-		old_buf = NULL;
-		pktlen = oob->ppi[i].pkt_len;
-		if (pktlen == 0)
-			break;
-
-		curr = rxq->buf_index;
-		rxbuf_oob = &rxq->rx_oobs[curr];
-		WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1);
-
-		if (unlikely(pktlen > rxq->datasize)) {
-			/* Increase it even if mana_rx_skb() isn't called. */
-			rxq->rx_cq.work_done++;
+		u32 pkt_hash;
 
-			++ndev->stats.rx_dropped;
-			netdev_warn_once(ndev,
-				"Dropped oversized RX packet: len=%u, datasize=%u\n",
-				pktlen, rxq->datasize);
-
-			/* Reuse the RX buffer since rxbuf_oob is unchanged. */
+		if (coalesced_8) {
+			/* 8-pkt mode: 2 packets per PPI entry */
+			pktlen = oob->ppi[i].pkt_len0;
+			pkt_hash = oob->ppi[i].pkt_hash0;
 		} else {
-
-			mana_refill_rx_oob(dev, rxq, rxbuf_oob, pktlen,
-					   &old_buf, &old_fp);
-
-			/* Unsuccessful refill will have old_buf == NULL.
-			 * In this case, mana_rx_skb() will drop the packet.
-			 */
-			mana_rx_skb(old_buf, old_fp, oob, rxq, i);
+			pktlen = oob->ppi[i].pkt_len;
+			pkt_hash = oob->ppi[i].pkt_hash;
 		}
+		if (pktlen == 0)
+			break;
 
-		mana_move_wq_tail(rxq->gdma_rq,
-				  rxbuf_oob->wqe_inf.wqe_size_in_bu);
-
-		mana_post_pkt_rxq(rxq);
+		mana_process_one_rx_pkt(dev, rxq, oob, pktlen, pkt_hash);
+		pkt_i++;
 
 		if (!coalesced)
 			break;
+
+		/* Process 2nd packet from the same PPI in 8-pkt mode */
+		if (coalesced_8) {
+			pktlen = oob->ppi[i].pkt_len1;
+			pkt_hash = oob->ppi[i].pkt_hash1;
+			if (pktlen == 0)
+				break;
+
+			mana_process_one_rx_pkt(dev, rxq, oob, pktlen,
+						pkt_hash);
+			pkt_i++;
+		}
 	}
 
 	/* Collect coalesced CQE count based on packets processed.
-	 * Coalesced CQEs have at least 2 packets, so index is i - 2.
+	 * Coalesced CQEs have at least 2 packets, so index is pkt_i - 2.
 	 */
-	if (i > 1) {
+	if (pkt_i > 1) {
 		u64_stats_update_begin(&rxq->stats.syncp);
-		rxq->stats.coalesced_cqe[i - 2]++;
+		rxq->stats.coalesced_cqe[pkt_i - 2]++;
 		u64_stats_update_end(&rxq->stats.syncp);
-	} else if (!i && !pktlen) {
+	} else if (!pkt_i && !pktlen) {
 		u64_stats_update_begin(&rxq->stats.syncp);
 		rxq->stats.pkt_len0_err++;
 		u64_stats_update_end(&rxq->stats.syncp);
@@ -3763,6 +3807,7 @@ static int mana_probe_port(struct mana_context *ac, int port_idx,
 	apc->port_idx = port_idx;
 	apc->link_cfg_error = 1;
 	apc->cqe_coalescing_enable = 0;
+	apc->cqe8_coalescing_enable = 0;
 
 	/* Initialize interrupt moderation settings if supported by HW */
 	if (gc->pf_cap_flags1 & GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION) {
diff --git a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
index 9e31e2595ae3..8743f7edecbb 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
@@ -169,7 +169,7 @@ static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
 		ethtool_sprintf(&data, "rx_%d_xdp_tx", i);
 		ethtool_sprintf(&data, "rx_%d_xdp_redirect", i);
 		ethtool_sprintf(&data, "rx_%d_pkt_len0_err", i);
-		for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
+		for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++)
 			ethtool_sprintf(&data, "rx_%d_coalesced_cqe_%d", i, j + 2);
 	}
 
@@ -205,7 +205,7 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
 	u64 xdp_drop;
 	u64 xdp_tx;
 	u64 pkt_len0_err;
-	u64 coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 1];
+	u64 coalesced_cqe[MANA_CQE_COAL_PKTS_8 - 1];
 	u64 tso_packets;
 	u64 tso_bytes;
 	u64 tso_inner_packets;
@@ -245,7 +245,7 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
 			xdp_tx = rx_stats->xdp_tx;
 			xdp_redirect = rx_stats->xdp_redirect;
 			pkt_len0_err = rx_stats->pkt_len0_err;
-			for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
+			for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++)
 				coalesced_cqe[j] = rx_stats->coalesced_cqe[j];
 		} while (u64_stats_fetch_retry(&rx_stats->syncp, start));
 
@@ -255,7 +255,7 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
 		data[i++] = xdp_tx;
 		data[i++] = xdp_redirect;
 		data[i++] = pkt_len0_err;
-		for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++)
+		for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++)
 			data[i++] = coalesced_cqe[j];
 	}
 
@@ -408,6 +408,7 @@ static int mana_get_coalesce(struct net_device *ndev,
 	struct mana_port_context *apc = netdev_priv(ndev);
 
 	kernel_coal->rx_cqe_frames =
+		apc->cqe8_coalescing_enable ? MANA_CQE_COAL_PKTS_8 :
 		apc->cqe_coalescing_enable ? MANA_RXCOMP_OOB_NUM_PPI : 1;
 
 	kernel_coal->rx_cqe_nsecs = apc->cqe_coalescing_timeout_ns;
@@ -443,6 +444,7 @@ static int mana_set_coalesce(struct net_device *ndev,
 		u16 intr_modr_tx_usec;
 		u16 intr_modr_tx_comp;
 		u8 cqe_coalescing_enable;
+		u8 cqe8_coalescing_enable;
 		bool rx_dim_enabled;
 		bool tx_dim_enabled;
 	} saved;
@@ -466,10 +468,12 @@ static int mana_set_coalesce(struct net_device *ndev,
 	}
 
 	if (kernel_coal->rx_cqe_frames != 1 &&
-	    kernel_coal->rx_cqe_frames != MANA_RXCOMP_OOB_NUM_PPI) {
+	    kernel_coal->rx_cqe_frames != MANA_RXCOMP_OOB_NUM_PPI &&
+	    kernel_coal->rx_cqe_frames != MANA_CQE_COAL_PKTS_8) {
 		NL_SET_ERR_MSG_FMT(extack,
-				   "rx-frames must be 1 or %u, got %u",
+				   "rx-frames must be 1, %u, or %u, got %u",
 				   MANA_RXCOMP_OOB_NUM_PPI,
+				   MANA_CQE_COAL_PKTS_8,
 				   kernel_coal->rx_cqe_frames);
 		return -EINVAL;
 	}
@@ -514,8 +518,15 @@ static int mana_set_coalesce(struct net_device *ndev,
 	saved.tx_dim_enabled = apc->tx_dim_enabled;
 
 	saved.cqe_coalescing_enable = apc->cqe_coalescing_enable;
+	saved.cqe8_coalescing_enable = apc->cqe8_coalescing_enable;
 	apc->cqe_coalescing_enable =
-		kernel_coal->rx_cqe_frames == MANA_RXCOMP_OOB_NUM_PPI;
+		kernel_coal->rx_cqe_frames >= MANA_RXCOMP_OOB_NUM_PPI;
+
+	if (gc->cqe8_coalescing_sup)
+		apc->cqe8_coalescing_enable =
+			kernel_coal->rx_cqe_frames == MANA_CQE_COAL_PKTS_8;
+	else if (kernel_coal->rx_cqe_frames == MANA_CQE_COAL_PKTS_8)
+		netdev_info(ndev, "Fall back to RX-CQE x4 coalescing\n");
 
 	if (!apc->port_is_up) {
 		WRITE_ONCE(apc->rx_dim_enabled, !!ec->use_adaptive_rx_coalesce);
@@ -523,7 +534,8 @@ static int mana_set_coalesce(struct net_device *ndev,
 		return 0;
 	}
 
-	if (apc->cqe_coalescing_enable != saved.cqe_coalescing_enable) {
+	if (apc->cqe_coalescing_enable != saved.cqe_coalescing_enable ||
+	    apc->cqe8_coalescing_enable != saved.cqe8_coalescing_enable) {
 		/* CQE coalescing setting is applied via RSS configuration. */
 		err = mana_config_rss(apc, TRI_STATE_TRUE, false, false);
 		if (err) {
@@ -531,6 +543,8 @@ static int mana_set_coalesce(struct net_device *ndev,
 				   err);
 			apc->cqe_coalescing_enable =
 				saved.cqe_coalescing_enable;
+			apc->cqe8_coalescing_enable =
+				saved.cqe8_coalescing_enable;
 			apc->intr_modr_rx_usec = saved.intr_modr_rx_usec;
 			apc->intr_modr_rx_comp = saved.intr_modr_rx_comp;
 			apc->intr_modr_tx_usec = saved.intr_modr_tx_usec;
diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h
index 8529cef0d7c4..70a7f1fee5d3 100644
--- a/include/net/mana/gdma.h
+++ b/include/net/mana/gdma.h
@@ -182,6 +182,7 @@ struct gdma_general_req {
 #define GDMA_MESSAGE_V2 2
 #define GDMA_MESSAGE_V3 3
 #define GDMA_MESSAGE_V4 4
+#define GDMA_MESSAGE_V5 5
 
 struct gdma_general_resp {
 	struct gdma_resp_hdr hdr;
@@ -428,6 +429,9 @@ struct gdma_context {
 	/* L2 MTU */
 	u16 adapter_mtu;
 
+	/* NIC supports CQE x8 coalescing */
+	bool cqe8_coalescing_sup;
+
 	/* This maps a CQ index to the queue structure. */
 	unsigned int		max_num_cqs;
 	struct gdma_queue	**cq_table;
diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h
index 4d041fb8437f..3ca1399a2faf 100644
--- a/include/net/mana/mana.h
+++ b/include/net/mana/mana.h
@@ -62,9 +62,12 @@ enum TRI_STATE {
 
 #define MAX_PORTS_IN_MANA_DEV 256
 
-/* Maximum number of packets per coalesced CQE */
+/* Maximum number of PPIs per coalesced CQE */
 #define MANA_RXCOMP_OOB_NUM_PPI 4
 
+/* 8-pkt mode packs up to 2 packets per PPI entry */
+#define MANA_CQE_COAL_PKTS_8 8
+
 /* Default/max interrupt moderation settings */
 #define MANA_INTR_MODR_USEC_DEF 0
 #define MANA_INTR_MODR_COMP_DEF 0
@@ -79,7 +82,7 @@ enum TRI_STATE {
 #define MANA_INTR_MODR_COMP_MASK   GENMASK(23, 16)
 
 /* Update this count whenever the respective structures are changed */
-#define MANA_STATS_RX_COUNT (6 + MANA_RXCOMP_OOB_NUM_PPI - 1)
+#define MANA_STATS_RX_COUNT (6 + MANA_CQE_COAL_PKTS_8 - 1)
 #define MANA_STATS_TX_COUNT 11
 
 #define MANA_RX_FRAG_ALIGNMENT 64
@@ -91,7 +94,7 @@ struct mana_stats_rx {
 	u64 xdp_tx;
 	u64 xdp_redirect;
 	u64 pkt_len0_err;
-	u64 coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 1];
+	u64 coalesced_cqe[MANA_CQE_COAL_PKTS_8 - 1];
 	struct u64_stats_sync syncp;
 };
 
@@ -202,6 +205,7 @@ enum mana_cqe_type {
 	CQE_RX_COALESCED_4		= 2,
 	CQE_RX_OBJECT_FENCE		= 3,
 	CQE_RX_TRUNCATED		= 4,
+	CQE_RX_COALESCED_8		= 7,
 
 	CQE_TX_OKAY			= 32,
 	CQE_TX_SA_DROP			= 33,
@@ -238,12 +242,25 @@ struct mana_cqe_header {
 #define MANA_HASH_L4                                                         \
 	(NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4 | NDIS_HASH_TCP_IPV6 |      \
 	 NDIS_HASH_UDP_IPV6 | NDIS_HASH_TCP_IPV6_EX | NDIS_HASH_UDP_IPV6_EX)
+#define MANA_HASH_ENABLE_SUPPORTED \
+	(NDIS_HASH_IPV4 | NDIS_HASH_TCP_IPV4 | NDIS_HASH_UDP_IPV4 | \
+	 NDIS_HASH_IPV6 | NDIS_HASH_TCP_IPV6 | NDIS_HASH_UDP_IPV6)
+
+union mana_rxcomp_perpkt_info {
+	struct {
+		u32 pkt_len : 16;
+		u32 reserved1 : 16;
+		u32 reserved2;
+		u32 pkt_hash;
+	};
 
-struct mana_rxcomp_perpkt_info {
-	u32 pkt_len	: 16;
-	u32 reserved1	: 16;
-	u32 reserved2;
-	u32 pkt_hash;
+	/* Up to two pkts per PPI entry */
+	struct {
+		u32 pkt_hash0;
+		u16 pkt_len0;
+		u16 pkt_len1;
+		u32 pkt_hash1;
+	};
 }; /* HW DATA */
 
 /* Receive completion OOB */
@@ -264,7 +281,7 @@ struct mana_rxcomp_oob {
 	u32 rx_udp_csum_fail		: 1;
 	u32 reserved2			: 1;
 
-	struct mana_rxcomp_perpkt_info ppi[MANA_RXCOMP_OOB_NUM_PPI];
+	union mana_rxcomp_perpkt_info ppi[MANA_RXCOMP_OOB_NUM_PPI];
 
 	u32 rx_wqe_offset;
 }; /* HW DATA */
@@ -607,6 +624,7 @@ struct mana_port_context {
 	bool port_st_save; /* Saved port state */
 
 	u8 cqe_coalescing_enable;
+	u8 cqe8_coalescing_enable;
 	u32 cqe_coalescing_timeout_ns;
 
 	/* Interrupt moderation settings */
@@ -751,6 +769,8 @@ struct mana_query_device_cfg_req {
 	u32 reserved;
 }; /* HW DATA */
 
+#define MANA_PF_FLAG_1_CQE_8_COALESCING_SUPPORTED BIT(5)
+
 struct mana_query_device_cfg_resp {
 	struct gdma_resp_hdr hdr;
 
@@ -986,7 +1006,10 @@ struct mana_cfg_rx_steer_req_v2 {
 	mana_handle_t default_rxobj;
 	u8 hashkey[MANA_HASH_KEY_SIZE];
 	u8 cqe_coalescing_enable;
-	u8 reserved2[7];
+	u8 reserved2[3];
+	u16 rss_hash_types;
+	u8 cqe8_coalescing_enable; /* v5 message */
+	u8 reserved3;
 	mana_handle_t indir_tab[] __counted_by(num_indir_entries);
 }; /* HW DATA */
 
-- 
2.34.1