[PATCH v2 net-next 4/4] ibmvnic: Make max subcrq indirect entries tunable via module param

[Mingming Cao <mmc@linux.ibm.com>]

This patch increases the default number of subcrq indirect entries from 16
to 128, a value supported on POWER9 and later systems. Increasing this limit
improves batching efficiency in hypervisor communication, enhancing throughput
under high-load conditions.

To maintain compatibility with older or constrained systems (e.g., some POWER8 platforms),
 a module parameter max_subcrq_indirect is introduced as a transitional mechanism.
This allows administrators to manually reduce the limit if needed.

The module parameter is not intended for dynamic runtime tuning, but rather
provides forward compatibility without requiring broader structural changes at this time.

Signed-off-by: Mingming Cao <mmc@linux.ibm.com>
Reviewed by: Rick Lindsley <ricklind@linux.ibm.com>
Reviewed by: Dave Marquardt <davemarq@linux.ibm.com>
Reviewed by: Brian King <bjking1@linux.ibm.com>
Reviewed by: Haren Myneni <haren@linux.ibm.com>
---
 drivers/net/ethernet/ibm/ibmvnic.c | 29 ++++++++++++++++++++++++-----
 drivers/net/ethernet/ibm/ibmvnic.h |  7 +++++--
 2 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c
index 9406ea06d9..33cdcae6a7 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.c
+++ b/drivers/net/ethernet/ibm/ibmvnic.c
@@ -154,6 +154,11 @@ static const struct ibmvnic_stat ibmvnic_stats[] = {
 	{"internal_mac_rx_errors", IBMVNIC_STAT_OFF(internal_mac_rx_errors)},
 };
 
+/* Module parameter for max_ind_descs */
+static unsigned int max_ind_descs = IBMVNIC_MAX_IND_DESCS_DEFAULT;
+module_param(max_ind_descs, uint, 0444);
+MODULE_PARM_DESC(max_ind_descs, "Max indirect subcrq descriptors (16 to 128, default 128)");
+
 static int send_crq_init_complete(struct ibmvnic_adapter *adapter)
 {
 	union ibmvnic_crq crq;
@@ -844,7 +849,7 @@ static void replenish_rx_pool(struct ibmvnic_adapter *adapter,
 		sub_crq->rx_add.len = cpu_to_be32(pool->buff_size << shift);
 
 		/* if send_subcrq_indirect queue is full, flush to VIOS */
-		if (ind_bufp->index == IBMVNIC_MAX_IND_DESCS ||
+		if (ind_bufp->index == max_ind_descs ||
 		    i == count - 1) {
 			lpar_rc =
 				send_subcrq_indirect(adapter, handle,
@@ -2599,7 +2604,7 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev)
 	tx_crq.v1.n_crq_elem = num_entries;
 	tx_buff->num_entries = num_entries;
 	/* flush buffer if current entry can not fit */
-	if (num_entries + ind_bufp->index > IBMVNIC_MAX_IND_DESCS) {
+	if (num_entries + ind_bufp->index > max_ind_descs) {
 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, true);
 		if (lpar_rc != H_SUCCESS)
 			goto tx_flush_err;
@@ -2612,7 +2617,7 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev)
 	ind_bufp->index += num_entries;
 	if (__netdev_tx_sent_queue(txq, skb->len,
 				   netdev_xmit_more() &&
-				   ind_bufp->index < IBMVNIC_MAX_IND_DESCS)) {
+				   ind_bufp->index < max_ind_descs)) {
 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, true);
 		if (lpar_rc != H_SUCCESS)
 			goto tx_err;
@@ -4015,7 +4020,7 @@ static void release_sub_crq_queue(struct ibmvnic_adapter *adapter,
 	}
 
 	dma_free_coherent(dev,
-			  IBMVNIC_IND_ARR_SZ,
+			  max_ind_descs * IBMVNIC_IND_DESC_SZ,
 			  scrq->ind_buf.indir_arr,
 			  scrq->ind_buf.indir_dma);
 
@@ -4072,7 +4077,7 @@ static struct ibmvnic_sub_crq_queue *init_sub_crq_queue(struct ibmvnic_adapter
 
 	scrq->ind_buf.indir_arr =
 		dma_alloc_coherent(dev,
-				   IBMVNIC_IND_ARR_SZ,
+				   max_ind_descs * IBMVNIC_IND_DESC_SZ,
 				   &scrq->ind_buf.indir_dma,
 				   GFP_KERNEL);
 
@@ -6734,6 +6739,20 @@ static int __init ibmvnic_module_init(void)
 {
 	int ret;
 
+	if (max_ind_descs < IBMVNIC_MAX_IND_DESC_MIN ||
+	    max_ind_descs > IBMVNIC_MAX_IND_DESC_MAX) {
+		pr_info("ibmvnic: max_ind_descs=%u, must be between %d and %d. default %u\n",
+			max_ind_descs,
+			IBMVNIC_MAX_IND_DESC_MIN,
+			IBMVNIC_MAX_IND_DESC_MAX,
+			IBMVNIC_MAX_IND_DESCS_DEFAULT);
+
+		pr_info("ibmvnic: resetting max_ind_descs to default\n");
+		max_ind_descs = IBMVNIC_MAX_IND_DESCS_DEFAULT;
+	}
+
+	pr_info("ibmvnic: max_ind_descs set to %u\n", max_ind_descs);
+
 	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "net/ibmvnic:online",
 				      ibmvnic_cpu_online,
 				      ibmvnic_cpu_down_prep);
diff --git a/drivers/net/ethernet/ibm/ibmvnic.h b/drivers/net/ethernet/ibm/ibmvnic.h
index e574eed97c..48c16e6f8a 100644
--- a/drivers/net/ethernet/ibm/ibmvnic.h
+++ b/drivers/net/ethernet/ibm/ibmvnic.h
@@ -29,8 +29,10 @@
 #define IBMVNIC_BUFFS_PER_POOL	100
 #define IBMVNIC_MAX_QUEUES	16
 #define IBMVNIC_MAX_QUEUE_SZ   4096
-#define IBMVNIC_MAX_IND_DESCS  16
-#define IBMVNIC_IND_ARR_SZ	(IBMVNIC_MAX_IND_DESCS * 32)
+#define IBMVNIC_IND_DESC_SZ	32
+#define IBMVNIC_MAX_IND_DESCS_DEFAULT	128
+#define IBMVNIC_MAX_IND_DESC_MAX	128
+#define IBMVNIC_MAX_IND_DESC_MIN	16
 
 #define IBMVNIC_TSO_BUF_SZ	65536
 #define IBMVNIC_TSO_BUFS	64
@@ -945,6 +947,7 @@ struct ibmvnic_adapter {
 	int replenish_task_cycles;
 	int tx_send_failed;
 	int tx_map_failed;
+	u32 max_ind_descs;
 
 	struct ibmvnic_tx_queue_stats *tx_stats_buffers;
 	struct ibmvnic_rx_queue_stats *rx_stats_buffers;
-- 
2.39.3 (Apple Git-146)