All of lore.kernel.org
 help / color / mirror / Atom feed
From: Jakub Kicinski <kuba@kernel.org>
To: davem@davemloft.net, michael.chan@broadcom.com
Cc: netdev@vger.kernel.org, edumazet@google.com, pabeni@redhat.com,
	andrew+netdev@lunn.ch, horms@kernel.org,
	pavan.chebbi@broadcom.com, Jakub Kicinski <kuba@kernel.org>
Subject: [PATCH net-next 6/6] eth: bnxt_en: restart Rx queues when their IRQ moves to another NUMA node
Date: Tue, 17 Nov 2026 18:39:05 -0800	[thread overview]
Message-ID: <20261118023905.3050661-7-kuba@kernel.org> (raw)
In-Reply-To: <20261118023905.3050661-1-kuba@kernel.org>

Rx queue memory is allocated on the node of the CPU servicing the queue
when the device is opened. On a running device it only follows IRQ
affinity changes if TPH is enabled, as a side effect of the queue
restart which applies the new TPH Steering Tag.

Schedule the affinity update work whenever the device supports the
queue API, not just when TPH Steering Tag changed. Queue restart places
all the memory it reallocates, other than the ring page tables, on
the page pool's node, so compare the page pool's nid to decide if
the queue has to move.

Since we need queue state of the currently allocated queue to compare
nid - check that the device is still up after errors.

Signed-off-by: Jakub Kicinski <kuba@kernel.org>
---
 drivers/net/ethernet/broadcom/bnxt/bnxt.h |  2 +-
 drivers/net/ethernet/broadcom/bnxt/bnxt.c | 46 ++++++++++++++---------
 2 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
index 86fc2e8a5816..4b80b20288ba 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h
@@ -2656,7 +2656,7 @@ struct bnxt {
 #define BNXT_RING_COAL_NOW_SP_EVENT	17
 #define BNXT_FW_RESET_NOTIFY_SP_EVENT	18
 #define BNXT_FW_EXCEPTION_SP_EVENT	19
-#define BNXT_TPH_UPDATE_SP_EVENT	20
+#define BNXT_IRQ_AFFINITY_SP_EVENT	20
 #define BNXT_LINK_CFG_CHANGE_SP_EVENT	21
 #define BNXT_THERMAL_THRESHOLD_SP_EVENT	22
 #define BNXT_FW_ECHO_REQUEST_SP_EVENT	23
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index 134b24842c4f..5f4a8db1f8b5 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -11931,18 +11931,17 @@ static void bnxt_irq_affinity_notify(struct irq_affinity_notify *notify,
 	}
 #endif
 
-	if (!irq->bp->tph_mode)
+	/* Only Rx queues can be restarted to follow the IRQ */
+	if (!BNXT_SUPPORTS_QUEUE_API(irq->bp) ||
+	    irq->ring_nr >= irq->bp->rx_nr_rings)
 		return;
 
-	if (irq->ring_nr >= irq->bp->rx_nr_rings)
-		return;
+	if (irq->bp->tph_mode &&
+	    !pcie_tph_get_cpu_st(irq->bp->pdev, TPH_MEM_TYPE_VM,
+				 cpumask_first(mask), &tag))
+		WRITE_ONCE(irq->new_tag, tag);
 
-	if (pcie_tph_get_cpu_st(irq->bp->pdev, TPH_MEM_TYPE_VM,
-				cpumask_first(mask), &tag))
-		return;
-
-	WRITE_ONCE(irq->new_tag, tag);
-	bnxt_queue_sp_work(irq->bp, BNXT_TPH_UPDATE_SP_EVENT);
+	bnxt_queue_sp_work(irq->bp, BNXT_IRQ_AFFINITY_SP_EVENT);
 }
 
 static void bnxt_irq_affinity_release(struct kref *ref)
@@ -14663,7 +14662,10 @@ static void bnxt_rtnl_unlock_sp(struct bnxt *bp)
 	rtnl_unlock();
 }
 
-static void bnxt_tph_update(struct bnxt *bp)
+/* Restart the Rx queues whose IRQ moved, if the move changed the TPH
+ * Steering Tag or the node the queue memory should be allocated on.
+ */
+static void bnxt_irq_affinity_update(struct bnxt *bp)
 {
 	struct net_device *dev = bp->dev;
 	int i;
@@ -14673,6 +14675,8 @@ static void bnxt_tph_update(struct bnxt *bp)
 		goto unlock;
 
 	for (i = 0; i < bp->rx_nr_rings; i++) {
+		struct bnxt_rx_ring_info *rxr = &bp->rx_ring[i];
+		bool new_tag = false, new_node = false;
 		struct bnxt_irq *irq;
 		int map_idx, err;
 		u16 tag;
@@ -14680,20 +14684,26 @@ static void bnxt_tph_update(struct bnxt *bp)
 		map_idx = bnxt_cp_num_to_irq_num(bp, i);
 		irq = &bp->irq_tbl[map_idx];
 		tag = READ_ONCE(irq->new_tag);
-		if (irq->tag == tag)
+		if (irq->tag != tag &&
+		    !pcie_tph_set_st_entry(bp->pdev, irq->msix_nr, tag))
+			new_tag = true;
+		if (rxr->page_pool->p.nid != bnxt_cp_num_to_node(bp, i))
+			new_node = true;
+		if (!new_tag && !new_node)
 			continue;
 
-		if (pcie_tph_set_st_entry(bp->pdev, irq->msix_nr, tag))
-			continue;
-
-		err = netdev_rx_queue_restart(dev, irq->ring_nr);
+		err = netdev_rx_queue_restart(dev, i);
 		if (err) {
 			netdev_err(dev, "RX queue restart failed: err=%d\n",
 				   err);
+			/* Failing to start a new queue may close the device */
+			if (!test_bit(BNXT_STATE_OPEN, &bp->state))
+				break;
 			continue;
 		}
 
-		irq->tag = tag;
+		if (new_tag)
+			irq->tag = tag;
 	}
 
 unlock:
@@ -15137,8 +15147,8 @@ static void bnxt_sp_task(struct work_struct *work)
 			bnxt_devlink_health_fw_report(bp);
 	}
 
-	if (test_and_clear_bit(BNXT_TPH_UPDATE_SP_EVENT, &bp->sp_event))
-		bnxt_tph_update(bp);
+	if (test_and_clear_bit(BNXT_IRQ_AFFINITY_SP_EVENT, &bp->sp_event))
+		bnxt_irq_affinity_update(bp);
 
 	smp_mb__before_atomic();
 	clear_bit(BNXT_STATE_IN_SP_TASK, &bp->state);
-- 
2.56.1


      parent reply	other threads:[~2026-11-18  2:39 UTC|newest]

Thread overview: 7+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-18  2:38 [PATCH net-next 0/6] eth: bnxt_en: allocate memory on the right NUMA node Jakub Kicinski
2026-11-18  2:39 ` [PATCH net-next 1/6] eth: bnxt_en: order event data before setting the sp_event bit Jakub Kicinski
2026-11-18  2:39 ` [PATCH net-next 2/6] eth: bnxt_en: follow explicitly set IRQ affinity when placing page pool Jakub Kicinski
2026-11-18  2:39 ` [PATCH net-next 3/6] eth: bnxt_en: allocate per-queue SW state on the queue's NUMA node Jakub Kicinski
2026-11-18  2:39 ` [PATCH net-next 4/6] eth: bnxt_en: explicitly keep device-only memory on the device's node Jakub Kicinski
2026-11-18  2:39 ` [PATCH net-next 5/6] eth: bnxt_en: allocate fast path DMA rings on the queue's NUMA node Jakub Kicinski
2026-11-18  2:39 ` Jakub Kicinski [this message]

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261118023905.3050661-7-kuba@kernel.org \
    --to=kuba@kernel.org \
    --cc=andrew+netdev@lunn.ch \
    --cc=davem@davemloft.net \
    --cc=edumazet@google.com \
    --cc=horms@kernel.org \
    --cc=michael.chan@broadcom.com \
    --cc=netdev@vger.kernel.org \
    --cc=pabeni@redhat.com \
    --cc=pavan.chebbi@broadcom.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.