[PATCH net-next v2 01/13] net: mana: add queue-set allocation and teardown helpers

[PATCH net-next v2 01/13] net: mana: add queue-set allocation and teardown helpers

[Long Li]

The ethtool reconfiguration paths (channel count, ring size, private
flags) and mana_change_mtu()/mana_xdp_set() all rebuild the queues by
calling mana_detach() followed by mana_attach(). That tears the vport
down and rebuilds it, so the RDMA driver can claim the vport while it
is released, and if mana_attach() fails the port is left down with no
way back except manual intervention.

Introduce the data model for replacing that with pre-allocate and swap.
struct mana_qset holds the queue-related fields of
mana_port_context that can be rebuilt independently of the vport: the
EQ/TXQ/RXQ arrays, the indirection and rxobj tables, and the
configuration those queues were built for.

mana_alloc_qset() builds a complete queue set and mana_free_qset()
destroys one. Both run against a scratch mana_port_context obtained
from mana_qset_scratch_alloc() - a heap copy that shares the port's
vport identity but owns no queues - so the live port context is never
made to point at queues that are still being built or freed.

Running the allocators against a scratch context rather than
temporarily clearing the live one is essential, not cosmetic.
mana_start_xmit() dereferences apc->tx_qp[] guarded only by
apc->port_is_up, and mana_alloc_qset() takes 50-300ms including
firmware calls. An earlier revision of this work cleared apc->tx_qp so
the existing allocators could build into the live context, and reliably
panicked under traffic:

  RIP: 0010:mana_start_xmit+0x138/0x1040   ; apc->tx_qp[txq_idx]
  RAX: 0000000000000000  CR2: 0000000000000000
  Kernel panic - not syncing: Fatal exception in interrupt

The scratch context also gets rxbufs_pre = NULL so it never consumes
the live set's preallocated RX buffers, and mana_port_debugfs =
ERR_PTR(-ENODEV). debugfs_start_creating() returns early on an IS_ERR()
parent and debugfs_remove() ignores IS_ERR_OR_NULL, which makes every
create and remove a no-op for swapped-in sets. Without this the two
live sets collide on the same names under vport%d.

mana_dealloc_queues()'s inline TX drain moves into mana_drain_txqs() so
the new teardown path gets it too: a retiring queue set can still hold
packets the device has not completed, and freeing the SQs and the SKB
queues underneath them would leak those SKBs and their DMA mappings.

While moving it, the reset that the drain falls back to when the
hardware stops responding becomes pci_try_reset_function() instead of
an open-coded pcie_flr(). pcie_flr() resets the function without saving
and restoring config space, so BARs, MSI-X state and bus master enable
are wiped while the PCI core still believes its cached values are
live. pci_try_reset_function() brackets the reset with
pci_dev_save_and_disable() and pci_dev_restore(). The trylock variant
is required rather than pci_reset_function(): this runs under RTNL,
while driver removal takes the device lock first and RTNL second, so
blocking on the device lock here could deadlock.

Other than that reset change, no functional change: nothing calls the
new helpers yet.

Signed-off-by: Long Li <longli@microsoft.com>
---
 .../net/ethernet/microsoft/mana/mana_bpf.c    |  31 ++
 drivers/net/ethernet/microsoft/mana/mana_en.c | 520 ++++++++++++++++--
 include/net/mana/mana.h                       |  64 +++
 3 files changed, 577 insertions(+), 38 deletions(-)

diff --git a/drivers/net/ethernet/microsoft/mana/mana_bpf.c b/drivers/net/ethernet/microsoft/mana/mana_bpf.c
index 53308e139cbe917b074dd381c83546fc74d7b79f..ca602e27044f92b87295cbc2de924adc71efa780 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_bpf.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_bpf.c
@@ -265,3 +265,34 @@ int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
 
 	return ret;
 }
+
+/* Read the XDP program a queue set is running, without changing anything. */
+struct bpf_prog *mana_chn_xdp_peek(struct mana_port_context *apc)
+{
+	ASSERT_RTNL();
+
+	if (!apc->rxqs || !apc->rxqs[0])
+		return NULL;
+
+	return rtnl_dereference(apc->rxqs[0]->bpf_prog);
+}
+
+/* Drop the per-queue references a retiring set holds on @prog.
+ *
+ * Kept separate from mana_chn_setxdp() so the pointers can stay in place
+ * until the queues stop polling: clearing them up front would let packets
+ * already sitting in a retiring RQ take the pass path and reach the stack
+ * without the program ever seeing them.
+ */
+void mana_chn_xdp_release(struct bpf_prog *prog, unsigned int num_queues)
+{
+	unsigned int i;
+
+	ASSERT_RTNL();
+
+	if (!prog)
+		return;
+
+	for (i = 0; i < num_queues; i++)
+		bpf_prog_put(prog);
+}
diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
index 3c96e6fc3d81dc16853cc458ef620b5150aa8988..bf15222deb77679257f46d36800fdc4006c612a0 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_en.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
@@ -2015,7 +2015,8 @@ static void mana_poll_tx_cq(struct mana_cq *cq)
 	/* Ensure checking txq_stopped before apc->port_is_up. */
 	smp_rmb();
 
-	if (txq_stopped && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) {
+	if (txq_stopped && !READ_ONCE(txq->retiring) && apc->port_is_up &&
+	    avail_space >= MAX_TX_WQE_SIZE) {
 		netif_tx_wake_queue(net_txq);
 		apc->eth_stats.wake_queue++;
 	}
@@ -2751,6 +2752,7 @@ static int mana_create_txq(struct mana_port_context *apc,
 		u64_stats_init(&txq->stats.syncp);
 		txq->ndev = net;
 		txq->net_txq = netdev_get_tx_queue(net, i);
+		txq->reset_gen = READ_ONCE(apc->ac->reset_gen);
 		txq->vp_offset = apc->tx_vp_offset;
 		txq->napi_initialized = false;
 		skb_queue_head_init(&txq->pending_skbs);
@@ -3006,11 +3008,14 @@ static int mana_push_wqe(struct mana_rxq *rxq)
 
 static int mana_create_page_pool(struct mana_rxq *rxq, struct gdma_context *gc)
 {
-	struct mana_port_context *mpc = netdev_priv(rxq->ndev);
 	struct page_pool_params pprm = {};
 	int ret;
 
-	pprm.pool_size = mpc->rx_queue_size / rxq->frag_count + 1;
+	/* Size the recycle ring from the queue being built, not from the live
+	 * port context: during a swap the queue may be sized for a ring the
+	 * running configuration does not use yet.
+	 */
+	pprm.pool_size = rxq->num_rx_buf / rxq->frag_count + 1;
 	pprm.nid = gc->numa_node;
 	pprm.napi = &rxq->rx_cq.napi;
 	pprm.netdev = rxq->ndev;
@@ -3676,15 +3681,143 @@ int mana_attach(struct net_device *ndev)
 	return 0;
 }
 
-static int mana_dealloc_queues(struct net_device *ndev)
+/* Drain the TX queues of a set that is about to be destroyed.
+ *
+ * No new packet can reach these queues: either the port is down
+ * (mana_dealloc_queues()) or the set has already been unpublished from the
+ * live port context (mana_free_qset()). Wait for the hardware to complete
+ * what it still owns, then release every SKB left mapped.
+ *
+ * A total timeout of 120 seconds is used across all the queues. This breaks
+ * the loop when the h/w is not responding; the device is then reset, because
+ * the buffers are about to be freed while it may still DMA into them. This
+ * value of 120 has been decided considering the max number of queues. If that
+ * reset also fails, the buffers are deliberately leaked rather than handed
+ * back to a device that can still reach them.
+ *
+ * Returns true if the device was successfully reset, which takes down every
+ * queue on the function, not just the ones being retired here. A reset that
+ * was attempted but failed returns false: nothing was taken down, so the
+ * other ports have no reason to rebuild.
+ */
+static bool mana_drain_txqs(struct mana_port_context *apc)
 {
-	struct mana_port_context *apc = netdev_priv(ndev);
 	unsigned long timeout = jiffies + 120 * HZ;
 	struct gdma_dev *gd = apc->ac->gdma_dev;
+	bool quiesced = true;
+	bool reset = false;
 	struct mana_txq *txq;
 	struct sk_buff *skb;
-	int i, err;
 	u32 tsleep;
+	int i, err;
+
+	if (!apc->tx_qp)
+		return false;
+
+	for (i = 0; i < apc->num_queues; i++) {
+		if (!apc->tx_qp[i])
+			continue;
+
+		txq = &apc->tx_qp[i]->txq;
+
+		/* The function has been reset since this queue was created, so
+		 * the device has already stopped touching its buffers. The
+		 * completions being waited for below can never arrive: without
+		 * this the port a reset took down would spend the full timeout
+		 * here, under RTNL, and then reset the function again on its
+		 * way out, taking every other port down with it in turn.
+		 */
+		if (READ_ONCE(apc->ac->reset_gen) != txq->reset_gen)
+			continue;
+
+		tsleep = 1000;
+		while (atomic_read(&txq->pending_sends) > 0 &&
+		       time_before(jiffies, timeout)) {
+			usleep_range(tsleep, tsleep + 1000);
+			tsleep <<= 1;
+		}
+		if (atomic_read(&txq->pending_sends)) {
+			/* The device still owns these buffers, so it has to be
+			 * reset before they are freed.
+			 *
+			 * pci_try_reset_function() rather than a bare
+			 * pcie_flr(): it saves and restores config space (BARs,
+			 * MSI-X, bus master) around the reset, which an
+			 * open-coded FLR wipes without the PCI core ever
+			 * knowing, leaving the device wedged. The trylock form
+			 * is deliberate: RTNL is held here, while the driver
+			 * remove path takes the device lock first and RTNL
+			 * second, so blocking on the device lock could deadlock.
+			 */
+			err = pci_try_reset_function(to_pci_dev(gd->gdma_context->dev));
+			if (err) {
+				netdev_err(apc->ndev,
+					   "function reset failed: %d, %d pkts pending in txq %u\n",
+					   err, atomic_read(&txq->pending_sends),
+					   txq->gdma_txq_id);
+				quiesced = false;
+			} else {
+				/* Every queue on the function is dead now,
+				 * including the ones this loop has not reached
+				 * and those of the other ports.
+				 */
+				WRITE_ONCE(apc->ac->reset_gen,
+					   apc->ac->reset_gen + 1);
+
+				/* Only a reset that actually happened takes the
+				 * other ports' queues down with it. Reporting
+				 * one that failed would send every sibling
+				 * through a needless down/up while their queues
+				 * are still perfectly good.
+				 */
+				reset = true;
+			}
+			break;
+		}
+	}
+
+	/* The reset is what makes freeing these buffers safe. Without it the
+	 * device still owns them: unmapping would fault the IOMMU on the next
+	 * descriptor it reads, and on a system without one it would go on
+	 * reading memory that has already been handed to someone else and put
+	 * whatever it finds there on the wire. Leaking is the lesser evil.
+	 *
+	 * What is leaked is bounded: at most one SQ ring's worth of skbs per
+	 * queue, since mana_can_tx() stops the txq once the ring is full. And
+	 * it can only recur after another full drain timeout followed by
+	 * another failed function reset. Reaching this point at all means the
+	 * device left TX outstanding for two minutes, so it is already broken;
+	 * pci_try_reset_function() then failing for a transient reason, such
+	 * as the trylock losing to a concurrent PCI operation, only decides
+	 * whether these particular buffers can be recovered, not whether the
+	 * device was healthy.
+	 */
+	if (!quiesced) {
+		netdev_err(apc->ndev,
+			   "device not quiesced, leaking pending TX buffers instead of unmapping memory it can still DMA from\n");
+		return reset;
+	}
+
+	for (i = 0; i < apc->num_queues; i++) {
+		if (!apc->tx_qp[i])
+			continue;
+
+		txq = &apc->tx_qp[i]->txq;
+		while ((skb = skb_dequeue(&txq->pending_skbs))) {
+			mana_unmap_skb(skb, apc);
+			dev_kfree_skb_any(skb);
+		}
+		atomic_set(&txq->pending_sends, 0);
+	}
+
+	return reset;
+}
+
+static int mana_dealloc_queues(struct net_device *ndev)
+{
+	struct mana_port_context *apc = netdev_priv(ndev);
+	struct gdma_dev *gd = apc->ac->gdma_dev;
+	int err;
 
 	if (apc->port_is_up)
 		return -EINVAL;
@@ -3702,41 +3835,27 @@ static int mana_dealloc_queues(struct net_device *ndev)
 	 * new packets due to apc->port_is_up being false.
 	 *
 	 * Drain all the in-flight TX packets.
-	 * A timeout of 120 seconds for all the queues is used.
-	 * This will break the while loop when h/w is not responding.
-	 * This value of 120 has been decided here considering max
-	 * number of queues.
+	 *
+	 * If the drain had to reset the function to get there, every other
+	 * port on the adapter lost its queues too, so schedule them for a
+	 * rebuild. This port is being torn down here and needs no such
+	 * treatment, and a port that is already down returns early from its
+	 * handler.
 	 */
+	if (mana_drain_txqs(apc)) {
+		struct mana_context *ac = apc->ac;
+		unsigned int i;
 
-	if (apc->tx_qp) {
-		for (i = 0; i < apc->num_queues; i++) {
-			txq = &apc->tx_qp[i]->txq;
-			tsleep = 1000;
-			while (atomic_read(&txq->pending_sends) > 0 &&
-			       time_before(jiffies, timeout)) {
-				usleep_range(tsleep, tsleep + 1000);
-				tsleep <<= 1;
-			}
-			if (atomic_read(&txq->pending_sends)) {
-				err =
-				    pcie_flr(to_pci_dev(gd->gdma_context->dev));
-				if (err) {
-					netdev_err(ndev, "flr failed %d with %d pkts pending in txq %u\n",
-						   err,
-					    atomic_read(&txq->pending_sends),
-					    txq->gdma_txq_id);
-				}
-				break;
-			}
-		}
+		for (i = 0; i < ac->num_ports; i++) {
+			struct mana_port_context *sib;
 
-		for (i = 0; i < apc->num_queues; i++) {
-			txq = &apc->tx_qp[i]->txq;
-			while ((skb = skb_dequeue(&txq->pending_skbs))) {
-				mana_unmap_skb(skb, apc);
-				dev_kfree_skb_any(skb);
-			}
-			atomic_set(&txq->pending_sends, 0);
+			if (!ac->ports[i] || ac->ports[i] == ndev)
+				continue;
+			sib = netdev_priv(ac->ports[i]);
+			netdev_err(ac->ports[i],
+				   "queues reset by a sibling port, scheduling rebuild\n");
+			queue_work(ac->per_port_queue_reset_wq,
+				   &sib->queue_reset_work);
 		}
 	}
 
@@ -3760,6 +3879,324 @@ static int mana_dealloc_queues(struct net_device *ndev)
 	return 0;
 }
 
+/*
+ * ---------------------------------------------------------------------------
+ * Pre-allocate + swap reconfiguration path.
+ *
+ * The detach/attach reconfigure path tears the vport down and rebuilds it,
+ * which lets RDMA grab the vport mid-flight and, if attach fails, leaves the
+ * port permanently broken.
+ *
+ * The swap path builds a *new* set of EQs/TXQs/RXQs while the current set
+ * keeps serving traffic. If allocation fails the current qset is untouched
+ * and we return the error; the user's requested value is never silently
+ * replaced by a fallback. Publishing a new set onto the live port
+ * context is added separately. The vport is never torn down: vport_use_count
+ * stays at 1 throughout, so RDMA cannot hijack it.
+ *
+ * Allocation and teardown run against a *scratch* mana_port_context rather
+ * than the live one. This is essential, not cosmetic: an earlier revision
+ * temporarily NULLed apc->tx_qp so the allocators could
+ * build into the live context, which reliably panicked in mana_start_xmit()
+ * under traffic (it dereferences apc->tx_qp[] guarded only by port_is_up).
+ * The live apc is now mutated only inside mana_publish_qset(), with TX
+ * disabled.
+ *
+ * Note that both sets are live between publish and free, so this peaks at
+ * old+new queues, and therefore at old+new MSI-X vectors. A later patch
+ * gives the port a shared EQ pool so only the queues, not the interrupts,
+ * are doubled up.
+ *
+ * Per-queue debugfs is suppressed for a set while it is being built or torn
+ * down (see mana_qset_scratch_alloc()): the directory names are derived from
+ * the queue index, so the incoming set would collide with the outgoing one
+ * under vport%d. Restoring it needs per-set subdirectories or a
+ * debugfs_rename() once the swap has completed.
+ * ---------------------------------------------------------------------------
+ */
+
+/* Snapshot the queue-set fields of @ctx into @out. */
+static void mana_qset_snapshot(const struct mana_port_context *ctx,
+			       struct mana_qset *out)
+{
+	out->eqs		= ctx->eqs;
+	out->tx_qp		= ctx->tx_qp;
+	out->rxqs		= ctx->rxqs;
+	out->indir_table	= ctx->indir_table;
+	out->indir_table_sz	= ctx->indir_table_sz;
+	out->rxobj_table	= ctx->rxobj_table;
+	out->default_rxobj	= ctx->default_rxobj;
+	out->num_queues		= ctx->num_queues;
+	out->rx_queue_size	= ctx->rx_queue_size;
+	out->tx_queue_size	= ctx->tx_queue_size;
+	out->priv_flags		= ctx->priv_flags;
+	out->mana_eqs_debugfs	= ctx->mana_eqs_debugfs;
+}
+
+/* Install @qset's fields onto @ctx. The vport (port_handle,
+ * vport_use_count) and the port-level debugfs dir are deliberately not
+ * touched: they outlive any individual queue set.
+ */
+static void mana_qset_install(struct mana_port_context *ctx,
+			      const struct mana_qset *qset)
+{
+	ctx->eqs		= qset->eqs;
+	ctx->tx_qp		= qset->tx_qp;
+	ctx->rxqs		= qset->rxqs;
+	ctx->indir_table	= qset->indir_table;
+	ctx->indir_table_sz	= qset->indir_table_sz;
+	ctx->rxobj_table	= qset->rxobj_table;
+	ctx->default_rxobj	= qset->default_rxobj;
+	ctx->num_queues		= qset->num_queues;
+	ctx->rx_queue_size	= qset->rx_queue_size;
+	ctx->tx_queue_size	= qset->tx_queue_size;
+	ctx->priv_flags		= qset->priv_flags;
+	ctx->mana_eqs_debugfs	= qset->mana_eqs_debugfs;
+}
+
+/**
+ * mana_qset_scratch_alloc - build a scratch port context for queue work
+ * @apc: the live port context to shadow
+ *
+ * Returns a heap copy of @apc that shares its vport identity (ac, ndev,
+ * port_handle, indir_table_sz, mac_addr, hashkey, ...) but owns no queues.
+ * The existing per-apc allocators and destroyers can then be run against
+ * it without ever touching the live context.
+ */
+struct mana_port_context *mana_qset_scratch_alloc(struct mana_port_context *apc)
+{
+	struct mana_port_context *scratch;
+
+	scratch = kvzalloc(sizeof(*scratch), GFP_KERNEL);
+	if (!scratch)
+		return NULL;
+
+	*scratch = *apc;
+
+	/* Owns no queues yet. */
+	scratch->eqs		= NULL;
+	scratch->tx_qp		= NULL;
+	scratch->rxqs		= NULL;
+	scratch->indir_table	= NULL;
+	scratch->rxobj_table	= NULL;
+	scratch->default_rxobj	= INVALID_MANA_HANDLE;
+	scratch->mana_eqs_debugfs = NULL;
+
+	/* Never consume the live set's pre-allocated RX buffers;
+	 * mana_get_rxbuf() falls back to normal allocation when these
+	 * are NULL, which is what we want since the swap path no longer
+	 * needs to de-risk post-teardown allocation.
+	 */
+	scratch->rxbufs_pre	= NULL;
+	scratch->das_pre	= NULL;
+	scratch->rxbpre_total	= 0;
+
+	/* Suppress debugfs for queues built through the scratch context:
+	 * two sets are alive at once and would collide on the same names
+	 * under vport%d. debugfs_start_creating() returns early on an
+	 * IS_ERR() parent, and debugfs_remove() ignores IS_ERR_OR_NULL,
+	 * so this makes every create/remove a clean no-op.
+	 */
+	scratch->mana_port_debugfs = ERR_PTR(-ENODEV);
+
+	return scratch;
+}
+
+void mana_qset_scratch_free(struct mana_port_context *scratch)
+{
+	kvfree(scratch);
+}
+
+/**
+ * mana_alloc_qset - build a complete queue set in @scratch
+ * @scratch:	   scratch context from mana_qset_scratch_alloc()
+ * @num_queues:	   number of queues in the new set
+ * @rx_queue_size: new RX ring size
+ * @tx_queue_size: new TX ring size
+ * @priv_flags:	   new priv-flag word (affects full-page RX)
+ * @out:	   output qset, populated on success
+ *
+ * The live port context is not referenced at all, so the currently
+ * running queue set keeps serving traffic throughout. On error nothing
+ * is left allocated.
+ */
+int mana_alloc_qset(struct mana_port_context *scratch, unsigned int num_queues,
+		    unsigned int rx_queue_size, unsigned int tx_queue_size,
+		    u32 priv_flags, struct mana_qset *out)
+{
+	struct net_device *ndev = scratch->ndev;
+	int err;
+
+	ASSERT_RTNL();
+
+	scratch->num_queues	= num_queues;
+	scratch->rx_queue_size	= rx_queue_size;
+	scratch->tx_queue_size	= tx_queue_size;
+	scratch->priv_flags	= priv_flags;
+
+	err = mana_init_port_context(scratch);
+	if (err)
+		goto out_err;
+
+	err = mana_rss_table_alloc(scratch);
+	if (err)
+		goto cleanup_rxq_array;
+
+	err = mana_create_eq(scratch);
+	if (err)
+		goto cleanup_rss;
+
+	err = mana_create_txq(scratch, ndev);
+	if (err)
+		goto cleanup_eq;
+
+	err = mana_add_rx_queues(scratch, ndev);
+	if (err)
+		goto cleanup_rxq;
+
+	mana_rss_table_init(scratch);
+
+	mana_qset_snapshot(scratch, out);
+	return 0;
+
+cleanup_rxq:
+	/* mana_add_rx_queues() may have created queues before failing; they
+	 * own RQ/CQ objects, NAPI state and page pools, so tear down whatever
+	 * made it into scratch->rxqs[] before dropping the array.
+	 */
+	mana_destroy_rxqs(scratch);
+	mana_destroy_txq(scratch);
+cleanup_eq:
+	mana_destroy_eq(scratch);
+cleanup_rss:
+	mana_cleanup_indir_table(scratch);
+cleanup_rxq_array:
+	kfree(scratch->rxqs);
+	scratch->rxqs = NULL;
+out_err:
+	netdev_err(ndev, "mana_alloc_qset(num_queues=%u) failed: %d\n",
+		   num_queues, err);
+	return err;
+}
+
+/**
+ * mana_free_qset - tear down all queues in @qset
+ * @scratch: scratch context from mana_qset_scratch_alloc()
+ * @qset:    queue set to destroy (must no longer be installed on the live apc)
+ *
+ * Runs the existing destroyers against @scratch so the live port context
+ * is never made to point at queues that are being freed.
+ */
+void mana_free_qset(struct mana_port_context *scratch, struct mana_qset *qset)
+{
+	struct bpf_prog *retiring_prog;
+	unsigned int retiring_queues;
+
+	ASSERT_RTNL();
+
+	if (!qset->rxqs && !qset->tx_qp && !qset->eqs)
+		return;
+
+	/* These queues are leaving. Stop their completions from touching the
+	 * shared netdev queues: net_txq is shared with whatever replaced them
+	 * at the same index, and a queue that is only draining always looks
+	 * like it has room, so it would wake a live queue that stopped itself
+	 * because its ring was full. The synchronize_net() below then retires
+	 * any poll that has not seen the flag yet.
+	 */
+	if (qset->tx_qp) {
+		unsigned int q;
+
+		for (q = 0; q < qset->num_queues; q++) {
+			if (qset->tx_qp[q])
+				WRITE_ONCE(qset->tx_qp[q]->txq.retiring, true);
+		}
+	}
+
+	/* The datapath gates on apc->port_is_up and then dereferences
+	 * apc->tx_qp[] / apc->rxqs[] with no lock. mana_publish_qset() drains
+	 * those readers before it installs the incoming set, which cannot
+	 * cover one that sampled the retiring pointers between that install
+	 * and the gate reopening. mana_xdp_xmit() is the case that matters:
+	 * it runs from a redirecting device's NAPI, so the napi_synchronize()
+	 * that mana_destroy_txq()/mana_destroy_rxq() do on this port's own
+	 * NAPIs never waits for it. Give any such reader a grace period to
+	 * finish before its queues are torn down under it. Every caller is a
+	 * reconfiguration path holding RTNL, so this is expedited.
+	 */
+	synchronize_net();
+
+	mana_qset_install(scratch, qset);
+
+	/* Note what this set owes the XDP program, but leave the queues
+	 * pointing at it. They are still polling, and a packet already in a
+	 * retiring RQ has to keep running the program rather than slip past
+	 * it into the stack. The references are dropped once the queues are
+	 * gone, below. XDP_TX from those polls is harmless here: it goes
+	 * through mana_start_xmit() on the live port context, so it reaches
+	 * the queue set that replaced this one, not the one being drained.
+	 */
+	retiring_prog = mana_chn_xdp_peek(scratch);
+	retiring_queues = scratch->num_queues;
+
+	/* The retiring TX queues may still hold packets the device has not
+	 * completed. Drain them before the SQs and the SKB queues go away,
+	 * or those SKBs and their DMA mappings are leaked.
+	 *
+	 * This runs before any RX teardown, the order mana_dealloc_queues()
+	 * uses. A device wedged badly enough to need the reset below is also
+	 * one whose RQ teardown will not complete, and unmapping RX buffers
+	 * first would leave it free to keep writing into them for as long as
+	 * the drain takes.
+	 */
+	if (mana_drain_txqs(scratch)) {
+		/* The drain had to reset the function to stop the device
+		 * touching those buffers. A function reset takes down every
+		 * port on the adapter, not just this one, so rebuild them all
+		 * - the same recovery mana_tx_timeout() relies on. A port that
+		 * is already down has nothing to rebuild and its handler
+		 * returns early.
+		 */
+		struct mana_port_context *apc = netdev_priv(scratch->ndev);
+		struct mana_context *ac = apc->ac;
+		unsigned int i;
+
+		netdev_err(scratch->ndev,
+			   "device reset while retiring a queue set, scheduling port reset\n");
+
+		for (i = 0; i < ac->num_ports; i++) {
+			if (!ac->ports[i])
+				continue;
+			queue_work(ac->per_port_queue_reset_wq,
+				   &((struct mana_port_context *)
+				     netdev_priv(ac->ports[i]))->queue_reset_work);
+		}
+	}
+
+	/* Traffic was still being steered at these queues moments ago, so
+	 * fence each retiring RQ before its buffers are unmapped, again the
+	 * order mana_dealloc_queues() uses. mana_destroy_rxq() does destroy
+	 * the hardware RQ before unmapping anything, but the fence is what
+	 * makes the device confirm it is done with the buffers first.
+	 */
+	mana_fence_rqs(scratch);
+
+	mana_destroy_rxqs(scratch);
+
+	/* The queues are gone, so nothing can run the program any more. */
+	mana_chn_xdp_release(retiring_prog, retiring_queues);
+
+	mana_destroy_txq(scratch);
+	mana_destroy_eq(scratch);
+	mana_cleanup_indir_table(scratch);
+	kfree(scratch->rxqs);
+	scratch->rxqs = NULL;
+
+	memset(qset, 0, sizeof(*qset));
+}
+
+/* --- end of pre-allocate + swap reconfiguration path ---------------------- */
+
 int mana_detach(struct net_device *ndev, bool from_close)
 {
 	struct mana_port_context *apc = netdev_priv(ndev);
@@ -4237,6 +4674,13 @@ void mana_remove(struct gdma_dev *gd, bool suspending)
 		unregister_netdevice(ndev);
 		mana_cleanup_indir_table(apc);
 
+		/* Clear the slot before the netdev goes away. A later port
+		 * whose teardown has to reset the function walks ac->ports[]
+		 * to schedule the rebuild, and would otherwise reach into the
+		 * port freed here.
+		 */
+		ac->ports[i] = NULL;
+
 		rtnl_unlock();
 
 		free_netdev(ndev);
diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h
index 83b7eff4646ead7aef1382c6ce565a573a940af4..4727c231bf0391bd9a1e9ff15c65815a6a3d01dd 100644
--- a/include/net/mana/mana.h
+++ b/include/net/mana/mana.h
@@ -143,6 +143,16 @@ struct mana_txq {
 
 	bool napi_initialized;
 
+	/* Value of mana_context.reset_gen when this queue was created. */
+	u32 reset_gen;
+
+	/* Set once this queue has been unpublished and is on its way out.
+	 * Its completions must not touch flow control any more: net_txq is
+	 * shared with the queue that replaced it at the same index, and a
+	 * draining queue always looks like it has room.
+	 */
+	bool retiring;
+
 	struct mana_stats_tx stats;
 };
 
@@ -537,6 +547,14 @@ struct mana_context {
 	u8 bm_hostmode;
 
 	struct mana_ethtool_hc_stats hc_stats;
+
+	/* Bumped every time the PCI function is reset to unstick a TX queue.
+	 * A queue created before the current value cannot be touched by the
+	 * device any more, so nothing has to be waited for before its buffers
+	 * are released. Written under RTNL, read locklessly.
+	 */
+	u32 reset_gen;
+
 	struct workqueue_struct *per_port_queue_reset_wq;
 	/* Workqueue for querying hardware stats */
 	struct delayed_work gf_stats_work;
@@ -661,6 +679,39 @@ struct mana_port_context {
 	u32 steer_cqe_coalescing;
 };
 
+/* struct mana_qset - a self-contained snapshot of the queue-related
+ * fields inside mana_port_context that can be swapped atomically.
+ *
+ * Prototype for the "pre-allocate + swap" reconfiguration path (as
+ * suggested by netdev maintainers): a new qset is allocated while the
+ * current one keeps serving traffic, then apc's queue fields are
+ * atomically switched to the new set and the old set is torn down.
+ * The vport (port_handle / vport_use_count) is *not* touched, so RDMA
+ * can never race in during reconfiguration.
+ */
+struct mana_qset {
+	struct mana_eq		*eqs;
+	struct mana_tx_qp	**tx_qp;
+	struct mana_rxq		**rxqs;
+
+	u32			*indir_table;
+	u32			indir_table_sz;
+	mana_handle_t		*rxobj_table;
+	mana_handle_t		default_rxobj;
+
+	unsigned int		num_queues;
+	unsigned int		rx_queue_size;
+	unsigned int		tx_queue_size;
+	u32			priv_flags;
+
+	/* Per-queue-set debugfs root ("EQs"). Owned by the qset: it is
+	 * recreated by mana_create_eq() for each new set and torn down
+	 * with that set, so it must travel with the qset rather than
+	 * staying on apc.
+	 */
+	struct dentry		*mana_eqs_debugfs;
+};
+
 netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev);
 int mana_config_rss(struct mana_port_context *ac, enum TRI_STATE rx,
 		    bool update_hash, bool update_tab);
@@ -670,6 +721,17 @@ int mana_alloc_queues(struct net_device *ndev);
 int mana_attach(struct net_device *ndev);
 int mana_detach(struct net_device *ndev, bool from_close);
 
+/* Pre-allocate + swap reconfiguration path. Allocation and teardown run
+ * against a scratch context so the live port context is never made to
+ * point at queues that are still being built or freed.
+ */
+struct mana_port_context *mana_qset_scratch_alloc(struct mana_port_context *apc);
+void mana_qset_scratch_free(struct mana_port_context *scratch);
+int mana_alloc_qset(struct mana_port_context *scratch, unsigned int num_queues,
+		    unsigned int rx_queue_size, unsigned int tx_queue_size,
+		    u32 priv_flags, struct mana_qset *out);
+void mana_free_qset(struct mana_port_context *scratch, struct mana_qset *qset);
+
 void mana_dim_change(struct mana_cq *cq, bool enable);
 
 int mana_probe(struct gdma_dev *gd, bool resuming);
@@ -685,6 +747,8 @@ u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq,
 		 struct xdp_buff *xdp, void *buf_va, uint pkt_len);
 struct bpf_prog *mana_xdp_get(struct mana_port_context *apc);
 void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog);
+struct bpf_prog *mana_chn_xdp_peek(struct mana_port_context *apc);
+void mana_chn_xdp_release(struct bpf_prog *prog, unsigned int num_queues);
 int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf);
 int mana_query_gf_stats(struct mana_context *ac);
 int mana_query_link_cfg(struct mana_port_context *apc);
-- 
2.43.0

[PATCH net-next v2 10/13] net: mana: release EQs left idle by a channel-count reduction

[Long Li]

The shared EQ pool introduced by the previous patch is grow-only, so it
sits at the high-water mark of every channel count the port has ever
used. After "ethtool -L ens1 combined 32" followed by "combined 4" the
port keeps 32 EQs and 32 MSI-X vectors while using four:

  # ethtool -L ens1 combined 4
  # grep -c mana /proc/interrupts
  33

Before the queue-set swap this did not happen: mana_detach() destroyed
every EQ and mana_attach() created exactly num_queues of them, so the
interrupt count tracked the configured channel count.

Add mana_shrink_eqs() and call it from mana_free_qset(), which is the
one place where a queue set has just been torn down and the live count
is already known. Both callers benefit: on success the retiring set is
the old one and the pool shrinks to the new count, and if
mana_publish_qset() failed and rolled back, the new set is discarded and
the pool shrinks back to the restored count.

Shrinking is only safe there. A CQ holds the gdma_queue pointer of its
parent EQ, so an EQ may only be destroyed once the set referencing it is
gone.

mana_free_qset() therefore takes the live port context as well, since
the pool belongs to it rather than to the scratch context.

While here, fix mana_create_eq_debugfs(), which stored the new dentry in
a stack copy of the mana_eq rather than in apc->eqs[i]. Nothing noticed
because mana_destroy_eq() removes the whole "EQs" directory recursively,
but per-EQ removal is needed to shrink without leaving stale nodes
behind, which would then collide when the pool grows again.

Signed-off-by: Long Li <longli@microsoft.com>
---
 .../net/ethernet/microsoft/mana/mana_bpf.c    |  4 +-
 drivers/net/ethernet/microsoft/mana/mana_en.c | 79 ++++++++++++++++---
 .../ethernet/microsoft/mana/mana_ethtool.c    | 12 +--
 include/net/mana/mana.h                       |  3 +-
 4 files changed, 78 insertions(+), 20 deletions(-)

diff --git a/drivers/net/ethernet/microsoft/mana/mana_bpf.c b/drivers/net/ethernet/microsoft/mana/mana_bpf.c
index 1bae4174e268f7f53b0880c5d1098cd0ef687f25..7031ecb4da2e0fd5b83aeb07aeb8d7ae56e16d65 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_bpf.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_bpf.c
@@ -235,7 +235,7 @@ static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog,
 		if (err) {
 			NL_SET_ERR_MSG_MOD(extack,
 					   "XDP: Re-config failed at publish");
-			mana_free_qset(scratch, &newq);
+			mana_free_qset(apc, scratch, &newq);
 			/* After the cleanup above: closing destroys the EQ pool
 			 * those queues' CQs were attached to.
 			 */
@@ -244,7 +244,7 @@ static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog,
 			return err;
 		}
 
-		mana_free_qset(scratch, &oldq);
+		mana_free_qset(apc, scratch, &oldq);
 		mana_qset_scratch_free(scratch);
 	} else {
 		/* No queues to rebuild; mana_open() will size the RX buffers
diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
index 71cbdebc5f3f7126495b2b11f0c673955fe0f8df..0d87440fbfbee7ac5729945d101eaa0c37745fbe 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_en.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
@@ -931,11 +931,11 @@ static int mana_change_mtu(struct net_device *ndev, int new_mtu)
 
 	err = mana_publish_qset(mpc, &newq, &oldq);
 	if (err) {
-		mana_free_qset(scratch, &newq);
+		mana_free_qset(mpc, scratch, &newq);
 		goto free_scratch;
 	}
 
-	mana_free_qset(scratch, &oldq);
+	mana_free_qset(mpc, scratch, &oldq);
 
 free_scratch:
 	/* After the caller-side cleanup above, so the EQ pool outlives the
@@ -1847,6 +1847,9 @@ void mana_destroy_eq(struct mana_port_context *apc)
 		msi = eq->eq.msix_index;
 		mana_gd_destroy_queue(gc, eq);
 		mana_gd_put_gic(gc, !gc->msi_sharing, msi);
+		apc->eqs[i].eq = NULL;
+		/* Freed with the parent by debugfs_remove_recursive() above. */
+		apc->eqs[i].mana_eq_debugfs = NULL;
 	}
 
 	kfree(apc->eqs);
@@ -1857,15 +1860,16 @@ EXPORT_SYMBOL_NS(mana_destroy_eq, "NET_MANA");
 
 static void mana_create_eq_debugfs(struct mana_port_context *apc, int i)
 {
-	struct mana_eq eq = apc->eqs[i];
+	struct mana_eq *eq = &apc->eqs[i];
 	char eqnum[32];
 
 	sprintf(eqnum, "eq%d", i);
-	eq.mana_eq_debugfs = debugfs_create_dir(eqnum, apc->mana_eqs_debugfs);
-	debugfs_create_u32("head", 0400, eq.mana_eq_debugfs, &eq.eq->head);
-	debugfs_create_u32("tail", 0400, eq.mana_eq_debugfs, &eq.eq->tail);
-	debugfs_create_u32("irq", 0400, eq.mana_eq_debugfs, &eq.eq->eq.irq);
-	debugfs_create_file("eq_dump", 0400, eq.mana_eq_debugfs, eq.eq, &mana_dbg_q_fops);
+	eq->mana_eq_debugfs = debugfs_create_dir(eqnum, apc->mana_eqs_debugfs);
+	debugfs_create_u32("head", 0400, eq->mana_eq_debugfs, &eq->eq->head);
+	debugfs_create_u32("tail", 0400, eq->mana_eq_debugfs, &eq->eq->tail);
+	debugfs_create_u32("irq", 0400, eq->mana_eq_debugfs, &eq->eq->eq.irq);
+	debugfs_create_file("eq_dump", 0400, eq->mana_eq_debugfs, eq->eq,
+			    &mana_dbg_q_fops);
 }
 
 int mana_create_eq(struct mana_port_context *apc)
@@ -1999,6 +2003,43 @@ static int mana_grow_eqs(struct mana_port_context *apc, unsigned int need)
 	return err;
 }
 
+/**
+ * mana_shrink_eqs - release EQs above @keep
+ * @apc:  port context
+ * @keep: number of EQs the live queue set still uses
+ *
+ * Returns the MSI-X vectors of the EQs a shrink left behind. Must only be
+ * called once no queue set references them, i.e. after the retiring set has
+ * been torn down by mana_free_qset(), otherwise a live CQ would be left
+ * pointing at a destroyed EQ.
+ */
+static void mana_shrink_eqs(struct mana_port_context *apc, unsigned int keep)
+{
+	struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
+	struct gdma_queue *eq;
+	unsigned int msi;
+	unsigned int i;
+
+	if (!apc->eqs || keep >= apc->num_eqs)
+		return;
+
+	for (i = keep; i < apc->num_eqs; i++) {
+		eq = apc->eqs[i].eq;
+		if (!eq)
+			continue;
+
+		debugfs_remove_recursive(apc->eqs[i].mana_eq_debugfs);
+		apc->eqs[i].mana_eq_debugfs = NULL;
+
+		msi = eq->eq.msix_index;
+		mana_gd_destroy_queue(gc, eq);
+		mana_gd_put_gic(gc, !gc->msi_sharing, msi);
+		apc->eqs[i].eq = NULL;
+	}
+
+	apc->num_eqs = keep;
+}
+
 static int mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *rxq)
 {
 	struct mana_fence_rq_resp resp = {};
@@ -4224,7 +4265,7 @@ void mana_qset_scratch_free(struct mana_port_context *scratch)
  * Every queue is built in @scratch, so the queue set currently installed on
  * @apc keeps serving traffic throughout. @apc is touched only to grow the
  * shared EQ pool, which both sets reference while they are both live. On
- * error no queue is left allocated.
+ * error no queue is left allocated, and any EQ this call added is released.
  */
 int mana_alloc_qset(struct mana_port_context *apc,
 		    struct mana_port_context *scratch, unsigned int num_queues,
@@ -4294,6 +4335,13 @@ int mana_alloc_qset(struct mana_port_context *apc,
 	kfree(scratch->rxqs);
 	scratch->rxqs = NULL;
 out_err:
+	/* Give back any EQ this attempt added to the shared pool rather than
+	 * holding its MSI-X vectors until some later teardown: the live set
+	 * still needs only apc->num_queues of them. Safe here because this
+	 * set's CQs have already been destroyed above.
+	 */
+	mana_shrink_eqs(apc, apc->num_queues);
+
 	netdev_err(ndev, "mana_alloc_qset(num_queues=%u) failed: %d\n",
 		   num_queues, err);
 	return err;
@@ -4655,13 +4703,15 @@ static void mana_qset_debugfs_publish(struct mana_port_context *apc)
 
 /**
  * mana_free_qset - tear down all queues in @qset
+ * @apc:     live port context (owns the shared EQ pool)
  * @scratch: scratch context from mana_qset_scratch_alloc()
  * @qset:    queue set to destroy (must no longer be installed on the live apc)
  *
  * Runs the existing destroyers against @scratch so the live port context
  * is never made to point at queues that are being freed.
  */
-void mana_free_qset(struct mana_port_context *scratch, struct mana_qset *qset)
+void mana_free_qset(struct mana_port_context *apc,
+		    struct mana_port_context *scratch, struct mana_qset *qset)
 {
 	struct bpf_prog *retiring_prog;
 	unsigned int retiring_queues;
@@ -4776,12 +4826,19 @@ void mana_free_qset(struct mana_port_context *scratch, struct mana_qset *qset)
 
 	memset(qset, 0, sizeof(*qset));
 
+	/* This set is gone, so any EQ above the live queue count is now
+	 * unreferenced. Release those vectors instead of holding them at the
+	 * high-water mark. Safe here and only here: the retiring set's CQs
+	 * have just been destroyed.
+	 */
+	mana_shrink_eqs(apc, apc->num_queues);
+
 	/* Queues built through a scratch context carry no debugfs nodes,
 	 * because both sets are alive during the swap and would collide on
 	 * the same names. The retiring set's nodes are gone now, so the
 	 * published queues can finally take those names.
 	 */
-	mana_qset_debugfs_publish(netdev_priv(scratch->ndev));
+	mana_qset_debugfs_publish(apc);
 }
 
 /* --- end of pre-allocate + swap reconfiguration path ---------------------- */
diff --git a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
index 03fe657334c49a69ebe9c2677b2b8268321a162d..a4d62ea8aec8f4aaa521af2c4fa4af6d928e9b4e 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
@@ -744,11 +744,11 @@ static int mana_set_channels(struct net_device *ndev,
 
 	err = mana_publish_qset(apc, &newq, &oldq);
 	if (err) {
-		mana_free_qset(scratch, &newq);
+		mana_free_qset(apc, scratch, &newq);
 		goto free_scratch;
 	}
 
-	mana_free_qset(scratch, &oldq);
+	mana_free_qset(apc, scratch, &oldq);
 
 free_scratch:
 	/* After the caller-side cleanup above, so the EQ pool outlives the
@@ -852,11 +852,11 @@ static int mana_set_ringparam(struct net_device *ndev,
 	if (err) {
 		NL_SET_ERR_MSG_FMT(extack, "failed to change ring params: %d",
 				   err);
-		mana_free_qset(scratch, &newq);
+		mana_free_qset(apc, scratch, &newq);
 		goto free_scratch;
 	}
 
-	mana_free_qset(scratch, &oldq);
+	mana_free_qset(apc, scratch, &oldq);
 
 free_scratch:
 	/* After the caller-side cleanup above, so the EQ pool outlives the
@@ -949,11 +949,11 @@ static int mana_set_priv_flags(struct net_device *ndev, u32 priv_flags)
 
 	err = mana_publish_qset(apc, &newq, &oldq);
 	if (err) {
-		mana_free_qset(scratch, &newq);
+		mana_free_qset(apc, scratch, &newq);
 		goto free_scratch;
 	}
 
-	mana_free_qset(scratch, &oldq);
+	mana_free_qset(apc, scratch, &oldq);
 
 free_scratch:
 	/* After the caller-side cleanup above, so the EQ pool outlives the
diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h
index d09bf2c7cec0be06c4190caab1f0723a5c4956d6..52f1c8f9c2b968ad14c90849fd3286b2c046d1d3 100644
--- a/include/net/mana/mana.h
+++ b/include/net/mana/mana.h
@@ -776,7 +776,8 @@ int mana_alloc_qset(struct mana_port_context *apc,
 int mana_publish_qset(struct mana_port_context *apc, struct mana_qset *newq,
 		      struct mana_qset *out_old);
 void mana_publish_close_if_needed(struct mana_port_context *apc);
-void mana_free_qset(struct mana_port_context *scratch, struct mana_qset *qset);
+void mana_free_qset(struct mana_port_context *apc,
+		    struct mana_port_context *scratch, struct mana_qset *qset);
 
 void mana_dim_change(struct mana_cq *cq, bool enable);
 
-- 
2.43.0

[PATCH net-next v2 11/13] net: mana: keep a user-configured RSS table across a queue rebuild

[Long Li]

mana_rss_table_init() overwrites the indirection table with the driver
default every time the queues are rebuilt. Both rebuild paths do it:
mana_alloc_qset() for the ethtool/MTU/XDP queue-set swap, and
mana_alloc_queues() for ndo_open and for the TX-timeout reset.

A table the user installed with "ethtool -X" is therefore lost by
operations that have nothing to do with RSS. Resizing the rings, changing
the MTU, toggling a private flag, attaching an XDP program, or simply
taking the port down and up again all silently reset the steering:

  # ethtool -X ens1 equal 1        # everything to queue 0
  # ethtool -G ens1 rx 1024
  # ethtool -x ens1                # back to 0..15, silently

The entries are queue indices, so they stay meaningful as long as the
queue count does not change, and mana_config_rss() already maps them onto
whichever RX objects the new set has. Carry the table over instead of
regenerating it.

Only a user-configured table is preserved, which netif_is_rxfh_configured()
reports: a driver-generated table must still be rebuilt so that it spreads
over all the queues of the new set. ethtool_check_max_channel() refuses a
channel-count reduction that would leave a user table pointing past the
last queue, so the entries are in range by construction; the bounds check
is a safety net for the rebuild paths that do not come from ethtool, and
reports the table as lost rather than steering to a queue that is gone.

Signed-off-by: Long Li <longli@microsoft.com>
---
 drivers/net/ethernet/microsoft/mana/mana_en.c | 50 ++++++++++++++++++-
 1 file changed, 48 insertions(+), 2 deletions(-)

diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
index 0d87440fbfbee7ac5729945d101eaa0c37745fbe..4cab3f658f2487671d26243d4e91f834580b3c5c 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_en.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
@@ -3545,6 +3545,38 @@ static void mana_rss_table_init(struct mana_port_context *apc)
 			ethtool_rxfh_indir_default(i, apc->num_queues);
 }
 
+/* Decide whether @apc's indirection table can be carried over to a queue set
+ * with @num_queues queues, instead of being rebuilt from the driver default.
+ *
+ * Only a table the user installed with "ethtool -X" is worth preserving: a
+ * driver-generated one has to be rebuilt so that it spreads over all the
+ * queues the new set actually has.
+ *
+ * ethtool_check_max_channel() already refuses a channel-count reduction that
+ * would leave a user-configured table pointing past the last queue, so the
+ * bounds check below is only a safety net for the rebuild paths that do not
+ * originate from ethtool. If it ever trips, the table cannot be honoured for
+ * the new queue count, so tell the core the user's table is gone rather than
+ * silently steering to queues that no longer exist.
+ */
+static bool mana_rss_table_keep(struct mana_port_context *apc,
+				unsigned int num_queues)
+{
+	u32 i;
+
+	if (!netif_is_rxfh_configured(apc->ndev))
+		return false;
+
+	for (i = 0; i < apc->indir_table_sz; i++) {
+		if (apc->indir_table[i] >= num_queues) {
+			ethtool_rxfh_indir_lost(apc->ndev);
+			return false;
+		}
+	}
+
+	return true;
+}
+
 int mana_disable_vport_rx(struct mana_port_context *apc)
 {
 	return mana_cfg_vport_steering(apc, TRI_STATE_FALSE, false, false,
@@ -3860,7 +3892,12 @@ int mana_alloc_queues(struct net_device *ndev)
 		goto destroy_rxq;
 	}
 
-	mana_rss_table_init(apc);
+	/* Keep a user-configured RSS table across a rebuild; the entries are
+	 * queue indices, so they stay meaningful as long as the queue count
+	 * is unchanged. Only a driver-generated table is regenerated here.
+	 */
+	if (!mana_rss_table_keep(apc, apc->num_queues))
+		mana_rss_table_init(apc);
 
 	err = mana_config_rss(apc, TRI_STATE_TRUE, true, true);
 	if (err) {
@@ -4317,7 +4354,16 @@ int mana_alloc_qset(struct mana_port_context *apc,
 	if (err)
 		goto cleanup_rxq;
 
-	mana_rss_table_init(scratch);
+	/* Carry a user-configured RSS table over to the new set. The entries
+	 * are queue indices, so mana_config_rss() in mana_publish_qset() maps
+	 * them onto the new set's RX objects. A driver-generated table is
+	 * rebuilt instead, so it covers every queue of the new set.
+	 */
+	if (mana_rss_table_keep(apc, num_queues))
+		memcpy(scratch->indir_table, apc->indir_table,
+		       apc->indir_table_sz * sizeof(*apc->indir_table));
+	else
+		mana_rss_table_init(scratch);
 
 	mana_qset_snapshot(scratch, out);
 	return 0;
-- 
2.43.0

[PATCH net-next v2 12/13] net: mana: keep the surviving queues when the channel count is reduced

[Long Li]

A channel-count reduction currently builds a whole new queue set and then
destroys the old one, even though the queues being kept do not change.
Shrinking 32 channels to 8 creates 8 TX and 8 RX queues, each with its own
DMA rings and hardware WQ object, and then destroys all 32 of each:

  set_channels 16 -> 8   created SQ=8  RQ=8  | destroyed SQ=16 RQ=16

None of that work is necessary. mana_create_txq() derives a TX queue only
from apc->tx_queue_size and apc->eqs[i], and mana_create_rxq() derives an
RX queue only from apc->rx_queue_size and the MTU/priv-flag/XDP buffer
layout. Neither depends on how many queues there are, and queue i is bound
to EQ i at any count, so queue i is configured identically before and
after. This is specific to the channel count: a ring resize, MTU change,
priv-flag toggle or XDP attach changes a property of every queue, so those
still rebuild.

Add mana_split_qset(), which carves the live set into the prefix to keep
and the tail to retire. It allocates two pointer arrays and a steering
table and moves the existing queue pointers into them; no DMA ring is
allocated and no WQ object is created. Both halves are then handed to the
existing mana_publish_qset() and mana_free_qset(), so the swap ordering,
the TX quiesce, the rollback and the TX drain are unchanged - the live
context is never mutated in place, which is what makes this safe under
traffic.

Two details worth noting. The steering table is rebuilt for the smaller
set rather than edited in place, because mana_config_rss() would otherwise
index the shorter rxqs[] with entries still referring to retired queues.
And the tail carries apc->bpf_prog so that mana_free_qset()'s
mana_chn_setxdp(NULL) drops exactly the tail's per-queue program
references; the kept queues are left alone because mana_chn_setxdp()
returns early when the program is unchanged.

A reduction now allocates two small arrays and a steering table instead of
a full queue set, so the peak stays at one set rather than two, and the
surviving queues keep their page pools, posted RX buffers and NAPI state
instead of being drained and refilled.

This completes the conversion, so advertise it to the firmware. Every
queue resize path - channel count, ring size, MTU, the full-page RX
private flag and XDP attach - now builds the new queue set before
retiring the old one and keeps the old one running if that fails, so a
failed resize no longer needs external intervention to restore the port.
Add GDMA_DRV_CAP_FLAG_1_SELF_RECOVERY_ON_QUEUE_RESIZE_FAILURE and set it
in GDMA_DRV_CAP_FLAGS1.

Signed-off-by: Long Li <longli@microsoft.com>
---
 drivers/net/ethernet/microsoft/mana/mana_en.c | 146 ++++++++++++++++++
 .../ethernet/microsoft/mana/mana_ethtool.c    |  35 +++++
 include/net/mana/gdma.h                       |  11 +-
 include/net/mana/mana.h                       |   4 +
 4 files changed, 195 insertions(+), 1 deletion(-)

diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
index 4cab3f658f2487671d26243d4e91f834580b3c5c..7cd2fd9ea050f10604dc0adb368c0aa4e7b6bb10 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_en.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
@@ -4287,6 +4287,152 @@ void mana_qset_scratch_free(struct mana_port_context *scratch)
 	kvfree(scratch);
 }
 
+/**
+ * mana_split_qset - carve the live queue set into a kept prefix and a tail
+ * @apc:       live port context
+ * @scratch:   scratch context, used to allocate the new steering table
+ * @new_count: number of queues to keep
+ * @out_new:   filled with the set to publish, queues [0, @new_count)
+ * @out_tail:  filled with the set to retire, queues [@new_count, num_queues)
+ *
+ * A channel-count reduction is the one reconfiguration where the surviving
+ * queues need no rebuilding at all. mana_create_txq() derives a TX queue only
+ * from apc->tx_queue_size and apc->eqs[i], and mana_create_rxq() derives an RX
+ * queue only from apc->rx_queue_size and the MTU/priv-flag/XDP buffer layout;
+ * neither depends on how many queues exist, and queue i keeps EQ i at any
+ * count. Queue i is therefore configured identically before and after, so the
+ * low queues can be carried over and only the tail destroyed.
+ *
+ * All this allocates is two pointer arrays and a steering table. No DMA ring
+ * is allocated and no hardware WQ object is created, so the peak stays at one
+ * set instead of two, and the surviving queues keep their page pools, their
+ * posted RX buffers and their NAPI state instead of being drained and refilled.
+ *
+ * Nothing in @apc is modified here. Both sets are handed back for the existing
+ * mana_publish_qset() and mana_free_qset() to install and retire, so the swap
+ * ordering, the rollback and the TX drain all keep working the way they do for
+ * a rebuilt set.
+ *
+ * On success the caller owns both sets. On failure @apc is untouched.
+ */
+int mana_split_qset(struct mana_port_context *apc,
+		    struct mana_port_context *scratch, unsigned int new_count,
+		    struct mana_qset *out_new, struct mana_qset *out_tail)
+{
+	unsigned int old_count = apc->num_queues;
+	struct mana_tx_qp **new_tx, **tail_tx;
+	struct mana_rxq **new_rx, **tail_rx;
+	unsigned int tail_count;
+	unsigned int i;
+	int err;
+
+	ASSERT_RTNL();
+
+	if (WARN_ON(new_count == 0 || new_count >= old_count))
+		return -EINVAL;
+	if (WARN_ON(!apc->tx_qp || !apc->rxqs))
+		return -EINVAL;
+
+	tail_count = old_count - new_count;
+
+	/* Build the steering table for the smaller set separately, so the live
+	 * one keeps describing the live queues until mana_publish_qset() swaps
+	 * both over together. mana_config_rss() would otherwise index the new
+	 * (shorter) rxqs[] with entries that still refer to retired queues.
+	 */
+	scratch->num_queues = new_count;
+	err = mana_rss_table_alloc(scratch);
+	if (err)
+		return err;
+
+	if (mana_rss_table_keep(apc, new_count))
+		memcpy(scratch->indir_table, apc->indir_table,
+		       apc->indir_table_sz * sizeof(*apc->indir_table));
+	else
+		mana_rss_table_init(scratch);
+
+	new_tx = kzalloc_objs(struct mana_tx_qp *, new_count);
+	new_rx = kzalloc_objs(struct mana_rxq *, new_count);
+	tail_tx = kzalloc_objs(struct mana_tx_qp *, tail_count);
+	tail_rx = kzalloc_objs(struct mana_rxq *, tail_count);
+	if (!new_tx || !new_rx || !tail_tx || !tail_rx) {
+		err = -ENOMEM;
+		goto free_arrays;
+	}
+
+	for (i = 0; i < new_count; i++) {
+		new_tx[i] = apc->tx_qp[i];
+		new_rx[i] = apc->rxqs[i];
+	}
+	for (i = 0; i < tail_count; i++) {
+		tail_tx[i] = apc->tx_qp[new_count + i];
+		tail_rx[i] = apc->rxqs[new_count + i];
+	}
+
+	/* The kept prefix, with the new steering table. */
+	out_new->tx_qp		= new_tx;
+	out_new->rxqs		= new_rx;
+	out_new->indir_table	= scratch->indir_table;
+	out_new->indir_table_sz	= scratch->indir_table_sz;
+	out_new->rxobj_table	= scratch->rxobj_table;
+	out_new->default_rxobj	= apc->rxqs[0]->rxobj;
+	out_new->num_queues	= new_count;
+	out_new->rx_queue_size	= apc->rx_queue_size;
+	out_new->tx_queue_size	= apc->tx_queue_size;
+	out_new->priv_flags	= apc->priv_flags;
+	out_new->mtu		= apc->configured_mtu;
+	out_new->bpf_prog	= apc->bpf_prog;
+
+	/* Ownership of the table moved to @out_new. */
+	scratch->indir_table	= NULL;
+	scratch->rxobj_table	= NULL;
+
+	/* The tail. It owns no steering table; bpf_prog is carried so that
+	 * mana_free_qset()'s mana_chn_setxdp(NULL) drops exactly the tail's
+	 * per-queue program references and leaves the kept ones alone.
+	 */
+	memset(out_tail, 0, sizeof(*out_tail));
+	out_tail->tx_qp		= tail_tx;
+	out_tail->rxqs		= tail_rx;
+	out_tail->default_rxobj	= INVALID_MANA_HANDLE;
+	out_tail->num_queues	= tail_count;
+	out_tail->rx_queue_size	= apc->rx_queue_size;
+	out_tail->tx_queue_size	= apc->tx_queue_size;
+	out_tail->priv_flags	= apc->priv_flags;
+	out_tail->mtu		= apc->configured_mtu;
+	out_tail->bpf_prog	= apc->bpf_prog;
+
+	return 0;
+
+free_arrays:
+	kfree(new_tx);
+	kfree(new_rx);
+	kfree(tail_tx);
+	kfree(tail_rx);
+	mana_cleanup_indir_table(scratch);
+	return err;
+}
+
+/**
+ * mana_discard_split - drop the containers built by mana_split_qset()
+ * @newq:  set that was never published
+ * @tailq: matching tail
+ *
+ * Frees only the pointer arrays and the steering table. The queues they refer
+ * to are still owned by the live port context, so they must not be destroyed.
+ */
+void mana_discard_split(struct mana_qset *newq, struct mana_qset *tailq)
+{
+	kfree(newq->tx_qp);
+	kfree(newq->rxqs);
+	kfree(newq->indir_table);
+	kfree(newq->rxobj_table);
+	kfree(tailq->tx_qp);
+	kfree(tailq->rxqs);
+	memset(newq, 0, sizeof(*newq));
+	memset(tailq, 0, sizeof(*tailq));
+}
+
 /**
  * mana_alloc_qset - build a complete queue set in @scratch
  * @apc:	   live port context, owner of the shared EQ pool
diff --git a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
index a4d62ea8aec8f4aaa521af2c4fa4af6d928e9b4e..091b4a79ad238151b3a7f2014306b9551680c929 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
@@ -736,6 +736,41 @@ static int mana_set_channels(struct net_device *ndev,
 		goto clear_flag;
 	}
 
+	/* A reduction does not need new queues: the ones being kept are
+	 * configured identically before and after, so carry them over and
+	 * retire only the tail. This allocates no DMA ring and creates no
+	 * hardware WQ object, so it also avoids the old+new peak that a
+	 * rebuild has to pay.
+	 */
+	if (new_count < apc->num_queues) {
+		struct mana_qset tailq;
+
+		err = mana_split_qset(apc, scratch, new_count, &newq, &tailq);
+		if (err)
+			goto free_scratch; /* current qset untouched */
+
+		err = mana_publish_qset(apc, &newq, &oldq);
+		if (err) {
+			/* The old set is live again; drop only the containers
+			 * built above, never the queues they point at.
+			 */
+			mana_discard_split(&newq, &tailq);
+			goto free_scratch;
+		}
+
+		/* @oldq holds the original arrays and steering table. Every
+		 * queue they referenced is now owned by either the published
+		 * set or the tail, so only the containers are freed here.
+		 */
+		kfree(oldq.tx_qp);
+		kfree(oldq.rxqs);
+		kfree(oldq.indir_table);
+		kfree(oldq.rxobj_table);
+
+		mana_free_qset(apc, scratch, &tailq);
+		goto free_scratch;
+	}
+
 	err = mana_alloc_qset(apc, scratch, new_count, apc->rx_queue_size,
 			      apc->tx_queue_size, apc->priv_flags,
 			      apc->configured_mtu, apc->bpf_prog, &newq);
diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h
index 70a7f1fee5d3b0a6460cbd76159d01a369838d68..c54500700f6f2f4b432102364e64fd021e7c2c88 100644
--- a/include/net/mana/gdma.h
+++ b/include/net/mana/gdma.h
@@ -672,6 +672,14 @@ enum {
 /* Driver supports dynamic interrupt moderation - DIM */
 #define GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(28)
 
+/* Driver recovers by itself when a queue resize fails: a failed resize leaves
+ * the queues that were already serving traffic in place, so the host does not
+ * have to bring the port back. This covers the resize itself failing. It does
+ * not promise recovery when restoring the previous queue set fails too, which
+ * leaves the port administratively down for the admin to bring back up.
+ */
+#define GDMA_DRV_CAP_FLAG_1_SELF_RECOVERY_ON_QUEUE_RESIZE_FAILURE BIT(31)
+
 #define GDMA_DRV_CAP_FLAGS1 \
 	(GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \
 	 GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \
@@ -688,7 +696,8 @@ enum {
 	 GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY | \
 	 GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY | \
 	 GDMA_DRV_CAP_FLAG_1_EQ_MSI_UNSHARE_MULTI_VPORT | \
-	 GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION)
+	 GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION | \
+	 GDMA_DRV_CAP_FLAG_1_SELF_RECOVERY_ON_QUEUE_RESIZE_FAILURE)
 
 #define GDMA_DRV_CAP_FLAGS2 0
 
diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h
index 52f1c8f9c2b968ad14c90849fd3286b2c046d1d3..8603f66ded7c2a8745d257ba4b8b5801a1c289e3 100644
--- a/include/net/mana/mana.h
+++ b/include/net/mana/mana.h
@@ -773,6 +773,10 @@ int mana_alloc_qset(struct mana_port_context *apc,
 		    unsigned int rx_queue_size, unsigned int tx_queue_size,
 		    u32 priv_flags, int mtu, struct bpf_prog *bpf_prog,
 		    struct mana_qset *out);
+int mana_split_qset(struct mana_port_context *apc,
+		    struct mana_port_context *scratch, unsigned int new_count,
+		    struct mana_qset *out_new, struct mana_qset *out_tail);
+void mana_discard_split(struct mana_qset *newq, struct mana_qset *tailq);
 int mana_publish_qset(struct mana_port_context *apc, struct mana_qset *newq,
 		      struct mana_qset *out_old);
 void mana_publish_close_if_needed(struct mana_port_context *apc);
-- 
2.43.0

[PATCH net-next v2 13/13] net: mana: keep the existing queues when the channel count is raised

[Long Li]

The reduction path already carries its surviving queues over instead of
rebuilding them. An increase still builds a complete second set and
throws the running one away, even though it keeps every queue it already
had:

  set_channels 4 -> 8    created SQ=8  RQ=8  | destroyed SQ=4  RQ=4

The reasoning that makes a reduction safe is not directional. A TX queue
is built from apc->tx_queue_size and apc->eqs[i], an RX queue from
apc->rx_queue_size and the MTU/priv-flag/XDP buffer layout, and queue i
is bound to EQ i at any count. Nothing a queue is made of depends on how
many queues there are, so the queues that were already running are
configured identically before and after and can simply be carried over.

Add mana_grow_qset(), the mirror image of mana_split_qset(). It builds
the new set's pointer arrays with the running queues in [0, old) and
calls the builders for [old, new) only, so growing 4 channels to 8
creates 4 SQ/RQ pairs instead of 8 and never holds 12 of each against
the vport's advertised maximum. The queues that stay keep their page
pools, their posted RX buffers and their NAPI state. As with a reduction
the result is handed to the existing mana_publish_qset(), so the swap
ordering, the TX quiesce and the rollback are unchanged.

Building part of a set needs the builders to start at an index, so
mana_create_txq() and mana_add_rx_queues() take a first-queue argument,
and mana_destroy_txq_from() / mana_destroy_rxqs_from() tear down a range
without freeing the array the caller still needs. mana_create_txq()
allocates apc->tx_qp[] only when it is building from zero.

One thing does not carry over by itself. mana_chn_setxdp() decides what
to do by reading rxqs[0], which is now a queue that already has the
program, so the call in mana_publish_qset() returns early and would
leave the new queues without one. mana_grow_qset() therefore attaches
the program itself, addressing the new queues through a separate qset so
that exactly as many references are taken as there are new queues - the
same view mana_free_qset() uses to drop them again if the swap fails.

With this, mana_set_channels() no longer builds a set through
mana_alloc_qset(), and the four callers that remain - ring resize, MTU,
the full-page RX private flag and XDP attach - all rebuild at the
current queue count. mana_alloc_qset() can therefore never need a new
EQ, so drop its mana_grow_eqs() call and the queue count it was passed;
growing the pool is now something only mana_grow_qset() does.

Signed-off-by: Long Li <longli@microsoft.com>
---
 .../net/ethernet/microsoft/mana/mana_bpf.c    |   2 +-
 drivers/net/ethernet/microsoft/mana/mana_en.c | 287 +++++++++++++++---
 .../ethernet/microsoft/mana/mana_ethtool.c    |  52 +++-
 include/net/mana/mana.h                       |   7 +-
 4 files changed, 293 insertions(+), 55 deletions(-)

diff --git a/drivers/net/ethernet/microsoft/mana/mana_bpf.c b/drivers/net/ethernet/microsoft/mana/mana_bpf.c
index 7031ecb4da2e0fd5b83aeb07aeb8d7ae56e16d65..0edaea807c7fd21abb8ec0dc9d76894adbf77f92 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_bpf.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_bpf.c
@@ -220,7 +220,7 @@ static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog,
 			return -ENOMEM;
 		}
 
-		err = mana_alloc_qset(apc, scratch, apc->num_queues,
+		err = mana_alloc_qset(apc, scratch,
 				      apc->rx_queue_size, apc->tx_queue_size,
 				      apc->priv_flags, apc->configured_mtu,
 				      prog, &newq);
diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
index 7cd2fd9ea050f10604dc0adb368c0aa4e7b6bb10..ed0e1ac8a7dc07ed64174d221bb7b09fa72f4731 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_en.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
@@ -923,7 +923,7 @@ static int mana_change_mtu(struct net_device *ndev, int new_mtu)
 	if (!scratch)
 		return -ENOMEM;
 
-	err = mana_alloc_qset(mpc, scratch, mpc->num_queues, mpc->rx_queue_size,
+	err = mana_alloc_qset(mpc, scratch, mpc->rx_queue_size,
 			      mpc->tx_queue_size, mpc->priv_flags, new_mtu,
 			      mpc->bpf_prog, &newq);
 	if (err)
@@ -2874,7 +2874,12 @@ static void mana_deinit_txq(struct mana_port_context *apc, struct mana_txq *txq)
 	mana_gd_destroy_queue(gd->gdma_context, txq->gdma_sq);
 }
 
-static void mana_destroy_txq(struct mana_port_context *apc)
+/* Destroy the TX queues in [@first, apc->num_queues). The array itself is
+ * left in place: a partial teardown is used by the grow path, where the
+ * queues below @first are still live and still referenced by the array.
+ */
+static void mana_destroy_txq_from(struct mana_port_context *apc,
+				  unsigned int first)
 {
 	struct napi_struct *napi;
 	int i;
@@ -2882,7 +2887,7 @@ static void mana_destroy_txq(struct mana_port_context *apc)
 	if (!apc->tx_qp)
 		return;
 
-	for (i = 0; i < apc->num_queues; i++) {
+	for (i = first; i < apc->num_queues; i++) {
 		if (!apc->tx_qp[i])
 			continue;
 
@@ -2907,6 +2912,14 @@ static void mana_destroy_txq(struct mana_port_context *apc)
 
 		kvfree(apc->tx_qp[i]);
 	}
+}
+
+static void mana_destroy_txq(struct mana_port_context *apc)
+{
+	if (!apc->tx_qp)
+		return;
+
+	mana_destroy_txq_from(apc, 0);
 
 	kfree(apc->tx_qp);
 	apc->tx_qp = NULL;
@@ -2937,8 +2950,14 @@ static void mana_create_txq_debugfs(struct mana_port_context *apc, int idx)
 			    tx_qp->tx_cq.gdma_cq, &mana_dbg_q_fops);
 }
 
+/* Create the TX queues in [@first, apc->num_queues).
+ *
+ * @first is non-zero only for the grow path, which supplies an already
+ * allocated apc->tx_qp[] holding the queues that are being carried over.
+ * On error only the queues this call created are torn down.
+ */
 static int mana_create_txq(struct mana_port_context *apc,
-			   struct net_device *net)
+			   struct net_device *net, unsigned int first)
 {
 	struct mana_context *ac = apc->ac;
 	struct gdma_dev *gd = ac->gdma_dev;
@@ -2953,9 +2972,14 @@ static int mana_create_txq(struct mana_port_context *apc,
 	int err;
 	int i;
 
-	apc->tx_qp = kzalloc_objs(struct mana_tx_qp *, apc->num_queues);
-	if (!apc->tx_qp)
-		return -ENOMEM;
+	if (first) {
+		if (WARN_ON(!apc->tx_qp))
+			return -EINVAL;
+	} else {
+		apc->tx_qp = kzalloc_objs(struct mana_tx_qp *, apc->num_queues);
+		if (!apc->tx_qp)
+			return -ENOMEM;
+	}
 
 	/*  The minimum size of the WQE is 32 bytes, hence
 	 *  apc->tx_queue_size represents the maximum number of WQEs
@@ -2972,7 +2996,7 @@ static int mana_create_txq(struct mana_port_context *apc,
 
 	gc = gd->gdma_context;
 
-	for (i = 0; i < apc->num_queues; i++) {
+	for (i = first; i < apc->num_queues; i++) {
 		apc->tx_qp[i] = kvzalloc_obj(*apc->tx_qp[i]);
 		if (!apc->tx_qp[i]) {
 			err = -ENOMEM;
@@ -3080,7 +3104,10 @@ static int mana_create_txq(struct mana_port_context *apc,
 out:
 	netdev_err(net, "Failed to create %d TX queues, %d\n",
 		   apc->num_queues, err);
-	mana_destroy_txq(apc);
+	if (first)
+		mana_destroy_txq_from(apc, first);
+	else
+		mana_destroy_txq(apc);
 	return err;
 }
 
@@ -3436,14 +3463,19 @@ static void mana_create_rxq_debugfs(struct mana_port_context *apc, int idx)
 			    &mana_dbg_q_fops);
 }
 
+/* Create the RX queues in [@first, apc->num_queues). @first is non-zero only
+ * for the grow path; the slots below it already hold carried-over queues.
+ * Queues created before a failure are left in apc->rxqs[] for the caller to
+ * tear down.
+ */
 static int mana_add_rx_queues(struct mana_port_context *apc,
-			      struct net_device *ndev)
+			      struct net_device *ndev, unsigned int first)
 {
 	struct mana_rxq *rxq;
 	int err = 0;
 	int i;
 
-	for (i = 0; i < apc->num_queues; i++) {
+	for (i = first; i < apc->num_queues; i++) {
 		rxq = mana_create_rxq(apc, i, &apc->eqs[i], ndev);
 		if (IS_ERR(rxq)) {
 			err = PTR_ERR(rxq);
@@ -3462,14 +3494,18 @@ static int mana_add_rx_queues(struct mana_port_context *apc,
 	return err;
 }
 
-static void mana_destroy_rxqs(struct mana_port_context *apc)
+/* Destroy the RX queues in [@first, apc->num_queues). The array itself is
+ * left in place; see mana_destroy_txq_from().
+ */
+static void mana_destroy_rxqs_from(struct mana_port_context *apc,
+				   unsigned int first)
 {
 	struct mana_rxq *rxq;
 	u32 rxq_idx;
 
 	if (apc->rxqs) {
 
-		for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) {
+		for (rxq_idx = first; rxq_idx < apc->num_queues; rxq_idx++) {
 			rxq = apc->rxqs[rxq_idx];
 			if (!rxq)
 				continue;
@@ -3480,6 +3516,11 @@ static void mana_destroy_rxqs(struct mana_port_context *apc)
 	}
 }
 
+static void mana_destroy_rxqs(struct mana_port_context *apc)
+{
+	mana_destroy_rxqs_from(apc, 0);
+}
+
 static void mana_destroy_vport(struct mana_port_context *apc)
 {
 	struct gdma_dev *gd = apc->ac->gdma_dev;
@@ -3863,7 +3904,7 @@ int mana_alloc_queues(struct net_device *ndev)
 		goto destroy_vport;
 	}
 
-	err = mana_create_txq(apc, ndev);
+	err = mana_create_txq(apc, ndev, 0);
 	if (err) {
 		netdev_err(ndev, "Failed to create TXQ on vPort %u: %d\n",
 			   apc->port_idx, err);
@@ -3878,7 +3919,7 @@ int mana_alloc_queues(struct net_device *ndev)
 		goto destroy_txq;
 	}
 
-	err = mana_add_rx_queues(apc, ndev);
+	err = mana_add_rx_queues(apc, ndev, 0);
 	if (err)
 		goto destroy_rxq;
 
@@ -4433,11 +4474,181 @@ void mana_discard_split(struct mana_qset *newq, struct mana_qset *tailq)
 	memset(tailq, 0, sizeof(*tailq));
 }
 
+/**
+ * mana_grow_qset - extend the live queue set with freshly built queues
+ * @apc:       live port context, owner of the shared EQ pool
+ * @scratch:   scratch context from mana_qset_scratch_alloc()
+ * @new_count: number of queues the new set must have
+ * @out_new:   filled with the set to publish, queues [0, @new_count)
+ * @out_fresh: filled with just the queues this call created, for rollback
+ *
+ * The mirror image of mana_split_qset(). A channel-count increase does not
+ * change any property of the queues that already exist: queue i is built from
+ * apc->tx_queue_size / apc->rx_queue_size, the MTU/priv-flag/XDP buffer layout
+ * and apc->eqs[i], none of which depend on how many queues there are. So the
+ * running queues are carried over and only the [old, @new_count) tail is
+ * built, instead of constructing a second full set and throwing the running
+ * one away.
+ *
+ * That also keeps the peak at @new_count queues rather than old + new: growing
+ * 4 channels to 8 creates 4 SQ/RQ pairs instead of 8, and never has 12 of each
+ * outstanding against the vport's advertised maximum. The queues that stay
+ * keep their page pools, their posted RX buffers and their NAPI state.
+ *
+ * @out_fresh describes the same queues as the tail of @out_new, but as a
+ * standalone set, so that a failed mana_publish_qset() can hand it to
+ * mana_free_qset() and retire exactly the queues this call created.
+ *
+ * On success the caller owns both sets. On failure @apc is untouched.
+ */
+int mana_grow_qset(struct mana_port_context *apc,
+		   struct mana_port_context *scratch, unsigned int new_count,
+		   struct mana_qset *out_new, struct mana_qset *out_fresh)
+{
+	unsigned int old_count = apc->num_queues;
+	struct mana_tx_qp **new_tx, **fresh_tx;
+	struct mana_rxq **new_rx, **fresh_rx;
+	struct net_device *ndev = apc->ndev;
+	unsigned int fresh_count;
+	unsigned int i;
+	int err;
+
+	ASSERT_RTNL();
+
+	if (WARN_ON(new_count <= old_count))
+		return -EINVAL;
+	if (WARN_ON(!apc->tx_qp || !apc->rxqs))
+		return -EINVAL;
+
+	fresh_count = new_count - old_count;
+
+	new_tx = kzalloc_objs(struct mana_tx_qp *, new_count);
+	new_rx = kzalloc_objs(struct mana_rxq *, new_count);
+	fresh_tx = kzalloc_objs(struct mana_tx_qp *, fresh_count);
+	fresh_rx = kzalloc_objs(struct mana_rxq *, fresh_count);
+	if (!new_tx || !new_rx || !fresh_tx || !fresh_rx) {
+		err = -ENOMEM;
+		goto free_arrays;
+	}
+
+	for (i = 0; i < old_count; i++) {
+		new_tx[i] = apc->tx_qp[i];
+		new_rx[i] = apc->rxqs[i];
+	}
+
+	/* Build into @scratch, which now describes the merged set: the
+	 * builders fill in the [old_count, new_count) slots and leave the
+	 * carried-over ones alone. Only the queue count differs from @apc,
+	 * so every other property is inherited as-is.
+	 */
+	scratch->num_queues = new_count;
+	scratch->tx_qp = new_tx;
+	scratch->rxqs = new_rx;
+
+	err = mana_rss_table_alloc(scratch);
+	if (err)
+		goto free_arrays;
+
+	/* Same shared, port-owned EQ pool as a full rebuild; this only adds
+	 * the vectors the extra queues need.
+	 */
+	err = mana_grow_eqs(apc, new_count);
+	if (err)
+		goto cleanup_rss;
+
+	scratch->eqs = apc->eqs;
+	scratch->num_eqs = apc->num_eqs;
+
+	err = mana_create_txq(scratch, ndev, old_count);
+	if (err)
+		goto cleanup_rss; /* create_txq already undid its own work */
+
+	err = mana_add_rx_queues(scratch, ndev, old_count);
+	if (err)
+		goto cleanup_rxq;
+
+	if (mana_rss_table_keep(apc, new_count))
+		memcpy(scratch->indir_table, apc->indir_table,
+		       apc->indir_table_sz * sizeof(*apc->indir_table));
+	else
+		mana_rss_table_init(scratch);
+
+	mana_qset_snapshot(scratch, out_new);
+
+	for (i = 0; i < fresh_count; i++) {
+		fresh_tx[i] = new_tx[old_count + i];
+		fresh_rx[i] = new_rx[old_count + i];
+	}
+
+	memset(out_fresh, 0, sizeof(*out_fresh));
+	out_fresh->tx_qp	= fresh_tx;
+	out_fresh->rxqs		= fresh_rx;
+	out_fresh->default_rxobj = INVALID_MANA_HANDLE;
+	out_fresh->num_queues	= fresh_count;
+	out_fresh->rx_queue_size = apc->rx_queue_size;
+	out_fresh->tx_queue_size = apc->tx_queue_size;
+	out_fresh->priv_flags	= apc->priv_flags;
+	out_fresh->mtu		= apc->configured_mtu;
+	out_fresh->bpf_prog	= apc->bpf_prog;
+
+	/* mana_publish_qset() cannot attach the XDP program to these queues:
+	 * mana_chn_setxdp() decides from rxqs[0], which is a carried-over
+	 * queue that already holds the program, and returns early. Attach it
+	 * here instead, addressing only the new queues through @out_fresh so
+	 * exactly fresh_count references are taken - the same view that
+	 * mana_free_qset() uses to drop them again.
+	 */
+	mana_qset_install(scratch, out_fresh);
+	mana_chn_setxdp(scratch, mana_xdp_get(apc));
+
+	return 0;
+
+cleanup_rxq:
+	mana_destroy_rxqs_from(scratch, old_count);
+	mana_destroy_txq_from(scratch, old_count);
+cleanup_rss:
+	mana_cleanup_indir_table(scratch);
+free_arrays:
+	/* Only the containers: every queue they name is still live on @apc. */
+	scratch->tx_qp = NULL;
+	scratch->rxqs = NULL;
+	kfree(new_tx);
+	kfree(new_rx);
+	kfree(fresh_tx);
+	kfree(fresh_rx);
+
+	/* Give back any EQ this attempt added rather than holding its MSI-X
+	 * vectors: the live set still needs only apc->num_queues of them, and
+	 * every CQ this call created has been destroyed above.
+	 */
+	mana_shrink_eqs(apc, apc->num_queues);
+
+	netdev_err(ndev, "mana_grow_qset(num_queues=%u) failed: %d\n",
+		   new_count, err);
+	return err;
+}
+
+/**
+ * mana_discard_grow - drop the merged containers built by mana_grow_qset()
+ * @newq: set that was never published
+ *
+ * Frees only the pointer arrays and the steering table. The carried-over
+ * queues belong to the live port context and the fresh ones are retired
+ * through the matching @out_fresh set, so no queue is destroyed here.
+ */
+void mana_discard_grow(struct mana_qset *newq)
+{
+	kfree(newq->tx_qp);
+	kfree(newq->rxqs);
+	kfree(newq->indir_table);
+	kfree(newq->rxobj_table);
+	memset(newq, 0, sizeof(*newq));
+}
+
 /**
  * mana_alloc_qset - build a complete queue set in @scratch
  * @apc:	   live port context, owner of the shared EQ pool
  * @scratch:	   scratch context from mana_qset_scratch_alloc()
- * @num_queues:	   number of queues in the new set
  * @rx_queue_size: new RX ring size
  * @tx_queue_size: new TX ring size
  * @priv_flags:	   new priv-flag word (affects full-page RX)
@@ -4445,13 +4656,18 @@ void mana_discard_split(struct mana_qset *newq, struct mana_qset *tailq)
  * @bpf_prog:	   XDP program the new set is sized for, may be NULL
  * @out:	   output qset, populated on success
  *
+ * Rebuilds the port's queues at the current queue count for callers that
+ * change a per-queue property. Changing the count itself does not come
+ * through here: mana_set_channels() carries the queues it keeps over with
+ * mana_split_qset() or mana_grow_qset(), so this never has to add an EQ and
+ * the existing pool always covers the set being built.
+ *
  * Every queue is built in @scratch, so the queue set currently installed on
- * @apc keeps serving traffic throughout. @apc is touched only to grow the
- * shared EQ pool, which both sets reference while they are both live. On
- * error no queue is left allocated, and any EQ this call added is released.
+ * @apc keeps serving traffic throughout. On error no queue is left
+ * allocated.
  */
 int mana_alloc_qset(struct mana_port_context *apc,
-		    struct mana_port_context *scratch, unsigned int num_queues,
+		    struct mana_port_context *scratch,
 		    unsigned int rx_queue_size, unsigned int tx_queue_size,
 		    u32 priv_flags, int mtu, struct bpf_prog *bpf_prog,
 		    struct mana_qset *out)
@@ -4461,7 +4677,7 @@ int mana_alloc_qset(struct mana_port_context *apc,
 
 	ASSERT_RTNL();
 
-	scratch->num_queues	= num_queues;
+	scratch->num_queues	= apc->num_queues;
 	scratch->rx_queue_size	= rx_queue_size;
 	scratch->tx_queue_size	= tx_queue_size;
 	scratch->priv_flags	= priv_flags;
@@ -4481,22 +4697,19 @@ int mana_alloc_qset(struct mana_port_context *apc,
 	if (err)
 		goto cleanup_rxq_array;
 
-	/* Grow the port's shared EQ pool if this set needs more. The pool
-	 * belongs to @apc, not to either queue set, so both sets can be
-	 * live at once without double-booking MSI-X vectors.
+	/* The queue count is unchanged, so the port's shared EQ pool already
+	 * has an EQ for every queue this set will build. Both sets reference
+	 * the same pool while they are live, so a swap never needs old + new
+	 * MSI-X vectors.
 	 */
-	err = mana_grow_eqs(apc, num_queues);
-	if (err)
-		goto cleanup_rss;
-
 	scratch->eqs = apc->eqs;
 	scratch->num_eqs = apc->num_eqs;
 
-	err = mana_create_txq(scratch, ndev);
+	err = mana_create_txq(scratch, ndev, 0);
 	if (err)
 		goto cleanup_rss;
 
-	err = mana_add_rx_queues(scratch, ndev);
+	err = mana_add_rx_queues(scratch, ndev, 0);
 	if (err)
 		goto cleanup_rxq;
 
@@ -4505,7 +4718,7 @@ int mana_alloc_qset(struct mana_port_context *apc,
 	 * them onto the new set's RX objects. A driver-generated table is
 	 * rebuilt instead, so it covers every queue of the new set.
 	 */
-	if (mana_rss_table_keep(apc, num_queues))
+	if (mana_rss_table_keep(apc, scratch->num_queues))
 		memcpy(scratch->indir_table, apc->indir_table,
 		       apc->indir_table_sz * sizeof(*apc->indir_table));
 	else
@@ -4527,15 +4740,11 @@ int mana_alloc_qset(struct mana_port_context *apc,
 	kfree(scratch->rxqs);
 	scratch->rxqs = NULL;
 out_err:
-	/* Give back any EQ this attempt added to the shared pool rather than
-	 * holding its MSI-X vectors until some later teardown: the live set
-	 * still needs only apc->num_queues of them. Safe here because this
-	 * set's CQs have already been destroyed above.
+	/* No EQ to give back: this path never adds one, it reuses the pool
+	 * the live set is already using.
 	 */
-	mana_shrink_eqs(apc, apc->num_queues);
-
 	netdev_err(ndev, "mana_alloc_qset(num_queues=%u) failed: %d\n",
-		   num_queues, err);
+		   apc->num_queues, err);
 	return err;
 }
 
@@ -4873,7 +5082,7 @@ int mana_publish_qset(struct mana_port_context *apc, struct mana_qset *newq,
  * pointer from the scratch parent behind rather than NULL, so both have to
  * count as "no node". Must be called under RTNL.
  */
-static void mana_qset_debugfs_publish(struct mana_port_context *apc)
+void mana_qset_debugfs_publish(struct mana_port_context *apc)
 {
 	unsigned int i;
 
diff --git a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
index 091b4a79ad238151b3a7f2014306b9551680c929..408aa38fe64263446ca419b250370cf44bc8b4e4 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
@@ -650,12 +650,14 @@ static int mana_set_coalesce(struct net_device *ndev,
 
 /* mana_set_channels - change the number of queues on a port
  *
- * Uses the pre-allocate + swap path (mana_alloc_qset / mana_publish_qset
- * / mana_free_qset). If allocation of the new queue set fails, the
- * existing queues keep running unchanged and we simply return -ENOMEM;
- * the user's requested setting is never silently mutated to a fallback
- * value. The vport is never torn down, so RDMA cannot race in and take
- * ownership of it during the reconfiguration window.
+ * A channel-count change leaves every surviving queue configured exactly as
+ * it was, so neither direction rebuilds them: a reduction carries the kept
+ * queues over and retires the tail (mana_split_qset), an increase carries all
+ * of them over and builds only the queues being added (mana_grow_qset). If
+ * the operation fails, the existing queues keep running unchanged and we
+ * simply return the error; the user's requested setting is never silently
+ * mutated to a fallback value. The vport is never torn down, so RDMA cannot
+ * race in and take ownership of it during the reconfiguration window.
  */
 static int mana_set_channels(struct net_device *ndev,
 			     struct ethtool_channels *channels)
@@ -663,7 +665,7 @@ static int mana_set_channels(struct net_device *ndev,
 	struct mana_port_context *apc = netdev_priv(ndev);
 	unsigned int new_count = channels->combined_count;
 	struct mana_port_context *scratch;
-	struct mana_qset newq, oldq;
+	struct mana_qset newq, oldq, freshq;
 	int err;
 
 	if (new_count < 1 || new_count > apc->max_queues) {
@@ -771,19 +773,41 @@ static int mana_set_channels(struct net_device *ndev,
 		goto free_scratch;
 	}
 
-	err = mana_alloc_qset(apc, scratch, new_count, apc->rx_queue_size,
-			      apc->tx_queue_size, apc->priv_flags,
-			      apc->configured_mtu, apc->bpf_prog, &newq);
+	/* An increase does not change the queues that already exist either, so
+	 * carry them over as well and build only the queues being added. The
+	 * peak stays at the new count instead of old + new.
+	 */
+	err = mana_grow_qset(apc, scratch, new_count, &newq, &freshq);
 	if (err)
 		goto free_scratch; /* current qset untouched, nothing to undo */
 
 	err = mana_publish_qset(apc, &newq, &oldq);
 	if (err) {
-		mana_free_qset(apc, scratch, &newq);
+		/* The old set is live again. Retire the queues that were just
+		 * built - @freshq names exactly those - and then drop the
+		 * merged containers without touching the carried-over queues.
+		 */
+		mana_free_qset(apc, scratch, &freshq);
+		mana_discard_grow(&newq);
 		goto free_scratch;
 	}
 
-	mana_free_qset(apc, scratch, &oldq);
+	/* Nothing is retired by a grow: every queue @oldq referenced is now
+	 * part of the published set, and so is every queue in @freshq. Only
+	 * the containers of both are released here.
+	 */
+	kfree(oldq.tx_qp);
+	kfree(oldq.rxqs);
+	kfree(oldq.indir_table);
+	kfree(oldq.rxobj_table);
+	kfree(freshq.tx_qp);
+	kfree(freshq.rxqs);
+
+	/* A grow retires nothing, so mana_free_qset() never runs to hand out
+	 * the debugfs names. The queues that were just added are the only
+	 * ones missing a node, and no retiring set is holding their names.
+	 */
+	mana_qset_debugfs_publish(apc);
 
 free_scratch:
 	/* After the caller-side cleanup above, so the EQ pool outlives the
@@ -874,7 +898,7 @@ static int mana_set_ringparam(struct net_device *ndev,
 		goto clear_flag;
 	}
 
-	err = mana_alloc_qset(apc, scratch, apc->num_queues, new_rx, new_tx,
+	err = mana_alloc_qset(apc, scratch, new_rx, new_tx,
 			      apc->priv_flags, apc->configured_mtu,
 			      apc->bpf_prog, &newq);
 	if (err) {
@@ -976,7 +1000,7 @@ static int mana_set_priv_flags(struct net_device *ndev, u32 priv_flags)
 		goto clear_flag;
 	}
 
-	err = mana_alloc_qset(apc, scratch, apc->num_queues, apc->rx_queue_size,
+	err = mana_alloc_qset(apc, scratch, apc->rx_queue_size,
 			      apc->tx_queue_size, priv_flags,
 			      apc->configured_mtu, apc->bpf_prog, &newq);
 	if (err)
diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h
index 8603f66ded7c2a8745d257ba4b8b5801a1c289e3..3c286ba9d1376cf0268e268ff3802b32e29202b0 100644
--- a/include/net/mana/mana.h
+++ b/include/net/mana/mana.h
@@ -769,7 +769,7 @@ int mana_detach(struct net_device *ndev, bool from_close);
 struct mana_port_context *mana_qset_scratch_alloc(struct mana_port_context *apc);
 void mana_qset_scratch_free(struct mana_port_context *scratch);
 int mana_alloc_qset(struct mana_port_context *apc,
-		    struct mana_port_context *scratch, unsigned int num_queues,
+		    struct mana_port_context *scratch,
 		    unsigned int rx_queue_size, unsigned int tx_queue_size,
 		    u32 priv_flags, int mtu, struct bpf_prog *bpf_prog,
 		    struct mana_qset *out);
@@ -777,11 +777,16 @@ int mana_split_qset(struct mana_port_context *apc,
 		    struct mana_port_context *scratch, unsigned int new_count,
 		    struct mana_qset *out_new, struct mana_qset *out_tail);
 void mana_discard_split(struct mana_qset *newq, struct mana_qset *tailq);
+int mana_grow_qset(struct mana_port_context *apc,
+		   struct mana_port_context *scratch, unsigned int new_count,
+		   struct mana_qset *out_new, struct mana_qset *out_fresh);
+void mana_discard_grow(struct mana_qset *newq);
 int mana_publish_qset(struct mana_port_context *apc, struct mana_qset *newq,
 		      struct mana_qset *out_old);
 void mana_publish_close_if_needed(struct mana_port_context *apc);
 void mana_free_qset(struct mana_port_context *apc,
 		    struct mana_port_context *scratch, struct mana_qset *qset);
+void mana_qset_debugfs_publish(struct mana_port_context *apc);
 
 void mana_dim_change(struct mana_cq *cq, bool enable);
 
-- 
2.43.0