[RFC PATCH net-next 10/13] net/mlx5e: add knod XDP offload support

[Taehee Yoo <ap420073@gmail.com>]

Let mlx5e act as a knod NIC: register with the knod core, feed received
packets to the accelerator over the per-queue SPSC ring, and transmit
accelerator verdicts (XDP_TX) back through the XDP SQ.  Restricted to
inline-none TX and single-page MTU.

Signed-off-by: Taehee Yoo <ap420073@gmail.com>
(cherry picked from commit 218c3a122d6b53638b99441560b7a0da65f6d8c1)
---
 drivers/net/ethernet/mellanox/mlx5/core/en.h  |  10 +
 .../net/ethernet/mellanox/mlx5/core/en/xdp.c  | 351 ++++++++++++++++++
 .../net/ethernet/mellanox/mlx5/core/en/xdp.h  |  19 +-
 .../net/ethernet/mellanox/mlx5/core/en_main.c |  38 +-
 .../net/ethernet/mellanox/mlx5/core/en_rx.c   | 107 +++++-
 .../net/ethernet/mellanox/mlx5/core/en_txrx.c |  45 +++
 6 files changed, 564 insertions(+), 6 deletions(-)

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index d507289096c2..1401a24e1740 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -48,6 +48,7 @@
 #include <net/udp_tunnel.h>
 #include <net/switchdev.h>
 #include <net/psp/types.h>
+#include <net/knod.h>
 #include <net/xdp.h>
 #include <linux/dim.h>
 #include <linux/bits.h>
@@ -568,6 +569,8 @@ struct mlx5e_icosq {
 
 struct mlx5e_frag_page {
 	netmem_ref netmem;
+	struct page_pool *pp;
+	u32 page_idx;
 	u16 frags;
 };
 
@@ -739,6 +742,11 @@ struct mlx5e_rq {
 
 	struct mlx5e_xdp_buff mxbuf;
 
+	struct knod_dev *knodev;
+	struct knod_netdev *knetdev;
+	u32                    knod_spsc_prod_head;
+	bool                   knod_spsc_prod_valid;
+
 	/* AF_XDP zero-copy */
 	struct xsk_buff_pool  *xsk_pool;
 
@@ -985,6 +993,8 @@ struct mlx5e_priv {
 	struct dentry             *dfs_root;
 	struct mlx5_devcom_comp_dev *devcom;
 	struct ethtool_fec_hist_range *fec_ranges;
+	struct knod_dev *knodev;
+	struct knod_netdev *knetdev;
 };
 
 static inline u16 mlx5e_stats_nch_read(const struct mlx5e_priv *priv)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
index d8c7cb8837d7..82adfc9b7c1b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
@@ -35,8 +35,18 @@
 #include "en/xdp.h"
 #include "en/params.h"
 #include <linux/bitfield.h>
+#include <linux/module.h>
 #include <net/page_pool/helpers.h>
 
+INDIRECT_CALLABLE_SCOPE bool
+mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq, struct mlx5e_xmit_data *xdptxd,
+		     int check_result, struct xsk_tx_metadata *meta);
+
+static inline struct page_pool *mlx5e_knod_bd_pp(struct spsc_bd *bd)
+{
+	return likely(bd->pp) ? bd->pp : netmem_get_pp(bd->netmem);
+}
+
 int mlx5e_xdp_max_mtu(struct mlx5e_params *params,
 		      struct mlx5e_rq_opt_param *rqo)
 {
@@ -58,6 +68,36 @@ int mlx5e_xdp_max_mtu(struct mlx5e_params *params,
 	return MLX5E_HW2SW_MTU(params, SKB_MAX_HEAD(hr));
 }
 
+static inline bool mlx5e_xmit_xdp_offload_buff(struct mlx5e_xdpsq *sq,
+					       struct mlx5e_rq *rq,
+					       struct spsc_bd *bd)
+{
+	struct mlx5e_xmit_data_frags xdptxdf = {};
+	struct mlx5e_xmit_data *xdptxd;
+
+	/* attach is restricted to inline-none NICs, so the WQE inlines no
+	 * header and xdptxd->data is never read (left NULL here).
+	 */
+	xdptxd = &xdptxdf.xd;
+	xdptxd->len = bd->len;
+	xdptxd->has_frags = 0;
+	xdptxd->dma_addr = netmem_to_net_iov(bd->netmem)->desc.dma_addr +
+			   bd->off;
+
+	if (!mlx5e_xmit_xdp_frame(sq, xdptxd, 0, NULL))
+		return false;
+
+	mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+			     (union mlx5e_xdp_info) {
+				.mode = MLX5E_XDP_XMIT_MODE_OFFLOAD });
+	mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo,
+			     (union mlx5e_xdp_info) {
+				.offload.netmem = bd->netmem,
+				.offload.pp = mlx5e_knod_bd_pp(bd) });
+
+	return true;
+}
+
 static inline bool
 mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq,
 		    struct xdp_buff *xdp)
@@ -353,6 +393,154 @@ bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
 	}
 }
 
+static inline u16 mlx5e_xdpsq_get_avail(struct mlx5e_xdpsq *sq)
+{
+	if (sq->pc == sq->cc)
+		return sq->wq.fbc.sz_m1 + 1;
+
+	return sq->wq.fbc.sz_m1 & (sq->cc - sq->pc);
+}
+
+static inline u16 mlx5e_xdpsq_get_avail_after_poll(struct mlx5e_xdpsq *sq)
+{
+	u16 avail = mlx5e_xdpsq_get_avail(sq);
+
+	if (likely(avail))
+		return avail;
+
+	mlx5e_xmit_xdp_doorbell(sq);
+	mlx5e_poll_xdpsq_cq(&sq->cq);
+
+	return mlx5e_xdpsq_get_avail(sq);
+}
+
+struct mlx5e_knod_release_batch {
+	struct spsc_bd *bds[NAPI_POLL_WEIGHT];
+	struct spsc_pass_bd pass[NAPI_POLL_WEIGHT];
+};
+
+static struct mlx5e_knod_release_batch
+mlx5e_knod_release_batch[KNOD_SPSC_MAX];
+
+static noinline int
+mlx5e_rx_offload_release_pending(struct mlx5e_rq *rq,
+				 struct knod_work_priv *wpriv,
+				 bool flush, int budget)
+{
+	struct knod_dev *knodev = rq->knodev;
+	struct mlx5e_xdpsq *sq = rq->xdpsq;
+	struct mlx5e_knod_release_batch *batch =
+		&mlx5e_knod_release_batch[rq->ix];
+	struct spsc_bd **bds = batch->bds;
+	struct spsc_pass_bd *pass = batch->pass;
+	int cnt, i, done = 0;
+
+	while (done < budget) {
+		int pass_cnt = 0;
+
+		if (!mlx5e_xdpsq_get_avail_after_poll(sq))
+			break;
+		cnt = min(NAPI_POLL_WEIGHT, budget - done);
+
+		spsc_release(&wpriv->spsc_bds, (void **)bds, cnt, &cnt);
+		if (!cnt)
+			break;
+
+		for (i = 0; i < cnt; i++) {
+			switch ((u32)bds[i]->act) {
+			case KNOD_ACT_INFLIGHT:
+				fallthrough;
+			case KNOD_IPSEC_INFLIGHT:
+				goto stop_release;
+			case KNOD_IPSEC_PASS:
+				fallthrough;
+			case KNOD_IPSEC_DROP:
+				/* Finish worker has set the final verdict.
+				 * Safe to recycle the netmem page now.
+				 */
+				page_pool_recycle_direct_netmem(
+					mlx5e_knod_bd_pp(bds[i]),
+					bds[i]->netmem);
+				break;
+			case KNOD_TX:
+				if (!mlx5e_xmit_xdp_offload_buff(rq->xdpsq, rq,
+								 bds[i]))
+					goto stop_release;
+				break;
+			case XDP_DROP:
+				fallthrough;
+			case XDP_ABORTED:
+				rq->stats->xdp_drop++;
+				page_pool_recycle_direct_netmem(
+					mlx5e_knod_bd_pp(bds[i]),
+					bds[i]->netmem);
+				break;
+			case XDP_PASS:
+				/* Hand to the common device->host delivery:
+				 * accumulate here, flush to knod_d2h_copy
+				 * after the bd loop.  The source page is
+				 * recycled by knod_d2h_drain once the copy
+				 * has landed, so it is NOT recycled here.
+				 */
+				pass[pass_cnt].netmem = bds[i]->netmem;
+				pass[pass_cnt].page_idx = bds[i]->page_idx;
+				pass[pass_cnt].off = bds[i]->off;
+				pass[pass_cnt].len = bds[i]->len;
+				pass_cnt++;
+				break;
+			case XDP_REDIRECT:
+				/* No redirect delivery path yet; recycle. */
+				page_pool_recycle_direct_netmem(
+					mlx5e_knod_bd_pp(bds[i]),
+					bds[i]->netmem);
+				break;
+			default:
+				/* Unknown value: either the accel shader
+				 * did not stamp a verdict for this slot
+				 * (lane skip bug) or the slot never went
+				 * through an accel at all. Treat as DROP +
+				 * recycle + WARN so the ring keeps advancing.
+				 */
+				rq->stats->xdp_drop++;
+				pr_warn_ratelimited("mlx5 nod: invalid bd->act=0x%llx rq%d, treating as DROP\n",
+						    bds[i]->act, rq->ix);
+				page_pool_recycle_direct_netmem(
+					mlx5e_knod_bd_pp(bds[i]),
+					bds[i]->netmem);
+				break;
+			}
+		}
+stop_release:
+		spsc_release_commit(&wpriv->spsc_bds, i);
+		done += i;
+
+		/* Issue the device->host copies for this batch's PASS bds. */
+		if (pass_cnt)
+			knod_d2h_copy(knodev, rq->ix, pass, pass_cnt);
+
+		if (i < cnt)
+			break;
+	}
+
+	if (flush)
+		mlx5e_xmit_xdp_doorbell(sq);
+
+	return done;
+}
+
+int mlx5e_rx_offload_act_handler(struct mlx5e_rq *rq, bool flush, int budget)
+{
+	struct knod_work_priv *wpriv = &rq->knodev->wpriv[rq->ix];
+
+	if (!spsc_pending(&wpriv->spsc_bds)) {
+		if (flush)
+			mlx5e_xmit_xdp_doorbell(rq->xdpsq);
+		return 0;
+	}
+
+	return mlx5e_rx_offload_release_pending(rq, wpriv, flush, budget);
+}
+
 static u16 mlx5e_xdpsq_get_next_pi(struct mlx5e_xdpsq *sq, u16 size)
 {
 	struct mlx5_wq_cyc *wq = &sq->wq;
@@ -741,6 +929,18 @@ static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq,
 			(*xsk_frames)++;
 			break;
 		}
+		case MLX5E_XDP_XMIT_MODE_OFFLOAD: {
+			netmem_ref netmem;
+			struct page_pool *pp;
+
+			xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo);
+			netmem = xdpi.offload.netmem;
+			pp = xdpi.offload.pp;
+
+			page_pool_recycle_direct_netmem(pp, netmem);
+
+			break;
+		}
 		default:
 			WARN_ON_ONCE(true);
 		}
@@ -974,3 +1174,154 @@ void mlx5e_set_xmit_fp(struct mlx5e_xdpsq *sq, bool is_mpw)
 	sq->xmit_xdp_frame = is_mpw ?
 		mlx5e_xmit_xdp_frame_mpwqe : mlx5e_xmit_xdp_frame;
 }
+
+static int mlx5e_rx_offload_xdp_attach(struct knod_dev *knodev)
+{
+	struct mlx5e_priv *priv = netdev_priv(knodev->netdev);
+	struct mlx5e_params *params = &priv->channels.params;
+	int max_mtu = mlx5e_xdp_max_mtu(params, NULL);
+
+	if (knodev->netdev->mtu > max_mtu) {
+		netdev_warn(knodev->netdev,
+			    "MTU %u too big for single-page RX offload (max %d)\n",
+			    knodev->netdev->mtu, max_mtu);
+		return -EOPNOTSUPP;
+	}
+
+	/* The offload TX WQE carries the packet only in a data segment; the
+	 * eth header is not inlined (a zero dummy stands in). NICs that require
+	 * a minimum inline header (e.g. ConnectX-4) would transmit that dummy,
+	 * so only allow attach when the device needs no inline header.
+	 */
+	if (params->tx_min_inline_mode != MLX5_INLINE_MODE_NONE) {
+		netdev_warn(knodev->netdev,
+			    "knod offload requires a NIC with inline header mode 'none'\n");
+		return -EOPNOTSUPP;
+	}
+
+	pr_debug("Attaching XDP offload to netdev %s\n", knodev->netdev->name);
+	WRITE_ONCE(priv->knodev, knodev);
+
+	return 0;
+}
+
+static int mlx5e_rx_offload_xdp_detach(struct knod_dev *knodev)
+{
+	struct mlx5e_priv *priv = netdev_priv(knodev->netdev);
+
+	pr_debug("Detaching XDP offload from netdev %s\n",
+		 knodev->netdev->name);
+	WRITE_ONCE(priv->knodev, NULL);
+
+	return 0;
+}
+
+struct knod_nic_ops nic_ops = {
+	.attach = mlx5e_rx_offload_xdp_attach,
+	.detach = mlx5e_rx_offload_xdp_detach,
+};
+
+int mlx5e_knod_init(struct mlx5e_priv *priv)
+{
+	struct knod_netdev *knetdev;
+
+	knetdev = kzalloc_obj(struct knod_netdev, GFP_KERNEL);
+	if (!knetdev) {
+		pr_debug("Failed to allocate knetdev\n");
+		return -ENOMEM;
+	}
+
+	INIT_LIST_HEAD(&knetdev->list);
+	knetdev->dev = priv->netdev;
+	knetdev->priv = priv;
+	knetdev->nic_ops = &nic_ops;
+	knetdev->owner = THIS_MODULE;
+	knetdev->flags |= KNOD_FLAGS_XDP;
+	knod_netdev_register(knetdev);
+	priv->knetdev = knetdev;
+
+	return 0;
+}
+
+void mlx5e_knod_uninit(struct mlx5e_priv *priv)
+{
+	knod_netdev_unregister(priv->knetdev);
+	kfree(priv->knetdev);
+	priv->knetdev = NULL;
+	kfree(priv->knodev);
+	WRITE_ONCE(priv->knodev, NULL);
+}
+
+void mlx5e_rx_offload_set_napi(struct mlx5e_priv *priv)
+{
+	struct knod_dev *knodev = priv->knodev;
+	int i;
+
+	if (!knodev)
+		return;
+
+	for (i = 0; i < priv->channels.num; i++) {
+		struct mlx5e_channel *c = priv->channels.c[i];
+		struct spsc_ring *r = &knodev->wpriv[i].spsc_bds;
+
+		WRITE_ONCE(knodev->wpriv[i].napi, &c->napi);
+		c->rq.knod_spsc_prod_head = READ_ONCE(r->head);
+		c->rq.knod_spsc_prod_valid = true;
+	}
+}
+
+void mlx5e_rx_offload_clear_napi(struct mlx5e_priv *priv)
+{
+	struct knod_dev *knodev = priv->knodev;
+	int i;
+
+	if (!knodev)
+		return;
+
+	for (i = 0; i < KNOD_SPSC_MAX; i++)
+		WRITE_ONCE(knodev->wpriv[i].napi, NULL);
+}
+
+void mlx5e_rx_offload_start(struct mlx5e_priv *priv)
+{
+	if (!priv->knodev)
+		return;
+
+	knod_dev_start(priv->knodev);
+}
+
+void mlx5e_rx_offload_stop(struct mlx5e_priv *priv)
+{
+	struct knod_dev *knodev = priv->knodev;
+	int i;
+
+	if (!priv->knodev)
+		return;
+
+	knod_dev_stop(knodev);
+
+	synchronize_net();
+	for (i = 0; i < KNOD_SPSC_MAX; i++) {
+		struct spsc_bd *bd;
+
+		if (i < priv->channels.num) {
+			struct mlx5e_rq *rq = &priv->channels.c[i]->rq;
+
+			mlx5e_knod_spsc_flush(rq);
+			rq->knod_spsc_prod_head = 0;
+			rq->knod_spsc_prod_valid = false;
+		}
+
+		WRITE_ONCE(knodev->wpriv[i].napi, NULL);
+		/*
+		 * RX is quiesced now (worker stopped by knod_dev_stop, NAPI
+		 * drained by synchronize_net).  Return any frames the GPU
+		 * worker did not consume back to the page_pool before the RX
+		 * page_pool is torn down on interface down.
+		 */
+		spsc_rewind(&knodev->wpriv[i].spsc_bds);
+		while (!spsc_pop(&knodev->wpriv[i].spsc_bds, (void **)&bd))
+			page_pool_put_full_netmem(netmem_get_pp(bd->netmem),
+						  bd->netmem, true);
+	}
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
index 3c54f8962664..4de6babdafc5 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h
@@ -34,6 +34,7 @@
 
 #include <linux/indirect_call_wrapper.h>
 #include <net/xdp_sock.h>
+#include <net/knod.h>
 
 #include "en.h"
 #include "en/txrx.h"
@@ -64,6 +65,8 @@ enum mlx5e_xdp_xmit_mode {
 	 * page. The UMEM Completion Ring producer pointer has to be increased.
 	 */
 	MLX5E_XDP_XMIT_MODE_XSK,
+
+	MLX5E_XDP_XMIT_MODE_OFFLOAD,
 };
 
 /* xmit_mode entry is pushed to the fifo per packet, followed by multiple
@@ -78,6 +81,9 @@ enum mlx5e_xdp_xmit_mode {
  *
  * MLX5E_XDP_XMIT_MODE_XSK:
  *    frame.xsk_meta.
+ *
+ * MLX5E_XDP_XMIT_MODE_OFFLOAD:
+ *    offload.netmem.
  */
 #define MLX5E_XDP_FIFO_ENTRIES2DS_MAX_RATIO 4
 
@@ -93,6 +99,10 @@ union mlx5e_xdp_info {
 		struct page *page;
 	} page;
 	struct xsk_tx_metadata_compl xsk_meta;
+	struct {
+		netmem_ref netmem;
+		struct page_pool *pp;
+	} offload;
 };
 
 struct mlx5e_xsk_param;
@@ -100,14 +110,21 @@ int mlx5e_xdp_max_mtu(struct mlx5e_params *params,
 		      struct mlx5e_rq_opt_param *rqo);
 bool mlx5e_xdp_handle(struct mlx5e_rq *rq,
 		      struct bpf_prog *prog, struct mlx5e_xdp_buff *mlctx);
+int mlx5e_rx_offload_act_handler(struct mlx5e_rq *rq, bool flush, int budget);
+void mlx5e_knod_spsc_flush(struct mlx5e_rq *rq);
 void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq);
 bool mlx5e_poll_xdpsq_cq(struct mlx5e_cq *cq);
 void mlx5e_free_xdpsq_descs(struct mlx5e_xdpsq *sq);
 void mlx5e_set_xmit_fp(struct mlx5e_xdpsq *sq, bool is_mpw);
+int mlx5e_knod_init(struct mlx5e_priv *priv);
+void mlx5e_knod_uninit(struct mlx5e_priv *priv);
 void mlx5e_xdp_rx_poll_complete(struct mlx5e_rq *rq);
 int mlx5e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
 		   u32 flags);
-
+void mlx5e_rx_offload_start(struct mlx5e_priv *priv);
+void mlx5e_rx_offload_stop(struct mlx5e_priv *priv);
+void mlx5e_rx_offload_set_napi(struct mlx5e_priv *priv);
+void mlx5e_rx_offload_clear_napi(struct mlx5e_priv *priv);
 extern const struct xdp_metadata_ops mlx5e_xdp_metadata_ops;
 extern const struct xsk_tx_metadata_ops mlx5e_xsk_tx_metadata_ops;
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index aa8610cedaa8..cfc6ba6dc492 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -918,6 +918,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 	pool_size = 1 << params->log_rq_mtu_frames;
 
 	rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey);
+	rq->knodev = rq->priv->knodev;
 
 	switch (rq->wq_type) {
 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
@@ -1022,7 +1023,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params,
 		pp_params.queue_idx = rq->ix;
 
 		/* Shampo header data split allow for unreadable netmem */
-		if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
+		if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) || rq->knodev)
 			pp_params.flags |= PP_FLAG_ALLOW_UNREADABLE_NETMEM;
 
 		/* page_pool can be used even when there is no rq->xdp_prog,
@@ -2873,7 +2874,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
 	c->netdev   = priv->netdev;
 	c->mkey_be  = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey);
 	c->num_tc   = mlx5e_get_dcb_num_tc(params);
-	c->xdp      = !!params->xdp_prog;
+	c->xdp      = !!params->xdp_prog || !!priv->knodev;
 	c->stats    = &priv->channel_stats[ix]->ch;
 	c->aff_mask = irq_get_effective_affinity_mask(irq);
 	c->lag_port = mlx5e_enumerate_lag_port(mdev, ix);
@@ -3365,6 +3366,8 @@ void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
 
 	if (priv->rx_res)
 		mlx5e_rx_res_channels_activate(priv->rx_res, &priv->channels);
+
+	mlx5e_rx_offload_set_napi(priv);
 }
 
 static void mlx5e_cancel_tx_timeout_work(struct mlx5e_priv *priv)
@@ -3376,6 +3379,8 @@ static void mlx5e_cancel_tx_timeout_work(struct mlx5e_priv *priv)
 
 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
 {
+	mlx5e_rx_offload_clear_napi(priv);
+
 	if (priv->rx_res)
 		mlx5e_rx_res_channels_deactivate(priv->rx_res);
 
@@ -3592,6 +3597,7 @@ int mlx5e_open(struct net_device *netdev)
 		mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_UP);
 	mutex_unlock(&priv->state_lock);
 
+	mlx5e_rx_offload_start(priv);
 	return err;
 }
 
@@ -3623,6 +3629,7 @@ int mlx5e_close(struct net_device *netdev)
 	if (!netif_device_present(netdev))
 		return -ENODEV;
 
+	mlx5e_rx_offload_stop(priv);
 	mutex_lock(&priv->state_lock);
 	mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_DOWN);
 	err = mlx5e_close_locked(netdev);
@@ -4475,7 +4482,7 @@ void mlx5e_set_xdp_feature(struct mlx5e_priv *priv)
 	    params->packet_merge.type == MLX5E_PACKET_MERGE_NONE)
 		val = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
 		      NETDEV_XDP_ACT_XSK_ZEROCOPY |
-		      NETDEV_XDP_ACT_RX_SG;
+		      NETDEV_XDP_ACT_RX_SG | NETDEV_XDP_ACT_HW_OFFLOAD;
 
 	if (netdev->netdev_ops->ndo_xdp_xmit && params->xdp_prog)
 		val |= NETDEV_XDP_ACT_NDO_XMIT |
@@ -4738,6 +4745,14 @@ int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
 		goto out;
 	}
 
+	if (priv->knodev && new_mtu > mlx5e_xdp_max_mtu(&new_params, NULL)) {
+		netdev_warn(netdev,
+			    "MTU %d too big for single-page RX offload (max %d)\n",
+			    new_mtu, mlx5e_xdp_max_mtu(&new_params, NULL));
+		err = -EINVAL;
+		goto out;
+	}
+
 	if (priv->xsk.refcnt &&
 	    !mlx5e_xsk_validate_mtu(netdev, &priv->channels,
 				    &new_params, priv->mdev)) {
@@ -5261,15 +5276,30 @@ static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
 
 static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp)
 {
+	struct mlx5e_priv *priv = netdev_priv(dev);
+	struct knod_dev *knodev = priv->knodev;
+	int rc;
+
 	switch (xdp->command) {
 	case XDP_SETUP_PROG:
 		return mlx5e_xdp_set(dev, xdp->prog);
 	case XDP_SETUP_XSK_POOL:
 		return mlx5e_xsk_setup_pool(dev, xdp->xsk.pool,
 					    xdp->xsk.queue_id);
+	case XDP_SETUP_PROG_HW:
+	case BPF_OFFLOAD_MAP_ALLOC:
+	case BPF_OFFLOAD_MAP_FREE:
+		if (!knodev)
+			return -EOPNOTSUPP;
+
+		rc = knod_dev_xdp_install(knodev, xdp);
+		break;
+
 	default:
 		return -EINVAL;
 	}
+
+	return rc;
 }
 
 #ifdef CONFIG_MLX5_ESWITCH
@@ -6931,6 +6961,7 @@ static int _mlx5e_probe(struct auxiliary_device *adev)
 	mlx5e_dcbnl_init_app(priv);
 	mlx5_core_uplink_netdev_set(mdev, netdev);
 	mlx5e_params_print_info(mdev, &priv->channels.params);
+	mlx5e_knod_init(priv);
 	return 0;
 
 err_resume:
@@ -6982,6 +7013,7 @@ static void _mlx5e_remove(struct auxiliary_device *adev)
 	struct mlx5e_priv *priv = netdev_priv(netdev);
 	struct mlx5_core_dev *mdev = edev->mdev;
 
+	mlx5e_knod_uninit(priv);
 	mlx5_eswitch_safe_aux_devs_remove(mdev);
 	mlx5_core_uplink_netdev_set(mdev, NULL);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index 6fbc0441c4b8..ff4627a0895c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -276,15 +276,20 @@ static int mlx5e_page_alloc_fragmented(struct page_pool *pp,
 				       struct mlx5e_frag_page *frag_page)
 {
 	netmem_ref netmem = page_pool_dev_alloc_netmems(pp);
+	u32 page_idx = 0;
 
 	if (unlikely(!netmem))
 		return -ENOMEM;
 
 	page_pool_fragment_netmem(netmem, MLX5E_PAGECNT_BIAS_MAX);
+	if (netmem_is_net_iov(netmem))
+		page_idx = net_iov_binding_idx(netmem_to_net_iov(netmem));
 
 	*frag_page = (struct mlx5e_frag_page) {
-		.netmem	= netmem,
-		.frags	= 0,
+		.netmem		= netmem,
+		.pp		= pp,
+		.page_idx	= page_idx,
+		.frags		= 0,
 	};
 
 	return 0;
@@ -1572,19 +1577,92 @@ static void mlx5e_fill_mxbuf(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
 	mxbuf->rq = rq;
 }
 
+static inline int mlx5e_knod_spsc_produce_defer(struct mlx5e_rq *rq,
+						struct knod_work_priv *wpriv,
+						struct spsc_bd **bd)
+{
+	struct spsc_ring *r = &wpriv->spsc_bds;
+	unsigned int head;
+	unsigned int tail;
+
+	if (unlikely(!rq->knod_spsc_prod_valid)) {
+		rq->knod_spsc_prod_head = READ_ONCE(r->head);
+		rq->knod_spsc_prod_valid = true;
+	}
+
+	head = rq->knod_spsc_prod_head;
+	/* acquire tail to observe the slots the GPU worker has released */
+	tail = smp_load_acquire(&r->tail);
+	if (unlikely(head - tail > r->mask))
+		return -ENOSPC;
+
+	*bd = r->slots[head & r->mask];
+	rq->knod_spsc_prod_head = head + 1;
+
+	return 0;
+}
+
+void mlx5e_knod_spsc_flush(struct mlx5e_rq *rq)
+{
+	struct knod_work_priv *wpriv;
+	struct spsc_ring *r;
+	unsigned int head;
+
+	if (unlikely(!rq->knodev || !rq->knod_spsc_prod_valid))
+		return;
+
+	wpriv = &rq->knodev->wpriv[rq->ix];
+	r = &wpriv->spsc_bds;
+	head = rq->knod_spsc_prod_head;
+	if (head == READ_ONCE(r->head))
+		return;
+
+	/* drain WC descriptor stores before publishing the new head */
+	wmb();
+	/* release: publish the produced descriptors to the GPU worker */
+	smp_store_release(&r->head, head);
+}
+
 static struct sk_buff *
 mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *wi,
 			  struct mlx5_cqe64 *cqe, u32 cqe_bcnt)
 {
 	struct mlx5e_frag_page *frag_page = wi->frag_page;
 	u16 rx_headroom = rq->buff.headroom;
+	struct knod_work_priv *wpriv;
 	struct bpf_prog *prog;
 	struct sk_buff *skb;
+	struct spsc_bd *bd;
 	u32 metasize = 0;
 	void *va, *data;
 	dma_addr_t addr;
 	u32 frag_size;
 
+	if (likely(rq->knodev)) {
+		wpriv = &rq->knodev->wpriv[rq->ix];
+		if (unlikely(mlx5e_knod_spsc_produce_defer(rq, wpriv, &bd))) {
+			mlx5e_knod_spsc_flush(rq);
+			mlx5e_rx_offload_act_handler(rq, false, INT_MAX);
+			if (mlx5e_knod_spsc_produce_defer(rq, wpriv, &bd)) {
+				rq->stats->buff_alloc_err++;
+				return NULL;
+			}
+		}
+
+		bd->netmem = frag_page->netmem;
+		bd->pp = frag_page->pp;
+		bd->len = cqe_bcnt;
+		bd->off = wi->offset + rx_headroom;
+		bd->page_idx = frag_page->page_idx;
+		frag_page->frags++;
+		rq->stats->packets++;
+		rq->stats->bytes += cqe_bcnt;
+		pr_debug("mlx5_nod: spsc produce q=%d len=%u\n",
+			 rq->ix, cqe_bcnt);
+
+		return NULL;
+	}
+
 	va             = netmem_address(frag_page->netmem) + wi->offset;
 	data           = va + rx_headroom;
 	frag_size      = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
@@ -2101,8 +2179,10 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 {
 	struct mlx5e_frag_page *frag_page = &wi->alloc_units.frag_pages[page_idx];
 	u16 rx_headroom = rq->buff.headroom;
+	struct knod_work_priv *wpriv;
 	struct bpf_prog *prog;
 	struct sk_buff *skb;
+	struct spsc_bd *bd;
 	u32 metasize = 0;
 	void *va, *data;
 	dma_addr_t addr;
@@ -2114,6 +2194,29 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi,
 		return NULL;
 	}
 
+	if (likely(rq->knodev)) {
+		wpriv = &rq->knodev->wpriv[rq->ix];
+		if (unlikely(mlx5e_knod_spsc_produce_defer(rq, wpriv, &bd))) {
+			mlx5e_knod_spsc_flush(rq);
+			mlx5e_rx_offload_act_handler(rq, false, INT_MAX);
+			if (mlx5e_knod_spsc_produce_defer(rq, wpriv, &bd)) {
+				rq->stats->buff_alloc_err++;
+				return NULL;
+			}
+		}
+
+		bd->netmem = frag_page->netmem;
+		bd->pp = frag_page->pp;
+		bd->len = cqe_bcnt;
+		bd->off = head_offset + rx_headroom;
+		bd->page_idx = frag_page->page_idx;
+		frag_page->frags++;
+		rq->stats->packets++;
+		rq->stats->bytes += cqe_bcnt;
+
+		return NULL;
+	}
+
 	va             = netmem_address(frag_page->netmem) + head_offset;
 	data           = va + rx_headroom;
 	frag_size      = MLX5_SKB_FRAG_SZ(rx_headroom + cqe_bcnt);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
index 185105606469..10c2ffd93dab 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
@@ -177,6 +177,51 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
 	if (likely(budget - work_done))
 		work_done += mlx5e_poll_rx_cq(&rq->cq, budget - work_done);
 
+	if (likely(rq->knodev))
+		mlx5e_knod_spsc_flush(rq);
+
+	/* KNOD release can process thousands of completed verdicts and enqueue
+	 * XDP_TX MPWQEs. Refill RX WQEs first so the NIC is not left waiting
+	 * for descriptors while the release side drains. The normal post below
+	 * stays in place to publish pages recycled by this release pass.
+	 */
+	if (likely(rq->knodev))
+		busy |= INDIRECT_CALL_2(rq->post_wqes,
+					mlx5e_post_rx_mpwqes,
+					mlx5e_post_rx_wqes,
+					rq);
+
+	/* Drain SPSC bd ring + IPsec desc_ring unconditionally.
+	 * napi_schedule from the GPU finish_worker may wake us with
+	 * zero new CQEs, so act_handler (called per-CQE inside
+	 * poll_rx_cq) won't run.  Without this top-level call,
+	 * PASS/DROP-stamped bds are never recycled after traffic stops
+	 * and the SPSC ring fills up.
+	 */
+	if (likely(rq->knodev)) {
+		struct knod_work_priv *wpriv = &rq->knodev->wpriv[rq->ix];
+		struct napi_struct *napi;
+
+		work_done += mlx5e_rx_offload_act_handler(rq, true,
+							  budget - work_done);
+
+		/* KNOD direct XDP_TX keeps the RX netmem owned by the TX SQ
+		 * until the NIC reports TX completion.  The normal NAPI order
+		 * polls the XDP SQ before RX CQ processing, then the KNOD
+		 * release pass can enqueue and doorbell a large burst of
+		 * MPWQEs.  Poll once more here so completions that arrived
+		 * during RX/release processing are visible before the final
+		 * RX repost below.
+		 */
+		if (rq->xdpsq)
+			busy |= mlx5e_poll_xdpsq_cq(&rq->xdpsq->cq);
+
+		napi = READ_ONCE(wpriv->napi);
+		if (napi)
+			knod_dev_xdp_drain_pass(rq->knodev, napi, rq->ix,
+						budget);
+	}
+
 	busy |= work_done == budget;
 
 	mlx5e_poll_ico_cq(&c->icosq.cq);
-- 
2.43.0