[RFC PATCH net-next 04/13] net: knod: add offload device core and control plane

[Taehee Yoo <ap420073@gmail.com>]

Add the KNOD core framework and its genetlink control plane.

The core maintains three object types - a NIC-side netdev, an
accelerator, and a dynamically created offload device that binds them -
and drives the RX data path over per-queue SPSC rings backed by a
page_pool of accelerator memory.

The genetlink family ("knod") exposes attach/detach, device/nic/accel
inventory, and per-accelerator feature selection as a stable uAPI.

Signed-off-by: Taehee Yoo <ap420073@gmail.com>
(cherry picked from commit f17f82a39be852aa8c520aea7177ce0086f7e811)
---
 net/Kconfig            |    2 +
 net/Makefile           |    1 +
 net/knod/Kconfig       |    5 +
 net/knod/Makefile      |    8 +
 net/knod/knod-nl-gen.c |  121 ++++
 net/knod/knod-nl-gen.h |   31 +
 net/knod/knod.h        |   26 +
 net/knod/knod_core.c   | 1231 ++++++++++++++++++++++++++++++++++++++++
 net/knod/knod_nl.c     |  406 +++++++++++++
 9 files changed, 1831 insertions(+)
 create mode 100644 net/knod/Kconfig
 create mode 100644 net/knod/Makefile
 create mode 100644 net/knod/knod-nl-gen.c
 create mode 100644 net/knod/knod-nl-gen.h
 create mode 100644 net/knod/knod.h
 create mode 100644 net/knod/knod_core.c
 create mode 100644 net/knod/knod_nl.c

diff --git a/net/Kconfig b/net/Kconfig
index e38477393551..e7a2200e92cc 100644
--- a/net/Kconfig
+++ b/net/Kconfig
@@ -72,6 +72,8 @@ config NET_DEVMEM
 	depends on DMA_SHARED_BUFFER
 	depends on PAGE_POOL
 
+source "net/knod/Kconfig"
+
 config NET_SHAPER
 	bool
 
diff --git a/net/Makefile b/net/Makefile
index 5b2dd7f07a85..ea4ef5c5906d 100644
--- a/net/Makefile
+++ b/net/Makefile
@@ -19,6 +19,7 @@ obj-$(CONFIG_TLS)		+= tls/
 obj-$(CONFIG_XFRM)		+= xfrm/
 obj-$(CONFIG_UNIX)		+= unix/
 obj-$(CONFIG_INET_PSP)		+= psp/
+obj-$(CONFIG_KNOD)	+= knod/
 obj-y				+= ipv6/
 obj-$(CONFIG_PACKET)		+= packet/
 obj-$(CONFIG_NET_KEY)		+= key/
diff --git a/net/knod/Kconfig b/net/knod/Kconfig
new file mode 100644
index 000000000000..d5a6d76d33e3
--- /dev/null
+++ b/net/knod/Kconfig
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+config KNOD
+	def_bool y
+	depends on NET_DEVMEM
diff --git a/net/knod/Makefile b/net/knod/Makefile
new file mode 100644
index 000000000000..73abe5c99f8c
--- /dev/null
+++ b/net/knod/Makefile
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# Makefile for the KNOD (KFD network offload) control plane.
+#
+
+obj-$(CONFIG_KNOD) += knod.o
+
+knod-y := knod_core.o knod_nl.o knod-nl-gen.o
diff --git a/net/knod/knod-nl-gen.c b/net/knod/knod-nl-gen.c
new file mode 100644
index 000000000000..96042f0c15c9
--- /dev/null
+++ b/net/knod/knod-nl-gen.c
@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause)
+/* Do not edit directly, auto-generated from: */
+/*	Documentation/netlink/specs/knod.yaml */
+/* YNL-GEN kernel source */
+/* To regenerate run: tools/net/ynl/ynl-regen.sh */
+
+#include <net/netlink.h>
+#include <net/genetlink.h>
+
+#include "knod-nl-gen.h"
+
+#include <uapi/linux/knod.h>
+
+/* KNOD_CMD_ACCEL_GET - do */
+static const struct nla_policy knod_accel_get_nl_policy[KNOD_A_ACCEL_ID + 1] = {
+	[KNOD_A_ACCEL_ID] = NLA_POLICY_MIN(NLA_U32, 1),
+};
+
+/* KNOD_CMD_ACCEL_SET - do */
+static const struct nla_policy knod_accel_set_nl_policy[KNOD_A_ACCEL_FEATURE_ENA + 1] = {
+	[KNOD_A_ACCEL_ID] = NLA_POLICY_MIN(NLA_U32, 1),
+	[KNOD_A_ACCEL_FEATURE_ENA] = NLA_POLICY_MAX(NLA_U32, 2),
+};
+
+/* KNOD_CMD_NIC_GET - do */
+static const struct nla_policy knod_nic_get_nl_policy[KNOD_A_NIC_IFINDEX + 1] = {
+	[KNOD_A_NIC_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
+};
+
+/* KNOD_CMD_ATTACH - do */
+static const struct nla_policy knod_attach_nl_policy[KNOD_A_DEV_ACCEL_ID + 1] = {
+	[KNOD_A_DEV_NIC_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
+	[KNOD_A_DEV_ACCEL_ID] = NLA_POLICY_MIN(NLA_U32, 1),
+};
+
+/* KNOD_CMD_DETACH - do */
+static const struct nla_policy knod_detach_nl_policy[KNOD_A_DEV_NIC_IFINDEX + 1] = {
+	[KNOD_A_DEV_NIC_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
+};
+
+/* KNOD_CMD_DEV_GET - do */
+static const struct nla_policy knod_dev_get_nl_policy[KNOD_A_DEV_NIC_IFINDEX + 1] = {
+	[KNOD_A_DEV_NIC_IFINDEX] = NLA_POLICY_MIN(NLA_U32, 1),
+};
+
+/* Ops table for knod */
+static const struct genl_split_ops knod_nl_ops[] = {
+	{
+		.cmd		= KNOD_CMD_ACCEL_GET,
+		.doit		= knod_nl_accel_get_doit,
+		.policy		= knod_accel_get_nl_policy,
+		.maxattr	= KNOD_A_ACCEL_ID,
+		.flags		= GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd	= KNOD_CMD_ACCEL_GET,
+		.dumpit	= knod_nl_accel_get_dumpit,
+		.flags	= GENL_CMD_CAP_DUMP,
+	},
+	{
+		.cmd		= KNOD_CMD_ACCEL_SET,
+		.doit		= knod_nl_accel_set_doit,
+		.policy		= knod_accel_set_nl_policy,
+		.maxattr	= KNOD_A_ACCEL_FEATURE_ENA,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= KNOD_CMD_NIC_GET,
+		.doit		= knod_nl_nic_get_doit,
+		.policy		= knod_nic_get_nl_policy,
+		.maxattr	= KNOD_A_NIC_IFINDEX,
+		.flags		= GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd	= KNOD_CMD_NIC_GET,
+		.dumpit	= knod_nl_nic_get_dumpit,
+		.flags	= GENL_CMD_CAP_DUMP,
+	},
+	{
+		.cmd		= KNOD_CMD_ATTACH,
+		.doit		= knod_nl_attach_doit,
+		.policy		= knod_attach_nl_policy,
+		.maxattr	= KNOD_A_DEV_ACCEL_ID,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= KNOD_CMD_DETACH,
+		.doit		= knod_nl_detach_doit,
+		.policy		= knod_detach_nl_policy,
+		.maxattr	= KNOD_A_DEV_NIC_IFINDEX,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= KNOD_CMD_DEV_GET,
+		.doit		= knod_nl_dev_get_doit,
+		.policy		= knod_dev_get_nl_policy,
+		.maxattr	= KNOD_A_DEV_NIC_IFINDEX,
+		.flags		= GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd	= KNOD_CMD_DEV_GET,
+		.dumpit	= knod_nl_dev_get_dumpit,
+		.flags	= GENL_CMD_CAP_DUMP,
+	},
+};
+
+static const struct genl_multicast_group knod_nl_mcgrps[] = {
+	[KNOD_NLGRP_MGMT] = { "mgmt", },
+};
+
+struct genl_family knod_nl_family __ro_after_init = {
+	.name		= KNOD_FAMILY_NAME,
+	.version	= KNOD_FAMILY_VERSION,
+	.netnsok	= true,
+	.parallel_ops	= true,
+	.module		= THIS_MODULE,
+	.split_ops	= knod_nl_ops,
+	.n_split_ops	= ARRAY_SIZE(knod_nl_ops),
+	.mcgrps		= knod_nl_mcgrps,
+	.n_mcgrps	= ARRAY_SIZE(knod_nl_mcgrps),
+};
diff --git a/net/knod/knod-nl-gen.h b/net/knod/knod-nl-gen.h
new file mode 100644
index 000000000000..a12b6a174bed
--- /dev/null
+++ b/net/knod/knod-nl-gen.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/* Do not edit directly, auto-generated from: */
+/*	Documentation/netlink/specs/knod.yaml */
+/* YNL-GEN kernel header */
+/* To regenerate run: tools/net/ynl/ynl-regen.sh */
+
+#ifndef _LINUX_KNOD_GEN_H
+#define _LINUX_KNOD_GEN_H
+
+#include <net/netlink.h>
+#include <net/genetlink.h>
+
+#include <uapi/linux/knod.h>
+
+int knod_nl_accel_get_doit(struct sk_buff *skb, struct genl_info *info);
+int knod_nl_accel_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb);
+int knod_nl_accel_set_doit(struct sk_buff *skb, struct genl_info *info);
+int knod_nl_nic_get_doit(struct sk_buff *skb, struct genl_info *info);
+int knod_nl_nic_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb);
+int knod_nl_attach_doit(struct sk_buff *skb, struct genl_info *info);
+int knod_nl_detach_doit(struct sk_buff *skb, struct genl_info *info);
+int knod_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info);
+int knod_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb);
+
+enum {
+	KNOD_NLGRP_MGMT,
+};
+
+extern struct genl_family knod_nl_family;
+
+#endif /* _LINUX_KNOD_GEN_H */
diff --git a/net/knod/knod.h b/net/knod/knod.h
new file mode 100644
index 000000000000..eaff0e82ca23
--- /dev/null
+++ b/net/knod/knod.h
@@ -0,0 +1,26 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/* Copyright (c) 2021 Taehee Yoo <ap420073@gmail.com>
+ * Copyright (c) 2021 Hoyeon Lee <hoyeon.rhee@gmail.com>
+ */
+
+#ifndef _NET_KNOD_KNOD_H
+#define _NET_KNOD_KNOD_H
+
+#include <net/knod.h>
+
+#include "knod-nl-gen.h"
+
+struct net_device;
+
+/* knod_core.c */
+struct knod_netdev *knod_netdev_lookup(struct net_device *dev);
+struct knod_dev *knod_dev_lookup(struct net_device *dev);
+struct knod_accel *knod_accel_lookup(int id);
+int knod_dev_attach(struct knod_netdev *knetdev,
+			   struct knod_accel *accel);
+int knod_dev_detach(struct knod_dev *knodev);
+
+/* knod_nl.c */
+void knod_nl_notify_dev(struct knod_dev *knodev, u32 cmd);
+
+#endif /* _NET_KNOD_KNOD_H */
diff --git a/net/knod/knod_core.c b/net/knod/knod_core.c
new file mode 100644
index 000000000000..af50a69bca65
--- /dev/null
+++ b/net/knod/knod_core.c
@@ -0,0 +1,1231 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/* Copyright (c) 2021 Taehee Yoo <ap420073@gmail.com>
+ * Copyright (c) 2021 Hoyeon Lee <hoyeon.rhee@gmail.com>
+ */
+
+#include <net/knod.h>
+#include <net/spsc_ring.h>
+#include <net/page_pool/types.h>
+#include <net/page_pool/helpers.h>
+#include <net/page_pool/memory_provider.h>
+#include <net/xfrm.h>
+#include <linux/genalloc.h>
+#include <trace/events/page_pool.h>
+#include <net/devmem.h>
+
+#include "knod.h"
+
+DEFINE_MUTEX(knod_lock);
+LIST_HEAD(knod_dev_list);
+EXPORT_SYMBOL_GPL(knod_dev_list);
+LIST_HEAD(knod_netdev_list);
+EXPORT_SYMBOL_GPL(knod_netdev_list);
+LIST_HEAD(knod_accel_list);
+EXPORT_SYMBOL_GPL(knod_accel_list);
+
+void knod_dev_lock(void)
+{
+	mutex_lock(&knod_lock);
+}
+EXPORT_SYMBOL_GPL(knod_dev_lock);
+
+void knod_dev_unlock(void)
+{
+	mutex_unlock(&knod_lock);
+}
+EXPORT_SYMBOL_GPL(knod_dev_unlock);
+
+void knod_netdev_register(struct knod_netdev *knetdev)
+{
+	mutex_lock(&knod_lock);
+	knetdev->status = KNOD_STATUS_FREE;
+	list_add(&knetdev->list, &knod_netdev_list);
+	mutex_unlock(&knod_lock);
+}
+EXPORT_SYMBOL(knod_netdev_register);
+
+void knod_netdev_unregister(struct knod_netdev *knetdev)
+{
+	mutex_lock(&knod_lock);
+	list_del(&knetdev->list);
+	mutex_unlock(&knod_lock);
+}
+EXPORT_SYMBOL(knod_netdev_unregister);
+
+void knod_accel_register(struct knod_accel *accel)
+{
+	mutex_lock(&knod_lock);
+	accel->status = KNOD_STATUS_FREE;
+	list_add(&accel->list, &knod_accel_list);
+	mutex_unlock(&knod_lock);
+}
+EXPORT_SYMBOL(knod_accel_register);
+
+void knod_accel_unregister(struct knod_accel *accel)
+{
+	mutex_lock(&knod_lock);
+	list_del(&accel->list);
+	mutex_unlock(&knod_lock);
+}
+EXPORT_SYMBOL(knod_accel_unregister);
+
+struct knod_netdev *knod_netdev_lookup(struct net_device *dev)
+{
+	struct knod_netdev *knetdev;
+
+	list_for_each_entry(knetdev, &knod_netdev_list, list)
+		if (knetdev->dev == dev)
+			return knetdev;
+
+	return NULL;
+}
+
+struct knod_dev *knod_dev_lookup(struct net_device *dev)
+{
+	struct knod_dev *knodev;
+
+	list_for_each_entry(knodev, &knod_dev_list, list)
+		if (knodev->netdev == dev)
+			return knodev;
+
+	return NULL;
+}
+
+struct knod_accel *knod_accel_lookup(int id)
+{
+	struct knod_accel *accel;
+
+	list_for_each_entry(accel, &knod_accel_list, list)
+		if (accel->id == id)
+			return accel;
+
+	return NULL;
+}
+
+void knod_dev_start(struct knod_dev *knodev)
+{
+	/* Interface up: start the active feature's worker. */
+	knodev->started = true;
+	if (knodev->accel_ops->dev_start)
+		knodev->accel_ops->dev_start(knodev);
+}
+EXPORT_SYMBOL(knod_dev_start);
+
+void knod_dev_stop(struct knod_dev *knodev)
+{
+	/* Interface down: stop the worker + drain the GPU in-flight. */
+	if (knodev->accel_ops->dev_stop)
+		knodev->accel_ops->dev_stop(knodev);
+	knodev->started = false;
+}
+EXPORT_SYMBOL(knod_dev_stop);
+
+int knod_dev_xdp_install(struct knod_dev *knodev, struct netdev_bpf *xdp)
+{
+	if (!knodev->accel_ops->xdp_ops ||
+	    !knodev->accel_ops->xdp_ops->xdp_install)
+		return -EOPNOTSUPP;
+	return knodev->accel_ops->xdp_ops->xdp_install(knodev, xdp);
+}
+EXPORT_SYMBOL(knod_dev_xdp_install);
+
+void knod_dev_get_stats64(struct knod_dev *knodev,
+			  struct rtnl_link_stats64 *stats)
+{
+	struct knod_dev_stats *p;
+	u32 tx_dropped = 0, tx_errors = 0;
+	u64 tx_packets, tx_bytes;
+	unsigned int start;
+	int i;
+
+	for_each_possible_cpu(i) {
+		p = per_cpu_ptr(knodev->stats, i);
+		do {
+			start = u64_stats_fetch_begin(&p->syncp);
+			tx_packets      = u64_stats_read(&p->tx_packets);
+			tx_bytes        = u64_stats_read(&p->tx_bytes);
+		} while (u64_stats_fetch_retry(&p->syncp, start));
+
+		stats->tx_packets       += tx_packets;
+		stats->tx_bytes         += tx_bytes;
+		tx_dropped      += READ_ONCE(p->tx_dropped);
+		tx_errors	+= READ_ONCE(p->tx_errors);
+	}
+	stats->tx_dropped       += tx_dropped;
+	stats->tx_errors	+= tx_errors;
+}
+EXPORT_SYMBOL(knod_dev_get_stats64);
+
+/*
+ * Wrap a delivery-pool page as a zero-copy head_frag skb: the packet sits at
+ * @off (preserved headroom) for @len bytes.  Building it linear keeps
+ * skb->data on the packet so callers can edit headers in place.  The page
+ * recycles to @pool when the skb is freed.  On oversize or alloc failure the
+ * page is returned to @pool and NULL is returned.  @napi selects the NAPI skb
+ * cache; callers outside softirq pass false.
+ */
+struct sk_buff *knod_pass_build_skb(netmem_ref netmem, u16 off, u16 len,
+				    struct page_pool *pool, bool napi)
+{
+	struct page *pg = netmem_to_page(netmem);
+	struct sk_buff *skb;
+
+	/* Must fit alongside skb_shared_info at the page tail; MTU is capped
+	 * below this, so drop the rare oversized outlier.
+	 */
+	if (off + len > SKB_WITH_OVERHEAD(PAGE_SIZE)) {
+		if (pool)
+			page_pool_put_full_netmem(pool, netmem, false);
+		return NULL;
+	}
+
+	if (napi)
+		skb = napi_build_skb(page_address(pg), PAGE_SIZE);
+	else
+		skb = build_skb(page_address(pg), PAGE_SIZE);
+	if (unlikely(!skb)) {
+		if (pool)
+			page_pool_put_full_netmem(pool, netmem, false);
+		return NULL;
+	}
+
+	skb_mark_for_recycle(skb);
+	skb_reserve(skb, off);
+	skb_put(skb, len);
+	return skb;
+}
+EXPORT_SYMBOL(knod_pass_build_skb);
+
+/*
+ * Device->host copy: the NIC DD hands the PASS bds it accumulated during its
+ * single act traversal.  Allocate a delivery page per packet, issue the accel
+ * SDMA (GPU -> page) asynchronously, and queue a descriptor on the per-queue
+ * pending ring tagged with this batch's fence; knod_d2h_drain delivers them
+ * once the fence lands and recycles the source.  Sources that cannot be queued
+ * (bad len / ring full / pool empty) are recycled here.  Returns the count
+ * queued.  Producer and the drain consumer run on the same per-queue NAPI, so
+ * the ring is single-threaded; @d2h_lock only guards the shared SDMA submit.
+ */
+int knod_d2h_copy(struct knod_dev *knodev, int napi_index,
+		  struct spsc_pass_bd *bds, int cnt)
+{
+	struct knod_accel_ops *ops = knodev->accel_ops;
+	struct knod_work_priv *wpriv;
+	struct page_pool *pool;
+	bool submitted = false;
+	int i, produced = 0;
+
+	if (napi_index < 0 || napi_index >= KNOD_SPSC_MAX || !ops->d2h_submit)
+		goto drop_all;
+	wpriv = &knodev->wpriv[napi_index];
+	pool = READ_ONCE(wpriv->pass_pool);
+	if (!pool || !wpriv->pass_pending.slots)
+		goto drop_all;
+
+	spin_lock(&knodev->d2h_lock);
+
+	for (i = 0; i < cnt; i++) {
+		netmem_ref src = bds[i].netmem;
+		struct knod_pass_desc *desc;
+		netmem_ref dst;
+		void *ptr;
+		u32 fv;
+		u16 off = bds[i].off;
+		u16 len = bds[i].len;
+
+		if (!len || off + len > SKB_WITH_OVERHEAD(PAGE_SIZE))
+			goto drop;
+		if (spsc_produce(&wpriv->pass_pending, &ptr))
+			goto drop;
+		dst = page_pool_dev_alloc_netmems(pool);
+		if (!dst)
+			goto drop;	/* slot left uncommitted, reused next */
+
+		fv = ops->d2h_submit(knodev,
+				     page_pool_get_dma_addr_netmem(dst) + off,
+				     napi_index, bds[i].page_idx, off, len);
+		if (!fv) {		/* SDMA ring full: backpressure drop */
+			page_pool_put_full_netmem(pool, dst, false);
+			goto drop;
+		}
+		submitted = true;
+
+		desc = ptr;
+		desc->netmem = dst;
+		desc->src = src;
+		desc->off = off;
+		desc->len = len;
+		desc->fence_val = fv;
+		desc->sdma_idx = 0;
+		spsc_produce_commit(&wpriv->pass_pending);
+		produced++;
+		continue;
+drop:
+		page_pool_recycle_direct_netmem(netmem_get_pp(src), src);
+	}
+
+	if (submitted)
+		ops->d2h_kick(knodev);
+	spin_unlock(&knodev->d2h_lock);
+
+	/* The copies are async; re-arm the NAPI so the drain runs once the
+	 * SDMA lands -- at low rate, RX traffic alone may not poll again soon.
+	 */
+	if (submitted)
+		knod_napi_kick(wpriv);
+	return produced;
+
+drop_all:
+	for (i = 0; i < cnt; i++)
+		page_pool_recycle_direct_netmem(netmem_get_pp(bds[i].netmem),
+						bds[i].netmem);
+	return 0;
+}
+EXPORT_SYMBOL(knod_d2h_copy);
+
+/*
+ * Drain the per-queue pending ring: deliver every descriptor whose batch
+ * fence has landed (accel_ops->d2h_fence) as a zero-copy head_frag skb from
+ * the delivery page, recycling the source RX page.  Stops at the first
+ * not-yet-landed descriptor -- the ring is in fence order.  Runs on the NIC
+ * NAPI, the same thread as the knod_d2h_copy producer.
+ */
+int knod_d2h_drain(struct knod_dev *knodev, int napi_index,
+		   struct napi_struct *napi, int budget)
+{
+	void *ptrs[KNOD_DEFAULT_PASS_SLOTS];
+	struct knod_work_priv *wpriv;
+	struct knod_pass_desc *d0;
+	struct page_pool *pool;
+	LIST_HEAD(deliver_list);
+	unsigned int got = 0, i, n = 0;
+	int delivered = 0;
+	u32 cur_fence;
+
+	if (napi_index < 0 || napi_index >= KNOD_SPSC_MAX ||
+	    !knodev->accel_ops->d2h_fence)
+		return 0;
+	wpriv = &knodev->wpriv[napi_index];
+	if (!wpriv->pass_pending.slots)
+		return 0;
+	pool = READ_ONCE(wpriv->pass_pool);
+
+	if (spsc_peek(&wpriv->pass_pending, ptrs,
+		      min_t(unsigned int, budget, KNOD_DEFAULT_PASS_SLOTS),
+		      &got) < 0 || got == 0)
+		return 0;
+
+	d0 = ptrs[0];
+	cur_fence = knodev->accel_ops->d2h_fence(knodev, d0->sdma_idx);
+
+	for (i = 0; i < got; i++) {
+		struct knod_pass_desc *desc = ptrs[i];
+		struct sk_buff *skb;
+
+		if ((s32)(cur_fence - desc->fence_val) < 0)
+			break;	/* not landed yet; later descs are newer */
+
+		/* bpf carries the RX page as src for post-copy recycle; ipsec
+		 * leaves it 0 (the NIC act handler recycles via the bd).
+		 */
+		if (desc->src)
+			page_pool_recycle_direct_netmem(
+				netmem_get_pp(desc->src), desc->src);
+		skb = knod_pass_build_skb(desc->netmem, desc->off, desc->len,
+					  pool, true);
+		n++;
+		if (!skb)
+			continue;
+		if (knodev->post_copy) {
+			if (!knodev->post_copy(knodev, skb, desc, napi_index)) {
+				kfree_skb(skb);
+				continue;
+			}
+		} else if (likely(skb->len >= ETH_HLEN)) {
+			skb->protocol = eth_type_trans(skb, knodev->netdev);
+		} else {
+			kfree_skb(skb);
+			continue;
+		}
+		list_add_tail(&skb->list, &deliver_list);
+		delivered++;
+	}
+
+	if (n) {
+		spsc_acquire(&wpriv->pass_pending, NULL, n, NULL);
+		spsc_release_commit(&wpriv->pass_pending, n);
+	}
+
+	/* Descriptors whose copy has not landed yet remain queued; re-arm so
+	 * we poll again instead of waiting for the next RX event.
+	 */
+	if (spsc_count(&wpriv->pass_pending))
+		knod_napi_kick(wpriv);
+
+	if (!list_empty(&deliver_list))
+		netif_receive_skb_list(&deliver_list);
+	return delivered;
+}
+EXPORT_SYMBOL(knod_d2h_drain);
+
+/* ======================================================================
+ * NOD IPsec proxy - bridges standard kernel xfrmdev_ops to
+ * knod_accel_ipsec_ops on the GPU accelerator.
+ * ======================================================================
+ */
+#if IS_ENABLED(CONFIG_XFRM_OFFLOAD)
+
+static struct knod_dev *knod_ipsec_find_xdev(struct net_device *dev)
+{
+	struct knod_dev *knodev;
+
+	list_for_each_entry(knodev, &knod_dev_list, list) {
+		if (knodev->netdev == dev)
+			return knodev;
+	}
+	return NULL;
+}
+
+static int knod_ipsec_xdo_state_add(struct net_device *dev,
+				    struct xfrm_state *x,
+				    struct netlink_ext_ack *extack)
+{
+	struct knod_dev *knodev = knod_ipsec_find_xdev(dev);
+
+	if (!knodev || !knodev->accel_ops->ipsec_ops ||
+	    !knodev->accel_ops->ipsec_ops->xdo_dev_state_add)
+		return -EOPNOTSUPP;
+	return knodev->accel_ops->ipsec_ops->xdo_dev_state_add(knodev, x,
+							       extack);
+}
+
+static void knod_ipsec_xdo_state_delete(struct net_device *dev,
+					struct xfrm_state *x)
+{
+	struct knod_dev *knodev = knod_ipsec_find_xdev(dev);
+
+	if (!knodev || !knodev->accel_ops->ipsec_ops ||
+	    !knodev->accel_ops->ipsec_ops->xdo_dev_state_delete)
+		return;
+	knodev->accel_ops->ipsec_ops->xdo_dev_state_delete(knodev, x);
+}
+
+static void knod_ipsec_xdo_state_free(struct net_device *dev,
+				      struct xfrm_state *x)
+{
+	struct knod_dev *knodev = knod_ipsec_find_xdev(dev);
+
+	if (!knodev || !knodev->accel_ops->ipsec_ops ||
+	    !knodev->accel_ops->ipsec_ops->xdo_dev_state_free)
+		return;
+	knodev->accel_ops->ipsec_ops->xdo_dev_state_free(knodev, x);
+}
+
+static bool knod_ipsec_xdo_offload_ok(struct sk_buff *skb,
+				      struct xfrm_state *x)
+{
+	struct knod_dev *knodev = knod_ipsec_find_xdev(x->xso.dev);
+
+	if (!knodev || !knodev->accel_ops->ipsec_ops ||
+	    !knodev->accel_ops->ipsec_ops->xdo_dev_offload_ok)
+		return false;
+	return knodev->accel_ops->ipsec_ops->xdo_dev_offload_ok(knodev, skb, x);
+}
+
+static void knod_ipsec_xdo_state_advance_esn(struct xfrm_state *x)
+{
+	struct knod_dev *knodev;
+
+	if (!x->xso.dev)
+		return;
+	knodev = knod_ipsec_find_xdev(x->xso.dev);
+	if (!knodev || !knodev->accel_ops->ipsec_ops ||
+	    !knodev->accel_ops->ipsec_ops->xdo_dev_state_advance_esn)
+		return;
+	knodev->accel_ops->ipsec_ops->xdo_dev_state_advance_esn(knodev, x);
+}
+
+static void knod_ipsec_xdo_state_update_stats(struct xfrm_state *x)
+{
+	struct knod_dev *knodev;
+
+	if (!x->xso.dev)
+		return;
+	knodev = knod_ipsec_find_xdev(x->xso.dev);
+	if (!knodev || !knodev->accel_ops->ipsec_ops ||
+	    !knodev->accel_ops->ipsec_ops->xdo_dev_state_update_stats)
+		return;
+	knodev->accel_ops->ipsec_ops->xdo_dev_state_update_stats(knodev, x);
+}
+
+static int knod_ipsec_xdo_policy_add(struct xfrm_policy *xp,
+				     struct netlink_ext_ack *extack)
+{
+	struct knod_dev *knodev;
+
+	if (!xp->xdo.dev)
+		return -EOPNOTSUPP;
+	knodev = knod_ipsec_find_xdev(xp->xdo.dev);
+	if (!knodev || !knodev->accel_ops->ipsec_ops ||
+	    !knodev->accel_ops->ipsec_ops->xdo_dev_policy_add)
+		return -EOPNOTSUPP;
+	return knodev->accel_ops->ipsec_ops->xdo_dev_policy_add(knodev, xp,
+								extack);
+}
+
+static void knod_ipsec_xdo_policy_delete(struct xfrm_policy *xp)
+{
+	struct knod_dev *knodev;
+
+	if (!xp->xdo.dev)
+		return;
+	knodev = knod_ipsec_find_xdev(xp->xdo.dev);
+	if (!knodev || !knodev->accel_ops->ipsec_ops ||
+	    !knodev->accel_ops->ipsec_ops->xdo_dev_policy_delete)
+		return;
+	knodev->accel_ops->ipsec_ops->xdo_dev_policy_delete(knodev, xp);
+}
+
+static void knod_ipsec_xdo_policy_free(struct xfrm_policy *xp)
+{
+	struct knod_dev *knodev;
+
+	if (!xp->xdo.dev)
+		return;
+	knodev = knod_ipsec_find_xdev(xp->xdo.dev);
+	if (!knodev || !knodev->accel_ops->ipsec_ops ||
+	    !knodev->accel_ops->ipsec_ops->xdo_dev_policy_free)
+		return;
+	knodev->accel_ops->ipsec_ops->xdo_dev_policy_free(knodev, xp);
+}
+
+static const struct xfrmdev_ops knod_ipsec_xfrmdev_ops = {
+	.xdo_dev_state_add          = knod_ipsec_xdo_state_add,
+	.xdo_dev_state_delete       = knod_ipsec_xdo_state_delete,
+	.xdo_dev_state_free         = knod_ipsec_xdo_state_free,
+	.xdo_dev_offload_ok         = knod_ipsec_xdo_offload_ok,
+	.xdo_dev_state_advance_esn  = knod_ipsec_xdo_state_advance_esn,
+	.xdo_dev_state_update_stats = knod_ipsec_xdo_state_update_stats,
+	.xdo_dev_policy_add         = knod_ipsec_xdo_policy_add,
+	.xdo_dev_policy_delete      = knod_ipsec_xdo_policy_delete,
+	.xdo_dev_policy_free        = knod_ipsec_xdo_policy_free,
+};
+
+int knod_dev_xdp_drain_pass(struct knod_dev *knodev, struct napi_struct *napi,
+			    int queue_idx, int budget)
+{
+	if (!knodev || !knodev->accel_ops)
+		return 0;
+	/* Common device->host delivery drain.  PASS bds were SDMA-copied by
+	 * knod_d2h_copy from the NIC act handler; deliver the ones whose copy
+	 * has landed.
+	 */
+	return knod_d2h_drain(knodev, queue_idx, napi, budget);
+}
+EXPORT_SYMBOL_GPL(knod_dev_xdp_drain_pass);
+
+int knod_ipsec_attach(struct knod_dev *knodev)
+{
+	struct knod_accel_ipsec_ops *ops;
+
+	if (!knodev->accel_ops->ipsec_ops)
+		return 0;
+
+	ops = knodev->accel_ops->ipsec_ops;
+	if (ops->activate) {
+		int err = ops->activate(knodev);
+
+		if (err) {
+			pr_err("nod: IPsec activate failed on %s (%d)\n",
+			       netdev_name(knodev->netdev), err);
+			return err;
+		}
+	}
+
+	/* Take over xfrmdev_ops, saving the NIC's original so detach can
+	 * restore it (and only clear NETIF_F_HW_ESP if we added it).
+	 */
+	knodev->ipsec_orig_xfrmdev_ops = knodev->netdev->xfrmdev_ops;
+	knodev->ipsec_added_hw_esp =
+		!(knodev->netdev->features & NETIF_F_HW_ESP);
+	knodev->netdev->xfrmdev_ops = &knod_ipsec_xfrmdev_ops;
+	knodev->netdev->features |= NETIF_F_HW_ESP;
+	knodev->netdev->hw_enc_features |= NETIF_F_HW_ESP;
+
+	pr_info("nod: IPsec offload enabled on %s\n",
+		netdev_name(knodev->netdev));
+	return 0;
+}
+EXPORT_SYMBOL(knod_ipsec_attach);
+
+void knod_ipsec_detach(struct knod_dev *knodev)
+{
+	struct knod_accel_ipsec_ops *ops = knodev->accel_ops->ipsec_ops;
+
+	if (!ops)
+		return;
+	if (knodev->netdev->xfrmdev_ops != &knod_ipsec_xfrmdev_ops)
+		return;
+
+	if (knodev->ipsec_added_hw_esp) {
+		knodev->netdev->features &= ~NETIF_F_HW_ESP;
+		knodev->netdev->hw_enc_features &= ~NETIF_F_HW_ESP;
+	}
+	knodev->netdev->xfrmdev_ops = knodev->ipsec_orig_xfrmdev_ops;
+
+	if (ops->deactivate)
+		ops->deactivate(knodev);
+
+	pr_info("nod: IPsec offload disabled on %s\n",
+		netdev_name(knodev->netdev));
+}
+EXPORT_SYMBOL(knod_ipsec_detach);
+
+#endif /* CONFIG_XFRM_OFFLOAD */
+
+/*
+ * Host-page page_pool memory provider for GPU->host delivery (NOD-private).
+ * Unlike the devmem/dma-buf providers (which hand out unreadable net_iov), this
+ * returns real host-readable pages (a GPU GTT buffer also mapped into system
+ * memory), so delivered data becomes skbs on the normal receive path.  The
+ * pages stay owned by the accel and are never returned to the buddy; the
+ * gen_pool is only the slow-path backing store (page_pool's cache/ring absorb
+ * the per-packet churn), and the device address it hands out doubles as the
+ * netmem dma_addr (the worker's SDMA destination).  Selected via
+ * page_pool_params.mp_ops in knod_pass_attach().
+ */
+static int hostmem_pp_init(struct page_pool *pool)
+{
+	struct page_pool_hostmem *hm = pool->mp_priv;
+	int err;
+
+	if (pool->p.order != 0)
+		return -E2BIG;
+	if (!hm || !hm->pages || !hm->count)
+		return -EINVAL;
+
+	hm->genpool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);
+	if (!hm->genpool)
+		return -ENOMEM;
+
+	err = gen_pool_add(hm->genpool, (unsigned long)hm->base_addr,
+			   (size_t)hm->count * PAGE_SIZE, NUMA_NO_NODE);
+	if (err) {
+		gen_pool_destroy(hm->genpool);
+		hm->genpool = NULL;
+		return err;
+	}
+
+	/* Device addresses are pre-set; page_pool must not DMA-sync them. */
+	pool->dma_sync = false;
+	pool->dma_sync_for_cpu = false;
+
+	return 0;
+}
+
+static netmem_ref hostmem_pp_alloc_netmems(struct page_pool *pool, gfp_t gfp)
+{
+	struct page_pool_hostmem *hm = pool->mp_priv;
+	unsigned long addr;
+	netmem_ref netmem;
+	unsigned int idx;
+
+	addr = gen_pool_alloc(hm->genpool, PAGE_SIZE);
+	if (!addr)
+		return 0;
+
+	idx = (addr - hm->base_addr) >> PAGE_SHIFT;
+	if (WARN_ON_ONCE(idx >= hm->count)) {
+		gen_pool_free(hm->genpool, addr, PAGE_SIZE);
+		return 0;
+	}
+
+	netmem = page_to_netmem(hm->pages[idx]);
+	page_pool_provider_set_netmem(pool, netmem, addr);
+
+	pool->pages_state_hold_cnt++;
+	trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt);
+
+	return netmem;
+}
+
+static bool hostmem_pp_release_netmem(struct page_pool *pool, netmem_ref netmem)
+{
+	struct page_pool_hostmem *hm = pool->mp_priv;
+	unsigned long addr = page_pool_get_dma_addr_netmem(netmem);
+
+	page_pool_clear_pp_info(netmem);
+	gen_pool_free(hm->genpool, addr, PAGE_SIZE);
+
+	/* Pages are accel-owned: never put_page() them to the buddy. */
+	return false;
+}
+
+static void hostmem_pp_destroy(struct page_pool *pool)
+{
+	struct page_pool_hostmem *hm = pool->mp_priv;
+
+	gen_pool_destroy(hm->genpool);
+	hm->genpool = NULL;
+
+	/* The pool has fully drained (inflight == 0); tell the owner it may now
+	 * release the backing pages.  The struct itself is owner-allocated.
+	 */
+	if (hm->freed)
+		hm->freed(hm->arg);
+}
+
+static int hostmem_pp_nl_fill(void *mp_priv, struct sk_buff *rsp,
+			      struct netdev_rx_queue *rxq)
+{
+	/* Nothing provider-specific to report; page_pool_user.c calls this
+	 * unconditionally once mp_ops is set, so it must not be NULL.
+	 */
+	return 0;
+}
+
+static const struct memory_provider_ops page_pool_hostmem_ops = {
+	.init		= hostmem_pp_init,
+	.alloc_netmems	= hostmem_pp_alloc_netmems,
+	.release_netmem	= hostmem_pp_release_netmem,
+	.destroy	= hostmem_pp_destroy,
+	.nl_fill	= hostmem_pp_nl_fill,
+	/*
+	 * .uninstall is only invoked for rxq-bound pools and is
+	 * NULL-checked.
+	 */
+};
+
+/* Provider drain callback: fires once a delivery pool has no inflight pages. */
+static void knod_pass_drained(void *arg)
+{
+	struct knod_dev *knodev = arg;
+
+	if (atomic_dec_and_test(&knodev->pp_live))
+		complete(&knodev->pp_drained);
+}
+
+/*
+ * Create one GPU->host delivery page_pool per RX queue, backed by a single
+ * accel-allocated GTT buffer.  Mirrors the spsc-pool setup: alloc_mem() hands
+ * back the buffer (kaddr/gaddr/pages/priv); the framework owns the page_pools
+ * and the drain barrier.  The hostmem provider hands out the real GTT pages
+ * with their device address as dma_addr, so the worker's SDMA lands directly
+ * in the page that later becomes the skb frag.  page_pool inflight accounting
+ * then keeps the buffer alive until every delivered skb has drained.
+ */
+static int knod_pass_attach(struct knod_dev *knodev)
+{
+	struct page_pool_params pp = {
+		.order		= 0,
+		.pool_size	= KNOD_PASS_SLOTS,
+		.nid		= NUMA_NO_NODE,
+		.dev		= knodev->netdev->dev.parent,
+		.dma_dir	= DMA_FROM_DEVICE,
+		.max_len	= PAGE_SIZE,
+		.flags		= PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV |
+				  PP_FLAG_CUSTOM_MEMORY_PROVIDER,
+		.mp_ops		= &page_pool_hostmem_ops,
+	};
+	unsigned int nqueues = min(knodev->netdev->num_rx_queues,
+				   KNOD_SPSC_MAX);
+	struct page **pages;
+	void *pass_priv;
+	u64 base_gaddr;
+	size_t total;
+	void *kaddr;
+	int qi;
+
+	/* Accels without GPU memory simply run without zero-copy delivery. */
+	if (!knodev->accel_ops->alloc_mem)
+		return 0;
+
+	total = nqueues * KNOD_PASS_SLOTS * PAGE_SIZE;
+	kaddr = knodev->accel_ops->alloc_mem(knodev, total, &base_gaddr, &pages,
+					   &pass_priv);
+	if (!kaddr) {
+		pr_err("%s: delivery alloc_mem failed\n", __func__);
+		return -ENOMEM;
+	}
+	if (!pages) {
+		knodev->accel_ops->free_mem(knodev, pass_priv);
+		return -ENOMEM;
+	}
+
+	init_completion(&knodev->pp_drained);
+	atomic_set(&knodev->pp_live, 0);
+	spin_lock_init(&knodev->d2h_lock);
+
+	for (qi = 0; qi < nqueues; qi++) {
+		struct knod_work_priv *wpriv = &knodev->wpriv[qi];
+		int base = qi * KNOD_PASS_SLOTS;
+
+		if (spsc_init(&wpriv->pass_pending,
+			      sizeof(struct knod_pass_desc),
+			      KNOD_PASS_SLOTS, GFP_KERNEL))
+			goto err_destroy;
+
+		wpriv->pass_hm.pages	 = &pages[base];
+		wpriv->pass_hm.count	 = KNOD_PASS_SLOTS;
+		wpriv->pass_hm.base_addr = base_gaddr + (u64)base * PAGE_SIZE;
+		wpriv->pass_hm.freed	 = knod_pass_drained;
+		wpriv->pass_hm.arg	 = knodev;
+		pp.mp_priv		 = &wpriv->pass_hm;
+
+		wpriv->pass_pool = page_pool_create(&pp);
+		if (IS_ERR(wpriv->pass_pool)) {
+			wpriv->pass_pool = NULL;
+			spsc_destroy(&wpriv->pass_pending);
+			goto err_destroy;
+		}
+		atomic_inc(&knodev->pp_live);
+	}
+
+	knodev->pass_priv = pass_priv;
+	return 0;
+
+err_destroy:
+	while (qi-- > 0) {
+		spsc_destroy(&knodev->wpriv[qi].pass_pending);
+		page_pool_destroy(knodev->wpriv[qi].pass_pool);
+		knodev->wpriv[qi].pass_pool = NULL;
+	}
+	if (atomic_read(&knodev->pp_live))
+		wait_for_completion_timeout(&knodev->pp_drained,
+					    msecs_to_jiffies(5000));
+	knodev->accel_ops->free_mem(knodev, pass_priv);
+	return -ENOMEM;
+}
+
+/*
+ * Drain a queue's pass_pending ring on teardown.  The interface is down so no
+ * new copies are submitted; wait for each queued copy to land, then return its
+ * pages to the pool instead of delivering.  Without this, page_pool_destroy()
+ * stalls on the pages a detach raced against in-flight d2h copies.
+ */
+static void knod_pass_flush(struct knod_dev *knodev, unsigned int qi)
+{
+	struct knod_work_priv *wpriv = &knodev->wpriv[qi];
+	struct page_pool *pool = wpriv->pass_pool;
+	void *ptrs[KNOD_DEFAULT_PASS_SLOTS];
+	unsigned int got, i;
+
+	if (!wpriv->pass_pending.slots || !pool)
+		return;
+
+	while (spsc_peek(&wpriv->pass_pending, ptrs,
+			 KNOD_DEFAULT_PASS_SLOTS, &got) >= 0 && got) {
+		for (i = 0; i < got; i++) {
+			struct knod_pass_desc *desc = ptrs[i];
+			int spins = 1000000;
+
+			while (knodev->accel_ops->d2h_fence && spins-- &&
+			       (s32)(knodev->accel_ops->d2h_fence(knodev,
+					desc->sdma_idx) - desc->fence_val) < 0)
+				cpu_relax();
+
+			WARN_ONCE(knodev->accel_ops->d2h_fence &&
+				  (s32)(knodev->accel_ops->d2h_fence(knodev,
+					desc->sdma_idx) - desc->fence_val) < 0,
+				  "knod: d2h fence timeout on pass flush q%u idx%u\n",
+				  qi, desc->sdma_idx);
+
+			if (desc->src) {
+				struct page_pool *src_pp =
+					netmem_get_pp(desc->src);
+
+				page_pool_recycle_direct_netmem(src_pp,
+								desc->src);
+			}
+			page_pool_put_full_netmem(pool, desc->netmem, false);
+		}
+		spsc_acquire(&wpriv->pass_pending, NULL, got, NULL);
+		spsc_release_commit(&wpriv->pass_pending, got);
+	}
+}
+
+/*
+ * Tear down the delivery pools and free the backing buffer.  page_pool_destroy
+ * is async, so wait for every pool to drain (knod_pass_drained) before handing
+ * the buffer back to the accel -- otherwise an inflight skb frag would outlive
+ * the BO.  Idempotent: a no-op for accels that never allocated.
+ */
+static void knod_pass_detach(struct knod_dev *knodev)
+{
+	unsigned int qi;
+
+	if (!knodev->pass_priv)
+		return;
+
+	/* Iterate the full array (like the spsc teardown): destroy every pool
+	 * that was created, so the drain barrier is guaranteed to reach zero.
+	 */
+	for (qi = 0; qi < KNOD_SPSC_MAX; qi++) {
+		if (!knodev->wpriv[qi].pass_pool)
+			continue;
+		/* Return any queued (now-landed) d2h pages before destroying
+		 * the pool, or page_pool_destroy() stalls on the inflight
+		 * count a detach raced against in-flight d2h copies.
+		 */
+		knod_pass_flush(knodev, qi);
+		spsc_destroy(&knodev->wpriv[qi].pass_pending);
+		page_pool_destroy(knodev->wpriv[qi].pass_pool);
+		knodev->wpriv[qi].pass_pool = NULL;
+	}
+	if (atomic_read(&knodev->pp_live))
+		wait_for_completion_timeout(&knodev->pp_drained,
+					    msecs_to_jiffies(5000));
+	knodev->accel_ops->free_mem(knodev, knodev->pass_priv);
+	knodev->pass_priv = NULL;
+}
+
+static void knod_dmabuf_move_notify(struct dma_buf_attachment *attach)
+{
+}
+
+static const struct dma_buf_attach_ops knod_dmabuf_attach_ops = {
+	.allow_peer2peer = true,
+	.invalidate_mappings = knod_dmabuf_move_notify,
+};
+
+static int knod_dmabuf_attach(struct knod_dev *knodev)
+{
+	struct net_device *dev = knodev->netdev;
+	struct net_devmem_dmabuf_binding *binding;
+	struct dma_buf *dmabuf;
+	int i, err;
+
+	for (i = 0; i < min(dev->num_rx_queues, KNOD_SPSC_MAX); i++) {
+		dmabuf = knodev->wpriv[i].dmabuf;
+		if (!dmabuf)
+			continue;
+
+		/* The binding owns a dmabuf reference that
+		 * __net_devmem_dmabuf_binding_free() drops on unbind, mirroring
+		 * the dma_buf_get(fd) in the fd-based net_devmem_bind_dmabuf().
+		 * We hand it the BO's dmabuf directly, so take that reference
+		 * here -- otherwise the unbind put races the BO free's put and
+		 * underflows the dmabuf file refcount.
+		 */
+		get_dma_buf(dmabuf);
+		binding = __net_devmem_binding_create(dev, dev->dev.parent,
+						      dmabuf, DMA_BIDIRECTIONAL,
+						      &knod_dmabuf_attach_ops,
+						      NULL);
+		if (IS_ERR(binding)) {
+			dma_buf_put(dmabuf);
+			err = PTR_ERR(binding);
+			goto err_unwind;
+		}
+
+		err = net_devmem_bind_dmabuf_to_queue_direct(dev, i, binding);
+		if (err) {
+			net_devmem_unbind_dmabuf_direct(binding);
+			goto err_unwind;
+		}
+
+		knodev->bindings[i] = binding;
+	}
+
+	return 0;
+
+err_unwind:
+	while (i-- > 0) {
+		if (knodev->bindings[i]) {
+			net_devmem_unbind_dmabuf_direct(knodev->bindings[i]);
+			knodev->bindings[i] = NULL;
+		}
+	}
+	return err;
+}
+
+static void knod_dmabuf_detach(struct knod_dev *knodev)
+{
+	int i;
+
+	for (i = 0; i < KNOD_SPSC_MAX; i++) {
+		if (!knodev->bindings[i])
+			continue;
+
+		net_devmem_unbind_dmabuf_direct(knodev->bindings[i]);
+		knodev->bindings[i] = NULL;
+	}
+}
+
+int knod_dev_attach(struct knod_netdev *knetdev, struct knod_accel *accel)
+{
+	struct knod_dev *knodev;
+	int err = -EINVAL, i;
+
+	if (knetdev->status == KNOD_STATUS_USED ||
+	    accel->status == KNOD_STATUS_USED) {
+		pr_err("knod: %s already attached\n",
+		       netdev_name(knetdev->dev));
+		return -EINVAL;
+	}
+
+	knodev = kzalloc(sizeof(struct knod_dev), GFP_KERNEL);
+	if (!knodev)
+		return -ENOMEM;
+
+	if (!try_module_get(knetdev->owner)) {
+		pr_err("knod: NIC driver for %s is unloading\n",
+		       netdev_name(knetdev->dev));
+		kfree(knodev);
+		return -ENODEV;
+	}
+	if (!try_module_get(accel->owner)) {
+		pr_err("knod: accelerator driver is unloading\n");
+		module_put(knetdev->owner);
+		kfree(knodev);
+		return -ENODEV;
+	}
+
+	knetdev->accel = accel;
+	knetdev->knodev = knodev;
+	accel->knetdev = knetdev;
+	accel->knodev = knodev;
+	knodev->knetdev = knetdev;
+	knodev->accel = accel;
+	knodev->netdev = knetdev->dev;
+	knodev->accel_ops = accel->accel_ops;
+	knodev->nic_ops = knetdev->nic_ops;
+	mutex_init(&knodev->lock);
+
+	knodev->stats = netdev_alloc_pcpu_stats(struct knod_dev_stats);
+	if (!knodev->stats) {
+		err = -ENOMEM;
+		pr_err("knod: failed to allocate stats for %s\n",
+		       netdev_name(knetdev->dev));
+		goto free_xdev;
+	}
+
+	knodev->wpriv = kmalloc_array(KNOD_SPSC_MAX,
+				    sizeof(struct knod_work_priv),
+				    GFP_KERNEL | __GFP_ZERO);
+	if (!knodev->wpriv) {
+		pr_err("knod: failed to allocate work priv for %s\n",
+		       netdev_name(knetdev->dev));
+		err = -ENOMEM;
+		goto free_percpu;
+	}
+
+	netdev_lock(knodev->netdev);
+	err = knodev->nic_ops->attach(knodev);
+	if (err) {
+		err = -ENOMEM;
+		pr_err("knod: NIC attach failed on %s\n",
+		       netdev_name(knetdev->dev));
+		goto unlock;
+	}
+
+	err = knodev->accel_ops->attach(knodev);
+	if (err) {
+		err = -ENOMEM;
+		pr_err("knod: accelerator attach failed on %s\n",
+		       netdev_name(knetdev->dev));
+		goto nic_detach;
+	}
+
+	{
+		unsigned int nqueues = min(knodev->netdev->num_rx_queues,
+					  KNOD_SPSC_MAX);
+		unsigned int stride = ALIGN(sizeof(struct spsc_bd),
+					    SMP_CACHE_BYTES);
+		unsigned int cap = roundup_pow_of_two(KNOD_SPSC_ELEMS_MAX);
+		size_t pool_size = (size_t)stride * cap;
+
+		if (knodev->accel_ops->alloc_mem) {
+			size_t total = pool_size * nqueues;
+			u64 base_gaddr;
+			void *base_pool;
+			void *pool_priv;
+
+			base_pool = knodev->accel_ops->alloc_mem(knodev, total,
+					&base_gaddr, NULL, &pool_priv);
+			if (!base_pool) {
+				err = -ENOMEM;
+				pr_err("%s: alloc_mem failed\n", __func__);
+				goto free_spsc;
+			}
+			memset(base_pool, 0, total);
+
+			/* First queue owns the BO, others reference it */
+			knodev->wpriv[0].spsc_pool_priv = pool_priv;
+			for (i = 0; i < nqueues; i++) {
+				void *pool = base_pool +
+					     (unsigned long)i * pool_size;
+
+				knodev->wpriv[i].spsc_pool_gaddr =
+					base_gaddr + (u64)i * pool_size;
+				err = __spsc_init(&knodev->wpriv[i].spsc_bds,
+						  sizeof(struct spsc_bd),
+						  KNOD_SPSC_ELEMS_MAX, pool,
+						  GFP_KERNEL);
+				if (err) {
+					pr_err("%s: spsc_init failed q%d\n",
+					       __func__, i);
+					goto free_spsc;
+				}
+			}
+		} else {
+			for (i = 0; i < nqueues; i++) {
+				err = spsc_init(&knodev->wpriv[i].spsc_bds,
+						sizeof(struct spsc_bd),
+						KNOD_SPSC_ELEMS_MAX,
+						GFP_KERNEL);
+				if (err) {
+					pr_err("%s: spsc_init failed q%d\n",
+					       __func__, i);
+					goto free_spsc;
+				}
+			}
+		}
+	}
+	goto spsc_done;
+
+free_spsc:
+	for (i--; i >= 0; i--)
+		spsc_destroy(&knodev->wpriv[i].spsc_bds);
+	if (knodev->wpriv[0].spsc_pool_priv)
+		knodev->accel_ops->free_mem(knodev,
+				knodev->wpriv[0].spsc_pool_priv);
+	knodev->accel_ops->detach(knodev);
+	goto nic_detach;
+spsc_done:
+
+	err = knod_pass_attach(knodev);
+	if (err) {
+		pr_err("%s: knod_pass_attach failed (%d)\n", __func__, err);
+		goto accel_detach;
+	}
+
+	err = knod_dmabuf_attach(knodev);
+	if (err) {
+		err = -ENOMEM;
+		pr_err("knod: dmabuf attach failed on %s\n",
+		       netdev_name(knetdev->dev));
+		goto accel_detach;
+	}
+
+	pr_info("knod: %s attached to accel %d\n",
+		netdev_name(knodev->netdev), accel->id);
+	list_add(&knodev->list, &knod_dev_list);
+	knetdev->status = KNOD_STATUS_USED;
+	accel->status = KNOD_STATUS_USED;
+
+	if (knodev->accel_ops->xdp_ops && knodev->accel_ops->xdp_ops->init) {
+		err = knodev->accel_ops->xdp_ops->init(knodev);
+		if (err) {
+			pr_err("knod: XDP init failed on %s\n",
+			       netdev_name(knetdev->dev));
+			goto xdp_err;
+		}
+	}
+
+	/*
+	 * Feature offloads (BPF, IPsec) allocate their GPU resources and
+	 * advertise their netdev capabilities only when the feature is
+	 * selected via knod_accel_feature_set(), not at attach.
+	 */
+	netdev_unlock(knodev->netdev);
+
+	if (knodev->accel_ops->mp_map) {
+		err = knodev->accel_ops->mp_map(knodev);
+		if (err) {
+			pr_err("knod: mp_map failed (%d)\n", err);
+			goto dmabuf_detach;
+		}
+	}
+
+	return err;
+
+dmabuf_detach:
+	netdev_lock(knodev->netdev);
+xdp_err:
+	list_del(&knodev->list);
+	knetdev->status = KNOD_STATUS_FREE;
+	accel->status = KNOD_STATUS_FREE;
+	knod_dmabuf_detach(knodev);
+accel_detach:
+	knod_pass_detach(knodev);
+	for (i = 0; i < KNOD_SPSC_MAX; i++)
+		spsc_destroy(&knodev->wpriv[i].spsc_bds);
+	if (knodev->wpriv[0].spsc_pool_priv)
+		knodev->accel_ops->free_mem(knodev,
+				knodev->wpriv[0].spsc_pool_priv);
+	knodev->accel_ops->detach(knodev);
+nic_detach:
+	knodev->nic_ops->detach(knodev);
+unlock:
+	netdev_unlock(knodev->netdev);
+	kfree(knodev->wpriv);
+free_percpu:
+	free_percpu(knodev->stats);
+free_xdev:
+	/* Drop the accel<->knetdev<->knodev links set above before freeing
+	 * knodev, or a reader (e.g. knod_default_worker via accel->knodev)
+	 * dereferences a dangling pointer after a failed attach.
+	 */
+	accel->knodev = NULL;
+	accel->knetdev = NULL;
+	knetdev->knodev = NULL;
+	knetdev->accel = NULL;
+	module_put(accel->owner);
+	module_put(knetdev->owner);
+	kfree(knodev);
+	return err;
+}
+
+int knod_dev_detach(struct knod_dev *knodev)
+{
+	struct knod_accel *accel = knodev->accel;
+	struct knod_netdev *knetdev = knodev->knetdev;
+	int i;
+
+	if (netif_running(knodev->netdev)) {
+		pr_err("knod_dev: interface is up\n");
+		return -EINVAL;
+	}
+
+	netdev_lock(knodev->netdev);
+	/*
+	 * pre_detach() runs knod_feature_stop()+knod_feature_deactivate(),
+	 * which tears down whatever feature is active and frees its
+	 * resources, so no per-feature teardown is needed here.
+	 */
+	if (knodev->accel_ops->pre_detach)
+		knodev->accel_ops->pre_detach(knodev);
+	list_del(&knodev->list);
+	knod_dmabuf_detach(knodev);
+	knod_pass_detach(knodev);
+	for (i = 0; i < KNOD_SPSC_MAX; i++)
+		spsc_destroy(&knodev->wpriv[i].spsc_bds);
+	if (knodev->wpriv[0].spsc_pool_priv)
+		knodev->accel_ops->free_mem(knodev,
+				knodev->wpriv[0].spsc_pool_priv);
+	knodev->nic_ops->detach(knodev);
+	knodev->accel_ops->detach(knodev);
+	netdev_unlock(knodev->netdev);
+	knetdev->status = KNOD_STATUS_FREE;
+	knetdev->accel = NULL;
+	knetdev->knodev = NULL;
+	accel->knetdev = NULL;
+	accel->knodev = NULL;
+	accel->status = KNOD_STATUS_FREE;
+	kfree(knodev->wpriv);
+	free_percpu(knodev->stats);
+	kfree(knodev);
+
+	module_put(accel->owner);
+	module_put(knetdev->owner);
+	return 0;
+}
+
+static int __init knod_dev_init(void)
+{
+	return genl_register_family(&knod_nl_family);
+}
+
+core_initcall(knod_dev_init);
+
diff --git a/net/knod/knod_nl.c b/net/knod/knod_nl.c
new file mode 100644
index 000000000000..77799202abcf
--- /dev/null
+++ b/net/knod/knod_nl.c
@@ -0,0 +1,406 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/* Copyright (c) 2021 Taehee Yoo <ap420073@gmail.com>
+ * Copyright (c) 2021 Hoyeon Lee <hoyeon.rhee@gmail.com>
+ */
+
+#include <linux/netdevice.h>
+#include <linux/rtnetlink.h>
+#include <net/genetlink.h>
+#include <net/knod.h>
+#include <net/sock.h>
+
+#include "knod-nl-gen.h"
+#include "knod.h"
+
+/* ---- accel ---- */
+
+static int knod_nl_accel_fill(struct knod_accel *accel,
+			      struct sk_buff *rsp, const struct genl_info *info)
+{
+	u32 ena = 0, cap = 0;
+	void *hdr;
+
+	if (accel->accel_ops->feature_get)
+		accel->accel_ops->feature_get(accel, &ena, &cap);
+
+	hdr = genlmsg_iput(rsp, info);
+	if (!hdr)
+		return -EMSGSIZE;
+
+	if (nla_put_u32(rsp, KNOD_A_ACCEL_ID, accel->id) ||
+	    nla_put_string(rsp, KNOD_A_ACCEL_NAME, accel->name) ||
+	    nla_put_u32(rsp, KNOD_A_ACCEL_TYPE, accel->type) ||
+	    nla_put_u32(rsp, KNOD_A_ACCEL_FEATURE_CAP, cap) ||
+	    nla_put_u32(rsp, KNOD_A_ACCEL_FEATURE_ENA, ena)) {
+		genlmsg_cancel(rsp, hdr);
+		return -EMSGSIZE;
+	}
+
+	genlmsg_end(rsp, hdr);
+	return 0;
+}
+
+int knod_nl_accel_get_doit(struct sk_buff *skb, struct genl_info *info)
+{
+	struct knod_accel *accel;
+	struct sk_buff *rsp;
+	int err;
+
+	if (GENL_REQ_ATTR_CHECK(info, KNOD_A_ACCEL_ID))
+		return -EINVAL;
+
+	rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!rsp)
+		return -ENOMEM;
+
+	mutex_lock(&knod_lock);
+	accel = knod_accel_lookup(nla_get_u32(info->attrs[KNOD_A_ACCEL_ID]));
+	if (!accel) {
+		mutex_unlock(&knod_lock);
+		err = -ENODEV;
+		goto err_free;
+	}
+	err = knod_nl_accel_fill(accel, rsp, info);
+	mutex_unlock(&knod_lock);
+	if (err)
+		goto err_free;
+
+	return genlmsg_reply(rsp, info);
+
+err_free:
+	nlmsg_free(rsp);
+	return err;
+}
+
+int knod_nl_accel_get_dumpit(struct sk_buff *rsp, struct netlink_callback *cb)
+{
+	struct knod_accel *accel;
+	int idx = 0, s_idx = cb->args[0];
+
+	mutex_lock(&knod_lock);
+	list_for_each_entry(accel, &knod_accel_list, list) {
+		if (idx < s_idx)
+			goto cont;
+		if (knod_nl_accel_fill(accel, rsp, genl_info_dump(cb)) < 0)
+			break;
+cont:
+		idx++;
+	}
+	mutex_unlock(&knod_lock);
+
+	cb->args[0] = idx;
+	return rsp->len;
+}
+
+int knod_nl_accel_set_doit(struct sk_buff *skb, struct genl_info *info)
+{
+	struct knod_accel *accel;
+	u32 feature;
+	int err;
+
+	if (GENL_REQ_ATTR_CHECK(info, KNOD_A_ACCEL_ID) ||
+	    GENL_REQ_ATTR_CHECK(info, KNOD_A_ACCEL_FEATURE_ENA))
+		return -EINVAL;
+
+	feature = nla_get_u32(info->attrs[KNOD_A_ACCEL_FEATURE_ENA]);
+
+	/*
+	 * rtnl serialises the feature switch against attach/detach (which
+	 * also take rtnl) and against the NIC driver's interface up/down
+	 * (knod_dev_start/stop), and lets feature_set touch netdev->features
+	 * and run an expedited synchronize_net().  knod_lock still guards the
+	 * accel list walk (lock order: rtnl -> knod_lock).
+	 */
+	rtnl_lock();
+	mutex_lock(&knod_lock);
+	accel = knod_accel_lookup(nla_get_u32(info->attrs[KNOD_A_ACCEL_ID]));
+	if (!accel) {
+		err = -ENODEV;
+		goto unlock;
+	}
+	if (!accel->accel_ops->feature_set) {
+		err = -EOPNOTSUPP;
+		goto unlock;
+	}
+	err = accel->accel_ops->feature_set(accel, feature, info->extack);
+unlock:
+	mutex_unlock(&knod_lock);
+	rtnl_unlock();
+	return err;
+}
+
+/* ---- nic ---- */
+
+static int knod_nl_nic_fill(struct knod_netdev *knetdev,
+			    struct sk_buff *rsp, const struct genl_info *info)
+{
+	void *hdr;
+
+	hdr = genlmsg_iput(rsp, info);
+	if (!hdr)
+		return -EMSGSIZE;
+
+	if (nla_put_u32(rsp, KNOD_A_NIC_IFINDEX, knetdev->dev->ifindex) ||
+	    nla_put_string(rsp, KNOD_A_NIC_NAME, netdev_name(knetdev->dev))) {
+		genlmsg_cancel(rsp, hdr);
+		return -EMSGSIZE;
+	}
+
+	genlmsg_end(rsp, hdr);
+	return 0;
+}
+
+int knod_nl_nic_get_doit(struct sk_buff *skb, struct genl_info *info)
+{
+	struct knod_netdev *knetdev;
+	struct net_device *dev;
+	struct sk_buff *rsp;
+	int err;
+
+	if (GENL_REQ_ATTR_CHECK(info, KNOD_A_NIC_IFINDEX))
+		return -EINVAL;
+
+	rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!rsp)
+		return -ENOMEM;
+
+	rtnl_lock();
+	mutex_lock(&knod_lock);
+	dev = __dev_get_by_index(genl_info_net(info),
+				 nla_get_u32(info->attrs[KNOD_A_NIC_IFINDEX]));
+	knetdev = dev ? knod_netdev_lookup(dev) : NULL;
+	if (!knetdev) {
+		mutex_unlock(&knod_lock);
+		rtnl_unlock();
+		err = -ENODEV;
+		goto err_free;
+	}
+	err = knod_nl_nic_fill(knetdev, rsp, info);
+	mutex_unlock(&knod_lock);
+	rtnl_unlock();
+	if (err)
+		goto err_free;
+
+	return genlmsg_reply(rsp, info);
+
+err_free:
+	nlmsg_free(rsp);
+	return err;
+}
+
+int knod_nl_nic_get_dumpit(struct sk_buff *rsp, struct netlink_callback *cb)
+{
+	struct net *net = sock_net(rsp->sk);
+	struct knod_netdev *knetdev;
+	int idx = 0, s_idx = cb->args[0];
+
+	rtnl_lock();
+	mutex_lock(&knod_lock);
+	list_for_each_entry(knetdev, &knod_netdev_list, list) {
+		if (idx < s_idx)
+			goto cont;
+		if (dev_net(knetdev->dev) != net)
+			goto cont;
+		if (knod_nl_nic_fill(knetdev, rsp, genl_info_dump(cb)) < 0)
+			break;
+cont:
+		idx++;
+	}
+	mutex_unlock(&knod_lock);
+	rtnl_unlock();
+
+	cb->args[0] = idx;
+	return rsp->len;
+}
+
+/* ---- dev (NIC <-> accel binding) ---- */
+
+static int knod_nl_dev_fill(struct knod_dev *knodev, struct sk_buff *rsp,
+			    const struct genl_info *info)
+{
+	void *hdr;
+
+	hdr = genlmsg_iput(rsp, info);
+	if (!hdr)
+		return -EMSGSIZE;
+
+	if (nla_put_u32(rsp, KNOD_A_DEV_NIC_IFINDEX, knodev->netdev->ifindex) ||
+	    nla_put_u32(rsp, KNOD_A_DEV_ACCEL_ID, knodev->accel->id)) {
+		genlmsg_cancel(rsp, hdr);
+		return -EMSGSIZE;
+	}
+
+	genlmsg_end(rsp, hdr);
+	return 0;
+}
+
+void knod_nl_notify_dev(struct knod_dev *knodev, u32 cmd)
+{
+	struct net *net = dev_net(knodev->netdev);
+	struct genl_info info;
+	struct sk_buff *ntf;
+
+	if (!genl_has_listeners(&knod_nl_family, net, KNOD_NLGRP_MGMT))
+		return;
+
+	ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!ntf)
+		return;
+
+	genl_info_init_ntf(&info, &knod_nl_family, cmd);
+	if (knod_nl_dev_fill(knodev, ntf, &info)) {
+		nlmsg_free(ntf);
+		return;
+	}
+
+	genlmsg_multicast_netns(&knod_nl_family, net, ntf, 0,
+				KNOD_NLGRP_MGMT, GFP_KERNEL);
+}
+
+int knod_nl_attach_doit(struct sk_buff *skb, struct genl_info *info)
+{
+	struct knod_netdev *knetdev;
+	struct knod_accel *accel;
+	struct net_device *dev;
+	int err;
+
+	if (GENL_REQ_ATTR_CHECK(info, KNOD_A_DEV_NIC_IFINDEX) ||
+	    GENL_REQ_ATTR_CHECK(info, KNOD_A_DEV_ACCEL_ID))
+		return -EINVAL;
+
+	rtnl_lock();
+	mutex_lock(&knod_lock);
+
+	dev = __dev_get_by_index(genl_info_net(info),
+				 nla_get_u32(
+				 info->attrs[KNOD_A_DEV_NIC_IFINDEX]));
+	if (!dev) {
+		NL_SET_ERR_MSG(info->extack, "no such netdevice");
+		err = -ENODEV;
+		goto unlock;
+	}
+
+	knetdev = knod_netdev_lookup(dev);
+	if (!knetdev) {
+		NL_SET_ERR_MSG(info->extack, "netdevice not registered with knod");
+		err = -ENODEV;
+		goto unlock;
+	}
+
+	if (netif_running(dev)) {
+		NL_SET_ERR_MSG(info->extack, "bring the netdevice down first");
+		err = -EBUSY;
+		goto unlock;
+	}
+
+	accel = knod_accel_lookup(
+			nla_get_u32(info->attrs[KNOD_A_DEV_ACCEL_ID]));
+	if (!accel) {
+		NL_SET_ERR_MSG(info->extack, "no such accelerator");
+		err = -ENODEV;
+		goto unlock;
+	}
+
+	err = knod_dev_attach(knetdev, accel);
+	if (!err)
+		knod_nl_notify_dev(knetdev->knodev, KNOD_CMD_DEV_ADD_NTF);
+
+unlock:
+	mutex_unlock(&knod_lock);
+	rtnl_unlock();
+	return err;
+}
+
+int knod_nl_detach_doit(struct sk_buff *skb, struct genl_info *info)
+{
+	struct knod_dev *knodev;
+	struct net_device *dev;
+	int err;
+
+	if (GENL_REQ_ATTR_CHECK(info, KNOD_A_DEV_NIC_IFINDEX))
+		return -EINVAL;
+
+	rtnl_lock();
+
+	dev = __dev_get_by_index(genl_info_net(info),
+				 nla_get_u32(
+				 info->attrs[KNOD_A_DEV_NIC_IFINDEX]));
+	knodev = dev ? knod_dev_lookup(dev) : NULL;
+	if (!knodev) {
+		NL_SET_ERR_MSG(info->extack, "netdevice not attached");
+		err = -ENODEV;
+		goto unlock;
+	}
+
+	if (netif_running(dev)) {
+		NL_SET_ERR_MSG(info->extack, "bring the netdevice down first");
+		err = -EBUSY;
+		goto unlock;
+	}
+
+	knod_nl_notify_dev(knodev, KNOD_CMD_DEV_DEL_NTF);
+	err = knod_dev_detach(knodev);
+
+unlock:
+	rtnl_unlock();
+	return err;
+}
+
+int knod_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info)
+{
+	struct knod_dev *knodev;
+	struct net_device *dev;
+	struct sk_buff *rsp;
+	int err;
+
+	if (GENL_REQ_ATTR_CHECK(info, KNOD_A_DEV_NIC_IFINDEX))
+		return -EINVAL;
+
+	rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!rsp)
+		return -ENOMEM;
+
+	rtnl_lock();
+	dev = __dev_get_by_index(genl_info_net(info),
+				 nla_get_u32(
+				 info->attrs[KNOD_A_DEV_NIC_IFINDEX]));
+	knodev = dev ? knod_dev_lookup(dev) : NULL;
+	if (!knodev) {
+		rtnl_unlock();
+		err = -ENODEV;
+		goto err_free;
+	}
+	err = knod_nl_dev_fill(knodev, rsp, info);
+	rtnl_unlock();
+	if (err)
+		goto err_free;
+
+	return genlmsg_reply(rsp, info);
+
+err_free:
+	nlmsg_free(rsp);
+	return err;
+}
+
+int knod_nl_dev_get_dumpit(struct sk_buff *rsp, struct netlink_callback *cb)
+{
+	struct net *net = sock_net(rsp->sk);
+	struct knod_dev *knodev;
+	int idx = 0, s_idx = cb->args[0];
+
+	rtnl_lock();
+	list_for_each_entry(knodev, &knod_dev_list, list) {
+		if (idx < s_idx)
+			goto cont;
+		if (dev_net(knodev->netdev) != net)
+			goto cont;
+		if (knod_nl_dev_fill(knodev, rsp, genl_info_dump(cb)) < 0)
+			break;
+cont:
+		idx++;
+	}
+	rtnl_unlock();
+
+	cb->args[0] = idx;
+	return rsp->len;
+}
-- 
2.43.0