* [PATCH v4 net-next 1/3] net: introduce page_pool pointer in softnet_data percpu struct
2023-12-12 10:06 [PATCH v4 net-next 0/3] add multi-buff support for xdp running in generic mode Lorenzo Bianconi
@ 2023-12-12 10:06 ` Lorenzo Bianconi
2023-12-12 21:46 ` kernel test robot
2023-12-12 21:56 ` kernel test robot
2023-12-12 10:06 ` [PATCH v4 net-next 2/3] xdp: rely on skb pointer reference in do_xdp_generic and netif_receive_generic_xdp Lorenzo Bianconi
2023-12-12 10:06 ` [PATCH v4 net-next 3/3] xdp: add multi-buff support for xdp running in generic mode Lorenzo Bianconi
2 siblings, 2 replies; 7+ messages in thread
From: Lorenzo Bianconi @ 2023-12-12 10:06 UTC (permalink / raw)
To: netdev
Cc: lorenzo.bianconi, davem, kuba, edumazet, pabeni, bpf, hawk, toke,
willemdebruijn.kernel, jasowang, sdf
Allocate percpu page_pools in softnet_data.
Moreover add cpuid filed in page_pool struct in order to recycle the
page in the page_pool "hot" cache if napi_pp_put_page() is running on
the same cpu.
This is a preliminary patch to add xdp multi-buff support for xdp running
in generic mode.
Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
---
include/linux/netdevice.h | 1 +
include/net/page_pool/helpers.h | 5 +++++
include/net/page_pool/types.h | 2 +-
net/core/dev.c | 28 +++++++++++++++++++++++++++-
net/core/skbuff.c | 5 +++--
5 files changed, 37 insertions(+), 4 deletions(-)
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 1b935ee341b4..30b6a3f601fe 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -3319,6 +3319,7 @@ struct softnet_data {
int defer_count;
int defer_ipi_scheduled;
struct sk_buff *defer_list;
+ struct page_pool *page_pool;
call_single_data_t defer_csd;
};
diff --git a/include/net/page_pool/helpers.h b/include/net/page_pool/helpers.h
index 7dc65774cde5..b0da1fdb62dc 100644
--- a/include/net/page_pool/helpers.h
+++ b/include/net/page_pool/helpers.h
@@ -393,4 +393,9 @@ static inline void page_pool_nid_changed(struct page_pool *pool, int new_nid)
page_pool_update_nid(pool, new_nid);
}
+static inline void page_pool_set_cpuid(struct page_pool *pool, int cpuid)
+{
+ pool->cpuid = cpuid;
+}
+
#endif /* _NET_PAGE_POOL_HELPERS_H */
diff --git a/include/net/page_pool/types.h b/include/net/page_pool/types.h
index ac286ea8ce2d..75396f107b20 100644
--- a/include/net/page_pool/types.h
+++ b/include/net/page_pool/types.h
@@ -127,7 +127,7 @@ struct page_pool_stats {
struct page_pool {
struct page_pool_params_fast p;
-
+ int cpuid;
bool has_init_callback;
long frag_users;
diff --git a/net/core/dev.c b/net/core/dev.c
index 0432b04cf9b0..4c3d82fcf5b5 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -153,6 +153,8 @@
#include <linux/prandom.h>
#include <linux/once_lite.h>
#include <net/netdev_rx_queue.h>
+#include <net/page_pool/types.h>
+#include <net/page_pool/helpers.h>
#include "dev.h"
#include "net-sysfs.h"
@@ -11670,12 +11672,14 @@ static void __init net_dev_struct_check(void)
*
*/
+#define SD_PAGE_POOL_RING_SIZE 256
/*
* This is called single threaded during boot, so no need
* to take the rtnl semaphore.
*/
static int __init net_dev_init(void)
{
+ struct softnet_data *sd;
int i, rc = -ENOMEM;
BUG_ON(!dev_boot_phase);
@@ -11701,10 +11705,14 @@ static int __init net_dev_init(void)
for_each_possible_cpu(i) {
struct work_struct *flush = per_cpu_ptr(&flush_works, i);
- struct softnet_data *sd = &per_cpu(softnet_data, i);
+ struct page_pool_params page_pool_params = {
+ .pool_size = SD_PAGE_POOL_RING_SIZE,
+ .nid = NUMA_NO_NODE,
+ };
INIT_WORK(flush, flush_backlog);
+ sd = &per_cpu(softnet_data, i);
skb_queue_head_init(&sd->input_pkt_queue);
skb_queue_head_init(&sd->process_queue);
#ifdef CONFIG_XFRM_OFFLOAD
@@ -11722,6 +11730,13 @@ static int __init net_dev_init(void)
init_gro_hash(&sd->backlog);
sd->backlog.poll = process_backlog;
sd->backlog.weight = weight_p;
+
+ sd->page_pool = page_pool_create(&page_pool_params);
+ if (IS_ERR(sd->page_pool)) {
+ sd->page_pool = NULL;
+ goto out;
+ }
+ page_pool_set_cpuid(sd->page_pool, i);
}
dev_boot_phase = 0;
@@ -11749,6 +11764,17 @@ static int __init net_dev_init(void)
WARN_ON(rc < 0);
rc = 0;
out:
+ if (rc < 0) {
+ for_each_possible_cpu(i) {
+ sd = &per_cpu(softnet_data, i);
+ if (!sd->page_pool)
+ continue;
+
+ page_pool_destroy(sd->page_pool);
+ sd->page_pool = NULL;
+ }
+ }
+
return rc;
}
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index b157efea5dea..4bc0a7f98241 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -918,9 +918,10 @@ bool napi_pp_put_page(struct page *page, bool napi_safe)
*/
if (napi_safe || in_softirq()) {
const struct napi_struct *napi = READ_ONCE(pp->p.napi);
+ unsigned int cpuid = smp_processor_id();
- allow_direct = napi &&
- READ_ONCE(napi->list_owner) == smp_processor_id();
+ allow_direct = napi && READ_ONCE(napi->list_owner) == cpuid;
+ allow_direct |= (pp->cpuid == cpuid);
}
/* Driver set this to memory recycling info. Reset it on recycle.
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: [PATCH v4 net-next 1/3] net: introduce page_pool pointer in softnet_data percpu struct
2023-12-12 10:06 ` [PATCH v4 net-next 1/3] net: introduce page_pool pointer in softnet_data percpu struct Lorenzo Bianconi
@ 2023-12-12 21:46 ` kernel test robot
2023-12-12 21:56 ` kernel test robot
1 sibling, 0 replies; 7+ messages in thread
From: kernel test robot @ 2023-12-12 21:46 UTC (permalink / raw)
To: Lorenzo Bianconi, netdev
Cc: llvm, oe-kbuild-all, lorenzo.bianconi, davem, kuba, edumazet,
pabeni, bpf, hawk, toke, willemdebruijn.kernel, jasowang, sdf
Hi Lorenzo,
kernel test robot noticed the following build errors:
[auto build test ERROR on net-next/main]
url: https://github.com/intel-lab-lkp/linux/commits/Lorenzo-Bianconi/net-introduce-page_pool-pointer-in-softnet_data-percpu-struct/20231212-181103
base: net-next/main
patch link: https://lore.kernel.org/r/2a267c8f331996de0e26568472c45fe78eb67e1d.1702375338.git.lorenzo%40kernel.org
patch subject: [PATCH v4 net-next 1/3] net: introduce page_pool pointer in softnet_data percpu struct
config: um-allnoconfig (https://download.01.org/0day-ci/archive/20231213/202312130546.Kst7VY7F-lkp@intel.com/config)
compiler: clang version 17.0.0 (https://github.com/llvm/llvm-project.git 4a5ac14ee968ff0ad5d2cc1ffa0299048db4c88a)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20231213/202312130546.Kst7VY7F-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202312130546.Kst7VY7F-lkp@intel.com/
All errors (new ones prefixed by >>):
/usr/bin/ld: init/main.o: warning: relocation in read-only section `.ref.text'
/usr/bin/ld: warning: .tmp_vmlinux.kallsyms1 has a LOAD segment with RWX permissions
/usr/bin/ld: net/core/dev.o: in function `net_dev_init':
>> net/core/dev.c:11734: undefined reference to `page_pool_create'
/usr/bin/ld: warning: creating DT_TEXTREL in a PIE
clang: error: linker command failed with exit code 1 (use -v to see invocation)
vim +11734 net/core/dev.c
11667
11668 /*
11669 * Initialize the DEV module. At boot time this walks the device list and
11670 * unhooks any devices that fail to initialise (normally hardware not
11671 * present) and leaves us with a valid list of present and active devices.
11672 *
11673 */
11674
11675 #define SD_PAGE_POOL_RING_SIZE 256
11676 /*
11677 * This is called single threaded during boot, so no need
11678 * to take the rtnl semaphore.
11679 */
11680 static int __init net_dev_init(void)
11681 {
11682 struct softnet_data *sd;
11683 int i, rc = -ENOMEM;
11684
11685 BUG_ON(!dev_boot_phase);
11686
11687 net_dev_struct_check();
11688
11689 if (dev_proc_init())
11690 goto out;
11691
11692 if (netdev_kobject_init())
11693 goto out;
11694
11695 INIT_LIST_HEAD(&ptype_all);
11696 for (i = 0; i < PTYPE_HASH_SIZE; i++)
11697 INIT_LIST_HEAD(&ptype_base[i]);
11698
11699 if (register_pernet_subsys(&netdev_net_ops))
11700 goto out;
11701
11702 /*
11703 * Initialise the packet receive queues.
11704 */
11705
11706 for_each_possible_cpu(i) {
11707 struct work_struct *flush = per_cpu_ptr(&flush_works, i);
11708 struct page_pool_params page_pool_params = {
11709 .pool_size = SD_PAGE_POOL_RING_SIZE,
11710 .nid = NUMA_NO_NODE,
11711 };
11712
11713 INIT_WORK(flush, flush_backlog);
11714
11715 sd = &per_cpu(softnet_data, i);
11716 skb_queue_head_init(&sd->input_pkt_queue);
11717 skb_queue_head_init(&sd->process_queue);
11718 #ifdef CONFIG_XFRM_OFFLOAD
11719 skb_queue_head_init(&sd->xfrm_backlog);
11720 #endif
11721 INIT_LIST_HEAD(&sd->poll_list);
11722 sd->output_queue_tailp = &sd->output_queue;
11723 #ifdef CONFIG_RPS
11724 INIT_CSD(&sd->csd, rps_trigger_softirq, sd);
11725 sd->cpu = i;
11726 #endif
11727 INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd);
11728 spin_lock_init(&sd->defer_lock);
11729
11730 init_gro_hash(&sd->backlog);
11731 sd->backlog.poll = process_backlog;
11732 sd->backlog.weight = weight_p;
11733
11734 sd->page_pool = page_pool_create(&page_pool_params);
11735 if (IS_ERR(sd->page_pool)) {
11736 sd->page_pool = NULL;
11737 goto out;
11738 }
11739 page_pool_set_cpuid(sd->page_pool, i);
11740 }
11741
11742 dev_boot_phase = 0;
11743
11744 /* The loopback device is special if any other network devices
11745 * is present in a network namespace the loopback device must
11746 * be present. Since we now dynamically allocate and free the
11747 * loopback device ensure this invariant is maintained by
11748 * keeping the loopback device as the first device on the
11749 * list of network devices. Ensuring the loopback devices
11750 * is the first device that appears and the last network device
11751 * that disappears.
11752 */
11753 if (register_pernet_device(&loopback_net_ops))
11754 goto out;
11755
11756 if (register_pernet_device(&default_device_ops))
11757 goto out;
11758
11759 open_softirq(NET_TX_SOFTIRQ, net_tx_action);
11760 open_softirq(NET_RX_SOFTIRQ, net_rx_action);
11761
11762 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
11763 NULL, dev_cpu_dead);
11764 WARN_ON(rc < 0);
11765 rc = 0;
11766 out:
11767 if (rc < 0) {
11768 for_each_possible_cpu(i) {
11769 sd = &per_cpu(softnet_data, i);
11770 if (!sd->page_pool)
11771 continue;
11772
11773 page_pool_destroy(sd->page_pool);
11774 sd->page_pool = NULL;
11775 }
11776 }
11777
11778 return rc;
11779 }
11780
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply [flat|nested] 7+ messages in thread* Re: [PATCH v4 net-next 1/3] net: introduce page_pool pointer in softnet_data percpu struct
2023-12-12 10:06 ` [PATCH v4 net-next 1/3] net: introduce page_pool pointer in softnet_data percpu struct Lorenzo Bianconi
2023-12-12 21:46 ` kernel test robot
@ 2023-12-12 21:56 ` kernel test robot
1 sibling, 0 replies; 7+ messages in thread
From: kernel test robot @ 2023-12-12 21:56 UTC (permalink / raw)
To: Lorenzo Bianconi, netdev
Cc: oe-kbuild-all, lorenzo.bianconi, davem, kuba, edumazet, pabeni,
bpf, hawk, toke, willemdebruijn.kernel, jasowang, sdf
Hi Lorenzo,
kernel test robot noticed the following build errors:
[auto build test ERROR on net-next/main]
url: https://github.com/intel-lab-lkp/linux/commits/Lorenzo-Bianconi/net-introduce-page_pool-pointer-in-softnet_data-percpu-struct/20231212-181103
base: net-next/main
patch link: https://lore.kernel.org/r/2a267c8f331996de0e26568472c45fe78eb67e1d.1702375338.git.lorenzo%40kernel.org
patch subject: [PATCH v4 net-next 1/3] net: introduce page_pool pointer in softnet_data percpu struct
config: arc-defconfig (https://download.01.org/0day-ci/archive/20231213/202312130531.EIYsEYp0-lkp@intel.com/config)
compiler: arc-elf-gcc (GCC) 13.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20231213/202312130531.EIYsEYp0-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202312130531.EIYsEYp0-lkp@intel.com/
All errors (new ones prefixed by >>):
arc-elf-ld: net/core/dev.o: in function `net_dev_init':
>> dev.c:(.init.text+0x180): undefined reference to `page_pool_create'
>> arc-elf-ld: dev.c:(.init.text+0x180): undefined reference to `page_pool_create'
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply [flat|nested] 7+ messages in thread
* [PATCH v4 net-next 2/3] xdp: rely on skb pointer reference in do_xdp_generic and netif_receive_generic_xdp
2023-12-12 10:06 [PATCH v4 net-next 0/3] add multi-buff support for xdp running in generic mode Lorenzo Bianconi
2023-12-12 10:06 ` [PATCH v4 net-next 1/3] net: introduce page_pool pointer in softnet_data percpu struct Lorenzo Bianconi
@ 2023-12-12 10:06 ` Lorenzo Bianconi
2023-12-12 10:06 ` [PATCH v4 net-next 3/3] xdp: add multi-buff support for xdp running in generic mode Lorenzo Bianconi
2 siblings, 0 replies; 7+ messages in thread
From: Lorenzo Bianconi @ 2023-12-12 10:06 UTC (permalink / raw)
To: netdev
Cc: lorenzo.bianconi, davem, kuba, edumazet, pabeni, bpf, hawk, toke,
willemdebruijn.kernel, jasowang, sdf
Rely on skb pointer reference instead of the skb pointer in do_xdp_generic and
netif_receive_generic_xdp routine signatures. This is a preliminary patch to add
multi-buff support for xdp running in generic mode.
Acked-by: Jesper Dangaard Brouer <hawk@kernel.org>
Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
---
drivers/net/tun.c | 4 ++--
include/linux/netdevice.h | 2 +-
net/core/dev.c | 16 +++++++++-------
3 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index afa5497f7c35..206adddff699 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -1921,7 +1921,7 @@ static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
rcu_read_lock();
xdp_prog = rcu_dereference(tun->xdp_prog);
if (xdp_prog) {
- ret = do_xdp_generic(xdp_prog, skb);
+ ret = do_xdp_generic(xdp_prog, &skb);
if (ret != XDP_PASS) {
rcu_read_unlock();
local_bh_enable();
@@ -2511,7 +2511,7 @@ static int tun_xdp_one(struct tun_struct *tun,
skb_record_rx_queue(skb, tfile->queue_index);
if (skb_xdp) {
- ret = do_xdp_generic(xdp_prog, skb);
+ ret = do_xdp_generic(xdp_prog, &skb);
if (ret != XDP_PASS) {
ret = 0;
goto out;
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 30b6a3f601fe..d850ead7f9cd 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -3953,7 +3953,7 @@ static inline void dev_consume_skb_any(struct sk_buff *skb)
u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
struct bpf_prog *xdp_prog);
void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog);
-int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb);
+int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb);
int netif_rx(struct sk_buff *skb);
int __netif_rx(struct sk_buff *skb);
diff --git a/net/core/dev.c b/net/core/dev.c
index 4c3d82fcf5b5..14554698b1fa 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4916,10 +4916,11 @@ u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
return act;
}
-static u32 netif_receive_generic_xdp(struct sk_buff *skb,
+static u32 netif_receive_generic_xdp(struct sk_buff **pskb,
struct xdp_buff *xdp,
struct bpf_prog *xdp_prog)
{
+ struct sk_buff *skb = *pskb;
u32 act = XDP_DROP;
/* Reinjected packets coming from act_mirred or similar should
@@ -5000,24 +5001,24 @@ void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog)
static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
-int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
+int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff **pskb)
{
if (xdp_prog) {
struct xdp_buff xdp;
u32 act;
int err;
- act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
+ act = netif_receive_generic_xdp(pskb, &xdp, xdp_prog);
if (act != XDP_PASS) {
switch (act) {
case XDP_REDIRECT:
- err = xdp_do_generic_redirect(skb->dev, skb,
+ err = xdp_do_generic_redirect((*pskb)->dev, *pskb,
&xdp, xdp_prog);
if (err)
goto out_redir;
break;
case XDP_TX:
- generic_xdp_tx(skb, xdp_prog);
+ generic_xdp_tx(*pskb, xdp_prog);
break;
}
return XDP_DROP;
@@ -5025,7 +5026,7 @@ int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
}
return XDP_PASS;
out_redir:
- kfree_skb_reason(skb, SKB_DROP_REASON_XDP);
+ kfree_skb_reason(*pskb, SKB_DROP_REASON_XDP);
return XDP_DROP;
}
EXPORT_SYMBOL_GPL(do_xdp_generic);
@@ -5348,7 +5349,8 @@ static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
int ret2;
migrate_disable();
- ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
+ ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog),
+ &skb);
migrate_enable();
if (ret2 != XDP_PASS) {
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH v4 net-next 3/3] xdp: add multi-buff support for xdp running in generic mode
2023-12-12 10:06 [PATCH v4 net-next 0/3] add multi-buff support for xdp running in generic mode Lorenzo Bianconi
2023-12-12 10:06 ` [PATCH v4 net-next 1/3] net: introduce page_pool pointer in softnet_data percpu struct Lorenzo Bianconi
2023-12-12 10:06 ` [PATCH v4 net-next 2/3] xdp: rely on skb pointer reference in do_xdp_generic and netif_receive_generic_xdp Lorenzo Bianconi
@ 2023-12-12 10:06 ` Lorenzo Bianconi
2023-12-12 23:10 ` kernel test robot
2 siblings, 1 reply; 7+ messages in thread
From: Lorenzo Bianconi @ 2023-12-12 10:06 UTC (permalink / raw)
To: netdev
Cc: lorenzo.bianconi, davem, kuba, edumazet, pabeni, bpf, hawk, toke,
willemdebruijn.kernel, jasowang, sdf
Similar to native xdp, do not always linearize the skb in
netif_receive_generic_xdp routine but create a non-linear xdp_buff to be
processed by the eBPF program. This allow to add multi-buffer support
for xdp running in generic mode.
Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org>
---
net/core/dev.c | 121 +++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 111 insertions(+), 10 deletions(-)
diff --git a/net/core/dev.c b/net/core/dev.c
index 14554698b1fa..3826e0ea08e3 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4854,6 +4854,12 @@ u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
skb_headlen(skb) + mac_len, true);
+ if (skb_is_nonlinear(skb)) {
+ skb_shinfo(skb)->xdp_frags_size = skb->data_len;
+ xdp_buff_set_frags_flag(xdp);
+ } else {
+ xdp_buff_clear_frags_flag(xdp);
+ }
orig_data_end = xdp->data_end;
orig_data = xdp->data;
@@ -4883,6 +4889,14 @@ u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
skb->len += off; /* positive on grow, negative on shrink */
}
+ /* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers
+ * (e.g. bpf_xdp_adjust_tail), we need to update data_len here.
+ */
+ if (xdp_buff_has_frags(xdp))
+ skb->data_len = skb_shinfo(skb)->xdp_frags_size;
+ else
+ skb->data_len = 0;
+
/* check if XDP changed eth hdr such SKB needs update */
eth = (struct ethhdr *)xdp->data;
if ((orig_eth_type != eth->h_proto) ||
@@ -4916,12 +4930,84 @@ u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
return act;
}
+static int netif_skb_segment_for_xdp(struct sk_buff **pskb)
+{
+ struct softnet_data *sd = this_cpu_ptr(&softnet_data);
+ u32 size, truesize, len, max_head_size, off;
+ struct sk_buff *skb = *pskb, *nskb;
+ int err, i, head_off;
+ void *data;
+
+ max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE - XDP_PACKET_HEADROOM);
+ if (skb->len > max_head_size + MAX_SKB_FRAGS * PAGE_SIZE)
+ return -ENOMEM;
+
+ size = min_t(u32, skb->len, max_head_size);
+ truesize = SKB_HEAD_ALIGN(size) + XDP_PACKET_HEADROOM;
+ data = page_pool_dev_alloc_va(sd->page_pool, &truesize);
+ if (!data)
+ return -ENOMEM;
+
+ nskb = napi_build_skb(data, truesize);
+ if (!nskb) {
+ page_pool_free_va(sd->page_pool, data, true);
+ return -ENOMEM;
+ }
+
+ skb_reserve(nskb, XDP_PACKET_HEADROOM);
+ skb_copy_header(nskb, skb);
+ skb_mark_for_recycle(nskb);
+
+ err = skb_copy_bits(skb, 0, nskb->data, size);
+ if (err) {
+ consume_skb(nskb);
+ return err;
+ }
+ skb_put(nskb, size);
+
+ head_off = skb_headroom(nskb) - skb_headroom(skb);
+ skb_headers_offset_update(nskb, head_off);
+
+ off = size;
+ len = skb->len - off;
+ for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) {
+ struct page *page;
+ u32 page_off;
+
+ size = min_t(u32, len, PAGE_SIZE);
+ truesize = size;
+
+ page = page_pool_dev_alloc(sd->page_pool, &page_off,
+ &truesize);
+ if (!data) {
+ consume_skb(nskb);
+ return -ENOMEM;
+ }
+
+ skb_add_rx_frag(nskb, i, page, page_off, size, truesize);
+ err = skb_copy_bits(skb, off, page_address(page) + page_off,
+ size);
+ if (err) {
+ consume_skb(nskb);
+ return err;
+ }
+
+ len -= size;
+ off += size;
+ }
+
+ consume_skb(skb);
+ *pskb = nskb;
+
+ return 0;
+}
+
static u32 netif_receive_generic_xdp(struct sk_buff **pskb,
struct xdp_buff *xdp,
struct bpf_prog *xdp_prog)
{
struct sk_buff *skb = *pskb;
- u32 act = XDP_DROP;
+ u32 mac_len, act = XDP_DROP;
/* Reinjected packets coming from act_mirred or similar should
* not get XDP generic processing.
@@ -4929,12 +5015,10 @@ static u32 netif_receive_generic_xdp(struct sk_buff **pskb,
if (skb_is_redirected(skb))
return XDP_PASS;
- /* XDP packets must be linear and must have sufficient headroom
- * of XDP_PACKET_HEADROOM bytes. This is the guarantee that also
- * native XDP provides, thus we need to do it here as well.
+ /* XDP does not support fraglist so we need to linearize
+ * the skb.
*/
- if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
- skb_headroom(skb) < XDP_PACKET_HEADROOM) {
+ if (skb_has_frag_list(skb) || !xdp_prog->aux->xdp_has_frags) {
int hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
int troom = skb->tail + skb->data_len - skb->end;
@@ -4947,23 +5031,40 @@ static u32 netif_receive_generic_xdp(struct sk_buff **pskb,
goto do_drop;
if (skb_linearize(skb))
goto do_drop;
+
+ goto do_xdp;
}
- act = bpf_prog_run_generic_xdp(skb, xdp, xdp_prog);
+ /* XDP packets must have sufficient headroom of XDP_PACKET_HEADROOM
+ * bytes. This is the guarantee that also native XDP provides,
+ * thus we need to do it here as well.
+ */
+ mac_len = skb->data - skb_mac_header(skb);
+ __skb_push(skb, mac_len);
+
+ if (skb_cloned(skb) || skb_shinfo(skb)->nr_frags ||
+ skb_headroom(skb) < XDP_PACKET_HEADROOM) {
+ if (netif_skb_segment_for_xdp(pskb))
+ goto do_drop;
+ }
+
+ __skb_pull(*pskb, mac_len);
+do_xdp:
+ act = bpf_prog_run_generic_xdp(*pskb, xdp, xdp_prog);
switch (act) {
case XDP_REDIRECT:
case XDP_TX:
case XDP_PASS:
break;
default:
- bpf_warn_invalid_xdp_action(skb->dev, xdp_prog, act);
+ bpf_warn_invalid_xdp_action((*pskb)->dev, xdp_prog, act);
fallthrough;
case XDP_ABORTED:
- trace_xdp_exception(skb->dev, xdp_prog, act);
+ trace_xdp_exception((*pskb)->dev, xdp_prog, act);
fallthrough;
case XDP_DROP:
do_drop:
- kfree_skb(skb);
+ kfree_skb(*pskb);
break;
}
--
2.43.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* Re: [PATCH v4 net-next 3/3] xdp: add multi-buff support for xdp running in generic mode
2023-12-12 10:06 ` [PATCH v4 net-next 3/3] xdp: add multi-buff support for xdp running in generic mode Lorenzo Bianconi
@ 2023-12-12 23:10 ` kernel test robot
0 siblings, 0 replies; 7+ messages in thread
From: kernel test robot @ 2023-12-12 23:10 UTC (permalink / raw)
To: Lorenzo Bianconi, netdev
Cc: oe-kbuild-all, lorenzo.bianconi, davem, kuba, edumazet, pabeni,
bpf, hawk, toke, willemdebruijn.kernel, jasowang, sdf
Hi Lorenzo,
kernel test robot noticed the following build errors:
[auto build test ERROR on net-next/main]
url: https://github.com/intel-lab-lkp/linux/commits/Lorenzo-Bianconi/net-introduce-page_pool-pointer-in-softnet_data-percpu-struct/20231212-181103
base: net-next/main
patch link: https://lore.kernel.org/r/2d0f9388c6509192d88e359a402517a73124b50e.1702375338.git.lorenzo%40kernel.org
patch subject: [PATCH v4 net-next 3/3] xdp: add multi-buff support for xdp running in generic mode
config: sh-edosk7760_defconfig (https://download.01.org/0day-ci/archive/20231213/202312130625.4PfR5846-lkp@intel.com/config)
compiler: sh4-linux-gcc (GCC) 13.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20231213/202312130625.4PfR5846-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202312130625.4PfR5846-lkp@intel.com/
All errors (new ones prefixed by >>):
sh4-linux-ld: net/core/dev.o: in function `netif_skb_segment_for_xdp':
>> net/core/dev.c:5003:(.text+0x2860): undefined reference to `page_pool_alloc_frag'
>> sh4-linux-ld: net/core/dev.c:5003:(.text+0x28fc): undefined reference to `page_pool_alloc_pages'
sh4-linux-ld: net/core/dev.o: in function `net_dev_init':
net/core/dev.c:11882:(.init.text+0x198): undefined reference to `page_pool_create'
vim +5003 net/core/dev.c
4932
4933 static int netif_skb_segment_for_xdp(struct sk_buff **pskb)
4934 {
4935 struct softnet_data *sd = this_cpu_ptr(&softnet_data);
4936 u32 size, truesize, len, max_head_size, off;
4937 struct sk_buff *skb = *pskb, *nskb;
4938 int err, i, head_off;
4939 void *data;
4940
4941 max_head_size = SKB_WITH_OVERHEAD(PAGE_SIZE - XDP_PACKET_HEADROOM);
4942 if (skb->len > max_head_size + MAX_SKB_FRAGS * PAGE_SIZE)
4943 return -ENOMEM;
4944
4945 size = min_t(u32, skb->len, max_head_size);
4946 truesize = SKB_HEAD_ALIGN(size) + XDP_PACKET_HEADROOM;
4947 data = page_pool_dev_alloc_va(sd->page_pool, &truesize);
4948 if (!data)
4949 return -ENOMEM;
4950
4951 nskb = napi_build_skb(data, truesize);
4952 if (!nskb) {
4953 page_pool_free_va(sd->page_pool, data, true);
4954 return -ENOMEM;
4955 }
4956
4957 skb_reserve(nskb, XDP_PACKET_HEADROOM);
4958 skb_copy_header(nskb, skb);
4959 skb_mark_for_recycle(nskb);
4960
4961 err = skb_copy_bits(skb, 0, nskb->data, size);
4962 if (err) {
4963 consume_skb(nskb);
4964 return err;
4965 }
4966 skb_put(nskb, size);
4967
4968 head_off = skb_headroom(nskb) - skb_headroom(skb);
4969 skb_headers_offset_update(nskb, head_off);
4970
4971 off = size;
4972 len = skb->len - off;
4973 for (i = 0; i < MAX_SKB_FRAGS && off < skb->len; i++) {
4974 struct page *page;
4975 u32 page_off;
4976
4977 size = min_t(u32, len, PAGE_SIZE);
4978 truesize = size;
4979
4980 page = page_pool_dev_alloc(sd->page_pool, &page_off,
4981 &truesize);
4982 if (!data) {
4983 consume_skb(nskb);
4984 return -ENOMEM;
4985 }
4986
4987 skb_add_rx_frag(nskb, i, page, page_off, size, truesize);
4988 err = skb_copy_bits(skb, off, page_address(page) + page_off,
4989 size);
4990 if (err) {
4991 consume_skb(nskb);
4992 return err;
4993 }
4994
4995 len -= size;
4996 off += size;
4997 }
4998
4999 consume_skb(skb);
5000 *pskb = nskb;
5001
5002 return 0;
> 5003 }
5004
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply [flat|nested] 7+ messages in thread