[PATCH RFC net-next v4 14/14] xsk: optimize xsk_build_skb for batch copy-mode fast path

Jason Xing
2026-04-15T08:28:37Z

Three targeted optimizations for the batch copy-mode TX hot path:

Replace skb_store_bits() with memcpy() for single-buffer first-desc
path. After skb_reserve() + skb_put(), the SKB is freshly allocated
with all data in the linear area and no frags, so skb_store_bits()
degenerates to memcpy(skb->data, buffer, len) but carries unnecessary
function call overhead, offset validation, and frag iteration logic.

Inline UMEM address computation in Phase 3 and pass the pre-computed
buffer pointer to xsk_build_skb(), avoiding the per-packet non-inlined
xp_raw_get_data() (EXPORT_SYMBOL) call chain:
xsk_buff_raw_get_data -> xp_raw_get_data -> __xp_raw_get_addr +
__xp_raw_get_data.
In the batch loop the pool->addrs and pool->unaligned are invariant,
so we cache them once and compute each buffer address inline.

Prefetch the *next* descriptor's UMEM data buffer at the top of the
Phase 3 loop, hiding the memory latency of the upcoming memcpy.

It improves 3-4% performance stably.

Signed-off-by: Jason Xing <kernelxing@tencent.com>
---
include/net/xdp_sock.h | 3 ++-
net/core/skbuff.c | 18 ++++++++++++++++--
net/xdp/xsk.c | 15 ++++++---------
3 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
index 0609e3b04279..5e05236c7fba 100644
--- a/include/net/xdp_sock.h
+++ b/include/net/xdp_sock.h
@@ -139,7 +139,8 @@ void __xsk_map_flush(struct list_head *flush_list);
INDIRECT_CALLABLE_DECLARE(void xsk_destruct_skb(struct sk_buff *));
struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
struct sk_buff *allocated_skb,
- struct xdp_desc *desc);
+ struct xdp_desc *desc,
+ void *buffer);
int xsk_alloc_batch_skb(struct xdp_sock *xs, u32 nb_pkts, u32 nb_descs, int *err);
int xsk_direct_xmit_batch(struct xdp_sock *xs, struct net_device *dev);

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 5726b1566b2b..bef5270e6332 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -752,14 +752,28 @@ int xsk_alloc_batch_skb(struct xdp_sock *xs, u32 nb_pkts, u32 nb_descs, int *err
if (total_truesize)
refcount_add(total_truesize, &xs->sk.sk_wmem_alloc);

- /* Phase 3: Build SKBs with packet data */
+ /* Phase 3: Build SKBs with packet data. */
+ struct xsk_buff_pool *pool = xs->pool;
+ void *pool_addrs = pool->addrs;
+ bool unaligned = pool->unaligned;
+
for (j = 0; j < alloc_descs; j++) {
+ u64 addr = descs[j].addr;
+ void *buffer;
+
+ if (unaligned)
+ addr = xp_unaligned_add_offset_to_addr(addr);
+ buffer = pool_addrs + addr;
+
+ if (j + 1 < alloc_descs)
+ prefetch(pool_addrs + descs[j + 1].addr);
+
if (!xs->skb) {
skb = skbs[skb_count - 1 - k];
k++;
}

- skb = xsk_build_skb(xs, skb, &descs[j]);
+ skb = xsk_build_skb(xs, skb, &descs[j], buffer);
if (IS_ERR(skb)) {
*err = PTR_ERR(skb);
break;
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index be341290e42c..3bf81b838075 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -811,7 +811,8 @@ static struct sk_buff *xsk_build_skb_zerocopy(struct xdp_sock *xs,

struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
struct sk_buff *allocated_skb,
- struct xdp_desc *desc)
+ struct xdp_desc *desc,
+ void *buffer)
{
struct net_device *dev = xs->dev;
struct sk_buff *skb = xs->skb;
@@ -825,11 +826,10 @@ struct sk_buff *xsk_build_skb(struct xdp_sock *xs,
goto free_err;
}
} else {
- u32 hr, tr, len;
- void *buffer;
+ u32 hr, tr, len = desc->len;

- buffer = xsk_buff_raw_get_data(xs->pool, desc->addr);
- len = desc->len;
+ if (!buffer)
+ buffer = xsk_buff_raw_get_data(xs->pool, desc->addr);

if (!skb) {
hr = max(NET_SKB_PAD, L1_CACHE_ALIGN(dev->needed_headroom));
@@ -844,10 +844,7 @@ struct sk_buff *xsk_build_skb(struct xdp_sock *xs,

skb_reserve(skb, hr);
skb_put(skb, len);
-
- err = skb_store_bits(skb, 0, buffer, len);
- if (unlikely(err))
- goto free_err;
+ memcpy(skb->data, buffer, len);

xsk_skb_init_misc(skb, xs, desc->addr);
if (desc->options & XDP_TX_METADATA) {
--
2.41.3