[PATCH v2] skbuff: Improve the sending efficiency of __skb_send_sock

[PATCH v2] skbuff: Improve the sending efficiency of __skb_send_sock

[Feng Yang]

From: Feng Yang <yangfeng@kylinos.cn>

By aggregating skb data into a bvec array for transmission, when using sockmap to forward large packets,
what previously required multiple transmissions now only needs a single transmission, which significantly enhances performance.
For small packets, the performance remains comparable to the original level.

When using sockmap for forwarding, the average latency for different packet sizes
after sending 10,000 packets is as follows:
size	old(us)		new(us)
512	56		55
1472	58		58
1600	106		79
3000	145		108
5000	182		123

Signed-off-by: Feng Yang <yangfeng@kylinos.cn>
---
Changes in v2:
- Delete dynamic memory allocation, thanks: Paolo Abeni,Stanislav Fomichev.
- Link to v1: https://lore.kernel.org/all/20250623084212.122284-1-yangfeng59949@163.com/
---
 net/core/skbuff.c | 145 ++++++++++++++++++++++------------------------
 1 file changed, 68 insertions(+), 77 deletions(-)

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 85fc82f72d26..aae5139cfb28 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -3231,104 +3231,95 @@ static int sendmsg_unlocked(struct sock *sk, struct msghdr *msg)
 	return sock_sendmsg(sock, msg);
 }
 
+#define MAX_SKB_SEND_BIOVEC_SIZE	16
 typedef int (*sendmsg_func)(struct sock *sk, struct msghdr *msg);
 static int __skb_send_sock(struct sock *sk, struct sk_buff *skb, int offset,
 			   int len, sendmsg_func sendmsg, int flags)
 {
-	unsigned int orig_len = len;
 	struct sk_buff *head = skb;
 	unsigned short fragidx;
-	int slen, ret;
-
-do_frag_list:
-
-	/* Deal with head data */
-	while (offset < skb_headlen(skb) && len) {
-		struct kvec kv;
-		struct msghdr msg;
-
-		slen = min_t(int, len, skb_headlen(skb) - offset);
-		kv.iov_base = skb->data + offset;
-		kv.iov_len = slen;
-		memset(&msg, 0, sizeof(msg));
-		msg.msg_flags = MSG_DONTWAIT | flags;
-
-		iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &kv, 1, slen);
-		ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
-				      sendmsg_unlocked, sk, &msg);
-		if (ret <= 0)
-			goto error;
-
-		offset += ret;
-		len -= ret;
-	}
-
-	/* All the data was skb head? */
-	if (!len)
-		goto out;
+	struct msghdr msg;
+	struct bio_vec bvec[MAX_SKB_SEND_BIOVEC_SIZE];
+	int ret, slen, total_len = 0;
+	int bvec_count = 0;
+	unsigned int copied = 0;
+
+	memset(&msg, 0, sizeof(msg));
+	msg.msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT | flags;
+
+	while (copied < len) {
+		/* Deal with head data */
+		if (offset < skb_headlen(skb) && bvec_count < MAX_SKB_SEND_BIOVEC_SIZE) {
+			struct page *page = virt_to_page(skb->data + offset);
+			unsigned int page_offset = offset_in_page(skb->data + offset);
+
+			if (!sendpage_ok(page))
+				msg.msg_flags &= ~MSG_SPLICE_PAGES;
+
+			slen = min_t(int, skb_headlen(skb) - offset, len - copied);
+			bvec_set_page(&bvec[bvec_count++], page, slen, page_offset);
+			copied += slen;
+			offset += slen;
+		}
 
-	/* Make offset relative to start of frags */
-	offset -= skb_headlen(skb);
+		/* Make offset relative to start of frags */
+		offset -= skb_headlen(skb);
 
-	/* Find where we are in frag list */
-	for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
-		skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
+		if (copied < len && bvec_count < MAX_SKB_SEND_BIOVEC_SIZE) {
+			for (fragidx = 0; fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
+				skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
+				unsigned int frag_size = skb_frag_size(frag);
 
-		if (offset < skb_frag_size(frag))
-			break;
+				/* Find where we are in frag list */
+				if (offset >= frag_size) {
+					offset -= frag_size;
+					continue;
+				}
 
-		offset -= skb_frag_size(frag);
-	}
+				slen = min_t(size_t, frag_size - offset, len - copied);
+				bvec_set_page(&bvec[bvec_count++], skb_frag_page(frag), slen,
+					      skb_frag_off(frag) + offset);
 
-	for (; len && fragidx < skb_shinfo(skb)->nr_frags; fragidx++) {
-		skb_frag_t *frag  = &skb_shinfo(skb)->frags[fragidx];
+				copied += slen;
+				offset = 0;
 
-		slen = min_t(size_t, len, skb_frag_size(frag) - offset);
+				if (copied >= len || bvec_count >= MAX_SKB_SEND_BIOVEC_SIZE)
+					break;
+			}
+		}
 
-		while (slen) {
-			struct bio_vec bvec;
-			struct msghdr msg = {
-				.msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT |
-					     flags,
-			};
+		if (copied < len && bvec_count < MAX_SKB_SEND_BIOVEC_SIZE) {
+			/* Process any frag lists */
+			if (skb == head) {
+				if (skb_has_frag_list(skb))
+					skb = skb_shinfo(skb)->frag_list;
+			} else if (skb->next) {
+				skb = skb->next;
+			}
+		}
 
-			bvec_set_page(&bvec, skb_frag_page(frag), slen,
-				      skb_frag_off(frag) + offset);
-			iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1,
-				      slen);
+		if (bvec_count == MAX_SKB_SEND_BIOVEC_SIZE || copied == len) {
+			iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, bvec, bvec_count, len);
+			ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked, sendmsg_unlocked, sk, &msg);
 
-			ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
-					      sendmsg_unlocked, sk, &msg);
-			if (ret <= 0)
-				goto error;
+			if (ret < 0)
+				return ret;
 
+			/* Statistical data */
 			len -= ret;
 			offset += ret;
-			slen -= ret;
+			total_len += ret;
+
+			/* Restore initial value */
+			memset(&msg, 0, sizeof(msg));
+			msg.msg_flags = MSG_SPLICE_PAGES | MSG_DONTWAIT | flags;
+			copied = 0;
+			bvec_count = 0;
+			skb = head;
 		}
-
-		offset = 0;
 	}
 
-	if (len) {
-		/* Process any frag lists */
-
-		if (skb == head) {
-			if (skb_has_frag_list(skb)) {
-				skb = skb_shinfo(skb)->frag_list;
-				goto do_frag_list;
-			}
-		} else if (skb->next) {
-			skb = skb->next;
-			goto do_frag_list;
-		}
-	}
-
-out:
-	return orig_len - len;
-
-error:
-	return orig_len == len ? ret : orig_len - len;
+	return total_len;
 }
 
 /* Send skb data on a socket. Socket must be locked. */
-- 
2.43.0

Re: [PATCH v2] skbuff: Improve the sending efficiency of __skb_send_sock

[Eric Dumazet]

On Fri, Jun 27, 2025 at 2:44 AM Feng Yang <yangfeng59949@163.com> wrote:
>
> From: Feng Yang <yangfeng@kylinos.cn>
>
> By aggregating skb data into a bvec array for transmission, when using sockmap to forward large packets,
> what previously required multiple transmissions now only needs a single transmission, which significantly enhances performance.
> For small packets, the performance remains comparable to the original level.
>
> When using sockmap for forwarding, the average latency for different packet sizes
> after sending 10,000 packets is as follows:
> size    old(us)         new(us)
> 512     56              55
> 1472    58              58
> 1600    106             79
> 3000    145             108
> 5000    182             123
>
> Signed-off-by: Feng Yang <yangfeng@kylinos.cn>

Instead of changing everything, have you tried strategically adding
MSG_MORE in this function ?

Re: [PATCH v2] skbuff: Improve the sending efficiency of __skb_send_sock

[Eric Dumazet]

On Fri, Jun 27, 2025 at 3:19 AM Eric Dumazet <edumazet@google.com> wrote:
>
> On Fri, Jun 27, 2025 at 2:44 AM Feng Yang <yangfeng59949@163.com> wrote:
> >
> > From: Feng Yang <yangfeng@kylinos.cn>
> >
> > By aggregating skb data into a bvec array for transmission, when using sockmap to forward large packets,
> > what previously required multiple transmissions now only needs a single transmission, which significantly enhances performance.
> > For small packets, the performance remains comparable to the original level.
> >
> > When using sockmap for forwarding, the average latency for different packet sizes
> > after sending 10,000 packets is as follows:
> > size    old(us)         new(us)
> > 512     56              55
> > 1472    58              58
> > 1600    106             79
> > 3000    145             108
> > 5000    182             123
> >
> > Signed-off-by: Feng Yang <yangfeng@kylinos.cn>
>
> Instead of changing everything, have you tried strategically adding
> MSG_MORE in this function ?

Untested patch:

diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index d6420b74ea9c6a9c53a7c16634cce82a1cd1bbd3..b0f5e8898fdf450129948d829240b570f3cbf9eb
100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -3252,6 +3252,8 @@ static int __skb_send_sock(struct sock *sk,
struct sk_buff *skb, int offset,
                kv.iov_len = slen;
                memset(&msg, 0, sizeof(msg));
                msg.msg_flags = MSG_DONTWAIT | flags;
+               if (slen < len)
+                       msg.msg_flags |= MSG_MORE;

                iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &kv, 1, slen);
                ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
@@ -3292,6 +3294,8 @@ static int __skb_send_sock(struct sock *sk,
struct sk_buff *skb, int offset,
                                             flags,
                        };

+                       if (slen < len)
+                               msg.msg_flags |= MSG_MORE;
                        bvec_set_page(&bvec, skb_frag_page(frag), slen,
                                      skb_frag_off(frag) + offset);
                        iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1,

Re: [PATCH v2] skbuff: Improve the sending efficiency of __skb_send_sock

[David Laight]

On Fri, 27 Jun 2025 03:19:27 -0700
Eric Dumazet <edumazet@google.com> wrote:

> On Fri, Jun 27, 2025 at 2:44 AM Feng Yang <yangfeng59949@163.com> wrote:
> >
> > From: Feng Yang <yangfeng@kylinos.cn>
> >
> > By aggregating skb data into a bvec array for transmission, when using sockmap to forward large packets,
> > what previously required multiple transmissions now only needs a single transmission, which significantly enhances performance.
> > For small packets, the performance remains comparable to the original level.
> >
> > When using sockmap for forwarding, the average latency for different packet sizes
> > after sending 10,000 packets is as follows:
> > size    old(us)         new(us)
> > 512     56              55
> > 1472    58              58
> > 1600    106             79
> > 3000    145             108
> > 5000    182             123
> >
> > Signed-off-by: Feng Yang <yangfeng@kylinos.cn>  
> 
> Instead of changing everything, have you tried strategically adding
> MSG_MORE in this function ?
> 

Does (could) this code ever be used for protocols other than TCP?
For UDP setting MSG_MORE will generate a single datagram.
For SCTP all the data actually has to be sent as a single sendmsg()
in order to generate a single DATA chunk.

Prior to 6.5 the code used sock->ops->sendpage_locked() so had to do
a separate call per page.
But if all the overheads of 'ioc_iter' are being added it does seen
appropriate to make use of its features!

	David
 

Re: [PATCH v2] skbuff: Improve the sending efficiency of __skb_send_sock

[Feng Yang]

On Fri, 27 Jun 2025 03:23:24 -0700 Eric Dumazet <edumazet@google.com> wrote:

> On Fri, Jun 27, 2025 at 3:19 AM Eric Dumazet <edumazet@google.com> wrote:
> >
> > On Fri, Jun 27, 2025 at 2:44 AM Feng Yang <yangfeng59949@163.com> wrote:
> > >
> > > From: Feng Yang <yangfeng@kylinos.cn>
> > >
> > > By aggregating skb data into a bvec array for transmission, when using sockmap to forward large packets,
> > > what previously required multiple transmissions now only needs a single transmission, which significantly enhances performance.
> > > For small packets, the performance remains comparable to the original level.
> > >
> > > When using sockmap for forwarding, the average latency for different packet sizes
> > > after sending 10,000 packets is as follows:
> > > size    old(us)         new(us)
> > > 512     56              55
> > > 1472    58              58
> > > 1600    106             79
> > > 3000    145             108
> > > 5000    182             123
> > >
> > > Signed-off-by: Feng Yang <yangfeng@kylinos.cn>
> >
> > Instead of changing everything, have you tried strategically adding
> > MSG_MORE in this function ?
> 
> Untested patch:
> 
> diff --git a/net/core/skbuff.c b/net/core/skbuff.c
> index d6420b74ea9c6a9c53a7c16634cce82a1cd1bbd3..b0f5e8898fdf450129948d829240b570f3cbf9eb
> 100644
> --- a/net/core/skbuff.c
> +++ b/net/core/skbuff.c
> @@ -3252,6 +3252,8 @@ static int __skb_send_sock(struct sock *sk,
> struct sk_buff *skb, int offset,
>                 kv.iov_len = slen;
>                 memset(&msg, 0, sizeof(msg));
>                 msg.msg_flags = MSG_DONTWAIT | flags;
> +               if (slen < len)
> +                       msg.msg_flags |= MSG_MORE;
> 
>                 iov_iter_kvec(&msg.msg_iter, ITER_SOURCE, &kv, 1, slen);
>                 ret = INDIRECT_CALL_2(sendmsg, sendmsg_locked,
> @@ -3292,6 +3294,8 @@ static int __skb_send_sock(struct sock *sk,
> struct sk_buff *skb, int offset,
>                                              flags,
>                         };
> 
> +                       if (slen < len)
> +                               msg.msg_flags |= MSG_MORE;
>                         bvec_set_page(&bvec, skb_frag_page(frag), slen,
>                                       skb_frag_off(frag) + offset);
>                         iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1,

After testing, there is a performance improvement for large packets in both TCP and UDP.
Thanks.