[Patch net v2] kcm: close race conditions on sk_receive_queue

[Patch net v2] kcm: close race conditions on sk_receive_queue

[Cong Wang]

From: Cong Wang <cong.wang@bytedance.com>

sk->sk_receive_queue is protected by skb queue lock, but for KCM
sockets its RX path takes mux->rx_lock to protect more than just
skb queue. However, kcm_recvmsg() still only grabs the skb queue
lock, so race conditions still exist.

We can teach kcm_recvmsg() to grab mux->rx_lock too but this would
introduce a potential performance regression as struct kcm_mux can
be shared by multiple KCM sockets. So we have to enforce skb queue
lock in requeue_rx_msgs() and handle skb peek case carefully in
kcm_wait_data(). Fortunately, skb_recv_datagram() already handles
it nicely and is widely used by other sockets, we can just switch
to skb_recv_datagram() after getting rid of the unnecessary sock
lock in kcm_recvmsg() and kcm_splice_read().

I ran the original syzbot reproducer for 30 min without seeing any
issue.

Fixes: ab7ac4eb9832 ("kcm: Kernel Connection Multiplexor module")
Reported-by: syzbot+278279efdd2730dd14bf@syzkaller.appspotmail.com
Reported-by: shaozhengchao <shaozhengchao@huawei.com>
Cc: Paolo Abeni <pabeni@redhat.com>
Cc: Tom Herbert <tom@herbertland.com>
Signed-off-by: Cong Wang <cong.wang@bytedance.com>
---
 net/kcm/kcmsock.c | 58 +++++------------------------------------------
 1 file changed, 6 insertions(+), 52 deletions(-)

diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c
index a5004228111d..890a2423f559 100644
--- a/net/kcm/kcmsock.c
+++ b/net/kcm/kcmsock.c
@@ -222,7 +222,7 @@ static void requeue_rx_msgs(struct kcm_mux *mux, struct sk_buff_head *head)
 	struct sk_buff *skb;
 	struct kcm_sock *kcm;
 
-	while ((skb = __skb_dequeue(head))) {
+	while ((skb = skb_dequeue(head))) {
 		/* Reset destructor to avoid calling kcm_rcv_ready */
 		skb->destructor = sock_rfree;
 		skb_orphan(skb);
@@ -1085,53 +1085,17 @@ static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
 	return err;
 }
 
-static struct sk_buff *kcm_wait_data(struct sock *sk, int flags,
-				     long timeo, int *err)
-{
-	struct sk_buff *skb;
-
-	while (!(skb = skb_peek(&sk->sk_receive_queue))) {
-		if (sk->sk_err) {
-			*err = sock_error(sk);
-			return NULL;
-		}
-
-		if (sock_flag(sk, SOCK_DONE))
-			return NULL;
-
-		if ((flags & MSG_DONTWAIT) || !timeo) {
-			*err = -EAGAIN;
-			return NULL;
-		}
-
-		sk_wait_data(sk, &timeo, NULL);
-
-		/* Handle signals */
-		if (signal_pending(current)) {
-			*err = sock_intr_errno(timeo);
-			return NULL;
-		}
-	}
-
-	return skb;
-}
-
 static int kcm_recvmsg(struct socket *sock, struct msghdr *msg,
 		       size_t len, int flags)
 {
 	struct sock *sk = sock->sk;
 	struct kcm_sock *kcm = kcm_sk(sk);
 	int err = 0;
-	long timeo;
 	struct strp_msg *stm;
 	int copied = 0;
 	struct sk_buff *skb;
 
-	timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
-
-	lock_sock(sk);
-
-	skb = kcm_wait_data(sk, flags, timeo, &err);
+	skb = skb_recv_datagram(sk, flags, &err);
 	if (!skb)
 		goto out;
 
@@ -1162,14 +1126,11 @@ static int kcm_recvmsg(struct socket *sock, struct msghdr *msg,
 			/* Finished with message */
 			msg->msg_flags |= MSG_EOR;
 			KCM_STATS_INCR(kcm->stats.rx_msgs);
-			skb_unlink(skb, &sk->sk_receive_queue);
-			kfree_skb(skb);
 		}
 	}
 
 out:
-	release_sock(sk);
-
+	skb_free_datagram(sk, skb);
 	return copied ? : err;
 }
 
@@ -1179,7 +1140,6 @@ static ssize_t kcm_splice_read(struct socket *sock, loff_t *ppos,
 {
 	struct sock *sk = sock->sk;
 	struct kcm_sock *kcm = kcm_sk(sk);
-	long timeo;
 	struct strp_msg *stm;
 	int err = 0;
 	ssize_t copied;
@@ -1187,11 +1147,7 @@ static ssize_t kcm_splice_read(struct socket *sock, loff_t *ppos,
 
 	/* Only support splice for SOCKSEQPACKET */
 
-	timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
-
-	lock_sock(sk);
-
-	skb = kcm_wait_data(sk, flags, timeo, &err);
+	skb = skb_recv_datagram(sk, flags, &err);
 	if (!skb)
 		goto err_out;
 
@@ -1219,13 +1175,11 @@ static ssize_t kcm_splice_read(struct socket *sock, loff_t *ppos,
 	 * finish reading the message.
 	 */
 
-	release_sock(sk);
-
+	skb_free_datagram(sk, skb);
 	return copied;
 
 err_out:
-	release_sock(sk);
-
+	skb_free_datagram(sk, skb);
 	return err;
 }
 
-- 
2.34.1

Re: [Patch net v2] kcm: close race conditions on sk_receive_queue

[Paolo Abeni]

Hello,
On Thu, 2022-11-03 at 11:46 -0700, Cong Wang wrote:
> @@ -1085,53 +1085,17 @@ static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
>  	return err;
>  }
>  
> -static struct sk_buff *kcm_wait_data(struct sock *sk, int flags,
> -				     long timeo, int *err)
> -{
> -	struct sk_buff *skb;
> -
> -	while (!(skb = skb_peek(&sk->sk_receive_queue))) {
> -		if (sk->sk_err) {
> -			*err = sock_error(sk);
> -			return NULL;
> -		}
> -
> -		if (sock_flag(sk, SOCK_DONE))
> -			return NULL;

It looks like skb_recv_datagram() ignores the SOCK_DONE flag, so this
change could potentially miss some wait_data end coditions. On the flip
side I don't see any place where the SOCK_DONE flag is set for the kcm
socket, so this should be safe, but could you please document this in
the commit message?

[...]

> @@ -1187,11 +1147,7 @@ static ssize_t kcm_splice_read(struct socket *sock, loff_t *ppos,
>  
>  	/* Only support splice for SOCKSEQPACKET */
>  
> -	timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
> -
> -	lock_sock(sk);
> -
> -	skb = kcm_wait_data(sk, flags, timeo, &err);
> +	skb = skb_recv_datagram(sk, flags, &err);
>  	if (!skb)
>  		goto err_out;
>  
> @@ -1219,13 +1175,11 @@ static ssize_t kcm_splice_read(struct socket *sock, loff_t *ppos,
>  	 * finish reading the message.
>  	 */
>  
> -	release_sock(sk);
> -
> +	skb_free_datagram(sk, skb);
>  	return copied;
>  
>  err_out:
> -	release_sock(sk);
> -
> +	skb_free_datagram(sk, skb);

We can reach here with skb == NULL and skb_free_datagram() ->
__kfree_skb() -> skb_release_all() does not deal correctly with NULL
skb, you need to check for skb explicitly here (or rearrange the error
paths in a suitable way).

Thanks!

Paolo

Re: [Patch net v2] kcm: close race conditions on sk_receive_queue

[Cong Wang]

On Tue, Nov 08, 2022 at 10:13:23AM +0100, Paolo Abeni wrote:
> Hello,
> On Thu, 2022-11-03 at 11:46 -0700, Cong Wang wrote:
> > @@ -1085,53 +1085,17 @@ static int kcm_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
> >  	return err;
> >  }
> >  
> > -static struct sk_buff *kcm_wait_data(struct sock *sk, int flags,
> > -				     long timeo, int *err)
> > -{
> > -	struct sk_buff *skb;
> > -
> > -	while (!(skb = skb_peek(&sk->sk_receive_queue))) {
> > -		if (sk->sk_err) {
> > -			*err = sock_error(sk);
> > -			return NULL;
> > -		}
> > -
> > -		if (sock_flag(sk, SOCK_DONE))
> > -			return NULL;
> 
> It looks like skb_recv_datagram() ignores the SOCK_DONE flag, so this
> change could potentially miss some wait_data end coditions. On the flip
> side I don't see any place where the SOCK_DONE flag is set for the kcm
> socket, so this should be safe, but could you please document this in
> the commit message?

Good catch. Indeed, this flags seems only used by stream sockets, I will
include this in the commit message.

> 
> [...]
> 
> > @@ -1187,11 +1147,7 @@ static ssize_t kcm_splice_read(struct socket *sock, loff_t *ppos,
> >  
> >  	/* Only support splice for SOCKSEQPACKET */
> >  
> > -	timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
> > -
> > -	lock_sock(sk);
> > -
> > -	skb = kcm_wait_data(sk, flags, timeo, &err);
> > +	skb = skb_recv_datagram(sk, flags, &err);
> >  	if (!skb)
> >  		goto err_out;
> >  
> > @@ -1219,13 +1175,11 @@ static ssize_t kcm_splice_read(struct socket *sock, loff_t *ppos,
> >  	 * finish reading the message.
> >  	 */
> >  
> > -	release_sock(sk);
> > -
> > +	skb_free_datagram(sk, skb);
> >  	return copied;
> >  
> >  err_out:
> > -	release_sock(sk);
> > -
> > +	skb_free_datagram(sk, skb);
> 
> We can reach here with skb == NULL and skb_free_datagram() ->
> __kfree_skb() -> skb_release_all() does not deal correctly with NULL
> skb, you need to check for skb explicitly here (or rearrange the error
> paths in a suitable way).
> 

Are you sure? skb_free_datagram() is just consume_skb() which is guarded
by skb_unref() which takes NULL and returns false.


1195 static inline bool skb_unref(struct sk_buff *skb)
1196 {
1197         if (unlikely(!skb))
1198                 return false;

1027 void consume_skb(struct sk_buff *skb)
1028 {
1029         if (!skb_unref(skb))
1030                 return;

320 void skb_free_datagram(struct sock *sk, struct sk_buff *skb)
321 {
322         consume_skb(skb);
323 }


Thanks.