[PATCH 0/2] net: Use synchronous wakeups selectively

[PATCH 0/2] net: Use synchronous wakeups selectively

[Srikar Dronamraju]

The scheduler assumes in several wakeup paths that a task using WF_SYNC
is likely to yield the CPU shortly. However several networking wakeup
paths unconditionally use synchronous wakeups even when the waking task
continues execution.

During wakeup, with WF_SYNC flag set, because of the assumption that
current thread is ready to give up, the wakee thread will be migrated
from any core within the chip to the current LLC. If these operations
are frequent, and wakers are actually not going away, then it will lead
to load imbalance and hurt performance. This is especially true in
architectures where LLCs are small and number of LLCs per chip are more.

Running vllm workload was run on Power10 system
                                  No patch      with patch %diff
Inference Time (sec)              17.84         16.35     -8.35%
llm query bandwidth (tokens/sec)  14.78         16.21     +9.68%

Lower inference time and higher tokens/sec is better.

Srikar Dronamraju (2):
  net/socket: Record preference for synchronous wakeups
  net/sock: Propagate WF_SYNC only when requested

 include/net/sock.h |  1 +
 net/core/sock.c    | 31 +++++++++++++++++++------
 net/sctp/socket.c  | 10 +++++++--
 net/smc/af_smc.c   |  4 ++--
 net/smc/smc_rx.c   | 10 +++++++--
 net/socket.c       | 56 +++++++++++++++++++++++++++++++++++++++-------
 net/tipc/socket.c  | 22 +++++++++++++-----
 net/unix/af_unix.c | 26 ++++++++++++++-------
 8 files changed, 126 insertions(+), 34 deletions(-)

-- 
2.51.0

[PATCH 1/2] net/socket: Record preference for synchronous wakeups

[Srikar Dronamraju]

Scheduler differentiates between affine and non-affine wakeups by the
way of sync flags. Scheduler prefers to pull the tasks towards the waker
if the sync flag is set.

In some cases, socket APIs are blindly requesting sync wakeups. This may
cause load-balance issues and non-optimal performance.

Record whether the most recent blocking socket operation could benefit
from synchronous wakeups. Subsequent readiness notifications use this
hint to determine whether WF_SYNC should be propagated.

The flag is advisory and affects only wakeup placement decisions.

Signed-off-by: Srikar Dronamraju <srikar@linux.ibm.com>
---
 include/net/sock.h |  1 +
 net/socket.c       | 56 +++++++++++++++++++++++++++++++++++++++-------
 2 files changed, 49 insertions(+), 8 deletions(-)

diff --git a/include/net/sock.h b/include/net/sock.h
index 51185222aac2..acc6b1976dc4 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -1022,6 +1022,7 @@ enum sock_flags {
 	SOCK_RCVMARK, /* Receive SO_MARK  ancillary data with packet */
 	SOCK_RCVPRIORITY, /* Receive SO_PRIORITY ancillary data with packet */
 	SOCK_TIMESTAMPING_ANY, /* Copy of sk_tsflags & TSFLAGS_ANY */
+	SOCK_SYNC_WAKEUP, /* Prefer synchronous socket wakeups */
 };
 
 #define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
diff --git a/net/socket.c b/net/socket.c
index 63c69a0fa74e..0bcb57ae490e 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -1198,15 +1198,27 @@ static void sock_splice_eof(struct file *file)
 		ops->splice_eof(sock);
 }
 
+static inline void sock_update_sync_wakeup(struct sock *sk, bool nonblock)
+{
+	if (unlikely(!sk))
+		return;
+
+	if (nonblock) {
+		if (sock_flag(sk, SOCK_SYNC_WAKEUP))
+			sock_reset_flag(sk, SOCK_SYNC_WAKEUP);
+	} else {
+		if (!sock_flag(sk, SOCK_SYNC_WAKEUP))
+			sock_set_flag(sk, SOCK_SYNC_WAKEUP);
+	}
+}
+
 static ssize_t sock_read_iter(struct kiocb *iocb, struct iov_iter *to)
 {
 	struct file *file = iocb->ki_filp;
 	struct socket *sock = file->private_data;
 	struct msghdr msg = {.msg_iter = *to};
 	ssize_t res;
-
-	if (file->f_flags & O_NONBLOCK || (iocb->ki_flags & IOCB_NOWAIT))
-		msg.msg_flags = MSG_DONTWAIT;
+	bool nonblock;
 
 	if (iocb->ki_pos != 0)
 		return -ESPIPE;
@@ -1214,6 +1226,11 @@ static ssize_t sock_read_iter(struct kiocb *iocb, struct iov_iter *to)
 	if (!iov_iter_count(to))	/* Match SYS5 behaviour */
 		return 0;
 
+	nonblock = (file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT);
+	if (nonblock)
+		msg.msg_flags = MSG_DONTWAIT;
+
+	sock_update_sync_wakeup(sock->sk, nonblock);
 	res = sock_recvmsg(sock, &msg, msg.msg_flags);
 	*to = msg.msg_iter;
 	return res;
@@ -1225,13 +1242,17 @@ static ssize_t sock_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	struct socket *sock = file->private_data;
 	struct msghdr msg = {.msg_iter = *from};
 	ssize_t res;
+	bool nonblock;
 
 	if (iocb->ki_pos != 0)
 		return -ESPIPE;
 
-	if (file->f_flags & O_NONBLOCK || (iocb->ki_flags & IOCB_NOWAIT))
+	nonblock = (file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT);
+	if (nonblock)
 		msg.msg_flags = MSG_DONTWAIT;
 
+	sock_update_sync_wakeup(sock->sk, nonblock);
+
 	if (sock->type == SOCK_SEQPACKET)
 		msg.msg_flags |= MSG_EOR;
 
@@ -2221,6 +2242,7 @@ int __sys_sendto(int fd, void __user *buff, size_t len, unsigned int flags,
 	struct sockaddr_storage address;
 	int err;
 	struct msghdr msg;
+	bool nonblock;
 
 	err = import_ubuf(ITER_SOURCE, buff, len, &msg.msg_iter);
 	if (unlikely(err))
@@ -2246,8 +2268,11 @@ int __sys_sendto(int fd, void __user *buff, size_t len, unsigned int flags,
 		msg.msg_namelen = addr_len;
 	}
 	flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
-	if (sock->file->f_flags & O_NONBLOCK)
+	nonblock = (sock->file->f_flags & O_NONBLOCK);
+	if (nonblock)
 		flags |= MSG_DONTWAIT;
+
+	sock_update_sync_wakeup(sock->sk, nonblock);
 	msg.msg_flags = flags;
 	return __sock_sendmsg(sock, &msg);
 }
@@ -2284,6 +2309,7 @@ int __sys_recvfrom(int fd, void __user *ubuf, size_t size, unsigned int flags,
 	};
 	struct socket *sock;
 	int err, err2;
+	bool nonblock;
 
 	err = import_ubuf(ITER_DEST, ubuf, size, &msg.msg_iter);
 	if (unlikely(err))
@@ -2297,8 +2323,11 @@ int __sys_recvfrom(int fd, void __user *ubuf, size_t size, unsigned int flags,
 	if (unlikely(!sock))
 		return -ENOTSOCK;
 
-	if (sock->file->f_flags & O_NONBLOCK)
+	nonblock = (sock->file->f_flags & O_NONBLOCK);
+	if (nonblock)
 		flags |= MSG_DONTWAIT;
+
+	sock_update_sync_wakeup(sock->sk, nonblock);
 	err = sock_recvmsg(sock, &msg, flags);
 
 	if (err >= 0 && addr != NULL) {
@@ -2634,6 +2663,7 @@ static int ____sys_sendmsg(struct socket *sock, struct msghdr *msg_sys,
 	unsigned char *ctl_buf = ctl;
 	int ctl_len;
 	ssize_t err;
+	bool nonblock;
 
 	err = -ENOBUFS;
 
@@ -2666,8 +2696,12 @@ static int ____sys_sendmsg(struct socket *sock, struct msghdr *msg_sys,
 	flags &= ~MSG_INTERNAL_SENDMSG_FLAGS;
 	msg_sys->msg_flags = flags;
 
-	if (sock->file->f_flags & O_NONBLOCK)
+	nonblock = (sock->file->f_flags & O_NONBLOCK);
+	if (nonblock)
 		msg_sys->msg_flags |= MSG_DONTWAIT;
+
+	sock_update_sync_wakeup(sock->sk, nonblock);
+
 	/*
 	 * If this is sendmmsg() and current destination address is same as
 	 * previously succeeded address, omit asking LSM's decision.
@@ -2887,6 +2921,7 @@ static int ____sys_recvmsg(struct socket *sock, struct msghdr *msg_sys,
 	unsigned long cmsg_ptr;
 	int len;
 	ssize_t err;
+	bool nonblock;
 
 	msg_sys->msg_name = &addr;
 	cmsg_ptr = (unsigned long)msg_sys->msg_control;
@@ -2895,9 +2930,12 @@ static int ____sys_recvmsg(struct socket *sock, struct msghdr *msg_sys,
 	/* We assume all kernel code knows the size of sockaddr_storage */
 	msg_sys->msg_namelen = 0;
 
-	if (sock->file->f_flags & O_NONBLOCK)
+	nonblock = (sock->file->f_flags & O_NONBLOCK);
+	if (nonblock)
 		flags |= MSG_DONTWAIT;
 
+	sock_update_sync_wakeup(sock->sk, nonblock);
+
 	if (unlikely(nosec))
 		err = sock_recvmsg_nosec(sock, msg_sys, flags);
 	else
@@ -3056,6 +3094,8 @@ static int do_recvmmsg(int fd, struct mmsghdr __user *mmsg,
 		if (flags & MSG_WAITFORONE)
 			flags |= MSG_DONTWAIT;
 
+		sock_update_sync_wakeup(sock->sk, flags & MSG_WAITFORONE);
+
 		if (timeout) {
 			ktime_get_ts64(&timeout64);
 			*timeout = timespec64_sub(end_time, timeout64);
-- 
2.52.0

[PATCH 2/2] net/sock: Propagate WF_SYNC only when requested

[Srikar Dronamraju]

Use SOCK_SYNC_WAKEUP to select between synchronous and asynchronous wakeup
wakeup APIs. This avoids propagating WF_SYNC when no blocking waiter is
expected. All wakeup locations in networking code that currently issue
synchronous poll-style wakeups unconditionally are updated.

Signed-off-by: Srikar Dronamraju <srikar@linux.ibm.com>
---
 net/core/sock.c    | 31 ++++++++++++++++++++++++-------
 net/sctp/socket.c  | 10 ++++++++--
 net/smc/af_smc.c   |  4 ++--
 net/smc/smc_rx.c   | 10 ++++++++--
 net/tipc/socket.c  | 22 +++++++++++++++++-----
 net/unix/af_unix.c | 26 ++++++++++++++++++--------
 6 files changed, 77 insertions(+), 26 deletions(-)

diff --git a/net/core/sock.c b/net/core/sock.c
index 8a59bfaa8096..a214e883b14b 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -3652,9 +3652,15 @@ void sock_def_readable(struct sock *sk)
 
 	rcu_read_lock();
 	wq = rcu_dereference(sk->sk_wq);
-	if (skwq_has_sleeper(wq))
-		wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
+	if (skwq_has_sleeper(wq)) {
+		if (sock_flag(sk, SOCK_SYNC_WAKEUP)) {
+			wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
+						EPOLLRDNORM | EPOLLRDBAND);
+		} else {
+			wake_up_interruptible_poll(&wq->wait, EPOLLIN | EPOLLPRI |
 						EPOLLRDNORM | EPOLLRDBAND);
+		}
+	}
 	sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN);
 	rcu_read_unlock();
 }
@@ -3670,9 +3676,15 @@ static void sock_def_write_space(struct sock *sk)
 	 */
 	if (sock_writeable(sk)) {
 		wq = rcu_dereference(sk->sk_wq);
-		if (skwq_has_sleeper(wq))
-			wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
+		if (skwq_has_sleeper(wq)) {
+			if (sock_flag(sk, SOCK_SYNC_WAKEUP)) {
+				wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
+						EPOLLWRNORM | EPOLLWRBAND);
+			} else {
+				wake_up_interruptible_poll(&wq->wait, EPOLLOUT |
 						EPOLLWRNORM | EPOLLWRBAND);
+			}
+		}
 
 		/* Should agree with poll, otherwise some programs break */
 		sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
@@ -3695,10 +3707,15 @@ static void sock_def_write_space_wfree(struct sock *sk, int wmem_alloc)
 
 		/* rely on refcount_sub from sock_wfree() */
 		smp_mb__after_atomic();
-		if (wq && waitqueue_active(&wq->wait))
-			wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
+		if (wq && waitqueue_active(&wq->wait)) {
+			if (sock_flag(sk, SOCK_SYNC_WAKEUP)) {
+				wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
 						EPOLLWRNORM | EPOLLWRBAND);
-
+			} else {
+				wake_up_interruptible_poll(&wq->wait, EPOLLOUT |
+						EPOLLWRNORM | EPOLLWRBAND);
+			}
+		}
 		/* Should agree with poll, otherwise some programs break */
 		sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
 	}
diff --git a/net/sctp/socket.c b/net/sctp/socket.c
index c7b9e325ec1c..9cb3432f065a 100644
--- a/net/sctp/socket.c
+++ b/net/sctp/socket.c
@@ -9348,9 +9348,15 @@ void sctp_data_ready(struct sock *sk)
 
 	rcu_read_lock();
 	wq = rcu_dereference(sk->sk_wq);
-	if (skwq_has_sleeper(wq))
-		wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN |
+	if (skwq_has_sleeper(wq)) {
+		if (sock_flag(sk, SOCK_SYNC_WAKEUP)) {
+			wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN |
+					EPOLLRDNORM | EPOLLRDBAND);
+		} else {
+			wake_up_interruptible_poll(&wq->wait, EPOLLIN |
 						EPOLLRDNORM | EPOLLRDBAND);
+		}
+	}
 	sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN);
 	rcu_read_unlock();
 }
diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index b5db69073e20..1a6ea2e30769 100644
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -819,10 +819,10 @@ static void smc_fback_wakeup_waitqueue(struct smc_sock *smc, void *key)
 		wake_up_interruptible_all(&wq->wait);
 	} else {
 		flags = key_to_poll(key);
-		if (flags & (EPOLLIN | EPOLLOUT))
+		if (flags & (EPOLLIN | EPOLLOUT) && sock_flag(&smc->sk, SOCK_SYNC_WAKEUP))
 			/* sk_data_ready or sk_write_space */
 			wake_up_interruptible_sync_poll(&wq->wait, flags);
-		else if (flags & EPOLLERR)
+		else
 			/* sk_error_report */
 			wake_up_interruptible_poll(&wq->wait, flags);
 	}
diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c
index c1d9b923938d..4e288a2364d2 100644
--- a/net/smc/smc_rx.c
+++ b/net/smc/smc_rx.c
@@ -39,9 +39,15 @@ static void smc_rx_wake_up(struct sock *sk)
 	/* called already in smc_listen_work() */
 	rcu_read_lock();
 	wq = rcu_dereference(sk->sk_wq);
-	if (skwq_has_sleeper(wq))
-		wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
+	if (skwq_has_sleeper(wq)) {
+		if (sock_flag(sk, SOCK_SYNC_WAKEUP)) {
+			wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
 						EPOLLRDNORM | EPOLLRDBAND);
+		} else {
+			wake_up_interruptible_poll(&wq->wait, EPOLLIN | EPOLLPRI |
+						EPOLLRDNORM | EPOLLRDBAND);
+		}
+	}
 	sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN);
 	if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
 	    (sk->sk_state == SMC_CLOSED))
diff --git a/net/tipc/socket.c b/net/tipc/socket.c
index e564341e0216..9fa83a89882c 100644
--- a/net/tipc/socket.c
+++ b/net/tipc/socket.c
@@ -2116,9 +2116,15 @@ static void tipc_write_space(struct sock *sk)
 
 	rcu_read_lock();
 	wq = rcu_dereference(sk->sk_wq);
-	if (skwq_has_sleeper(wq))
-		wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
+	if (skwq_has_sleeper(wq)) {
+		if (sock_flag(sk, SOCK_SYNC_WAKEUP)) {
+			wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
 						EPOLLWRNORM | EPOLLWRBAND);
+		} else {
+			wake_up_interruptible_poll(&wq->wait, EPOLLOUT |
+						EPOLLWRNORM | EPOLLWRBAND);
+		}
+	}
 	rcu_read_unlock();
 }
 
@@ -2134,9 +2140,15 @@ static void tipc_data_ready(struct sock *sk)
 
 	rcu_read_lock();
 	wq = rcu_dereference(sk->sk_wq);
-	if (skwq_has_sleeper(wq))
-		wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN |
-						EPOLLRDNORM | EPOLLRDBAND);
+	if (skwq_has_sleeper(wq)) {
+		if (sock_flag(sk, SOCK_SYNC_WAKEUP)) {
+			wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN |
+							EPOLLRDNORM | EPOLLRDBAND);
+		} else {
+			wake_up_interruptible_poll(&wq->wait, EPOLLIN |
+							EPOLLRDNORM | EPOLLRDBAND);
+		}
+	}
 	rcu_read_unlock();
 }
 
diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c
index f7a9d55eee8a..15ebcc2d9d58 100644
--- a/net/unix/af_unix.c
+++ b/net/unix/af_unix.c
@@ -601,9 +601,15 @@ static void unix_write_space(struct sock *sk)
 	rcu_read_lock();
 	if (unix_writable(sk, READ_ONCE(sk->sk_state))) {
 		wq = rcu_dereference(sk->sk_wq);
-		if (skwq_has_sleeper(wq))
-			wake_up_interruptible_sync_poll(&wq->wait,
-				EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND);
+		if (skwq_has_sleeper(wq)) {
+			if (sock_flag(sk, SOCK_SYNC_WAKEUP)) {
+				wake_up_interruptible_sync_poll(&wq->wait,
+					EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND);
+			} else {
+				wake_up_interruptible_poll(&wq->wait,
+					EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND);
+			}
+		}
 		sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
 	}
 	rcu_read_unlock();
@@ -2603,11 +2609,15 @@ int __unix_dgram_recvmsg(struct sock *sk, struct msghdr *msg, size_t size,
 		goto out;
 	}
 
-	if (wq_has_sleeper(&u->peer_wait))
-		wake_up_interruptible_sync_poll(&u->peer_wait,
-						EPOLLOUT | EPOLLWRNORM |
-						EPOLLWRBAND);
-
+	if (wq_has_sleeper(&u->peer_wait)) {
+		if (sock_flag(sk, SOCK_SYNC_WAKEUP)) {
+			wake_up_interruptible_sync_poll(&u->peer_wait,
+						EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND);
+		} else {
+			wake_up_interruptible_poll(&u->peer_wait,
+						EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND);
+		}
+	}
 	if (msg->msg_name) {
 		unix_copy_addr(msg, skb->sk);
 
-- 
2.52.0