[PATCH net-next 0/2] tcp: avoid indirect call in __sk_stream_memory_free()

[PATCH net-next 0/2] tcp: avoid indirect call in __sk_stream_memory_free()

[Eric Dumazet]

From: Eric Dumazet <edumazet@google.com>

Small improvement for CONFIG_RETPOLINE=y, when dealing with TCP sockets.

Eric Dumazet (2):
  tcp: uninline tcp_stream_memory_free()
  tcp: avoid indirect call to tcp_stream_memory_free()

 include/net/sock.h  |  8 ++++++--
 include/net/tcp.h   | 13 +------------
 net/ipv4/tcp_ipv4.c | 14 ++++++++++++++
 3 files changed, 21 insertions(+), 14 deletions(-)

-- 
2.29.2.299.gdc1121823c-goog

[PATCH net-next 1/2] tcp: uninline tcp_stream_memory_free()

[Eric Dumazet]

From: Eric Dumazet <edumazet@google.com>

Both IPv4 and IPv6 needs it via a function pointer.
Following patch will avoid the indirect call.

Signed-off-by: Eric Dumazet <edumazet@google.com>
---
 include/net/tcp.h   | 13 +------------
 net/ipv4/tcp_ipv4.c | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/include/net/tcp.h b/include/net/tcp.h
index 4aba0f069b05d68b2595ec9a3bf8da7786437fde..d643ee4e42495f0b28c87e79d6192c23f8cdba7f 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -1965,18 +1965,7 @@ static inline u32 tcp_notsent_lowat(const struct tcp_sock *tp)
 	return tp->notsent_lowat ?: net->ipv4.sysctl_tcp_notsent_lowat;
 }
 
-/* @wake is one when sk_stream_write_space() calls us.
- * This sends EPOLLOUT only if notsent_bytes is half the limit.
- * This mimics the strategy used in sock_def_write_space().
- */
-static inline bool tcp_stream_memory_free(const struct sock *sk, int wake)
-{
-	const struct tcp_sock *tp = tcp_sk(sk);
-	u32 notsent_bytes = READ_ONCE(tp->write_seq) -
-			    READ_ONCE(tp->snd_nxt);
-
-	return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
-}
+bool tcp_stream_memory_free(const struct sock *sk, int wake);
 
 #ifdef CONFIG_PROC_FS
 int tcp4_proc_init(void);
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index 7352c097ae48f7df6caba1ae5b98dccfecd79d1a..c2d5132c523c8d4258e407c4cc5112840b07805e 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -2740,6 +2740,20 @@ void tcp4_proc_exit(void)
 }
 #endif /* CONFIG_PROC_FS */
 
+/* @wake is one when sk_stream_write_space() calls us.
+ * This sends EPOLLOUT only if notsent_bytes is half the limit.
+ * This mimics the strategy used in sock_def_write_space().
+ */
+bool tcp_stream_memory_free(const struct sock *sk, int wake)
+{
+	const struct tcp_sock *tp = tcp_sk(sk);
+	u32 notsent_bytes = READ_ONCE(tp->write_seq) -
+			    READ_ONCE(tp->snd_nxt);
+
+	return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
+}
+EXPORT_SYMBOL(tcp_stream_memory_free);
+
 struct proto tcp_prot = {
 	.name			= "TCP",
 	.owner			= THIS_MODULE,
-- 
2.29.2.299.gdc1121823c-goog

[PATCH net-next 2/2] tcp: avoid indirect call to tcp_stream_memory_free()

[Eric Dumazet]

From: Eric Dumazet <edumazet@google.com>

Signed-off-by: Eric Dumazet <edumazet@google.com>
---
 include/net/sock.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/include/net/sock.h b/include/net/sock.h
index a5c6ae78df77d08e4b65e634e93040446486bdc2..1d29aeae74fdbbae3410def33ebc66758e034205 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -60,7 +60,7 @@
 #include <linux/rculist_nulls.h>
 #include <linux/poll.h>
 #include <linux/sockptr.h>
-
+#include <linux/indirect_call_wrapper.h>
 #include <linux/atomic.h>
 #include <linux/refcount.h>
 #include <net/dst.h>
@@ -1264,13 +1264,17 @@ static inline void sk_refcnt_debug_release(const struct sock *sk)
 #define sk_refcnt_debug_release(sk) do { } while (0)
 #endif /* SOCK_REFCNT_DEBUG */
 
+INDIRECT_CALLABLE_DECLARE(bool tcp_stream_memory_free(const struct sock *sk, int wake));
+
 static inline bool __sk_stream_memory_free(const struct sock *sk, int wake)
 {
 	if (READ_ONCE(sk->sk_wmem_queued) >= READ_ONCE(sk->sk_sndbuf))
 		return false;
 
 	return sk->sk_prot->stream_memory_free ?
-		sk->sk_prot->stream_memory_free(sk, wake) : true;
+		INDIRECT_CALL_1(sk->sk_prot->stream_memory_free,
+			        tcp_stream_memory_free,
+				sk, wake) : true;
 }
 
 static inline bool sk_stream_memory_free(const struct sock *sk)
-- 
2.29.2.299.gdc1121823c-goog

Re: [PATCH net-next 0/2] tcp: avoid indirect call in __sk_stream_memory_free()

[Jakub Kicinski]

On Fri, 13 Nov 2020 07:08:07 -0800 Eric Dumazet wrote:
> From: Eric Dumazet <edumazet@google.com>
> 
> Small improvement for CONFIG_RETPOLINE=y, when dealing with TCP sockets.

Applied, thank you!