[PATCH net-next 0/5] net: three additions to net_hotdata

[PATCH net-next 0/5] net: three additions to net_hotdata

[Eric Dumazet]

This series moves three fast path sysctls to net_hotdata.

To avoid <net/hotdata.h> inclusion from <net/sock.h>,
create <net/proto_memory.h> to hold proto memory definitions.

Eric Dumazet (5):
  net: move sysctl_max_skb_frags to net_hotdata
  net: move sysctl_skb_defer_max to net_hotdata
  tcp: move tcp_out_of_memory() to net/ipv4/tcp.c
  net: add <net/proto_memory.h>
  net: move sysctl_mem_pcpu_rsv to net_hotdata

 include/linux/skbuff.h     |  2 -
 include/net/hotdata.h      |  3 ++
 include/net/proto_memory.h | 83 ++++++++++++++++++++++++++++++++++++++
 include/net/sock.h         | 78 -----------------------------------
 include/net/tcp.h          | 10 +----
 net/core/dev.c             |  1 -
 net/core/dev.h             |  1 -
 net/core/hotdata.c         |  7 +++-
 net/core/skbuff.c          |  7 +---
 net/core/sock.c            |  2 +-
 net/core/sysctl_net_core.c |  7 ++--
 net/ipv4/proc.c            |  1 +
 net/ipv4/tcp.c             | 14 ++++++-
 net/ipv4/tcp_input.c       |  1 +
 net/ipv4/tcp_output.c      |  1 +
 net/mptcp/protocol.c       |  3 +-
 net/sctp/sm_statefuns.c    |  1 +
 17 files changed, 117 insertions(+), 105 deletions(-)
 create mode 100644 include/net/proto_memory.h

-- 
2.44.0.769.g3c40516874-goog

[PATCH net-next 1/5] net: move sysctl_max_skb_frags to net_hotdata

[Eric Dumazet]

sysctl_max_skb_frags is used in TCP and MPTCP fast paths,
move it to net_hodata for better cache locality.

Signed-off-by: Eric Dumazet <edumazet@google.com>
---
 include/linux/skbuff.h     | 2 --
 include/net/hotdata.h      | 1 +
 net/core/hotdata.c         | 1 +
 net/core/skbuff.c          | 5 +----
 net/core/sysctl_net_core.c | 2 +-
 net/ipv4/tcp.c             | 3 ++-
 net/mptcp/protocol.c       | 3 ++-
 7 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index f76825e5b92a334f7726d7f7c99aa60ec69a8e07..4f99a39db2eb2868d9a14334070d4a574ba1f9a3 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -353,8 +353,6 @@ struct sk_buff;
 
 #define MAX_SKB_FRAGS CONFIG_MAX_SKB_FRAGS
 
-extern int sysctl_max_skb_frags;
-
 /* Set skb_shinfo(skb)->gso_size to this in case you want skb_segment to
  * segment using its current segmentation instead.
  */
diff --git a/include/net/hotdata.h b/include/net/hotdata.h
index 003667a1efd6b63fc0f0d7cadd2c8472281331b0..a6cff65904267f338fbd258d23be79d46a062f9e 100644
--- a/include/net/hotdata.h
+++ b/include/net/hotdata.h
@@ -38,6 +38,7 @@ struct net_hotdata {
 	int			max_backlog;
 	int			dev_tx_weight;
 	int			dev_rx_weight;
+	int			sysctl_max_skb_frags;
 };
 
 #define inet_ehash_secret	net_hotdata.tcp_protocol.secret
diff --git a/net/core/hotdata.c b/net/core/hotdata.c
index c8a7a451c18a383d091e413a510d84d163473f2f..f17cbb4807b99937b272be12953f790c66cc2cd1 100644
--- a/net/core/hotdata.c
+++ b/net/core/hotdata.c
@@ -18,5 +18,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = {
 	.max_backlog = 1000,
 	.dev_tx_weight = 64,
 	.dev_rx_weight = 64,
+	.sysctl_max_skb_frags = MAX_SKB_FRAGS,
 };
 EXPORT_SYMBOL(net_hotdata);
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 0c8b82750000f63b097cb4b7b990c647c81019df..65779b8f0b126a1c039cf24a47474c0cb80ff6ae 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -109,9 +109,6 @@ static struct kmem_cache *skbuff_ext_cache __ro_after_init;
 #define SKB_SMALL_HEAD_HEADROOM						\
 	SKB_WITH_OVERHEAD(SKB_SMALL_HEAD_CACHE_SIZE)
 
-int sysctl_max_skb_frags __read_mostly = MAX_SKB_FRAGS;
-EXPORT_SYMBOL(sysctl_max_skb_frags);
-
 /* kcm_write_msgs() relies on casting paged frags to bio_vec to use
  * iov_iter_bvec(). These static asserts ensure the cast is valid is long as the
  * netmem is a page.
@@ -7040,7 +7037,7 @@ static void skb_splice_csum_page(struct sk_buff *skb, struct page *page,
 ssize_t skb_splice_from_iter(struct sk_buff *skb, struct iov_iter *iter,
 			     ssize_t maxsize, gfp_t gfp)
 {
-	size_t frag_limit = READ_ONCE(sysctl_max_skb_frags);
+	size_t frag_limit = READ_ONCE(net_hotdata.sysctl_max_skb_frags);
 	struct page *pages[8], **ppages = pages;
 	ssize_t spliced = 0, ret = 0;
 	unsigned int i;
diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index 903ab4a51c178a4705b7eb610958c81c5ebcfdf5..e75375d54b9e50673af28f6d6b3bb83fc74cb1f8 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -595,7 +595,7 @@ static struct ctl_table net_core_table[] = {
 	},
 	{
 		.procname	= "max_skb_frags",
-		.data		= &sysctl_max_skb_frags,
+		.data		= &net_hotdata.sysctl_max_skb_frags,
 		.maxlen		= sizeof(int),
 		.mode		= 0644,
 		.proc_handler	= proc_dointvec_minmax,
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 4ec0f4feee003d91fa5ae37ed4dd50e09c4c874a..388f6e115bf168e6f70b762096a984a2cacfa5c9 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -280,6 +280,7 @@
 #include <linux/uaccess.h>
 #include <asm/ioctls.h>
 #include <net/busy_poll.h>
+#include <net/hotdata.h>
 #include <net/rps.h>
 
 /* Track pending CMSGs. */
@@ -1188,7 +1189,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
 
 			if (!skb_can_coalesce(skb, i, pfrag->page,
 					      pfrag->offset)) {
-				if (i >= READ_ONCE(sysctl_max_skb_frags)) {
+				if (i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) {
 					tcp_mark_push(tp, skb);
 					goto new_segment;
 				}
diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c
index 4b13ca362efa395a2ef7a92e553a8c49feebec79..aff17597e6a71ecc4b9aef4d02f039318fa1fe7f 100644
--- a/net/mptcp/protocol.c
+++ b/net/mptcp/protocol.c
@@ -20,6 +20,7 @@
 #include <net/transp_v6.h>
 #endif
 #include <net/mptcp.h>
+#include <net/hotdata.h>
 #include <net/xfrm.h>
 #include <asm/ioctls.h>
 #include "protocol.h"
@@ -1272,7 +1273,7 @@ static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,
 
 		i = skb_shinfo(skb)->nr_frags;
 		can_coalesce = skb_can_coalesce(skb, i, dfrag->page, offset);
-		if (!can_coalesce && i >= READ_ONCE(sysctl_max_skb_frags)) {
+		if (!can_coalesce && i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) {
 			tcp_mark_push(tcp_sk(ssk), skb);
 			goto alloc_skb;
 		}
-- 
2.44.0.769.g3c40516874-goog

[PATCH net-next 2/5] net: move sysctl_skb_defer_max to net_hotdata

[Eric Dumazet]

sysctl_skb_defer_max is used in TCP fast path,
move it to net_hodata.

Signed-off-by: Eric Dumazet <edumazet@google.com>
---
 include/net/hotdata.h      | 1 +
 net/core/dev.c             | 1 -
 net/core/dev.h             | 1 -
 net/core/hotdata.c         | 1 +
 net/core/skbuff.c          | 2 +-
 net/core/sysctl_net_core.c | 2 +-
 6 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/include/net/hotdata.h b/include/net/hotdata.h
index a6cff65904267f338fbd258d23be79d46a062f9e..290499f72e18d45a8ed7bd9e8880a51f7ef2b94c 100644
--- a/include/net/hotdata.h
+++ b/include/net/hotdata.h
@@ -39,6 +39,7 @@ struct net_hotdata {
 	int			dev_tx_weight;
 	int			dev_rx_weight;
 	int			sysctl_max_skb_frags;
+	int			sysctl_skb_defer_max;
 };
 
 #define inet_ehash_secret	net_hotdata.tcp_protocol.secret
diff --git a/net/core/dev.c b/net/core/dev.c
index c9e59eff8ec841f6267c2749489fdc7fe0d03430..cd7ba50eac15341c1d3a2136f8b4a3584d5c2669 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4450,7 +4450,6 @@ EXPORT_SYMBOL(__dev_direct_xmit);
  *************************************************************************/
 static DEFINE_PER_CPU(struct task_struct *, backlog_napi);
 
-unsigned int sysctl_skb_defer_max __read_mostly = 64;
 int weight_p __read_mostly = 64;           /* old backlog weight */
 int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
 int dev_weight_tx_bias __read_mostly = 1;  /* bias for output_queue quota */
diff --git a/net/core/dev.h b/net/core/dev.h
index 8572d2c8dc4adce75c98868c888363e6a32e0f52..b7b518bc2be55aa42847fee3f21749a658d97f35 100644
--- a/net/core/dev.h
+++ b/net/core/dev.h
@@ -36,7 +36,6 @@ int dev_addr_init(struct net_device *dev);
 void dev_addr_check(struct net_device *dev);
 
 /* sysctls not referred to from outside net/core/ */
-extern unsigned int	sysctl_skb_defer_max;
 extern int		netdev_unregister_timeout_secs;
 extern int		weight_p;
 extern int		dev_weight_rx_bias;
diff --git a/net/core/hotdata.c b/net/core/hotdata.c
index f17cbb4807b99937b272be12953f790c66cc2cd1..a359ff160d54ad379eac7e56e1810a7e840f9275 100644
--- a/net/core/hotdata.c
+++ b/net/core/hotdata.c
@@ -19,5 +19,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = {
 	.dev_tx_weight = 64,
 	.dev_rx_weight = 64,
 	.sysctl_max_skb_frags = MAX_SKB_FRAGS,
+	.sysctl_skb_defer_max = 64,
 };
 EXPORT_SYMBOL(net_hotdata);
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 65779b8f0b126a1c039cf24a47474c0cb80ff6ae..5f382e94b4d1d2a1913867f28c639881fce962c0 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -6985,7 +6985,7 @@ nodefer:	kfree_skb_napi_cache(skb);
 	DEBUG_NET_WARN_ON_ONCE(skb->destructor);
 
 	sd = &per_cpu(softnet_data, cpu);
-	defer_max = READ_ONCE(sysctl_skb_defer_max);
+	defer_max = READ_ONCE(net_hotdata.sysctl_skb_defer_max);
 	if (READ_ONCE(sd->defer_count) >= defer_max)
 		goto nodefer;
 
diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index e75375d54b9e50673af28f6d6b3bb83fc74cb1f8..118c78615543852eabd3067bbb7920418b7da7b3 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -654,7 +654,7 @@ static struct ctl_table net_core_table[] = {
 	},
 	{
 		.procname	= "skb_defer_max",
-		.data		= &sysctl_skb_defer_max,
+		.data		= &net_hotdata.sysctl_skb_defer_max,
 		.maxlen		= sizeof(unsigned int),
 		.mode		= 0644,
 		.proc_handler	= proc_dointvec_minmax,
-- 
2.44.0.769.g3c40516874-goog

[PATCH net-next 3/5] tcp: move tcp_out_of_memory() to net/ipv4/tcp.c

[Eric Dumazet]

tcp_out_of_memory() has a single caller: tcp_check_oom().

Following patch will also make sk_memory_allocated()
not anymore visible from <net/sock.h> and <net/tcp.h>

Add const qualifier to sock argument of tcp_out_of_memory()
and tcp_check_oom().

Signed-off-by: Eric Dumazet <edumazet@google.com>
---
 include/net/tcp.h | 10 +---------
 net/ipv4/tcp.c    | 10 +++++++++-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/include/net/tcp.h b/include/net/tcp.h
index a9eb21251195c3707c892247bfba5b430e7bc1bb..39e36b9a5f1c11462b0ff01a2de7b08ee0fb9258 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -296,14 +296,6 @@ static inline bool between(__u32 seq1, __u32 seq2, __u32 seq3)
 	return seq3 - seq2 >= seq1 - seq2;
 }
 
-static inline bool tcp_out_of_memory(struct sock *sk)
-{
-	if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
-	    sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
-		return true;
-	return false;
-}
-
 static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
 {
 	sk_wmem_queued_add(sk, -skb->truesize);
@@ -316,7 +308,7 @@ static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
 
 void sk_forced_mem_schedule(struct sock *sk, int size);
 
-bool tcp_check_oom(struct sock *sk, int shift);
+bool tcp_check_oom(const struct sock *sk, int shift);
 
 
 extern struct proto tcp_prot;
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 388f6e115bf168e6f70b762096a984a2cacfa5c9..0a3aa30470837f27d7db1a0328228b2e3323bad0 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -2752,7 +2752,15 @@ static bool tcp_too_many_orphans(int shift)
 		READ_ONCE(sysctl_tcp_max_orphans);
 }
 
-bool tcp_check_oom(struct sock *sk, int shift)
+static bool tcp_out_of_memory(const struct sock *sk)
+{
+	if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
+	    sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
+		return true;
+	return false;
+}
+
+bool tcp_check_oom(const struct sock *sk, int shift)
 {
 	bool too_many_orphans, out_of_socket_memory;
 
-- 
2.44.0.769.g3c40516874-goog

[PATCH net-next 4/5] net: add <net/proto_memory.h>

[Eric Dumazet]

Move some proto memory definitions out of <net/sock.h>

Very few files need them, and following patch
will include <net/hotdata.h> from <net/proto_memory.h>

Signed-off-by: Eric Dumazet <edumazet@google.com>
---
 include/net/proto_memory.h | 83 ++++++++++++++++++++++++++++++++++++++
 include/net/sock.h         | 78 -----------------------------------
 net/core/sock.c            |  1 +
 net/core/sysctl_net_core.c |  1 +
 net/ipv4/proc.c            |  1 +
 net/ipv4/tcp.c             |  1 +
 net/ipv4/tcp_input.c       |  1 +
 net/ipv4/tcp_output.c      |  1 +
 net/sctp/sm_statefuns.c    |  1 +
 9 files changed, 90 insertions(+), 78 deletions(-)
 create mode 100644 include/net/proto_memory.h

diff --git a/include/net/proto_memory.h b/include/net/proto_memory.h
new file mode 100644
index 0000000000000000000000000000000000000000..41404d4bb6f08e84030838f0e63ef240587f65dd
--- /dev/null
+++ b/include/net/proto_memory.h
@@ -0,0 +1,83 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+#ifndef _PROTO_MEMORY_H
+#define _PROTO_MEMORY_H
+
+#include <net/sock.h>
+
+/* 1 MB per cpu, in page units */
+#define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT))
+extern int sysctl_mem_pcpu_rsv;
+
+static inline bool sk_has_memory_pressure(const struct sock *sk)
+{
+	return sk->sk_prot->memory_pressure != NULL;
+}
+
+static inline bool
+proto_memory_pressure(const struct proto *prot)
+{
+	if (!prot->memory_pressure)
+		return false;
+	return !!READ_ONCE(*prot->memory_pressure);
+}
+
+static inline bool sk_under_global_memory_pressure(const struct sock *sk)
+{
+	return proto_memory_pressure(sk->sk_prot);
+}
+
+static inline bool sk_under_memory_pressure(const struct sock *sk)
+{
+	if (!sk->sk_prot->memory_pressure)
+		return false;
+
+	if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
+	    mem_cgroup_under_socket_pressure(sk->sk_memcg))
+		return true;
+
+	return !!READ_ONCE(*sk->sk_prot->memory_pressure);
+}
+
+static inline long
+proto_memory_allocated(const struct proto *prot)
+{
+	return max(0L, atomic_long_read(prot->memory_allocated));
+}
+
+static inline long
+sk_memory_allocated(const struct sock *sk)
+{
+	return proto_memory_allocated(sk->sk_prot);
+}
+
+static inline void proto_memory_pcpu_drain(struct proto *proto)
+{
+	int val = this_cpu_xchg(*proto->per_cpu_fw_alloc, 0);
+
+	if (val)
+		atomic_long_add(val, proto->memory_allocated);
+}
+
+static inline void
+sk_memory_allocated_add(const struct sock *sk, int val)
+{
+	struct proto *proto = sk->sk_prot;
+
+	val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val);
+
+	if (unlikely(val >= READ_ONCE(sysctl_mem_pcpu_rsv)))
+		proto_memory_pcpu_drain(proto);
+}
+
+static inline void
+sk_memory_allocated_sub(const struct sock *sk, int val)
+{
+	struct proto *proto = sk->sk_prot;
+
+	val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val);
+
+	if (unlikely(val <= -READ_ONCE(sysctl_mem_pcpu_rsv)))
+		proto_memory_pcpu_drain(proto);
+}
+
+#endif /* _PROTO_MEMORY_H */
diff --git a/include/net/sock.h b/include/net/sock.h
index 48bcc845202f283c258803c7d35fb2b9a0fd6fe7..0450494a1766a0136fa7b446e63c7357afd4a6e6 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -1371,75 +1371,6 @@ static inline int sk_under_cgroup_hierarchy(struct sock *sk,
 #endif
 }
 
-static inline bool sk_has_memory_pressure(const struct sock *sk)
-{
-	return sk->sk_prot->memory_pressure != NULL;
-}
-
-static inline bool sk_under_global_memory_pressure(const struct sock *sk)
-{
-	return sk->sk_prot->memory_pressure &&
-		!!READ_ONCE(*sk->sk_prot->memory_pressure);
-}
-
-static inline bool sk_under_memory_pressure(const struct sock *sk)
-{
-	if (!sk->sk_prot->memory_pressure)
-		return false;
-
-	if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
-	    mem_cgroup_under_socket_pressure(sk->sk_memcg))
-		return true;
-
-	return !!READ_ONCE(*sk->sk_prot->memory_pressure);
-}
-
-static inline long
-proto_memory_allocated(const struct proto *prot)
-{
-	return max(0L, atomic_long_read(prot->memory_allocated));
-}
-
-static inline long
-sk_memory_allocated(const struct sock *sk)
-{
-	return proto_memory_allocated(sk->sk_prot);
-}
-
-/* 1 MB per cpu, in page units */
-#define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT))
-extern int sysctl_mem_pcpu_rsv;
-
-static inline void proto_memory_pcpu_drain(struct proto *proto)
-{
-	int val = this_cpu_xchg(*proto->per_cpu_fw_alloc, 0);
-
-	if (val)
-		atomic_long_add(val, proto->memory_allocated);
-}
-
-static inline void
-sk_memory_allocated_add(const struct sock *sk, int val)
-{
-	struct proto *proto = sk->sk_prot;
-
-	val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val);
-
-	if (unlikely(val >= READ_ONCE(sysctl_mem_pcpu_rsv)))
-		proto_memory_pcpu_drain(proto);
-}
-
-static inline void
-sk_memory_allocated_sub(const struct sock *sk, int val)
-{
-	struct proto *proto = sk->sk_prot;
-
-	val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val);
-
-	if (unlikely(val <= -READ_ONCE(sysctl_mem_pcpu_rsv)))
-		proto_memory_pcpu_drain(proto);
-}
-
 #define SK_ALLOC_PERCPU_COUNTER_BATCH 16
 
 static inline void sk_sockets_allocated_dec(struct sock *sk)
@@ -1466,15 +1397,6 @@ proto_sockets_allocated_sum_positive(struct proto *prot)
 	return percpu_counter_sum_positive(prot->sockets_allocated);
 }
 
-static inline bool
-proto_memory_pressure(struct proto *prot)
-{
-	if (!prot->memory_pressure)
-		return false;
-	return !!READ_ONCE(*prot->memory_pressure);
-}
-
-
 #ifdef CONFIG_PROC_FS
 #define PROTO_INUSE_NR	64	/* should be enough for the first time */
 struct prot_inuse {
diff --git a/net/core/sock.c b/net/core/sock.c
index fe9195186c13f51b113f8e8bc69ea25ea4a13aad..e0692b752369cc1cbf34253a40c06d18dcf350fe 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -127,6 +127,7 @@
 #include <net/net_namespace.h>
 #include <net/request_sock.h>
 #include <net/sock.h>
+#include <net/proto_memory.h>
 #include <linux/net_tstamp.h>
 #include <net/xfrm.h>
 #include <linux/ipsec.h>
diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index 118c78615543852eabd3067bbb7920418b7da7b3..a452a330d0ed649a2d1b8e65c81aaa3ff3d826f8 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -24,6 +24,7 @@
 #include <net/busy_poll.h>
 #include <net/pkt_sched.h>
 #include <net/hotdata.h>
+#include <net/proto_memory.h>
 #include <net/rps.h>
 
 #include "dev.h"
diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c
index 914bc9c35cc702395aee257fb010034294e501de..6c4664c681ca530d67b9ccc54f11cbb178d32c8e 100644
--- a/net/ipv4/proc.c
+++ b/net/ipv4/proc.c
@@ -33,6 +33,7 @@
 #include <net/protocol.h>
 #include <net/tcp.h>
 #include <net/mptcp.h>
+#include <net/proto_memory.h>
 #include <net/udp.h>
 #include <net/udplite.h>
 #include <linux/bottom_half.h>
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 0a3aa30470837f27d7db1a0328228b2e3323bad0..e1f0efbb29d614ed81d4485ae81613f0f97db54a 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -272,6 +272,7 @@
 #include <net/inet_common.h>
 #include <net/tcp.h>
 #include <net/mptcp.h>
+#include <net/proto_memory.h>
 #include <net/xfrm.h>
 #include <net/ip.h>
 #include <net/sock.h>
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index 384fa5e2f0655389ac678b5d13553949598a9c74..233c34c0b49461ddf863373f0b441e01068fb92f 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -72,6 +72,7 @@
 #include <linux/prefetch.h>
 #include <net/dst.h>
 #include <net/tcp.h>
+#include <net/proto_memory.h>
 #include <net/inet_common.h>
 #include <linux/ipsec.h>
 #include <asm/unaligned.h>
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index ea7ad7d99245c26f4045ebb344bc42cd188afe37..57edf66ff91b3aebbb808604678603792025318c 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -39,6 +39,7 @@
 
 #include <net/tcp.h>
 #include <net/mptcp.h>
+#include <net/proto_memory.h>
 
 #include <linux/compiler.h>
 #include <linux/gfp.h>
diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c
index 08fdf1251f46af2c7abb154b35d951913ae90fa1..5adf0c0a6c1acdb354a6815b4b77b63599e0166e 100644
--- a/net/sctp/sm_statefuns.c
+++ b/net/sctp/sm_statefuns.c
@@ -38,6 +38,7 @@
 #include <linux/inet.h>
 #include <linux/slab.h>
 #include <net/sock.h>
+#include <net/proto_memory.h>
 #include <net/inet_ecn.h>
 #include <linux/skbuff.h>
 #include <net/sctp/sctp.h>
-- 
2.44.0.769.g3c40516874-goog

[PATCH net-next 5/5] net: move sysctl_mem_pcpu_rsv to net_hotdata

[Eric Dumazet]

sysctl_mem_pcpu_rsv is used in TCP fast path,
move it to net_hodata for better cache locality.

Signed-off-by: Eric Dumazet <edumazet@google.com>
---
 include/net/hotdata.h      | 1 +
 include/net/proto_memory.h | 6 +++---
 net/core/hotdata.c         | 5 +++--
 net/core/sock.c            | 1 -
 net/core/sysctl_net_core.c | 2 +-
 5 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/include/net/hotdata.h b/include/net/hotdata.h
index 290499f72e18d45a8ed7bd9e8880a51f7ef2b94c..30e9570beb2afbcb4ece641d6042bdc0de80bd38 100644
--- a/include/net/hotdata.h
+++ b/include/net/hotdata.h
@@ -40,6 +40,7 @@ struct net_hotdata {
 	int			dev_rx_weight;
 	int			sysctl_max_skb_frags;
 	int			sysctl_skb_defer_max;
+	int			sysctl_mem_pcpu_rsv;
 };
 
 #define inet_ehash_secret	net_hotdata.tcp_protocol.secret
diff --git a/include/net/proto_memory.h b/include/net/proto_memory.h
index 41404d4bb6f08e84030838f0e63ef240587f65dd..a6ab2f4f5e28a21ab63acb38caf157f6a7415a0b 100644
--- a/include/net/proto_memory.h
+++ b/include/net/proto_memory.h
@@ -3,10 +3,10 @@
 #define _PROTO_MEMORY_H
 
 #include <net/sock.h>
+#include <net/hotdata.h>
 
 /* 1 MB per cpu, in page units */
 #define SK_MEMORY_PCPU_RESERVE (1 << (20 - PAGE_SHIFT))
-extern int sysctl_mem_pcpu_rsv;
 
 static inline bool sk_has_memory_pressure(const struct sock *sk)
 {
@@ -65,7 +65,7 @@ sk_memory_allocated_add(const struct sock *sk, int val)
 
 	val = this_cpu_add_return(*proto->per_cpu_fw_alloc, val);
 
-	if (unlikely(val >= READ_ONCE(sysctl_mem_pcpu_rsv)))
+	if (unlikely(val >= READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))
 		proto_memory_pcpu_drain(proto);
 }
 
@@ -76,7 +76,7 @@ sk_memory_allocated_sub(const struct sock *sk, int val)
 
 	val = this_cpu_sub_return(*proto->per_cpu_fw_alloc, val);
 
-	if (unlikely(val <= -READ_ONCE(sysctl_mem_pcpu_rsv)))
+	if (unlikely(val <= -READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv)))
 		proto_memory_pcpu_drain(proto);
 }
 
diff --git a/net/core/hotdata.c b/net/core/hotdata.c
index a359ff160d54ad379eac7e56e1810a7e840f9275..d0aaaaa556f229ded4e1997bf814a2b690b46920 100644
--- a/net/core/hotdata.c
+++ b/net/core/hotdata.c
@@ -1,9 +1,9 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
-#include <net/hotdata.h>
 #include <linux/cache.h>
 #include <linux/jiffies.h>
 #include <linux/list.h>
-
+#include <net/hotdata.h>
+#include <net/proto_memory.h>
 
 struct net_hotdata net_hotdata __cacheline_aligned = {
 	.offload_base = LIST_HEAD_INIT(net_hotdata.offload_base),
@@ -20,5 +20,6 @@ struct net_hotdata net_hotdata __cacheline_aligned = {
 	.dev_rx_weight = 64,
 	.sysctl_max_skb_frags = MAX_SKB_FRAGS,
 	.sysctl_skb_defer_max = 64,
+	.sysctl_mem_pcpu_rsv = SK_MEMORY_PCPU_RESERVE
 };
 EXPORT_SYMBOL(net_hotdata);
diff --git a/net/core/sock.c b/net/core/sock.c
index e0692b752369cc1cbf34253a40c06d18dcf350fe..8d6e638b5426daf84ff71d22101252943f9c5466 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -284,7 +284,6 @@ __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
 EXPORT_SYMBOL(sysctl_rmem_max);
 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
-int sysctl_mem_pcpu_rsv __read_mostly = SK_MEMORY_PCPU_RESERVE;
 
 int sysctl_tstamp_allow_data __read_mostly = 1;
 
diff --git a/net/core/sysctl_net_core.c b/net/core/sysctl_net_core.c
index a452a330d0ed649a2d1b8e65c81aaa3ff3d826f8..6da5995ac86a08a58e63af9a1e62c7a136c08dc9 100644
--- a/net/core/sysctl_net_core.c
+++ b/net/core/sysctl_net_core.c
@@ -416,7 +416,7 @@ static struct ctl_table net_core_table[] = {
 	},
 	{
 		.procname	= "mem_pcpu_rsv",
-		.data		= &sysctl_mem_pcpu_rsv,
+		.data		= &net_hotdata.sysctl_mem_pcpu_rsv,
 		.maxlen		= sizeof(int),
 		.mode		= 0644,
 		.proc_handler	= proc_dointvec_minmax,
-- 
2.44.0.769.g3c40516874-goog

Re: [PATCH net-next 0/5] net: three additions to net_hotdata

[David Ahern]

On 4/29/24 7:40 AM, Eric Dumazet wrote:
> This series moves three fast path sysctls to net_hotdata.
> 
> To avoid <net/hotdata.h> inclusion from <net/sock.h>,
> create <net/proto_memory.h> to hold proto memory definitions.
> 
> Eric Dumazet (5):
>   net: move sysctl_max_skb_frags to net_hotdata
>   net: move sysctl_skb_defer_max to net_hotdata
>   tcp: move tcp_out_of_memory() to net/ipv4/tcp.c
>   net: add <net/proto_memory.h>
>   net: move sysctl_mem_pcpu_rsv to net_hotdata
> 
>  include/linux/skbuff.h     |  2 -
>  include/net/hotdata.h      |  3 ++
>  include/net/proto_memory.h | 83 ++++++++++++++++++++++++++++++++++++++
>  include/net/sock.h         | 78 -----------------------------------
>  include/net/tcp.h          | 10 +----
>  net/core/dev.c             |  1 -
>  net/core/dev.h             |  1 -
>  net/core/hotdata.c         |  7 +++-
>  net/core/skbuff.c          |  7 +---
>  net/core/sock.c            |  2 +-
>  net/core/sysctl_net_core.c |  7 ++--
>  net/ipv4/proc.c            |  1 +
>  net/ipv4/tcp.c             | 14 ++++++-
>  net/ipv4/tcp_input.c       |  1 +
>  net/ipv4/tcp_output.c      |  1 +
>  net/mptcp/protocol.c       |  3 +-
>  net/sctp/sm_statefuns.c    |  1 +
>  17 files changed, 117 insertions(+), 105 deletions(-)
>  create mode 100644 include/net/proto_memory.h
> 

For the set:

Reviewed-by: David Ahern <dsahern@kernel.org>

Re: [PATCH net-next 0/5] net: three additions to net_hotdata

[patchwork-bot+netdevbpf]

Hello:

This series was applied to netdev/net-next.git (main)
by Jakub Kicinski <kuba@kernel.org>:

On Mon, 29 Apr 2024 13:40:20 +0000 you wrote:
> This series moves three fast path sysctls to net_hotdata.
> 
> To avoid <net/hotdata.h> inclusion from <net/sock.h>,
> create <net/proto_memory.h> to hold proto memory definitions.
> 
> Eric Dumazet (5):
>   net: move sysctl_max_skb_frags to net_hotdata
>   net: move sysctl_skb_defer_max to net_hotdata
>   tcp: move tcp_out_of_memory() to net/ipv4/tcp.c
>   net: add <net/proto_memory.h>
>   net: move sysctl_mem_pcpu_rsv to net_hotdata
> 
> [...]

Here is the summary with links:
  - [net-next,1/5] net: move sysctl_max_skb_frags to net_hotdata
    https://git.kernel.org/netdev/net-next/c/a86a0661b86f
  - [net-next,2/5] net: move sysctl_skb_defer_max to net_hotdata
    https://git.kernel.org/netdev/net-next/c/d480dc76d9f8
  - [net-next,3/5] tcp: move tcp_out_of_memory() to net/ipv4/tcp.c
    https://git.kernel.org/netdev/net-next/c/dda4d96acb20
  - [net-next,4/5] net: add <net/proto_memory.h>
    https://git.kernel.org/netdev/net-next/c/f3d93817fba3
  - [net-next,5/5] net: move sysctl_mem_pcpu_rsv to net_hotdata
    https://git.kernel.org/netdev/net-next/c/c204fef97ee6

You are awesome, thank you!
-- 
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html