* [PATCH 02/16 net-next v2] net: core: add IPv4 fallback stubs and guards for CONFIG_IPV4=n
[not found] <20260928193046.6698-1-fmancera@suse.de>
@ 2026-09-28 19:29 ` Fernando Fernandez Mancera
2026-09-28 19:30 ` [PATCH 04/16 net-next v2] tcp: move protocol agnostic TCP functions out of tcp_ipv4.c Fernando Fernandez Mancera
` (3 subsequent siblings)
4 siblings, 0 replies; 7+ messages in thread
From: Fernando Fernandez Mancera @ 2026-09-28 19:29 UTC (permalink / raw)
To: netdev
Cc: horms, kuba, pabeni, edumazet, davem, Fernando Fernandez Mancera,
Eric Dumazet, David Ahern, Ido Schimmel, Daniel Borkmann,
John Fastabend, Stanislav Fomichev, Martin KaFai Lau,
Alexei Starovoitov, Andrii Nakryiko, Eduard Zingerman,
Kumar Kartikeya Dwivedi, Song Liu, Yonghong Song, Jiri Olsa,
Emil Tsalapatis, Ihor Solodrai, Neal Cardwell, Kuniyuki Iwashima,
Kory Maincent, Vadim Fedorenko, Jacob Keller, Nicolai Buchwitz,
Kees Cook, Jiaming Zhang, Petr Machata, linux-kernel, bpf
To enable compiling the core network stack without IPv4, generic
networking code must avoid referencing missing IPv4 symbols.
This patch provides static inline stubs in include/net/ip.h and
include/net/route.h that return a proper error code when IPv4 is
disabled. This behaves as IPv6 code does. In addition, it introduces
guards around IPv4 specific code in generic implementations to prevent
undefined reference errors while linking.
Signed-off-by: Fernando Fernandez Mancera <fmancera@suse.de>
---
include/linux/indirect_call_wrapper.h | 8 +-
include/net/ip.h | 111 ++++++++++++++++++++++----
include/net/route.h | 8 ++
net/core/dev_ioctl.c | 3 +
net/core/filter.c | 20 ++---
net/core/neighbour.c | 4 +
net/ipv4/inet_hashtables.c | 3 +
7 files changed, 127 insertions(+), 30 deletions(-)
diff --git a/include/linux/indirect_call_wrapper.h b/include/linux/indirect_call_wrapper.h
index 0e4340ecd857..583efbc03446 100644
--- a/include/linux/indirect_call_wrapper.h
+++ b/include/linux/indirect_call_wrapper.h
@@ -57,16 +57,18 @@
* builtin, this macro simplify dealing with indirect calls with only ipv4/ipv6
* alternatives
*/
-#if IS_ENABLED(CONFIG_IPV6)
+#if IS_ENABLED(CONFIG_IPV4) && IS_ENABLED(CONFIG_IPV6)
#define INDIRECT_CALL_INET(f, f2, f1, ...) \
INDIRECT_CALL_2(f, f2, f1, __VA_ARGS__)
-#elif IS_ENABLED(CONFIG_INET)
+#elif IS_ENABLED(CONFIG_IPV4)
#define INDIRECT_CALL_INET(f, f2, f1, ...) INDIRECT_CALL_1(f, f1, __VA_ARGS__)
+#elif IS_ENABLED(CONFIG_IPV6)
+#define INDIRECT_CALL_INET(f, f2, f1, ...) INDIRECT_CALL_1(f, f2, __VA_ARGS__)
#else
#define INDIRECT_CALL_INET(f, f2, f1, ...) f(__VA_ARGS__)
#endif
-#if IS_ENABLED(CONFIG_INET)
+#if IS_ENABLED(CONFIG_IPV4)
#define INDIRECT_CALL_INET_1(f, f1, ...) INDIRECT_CALL_1(f, f1, __VA_ARGS__)
#else
#define INDIRECT_CALL_INET_1(f, f1, ...) f(__VA_ARGS__)
diff --git a/include/net/ip.h b/include/net/ip.h
index 6f602df72ee6..194a464c443a 100644
--- a/include/net/ip.h
+++ b/include/net/ip.h
@@ -171,8 +171,17 @@ int ip_mr_input(struct sk_buff *skb);
int ip_mr_output(struct net *net, struct sock *sk, struct sk_buff *skb);
int ip_output(struct net *net, struct sock *sk, struct sk_buff *skb);
int ip_mc_output(struct net *net, struct sock *sk, struct sk_buff *skb);
+#if IS_ENABLED(CONFIG_IPV4)
int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
int (*output)(struct net *, struct sock *, struct sk_buff *));
+#else
+static inline int ip_do_fragment(struct net *net, struct sock *sk, struct sk_buff *skb,
+ int (*output)(struct net *, struct sock *, struct sk_buff *))
+{
+ kfree_skb(skb);
+ return -EAFNOSUPPORT;
+}
+#endif
struct ip_fraglist_iter {
struct sk_buff *frag;
@@ -218,6 +227,7 @@ int ip_local_out(struct net *net, struct sock *sk, struct sk_buff *skb);
int __ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl,
__u8 tos);
void ip_init(void);
+#if IS_ENABLED(CONFIG_IPV4)
int ip_append_data(struct sock *sk, struct flowi4 *fl4,
int getfrag(void *from, char *to, int offset, int len,
int odd, struct sk_buff *skb),
@@ -225,27 +235,58 @@ int ip_append_data(struct sock *sk, struct flowi4 *fl4,
struct ipcm_cookie *ipc,
struct rtable **rt,
unsigned int flags);
-int ip_generic_getfrag(void *from, char *to, int offset, int len, int odd,
- struct sk_buff *skb);
-struct sk_buff *__ip_make_skb(struct sock *sk, struct flowi4 *fl4,
- struct sk_buff_head *queue,
- struct inet_cork *cork);
-int ip_send_skb(struct net *net, struct sk_buff *skb);
-int ip_push_pending_frames(struct sock *sk, struct flowi4 *fl4);
-void ip_flush_pending_frames(struct sock *sk);
struct sk_buff *ip_make_skb(struct sock *sk, struct flowi4 *fl4,
int getfrag(void *from, char *to, int offset,
int len, int odd, struct sk_buff *skb),
void *from, int length, int transhdrlen,
struct ipcm_cookie *ipc, struct rtable **rtp,
struct inet_cork *cork, unsigned int flags);
-
-int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl);
+struct sk_buff *__ip_make_skb(struct sock *sk, struct flowi4 *fl4,
+ struct sk_buff_head *queue,
+ struct inet_cork *cork);
+void ip_flush_pending_frames(struct sock *sk);
static inline struct sk_buff *ip_finish_skb(struct sock *sk, struct flowi4 *fl4)
{
return __ip_make_skb(sk, fl4, &sk->sk_write_queue, &inet_sk(sk)->cork.base);
}
+#else
+static inline int ip_append_data(struct sock *sk, struct flowi4 *fl4,
+ int getfrag(void *from, char *to, int offset, int len,
+ int odd, struct sk_buff *skb),
+ void *from, int len, int protolen,
+ struct ipcm_cookie *ipc,
+ struct rtable **rt,
+ unsigned int flags)
+{
+ return -EAFNOSUPPORT;
+}
+
+static inline struct sk_buff *ip_make_skb(struct sock *sk, struct flowi4 *fl4,
+ int getfrag(void *from, char *to, int offset,
+ int len, int odd, struct sk_buff *skb),
+ void *from, int length, int transhdrlen,
+ struct ipcm_cookie *ipc, struct rtable **rtp,
+ struct inet_cork *cork, unsigned int flags)
+{
+ return ERR_PTR(-EAFNOSUPPORT);
+}
+
+static inline struct sk_buff *ip_finish_skb(struct sock *sk, struct flowi4 *fl4)
+{
+ return ERR_PTR(-EAFNOSUPPORT);
+}
+
+static inline void ip_flush_pending_frames(struct sock *sk)
+{
+}
+#endif
+
+int ip_generic_getfrag(void *from, char *to, int offset, int len, int odd,
+ struct sk_buff *skb);
+int ip_send_skb(struct net *net, struct sk_buff *skb);
+int ip_push_pending_frames(struct sock *sk, struct flowi4 *fl4);
+int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl);
/* Get the route scope that should be used when sending a packet. */
static inline u8 ip_sendmsg_scope(const struct inet_sock *inet,
@@ -768,7 +809,7 @@ static inline bool ip_defrag_user_in_between(u32 user,
}
int ip_defrag(struct net *net, struct sk_buff *skb, u32 user);
-#ifdef CONFIG_INET
+#if IS_ENABLED(CONFIG_IPV4)
struct sk_buff *ip_check_defrag(struct net *net, struct sk_buff *skb, u32 user);
#else
static inline struct sk_buff *ip_check_defrag(struct net *net, struct sk_buff *skb, u32 user)
@@ -813,24 +854,60 @@ int ip_options_rcv_srr(struct sk_buff *skb, struct net_device *dev);
* Functions provided by ip_sockglue.c
*/
-void ipv4_pktinfo_prepare(const struct sock *sk, struct sk_buff *skb, bool drop_dst);
+#if IS_ENABLED(CONFIG_IPV4)
void ip_cmsg_recv_offset(struct msghdr *msg, struct sock *sk,
struct sk_buff *skb, int tlen, int offset);
+void ipv4_pktinfo_prepare(const struct sock *sk, struct sk_buff *skb, bool drop_dst);
int ip_cmsg_send(struct sock *sk, struct msghdr *msg,
struct ipcm_cookie *ipc, bool allow_ipv6);
+int ip_recv_error(struct sock *sk, struct msghdr *msg, int len);
+int ip_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
+ unsigned int optlen);
+int ip_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
+ int __user *optlen);
+#else
+static inline void ip_cmsg_recv_offset(struct msghdr *msg, struct sock *sk,
+ struct sk_buff *skb, int tlen, int offset)
+{
+}
+
+static inline void ipv4_pktinfo_prepare(const struct sock *sk,
+ struct sk_buff *skb,
+ bool drop_dst)
+{
+}
+
+static inline int ip_cmsg_send(struct sock *sk, struct msghdr *msg,
+ struct ipcm_cookie *ipc, bool allow_ipv6)
+{
+ return 0;
+}
+
+static inline int ip_recv_error(struct sock *sk, struct msghdr *msg, int len)
+{
+ return -EAFNOSUPPORT;
+}
+
+static inline int ip_setsockopt(struct sock *sk, int level, int optname,
+ sockptr_t optval, unsigned int optlen)
+{
+ return -EAFNOSUPPORT;
+}
+
+static inline int ip_getsockopt(struct sock *sk, int level, int optname,
+ char __user *optval, int __user *optlen)
+{
+ return -EAFNOSUPPORT;
+}
+#endif
DECLARE_STATIC_KEY_FALSE(ip4_min_ttl);
int do_ip_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
unsigned int optlen);
-int ip_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
- unsigned int optlen);
int do_ip_getsockopt(struct sock *sk, int level, int optname,
sockptr_t optval, sockptr_t optlen);
-int ip_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
- int __user *optlen);
int ip_ra_control(struct sock *sk, unsigned char on,
void (*destructor)(struct sock *));
-int ip_recv_error(struct sock *sk, struct msghdr *msg, int len);
void ip_icmp_error(struct sock *sk, struct sk_buff *skb, int err, __be16 port,
u32 info, u8 *payload);
void ip_local_error(struct sock *sk, int err, __be32 daddr, __be16 dport,
diff --git a/include/net/route.h b/include/net/route.h
index 6b55de2e4df8..8c21881eb5f4 100644
--- a/include/net/route.h
+++ b/include/net/route.h
@@ -169,8 +169,16 @@ static inline struct rtable *__ip_route_output_key(struct net *net,
return ip_route_output_key_hash(net, flp, NULL);
}
+#if IS_ENABLED(CONFIG_IPV4)
struct rtable *ip_route_output_flow(struct net *, struct flowi4 *flp,
const struct sock *sk);
+#else
+static inline struct rtable *ip_route_output_flow(struct net *, struct flowi4 *flp,
+ const struct sock *sk)
+{
+ return ERR_PTR(-EAFNOSUPPORT);
+}
+#endif
struct dst_entry *ipv4_blackhole_route(struct net *net,
struct dst_entry *dst_orig);
diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c
index 164643140a52..a10921aa3e23 100644
--- a/net/core/dev_ioctl.c
+++ b/net/core/dev_ioctl.c
@@ -44,6 +44,9 @@ int dev_ifconf(struct net *net, struct ifconf __user *uifc)
size_t size;
int len, total = 0, done;
+ if (!IS_ENABLED(CONFIG_IPV4))
+ return -EAFNOSUPPORT;
+
/* both the ifconf and the ifreq structures are slightly different */
if (in_compat_syscall()) {
struct compat_ifconf ifc32;
diff --git a/net/core/filter.c b/net/core/filter.c
index 70dc621672f2..86d531c63683 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -2318,7 +2318,7 @@ static int __bpf_redirect_neigh_v6(struct sk_buff *skb, struct net_device *dev,
}
#endif /* CONFIG_IPV6 */
-#if IS_ENABLED(CONFIG_INET)
+#if IS_ENABLED(CONFIG_IPV4)
static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb,
struct net_device *dev, struct bpf_nh_params *nh)
{
@@ -2427,7 +2427,7 @@ static int __bpf_redirect_neigh_v4(struct sk_buff *skb, struct net_device *dev,
kfree_skb(skb);
return NET_XMIT_DROP;
}
-#endif /* CONFIG_INET */
+#endif /* CONFIG_IPV4 */
static int __bpf_redirect_neigh(struct sk_buff *skb, struct net_device *dev,
struct bpf_nh_params *nh)
@@ -5781,7 +5781,7 @@ static int __bpf_setsockopt(struct sock *sk, int level, int optname,
if (level == SOL_SOCKET)
return sol_socket_sockopt(sk, optname, optval, &optlen, false);
- else if (IS_ENABLED(CONFIG_INET) && level == SOL_IP)
+ else if (IS_ENABLED(CONFIG_IPV4) && level == SOL_IP)
return sol_ip_sockopt(sk, optname, optval, &optlen, false);
else if (IS_ENABLED(CONFIG_IPV6) && level == SOL_IPV6)
return sol_ipv6_sockopt(sk, optname, optval, &optlen, false);
@@ -5818,7 +5818,7 @@ static int __bpf_getsockopt(struct sock *sk, int level, int optname,
err = sol_socket_sockopt(sk, optname, optval, &optlen, true);
else if (IS_ENABLED(CONFIG_INET) && level == SOL_TCP)
err = sol_tcp_sockopt(sk, optname, optval, &optlen, true);
- else if (IS_ENABLED(CONFIG_INET) && level == SOL_IP)
+ else if (IS_ENABLED(CONFIG_IPV4) && level == SOL_IP)
err = sol_ip_sockopt(sk, optname, optval, &optlen, true);
else if (IS_ENABLED(CONFIG_IPV6) && level == SOL_IPV6)
err = sol_ipv6_sockopt(sk, optname, optval, &optlen, true);
@@ -6346,7 +6346,7 @@ static struct net_device *bpf_fib_vlan_input_dev(struct net_device *dev,
}
#endif
-#if IS_ENABLED(CONFIG_INET)
+#if IS_ENABLED(CONFIG_IPV4)
static int bpf_ipv4_fib_lookup(struct net *net, struct bpf_fib_lookup *params,
u32 flags, bool check_mtu)
{
@@ -6678,7 +6678,7 @@ BPF_CALL_4(bpf_xdp_fib_lookup, struct xdp_buff *, ctx,
return -EINVAL;
switch (params->family) {
-#if IS_ENABLED(CONFIG_INET)
+#if IS_ENABLED(CONFIG_IPV4)
case AF_INET:
return bpf_ipv4_fib_lookup(dev_net(ctx->rxq->dev), params,
flags, true);
@@ -6722,7 +6722,7 @@ BPF_CALL_4(bpf_skb_fib_lookup, struct sk_buff *, skb,
check_mtu = true;
switch (params->family) {
-#if IS_ENABLED(CONFIG_INET)
+#if IS_ENABLED(CONFIG_IPV4)
case AF_INET:
rc = bpf_ipv4_fib_lookup(net, params, flags, check_mtu);
break;
@@ -12231,7 +12231,7 @@ BPF_CALL_1(bpf_skc_to_tcp_timewait_sock, struct sock *, sk)
BTF_TYPE_EMIT(struct inet_timewait_sock);
BTF_TYPE_EMIT(struct tcp_timewait_sock);
-#ifdef CONFIG_INET
+#ifdef CONFIG_IPV4
if (sk && sk->sk_prot == &tcp_prot && sk->sk_state == TCP_TIME_WAIT)
return (unsigned long)sk;
#endif
@@ -12254,7 +12254,7 @@ const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto = {
BPF_CALL_1(bpf_skc_to_tcp_request_sock, struct sock *, sk)
{
-#ifdef CONFIG_INET
+#ifdef CONFIG_IPV4
if (sk && sk->sk_prot == &tcp_prot && sk->sk_state == TCP_NEW_SYN_RECV)
return (unsigned long)sk;
#endif
@@ -12736,7 +12736,7 @@ __bpf_kfunc int bpf_icmp_send(struct __sk_buff *skb_ctx, int type, int code)
return -ENETUNREACH;
switch (skb->protocol) {
-#if IS_ENABLED(CONFIG_INET)
+#if IS_ENABLED(CONFIG_IPV4)
case htons(ETH_P_IP): {
struct sk_buff *nskb;
diff --git a/net/core/neighbour.c b/net/core/neighbour.c
index 6ed8eb075146..63c2a1b86118 100644
--- a/net/core/neighbour.c
+++ b/net/core/neighbour.c
@@ -3241,6 +3241,10 @@ int neigh_xmit(int index, struct net_device *dev,
if (index == NEIGH_ARP_TABLE) {
u32 key = *((u32 *)addr);
+ if (!IS_ENABLED(CONFIG_IPV4)) {
+ rcu_read_unlock();
+ goto out_kfree_skb;
+ }
neigh = __ipv4_neigh_lookup_noref(dev, key);
} else {
neigh = __neigh_lookup_noref(tbl, addr, dev);
diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
index 5abcb0debb27..ae9f45368571 100644
--- a/net/ipv4/inet_hashtables.c
+++ b/net/ipv4/inet_hashtables.c
@@ -596,6 +596,9 @@ static int __inet_check_established(struct inet_timewait_death_row *death_row,
struct sock *sk2;
spinlock_t *lock;
+ if (!IS_ENABLED(CONFIG_IPV4))
+ return -EAFNOSUPPORT;
+
if (rcu_lookup) {
sk_nulls_for_each(sk2, node, &head->chain) {
if (sk2->sk_hash != hash ||
--
2.55.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 04/16 net-next v2] tcp: move protocol agnostic TCP functions out of tcp_ipv4.c
[not found] <20260928193046.6698-1-fmancera@suse.de>
2026-09-28 19:29 ` [PATCH 02/16 net-next v2] net: core: add IPv4 fallback stubs and guards for CONFIG_IPV4=n Fernando Fernandez Mancera
@ 2026-09-28 19:30 ` Fernando Fernandez Mancera
2026-09-28 19:30 ` [PATCH 06/16 net-next v2] ipv4: udp: split IPv4 specific logic into udp_ipv4.c Fernando Fernandez Mancera
` (2 subsequent siblings)
4 siblings, 0 replies; 7+ messages in thread
From: Fernando Fernandez Mancera @ 2026-09-28 19:30 UTC (permalink / raw)
To: netdev
Cc: horms, kuba, pabeni, edumazet, davem, Fernando Fernandez Mancera,
Eric Dumazet, Neal Cardwell, Kuniyuki Iwashima, John Fastabend,
Jakub Sitnicki, Jiayuan Chen, Kees Cook, Willem de Bruijn,
linux-kernel, bpf
As the tcp_ipv4.c file isn't compile anymore with CONFIG_IPV4=n, the
generic TCP function must be moved out of tcp_ipv4.c file and they
should be placed in tcp.c file.
As part of this migration, rename tcp_v4_destroy_sock() to
tcp_destroy_sock() to reflect its shared usage by IPv4 and IPv6.
In addition, guard IPv4 code around IPv6 TCP implementation. This
ensures that IPv4-mapped IPv6 sockects are rejected with a proper error
code when IPv4 stack is disabled.
Please note that this commit also addresses how /proc/net/sockstat
output is handled for all protocols. The implementation now force the
inuse attrtibute to be zero when CONFIG_IPV4=n but the shared stats are
represented correctly.
Signed-off-by: Fernando Fernandez Mancera <fmancera@suse.de>
---
include/net/tcp.h | 16 +-
net/ipv4/af_inet.c | 93 +--
net/ipv4/inet_connection_sock.c | 2 +-
net/ipv4/proc.c | 45 +-
net/ipv4/tcp.c | 1300 +++++++++++++++++++++++++++++-
net/ipv4/tcp_bpf.c | 3 +-
net/ipv4/tcp_ipv4.c | 1307 +------------------------------
net/ipv6/tcp_ipv6.c | 23 +-
8 files changed, 1412 insertions(+), 1377 deletions(-)
diff --git a/include/net/tcp.h b/include/net/tcp.h
index 96a1a23e47fa..04d487a19b17 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -2437,7 +2437,7 @@ struct tcp_iter_state {
extern struct request_sock_ops tcp_request_sock_ops;
extern struct request_sock_ops tcp6_request_sock_ops;
-void tcp_v4_destroy_sock(struct sock *sk);
+void tcp_destroy_sock(struct sock *sk);
struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
netdev_features_t features);
@@ -2855,6 +2855,20 @@ struct sk_psock;
#ifdef CONFIG_BPF_SYSCALL
int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore);
void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
+
+union bpf_tcp_iter_batch_item {
+ struct sock *sk;
+ __u64 cookie;
+};
+
+struct bpf_tcp_iter_state {
+ struct tcp_iter_state state;
+ unsigned int cur_sk;
+ unsigned int end_sk;
+ unsigned int max_sk;
+ union bpf_tcp_iter_batch_item *batch;
+};
+
#ifdef CONFIG_BPF_STREAM_PARSER
struct strparser;
int tcp_bpf_strp_read_sock(struct strparser *strp, read_descriptor_t *desc,
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index cf00386933d6..6c7b0816f950 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1185,7 +1185,6 @@ static const struct net_proto_family inet_family_ops = {
.owner = THIS_MODULE,
};
-#if IS_ENABLED(CONFIG_IPV4)
/* Upon startup we insert all the elements in inetsw_array[] into
* the linked list inetsw.
*/
@@ -1226,7 +1225,6 @@ static struct inet_protosw inetsw_array[] =
};
#define INETSW_ARRAY_LEN ARRAY_SIZE(inetsw_array)
-#endif
void inet_register_protosw(struct inet_protosw *p)
{
@@ -1814,10 +1812,6 @@ static int __init init_ipv4_mibs(void)
return register_pernet_subsys(&ipv4_mib_ops);
}
-#if IS_ENABLED(CONFIG_IPV4)
-static int ipv4_proc_init(void);
-#endif
-
static __net_init int inet_init_net(struct net *net)
{
/*
@@ -1868,6 +1862,8 @@ static int __init init_inet_pernet_ops(void)
return register_pernet_subsys(&af_inet_ops);
}
+static int ipv4_proc_init(void);
+
/*
* IP protocol layer initialiser
*/
@@ -1912,25 +1908,18 @@ static int __init ipv4_offload_init(void)
fs_initcall(ipv4_offload_init);
-#if IS_ENABLED(CONFIG_IPV4)
static struct packet_type ip_packet_type __read_mostly = {
.type = cpu_to_be16(ETH_P_IP),
.func = ip_rcv,
.list_func = ip_list_rcv,
};
-#endif
-static int __init inet_init(void)
+static int __init inet4_init(void)
{
-#if IS_ENABLED(CONFIG_IPV4)
struct inet_protosw *q;
struct list_head *r;
int rc;
-#endif
- sock_skb_cb_check_size(sizeof(struct inet_skb_parm));
-
-#if IS_ENABLED(CONFIG_IPV4)
raw_hashinfo_init(&raw_v4_hashinfo);
rc = proto_register(&tcp_prot, 1);
@@ -2005,7 +1994,46 @@ static int __init inet_init(void)
*/
ip_init();
-#endif /* CONFIG_IPV4 */
+
+ return 0;
+out_unregister_raw_proto:
+ proto_unregister(&raw_prot);
+out_unregister_udp_proto:
+ proto_unregister(&udp_prot);
+out_unregister_tcp_proto:
+ proto_unregister(&tcp_prot);
+out:
+ return rc;
+}
+
+static void __init inet4_late_init(void)
+{
+ /*
+ * Initialise the multicast router
+ */
+#if defined(CONFIG_IP_MROUTE)
+ if (ip_mr_init())
+ pr_crit("%s: Cannot init ipv4 mroute\n", __func__);
+#endif
+ ipv4_proc_init();
+
+ ipfrag_init();
+
+ dev_add_pack(&ip_packet_type);
+}
+
+static int __init inet_init(void)
+{
+ int rc;
+
+ sock_skb_cb_check_size(sizeof(struct inet_skb_parm));
+
+ if (IS_ENABLED(CONFIG_IPV4)) {
+ rc = inet4_init();
+ if (rc)
+ return rc;
+ }
+
/* Initialise per-cpu ipv4 mibs */
if (init_ipv4_mibs())
panic("%s: Cannot init ipv4 mibs\n", __func__);
@@ -2029,42 +2057,24 @@ static int __init inet_init(void)
if (init_inet_pernet_ops())
pr_crit("%s: Cannot init ipv4 inet pernet ops\n", __func__);
-#if IS_ENABLED(CONFIG_IPV4)
- /*
- * Initialise the multicast router
- */
-#if defined(CONFIG_IP_MROUTE)
- if (ip_mr_init())
- pr_crit("%s: Cannot init ipv4 mroute\n", __func__);
-#endif
- ipv4_proc_init();
- ipfrag_init();
+#ifdef CONFIG_PROC_FS
+ if (ip_misc_proc_init())
+ pr_crit("%s: Cannot init ipv4 misc proc\n", __func__);
+#endif
- dev_add_pack(&ip_packet_type);
-#endif /* CONFIG_IPV4 */
+ if (IS_ENABLED(CONFIG_IPV4))
+ inet4_late_init();
ip_tunnel_core_init();
return 0;
-#if IS_ENABLED(CONFIG_IPV4)
-out:
- return rc;
-out_unregister_raw_proto:
- proto_unregister(&raw_prot);
-out_unregister_udp_proto:
- proto_unregister(&udp_prot);
-out_unregister_tcp_proto:
- proto_unregister(&tcp_prot);
- goto out;
-#endif
}
fs_initcall(inet_init);
/* ------------------------------------------------------------------------ */
-#if IS_ENABLED(CONFIG_IPV4)
#ifdef CONFIG_PROC_FS
static int __init ipv4_proc_init(void)
{
@@ -2078,12 +2088,8 @@ static int __init ipv4_proc_init(void)
goto out_udp;
if (ping_proc_init())
goto out_ping;
- if (ip_misc_proc_init())
- goto out_misc;
out:
return rc;
-out_misc:
- ping_proc_exit();
out_ping:
udp4_proc_exit();
out_udp:
@@ -2101,4 +2107,3 @@ static int __init ipv4_proc_init(void)
return 0;
}
#endif /* CONFIG_PROC_FS */
-#endif
diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c
index c688c454422a..94ade387ad81 100644
--- a/net/ipv4/inet_connection_sock.c
+++ b/net/ipv4/inet_connection_sock.c
@@ -1384,7 +1384,7 @@ static void inet_child_forget(struct sock *sk, struct request_sock *req,
* an inbound pkt destined for child is
* blocked by sock lock in tcp_v4_rcv().
* Also to satisfy an assertion in
- * tcp_v4_destroy_sock().
+ * tcp_destroy_sock().
*/
RCU_INIT_POINTER(tcp_sk(child)->fastopen_rsk, NULL);
}
diff --git a/net/ipv4/proc.c b/net/ipv4/proc.c
index bfc06d1713ec..169e0dd0d213 100644
--- a/net/ipv4/proc.c
+++ b/net/ipv4/proc.c
@@ -29,6 +29,7 @@
*/
#include <linux/types.h>
#include <net/net_namespace.h>
+#include <net/aligned_data.h>
#include <net/icmp.h>
#include <net/protocol.h>
#include <net/tcp.h>
@@ -50,25 +51,33 @@
*/
static int sockstat_seq_show(struct seq_file *seq, void *v)
{
+ int tcp_inuse = 0, udp_inuse = 0, raw_inuse = 0;
struct net *net = seq->private;
+ long tcp_mem, udp_mem;
int orphans, sockets;
orphans = tcp_orphan_count_sum();
- sockets = proto_sockets_allocated_sum_positive(&tcp_prot);
+ sockets = percpu_counter_sum_positive(&tcp_sockets_allocated);
+ tcp_mem = max(0L, atomic_long_read(&net_aligned_data.tcp_memory_allocated));
+ udp_mem = max(0L, atomic_long_read(&net_aligned_data.udp_memory_allocated));
+
+ if (IS_ENABLED(CONFIG_IPV4)) {
+ tcp_inuse = sock_prot_inuse_get(net, &tcp_prot);
+ udp_inuse = sock_prot_inuse_get(net, &udp_prot);
+ raw_inuse = sock_prot_inuse_get(net, &raw_prot);
+ }
socket_seq_show(seq);
seq_printf(seq, "TCP: inuse %d orphan %d tw %d alloc %d mem %ld\n",
- sock_prot_inuse_get(net, &tcp_prot), orphans,
+ tcp_inuse, orphans,
refcount_read(&net->ipv4.tcp_death_row.tw_refcount) - 1,
- sockets, proto_memory_allocated(&tcp_prot));
- seq_printf(seq, "UDP: inuse %d mem %ld\n",
- sock_prot_inuse_get(net, &udp_prot),
- proto_memory_allocated(&udp_prot));
- seq_printf(seq, "RAW: inuse %d\n",
- sock_prot_inuse_get(net, &raw_prot));
- seq_printf(seq, "FRAG: inuse %u memory %lu\n",
- atomic_read(&net->ipv4.fqdir->rhashtable.nelems),
- frag_mem_limit(net->ipv4.fqdir));
+ sockets, tcp_mem);
+ seq_printf(seq, "UDP: inuse %d mem %ld\n", udp_inuse, udp_mem);
+ seq_printf(seq, "RAW: inuse %d\n", raw_inuse);
+ if (IS_ENABLED(CONFIG_IPV4))
+ seq_printf(seq, "FRAG: inuse %u memory %lu\n",
+ atomic_read(&net->ipv4.fqdir->rhashtable.nelems),
+ frag_mem_limit(net->ipv4.fqdir));
return 0;
}
@@ -406,7 +415,7 @@ static int snmp_seq_show_ipstats(struct seq_file *seq, void *v)
return 0;
}
-static int snmp_seq_show_tcp_udp(struct seq_file *seq, void *v)
+static int snmp_seq_show_tcp_udp(struct seq_file *seq, void *v, bool first)
{
const int udp_cnt = ARRAY_SIZE(snmp4_udp_list);
const int tcp_cnt = ARRAY_SIZE(snmp4_tcp_list);
@@ -416,7 +425,7 @@ static int snmp_seq_show_tcp_udp(struct seq_file *seq, void *v)
memset(buff, 0, tcp_cnt * sizeof(unsigned long));
- seq_puts(seq, "\nTcp:");
+ seq_puts(seq, first ? "Tcp:" : "\nTcp:");
for (i = 0; i < tcp_cnt; i++)
seq_printf(seq, " %s", snmp4_tcp_list[i].name);
@@ -450,12 +459,14 @@ static int snmp_seq_show_tcp_udp(struct seq_file *seq, void *v)
static int snmp_seq_show(struct seq_file *seq, void *v)
{
- snmp_seq_show_ipstats(seq, v);
+ if (IS_ENABLED(CONFIG_IPV4)) {
+ snmp_seq_show_ipstats(seq, v);
- icmp_put(seq); /* RFC 2011 compatibility */
- icmpmsg_put(seq);
+ icmp_put(seq); /* RFC 2011 compatibility */
+ icmpmsg_put(seq);
+ }
- snmp_seq_show_tcp_udp(seq, v);
+ snmp_seq_show_tcp_udp(seq, v, !IS_ENABLED(CONFIG_IPV4));
return 0;
}
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 3ac485685279..1f0fc46136ea 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -267,6 +267,8 @@
#include <linux/errqueue.h>
#include <linux/static_key.h>
#include <linux/btf.h>
+#include <linux/proc_fs.h>
+#include <linux/skbuff_ref.h>
#include <net/icmp.h>
#include <net/inet_common.h>
@@ -308,6 +310,10 @@ DEFINE_STATIC_KEY_FALSE(tcp_have_smc);
EXPORT_SYMBOL(tcp_have_smc);
#endif
+struct inet_hashinfo tcp_hashinfo;
+
+static DEFINE_MUTEX(tcp_exit_batch_mutex);
+
/*
* Current number of TCP sockets.
*/
@@ -1458,6 +1464,1115 @@ int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
}
EXPORT_SYMBOL(tcp_sendmsg);
+#ifdef CONFIG_PROC_FS
+/* Proc filesystem TCP sock list dumping. */
+
+static unsigned short seq_file_family(const struct seq_file *seq);
+
+static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
+{
+ unsigned short family = seq_file_family(seq);
+
+ /* AF_UNSPEC is used as a match all */
+ return ((family == AF_UNSPEC || family == sk->sk_family) &&
+ net_eq(sock_net(sk), seq_file_net(seq)));
+}
+
+/* Find a non empty bucket (starting from st->bucket)
+ * and return the first sk from it.
+ */
+static void *listening_get_first(struct seq_file *seq)
+{
+ struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
+ struct tcp_iter_state *st = seq->private;
+
+ st->offset = 0;
+ for (; st->bucket <= hinfo->lhash2_mask; st->bucket++) {
+ struct inet_listen_hashbucket *ilb2;
+ struct hlist_nulls_node *node;
+ struct sock *sk;
+
+ ilb2 = &hinfo->lhash2[st->bucket];
+ if (hlist_nulls_empty(&ilb2->nulls_head))
+ continue;
+
+ spin_lock(&ilb2->lock);
+ sk_nulls_for_each(sk, node, &ilb2->nulls_head) {
+ if (seq_sk_match(seq, sk))
+ return sk;
+ }
+ spin_unlock(&ilb2->lock);
+ }
+
+ return NULL;
+}
+
+/* Find the next sk of "cur" within the same bucket (i.e. st->bucket).
+ * If "cur" is the last one in the st->bucket,
+ * call listening_get_first() to return the first sk of the next
+ * non empty bucket.
+ */
+static void *listening_get_next(struct seq_file *seq, void *cur)
+{
+ struct tcp_iter_state *st = seq->private;
+ struct inet_listen_hashbucket *ilb2;
+ struct hlist_nulls_node *node;
+ struct inet_hashinfo *hinfo;
+ struct sock *sk = cur;
+
+ ++st->num;
+ ++st->offset;
+
+ sk = sk_nulls_next(sk);
+ sk_nulls_for_each_from(sk, node) {
+ if (seq_sk_match(seq, sk))
+ return sk;
+ }
+
+ hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
+ ilb2 = &hinfo->lhash2[st->bucket];
+ spin_unlock(&ilb2->lock);
+ ++st->bucket;
+ return listening_get_first(seq);
+}
+
+static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
+{
+ struct tcp_iter_state *st = seq->private;
+ void *rc;
+
+ st->bucket = 0;
+ st->offset = 0;
+ rc = listening_get_first(seq);
+
+ while (rc && *pos) {
+ rc = listening_get_next(seq, rc);
+ --*pos;
+ }
+ return rc;
+}
+
+static inline bool empty_bucket(struct inet_hashinfo *hinfo,
+ const struct tcp_iter_state *st)
+{
+ return hlist_nulls_empty(&hinfo->ehash[st->bucket].chain);
+}
+
+/*
+ * Get first established socket starting from bucket given in st->bucket.
+ * If st->bucket is zero, the very first socket in the hash is returned.
+ */
+static void *established_get_first(struct seq_file *seq)
+{
+ struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
+ struct tcp_iter_state *st = seq->private;
+
+ st->offset = 0;
+ for (; st->bucket <= hinfo->ehash_mask; ++st->bucket) {
+ struct sock *sk;
+ struct hlist_nulls_node *node;
+ spinlock_t *lock = inet_ehash_lockp(hinfo, st->bucket);
+
+ cond_resched();
+
+ /* Lockless fast path for the common case of empty buckets */
+ if (empty_bucket(hinfo, st))
+ continue;
+
+ spin_lock_bh(lock);
+ sk_nulls_for_each(sk, node, &hinfo->ehash[st->bucket].chain) {
+ if (seq_sk_match(seq, sk))
+ return sk;
+ }
+ spin_unlock_bh(lock);
+ }
+
+ return NULL;
+}
+
+static void *established_get_next(struct seq_file *seq, void *cur)
+{
+ struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
+ struct tcp_iter_state *st = seq->private;
+ struct hlist_nulls_node *node;
+ struct sock *sk = cur;
+
+ ++st->num;
+ ++st->offset;
+
+ sk = sk_nulls_next(sk);
+
+ sk_nulls_for_each_from(sk, node) {
+ if (seq_sk_match(seq, sk))
+ return sk;
+ }
+
+ spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
+ ++st->bucket;
+ return established_get_first(seq);
+}
+
+static void *established_get_idx(struct seq_file *seq, loff_t pos)
+{
+ struct tcp_iter_state *st = seq->private;
+ void *rc;
+
+ st->bucket = 0;
+ rc = established_get_first(seq);
+
+ while (rc && pos) {
+ rc = established_get_next(seq, rc);
+ --pos;
+ }
+ return rc;
+}
+
+static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
+{
+ void *rc;
+ struct tcp_iter_state *st = seq->private;
+
+ st->state = TCP_SEQ_STATE_LISTENING;
+ rc = listening_get_idx(seq, &pos);
+
+ if (!rc) {
+ st->state = TCP_SEQ_STATE_ESTABLISHED;
+ rc = established_get_idx(seq, pos);
+ }
+
+ return rc;
+}
+
+static void *tcp_seek_last_pos(struct seq_file *seq)
+{
+ struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
+ struct tcp_iter_state *st = seq->private;
+ int bucket = st->bucket;
+ int offset = st->offset;
+ int orig_num = st->num;
+ void *rc = NULL;
+
+ switch (st->state) {
+ case TCP_SEQ_STATE_LISTENING:
+ if (st->bucket > hinfo->lhash2_mask)
+ break;
+ rc = listening_get_first(seq);
+ while (offset-- && rc && bucket == st->bucket)
+ rc = listening_get_next(seq, rc);
+ if (rc)
+ break;
+ st->bucket = 0;
+ st->state = TCP_SEQ_STATE_ESTABLISHED;
+ fallthrough;
+ case TCP_SEQ_STATE_ESTABLISHED:
+ if (st->bucket > hinfo->ehash_mask)
+ break;
+ rc = established_get_first(seq);
+ while (offset-- && rc && bucket == st->bucket)
+ rc = established_get_next(seq, rc);
+ }
+
+ st->num = orig_num;
+
+ return rc;
+}
+
+void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
+{
+ struct tcp_iter_state *st = seq->private;
+ void *rc;
+
+ if (*pos && *pos == st->last_pos) {
+ rc = tcp_seek_last_pos(seq);
+ if (rc)
+ goto out;
+ }
+
+ st->state = TCP_SEQ_STATE_LISTENING;
+ st->num = 0;
+ st->bucket = 0;
+ st->offset = 0;
+ rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
+
+out:
+ st->last_pos = *pos;
+ return rc;
+}
+
+void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+ struct tcp_iter_state *st = seq->private;
+ void *rc = NULL;
+
+ if (v == SEQ_START_TOKEN) {
+ rc = tcp_get_idx(seq, 0);
+ goto out;
+ }
+
+ switch (st->state) {
+ case TCP_SEQ_STATE_LISTENING:
+ rc = listening_get_next(seq, v);
+ if (!rc) {
+ st->state = TCP_SEQ_STATE_ESTABLISHED;
+ st->bucket = 0;
+ st->offset = 0;
+ rc = established_get_first(seq);
+ }
+ break;
+ case TCP_SEQ_STATE_ESTABLISHED:
+ rc = established_get_next(seq, v);
+ break;
+ }
+out:
+ ++*pos;
+ st->last_pos = *pos;
+ return rc;
+}
+
+void tcp_seq_stop(struct seq_file *seq, void *v)
+{
+ struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
+ struct tcp_iter_state *st = seq->private;
+
+ switch (st->state) {
+ case TCP_SEQ_STATE_LISTENING:
+ if (v != SEQ_START_TOKEN)
+ spin_unlock(&hinfo->lhash2[st->bucket].lock);
+ break;
+ case TCP_SEQ_STATE_ESTABLISHED:
+ if (v)
+ spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
+ break;
+ }
+}
+
+#ifdef CONFIG_BPF_SYSCALL
+struct bpf_iter__tcp {
+ __bpf_md_ptr(struct bpf_iter_meta *, meta);
+ __bpf_md_ptr(struct sock_common *, sk_common);
+ uid_t uid __aligned(8);
+};
+
+static int tcp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
+ struct sock_common *sk_common, uid_t uid)
+{
+ struct bpf_iter__tcp ctx;
+
+ meta->seq_num--; /* skip SEQ_START_TOKEN */
+ ctx.meta = meta;
+ ctx.sk_common = sk_common;
+ ctx.uid = uid;
+ return bpf_iter_run_prog(prog, &ctx);
+}
+
+static void bpf_iter_tcp_put_batch(struct bpf_tcp_iter_state *iter)
+{
+ union bpf_tcp_iter_batch_item *item;
+ unsigned int cur_sk = iter->cur_sk;
+ __u64 cookie;
+
+ /* Remember the cookies of the sockets we haven't seen yet, so we can
+ * pick up where we left off next time around.
+ */
+ while (cur_sk < iter->end_sk) {
+ item = &iter->batch[cur_sk++];
+ cookie = sock_gen_cookie(item->sk);
+ sock_gen_put(item->sk);
+ item->cookie = cookie;
+ }
+}
+
+static int bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state *iter,
+ unsigned int new_batch_sz, gfp_t flags)
+{
+ union bpf_tcp_iter_batch_item *new_batch;
+
+ new_batch = kvmalloc_objs(*new_batch, new_batch_sz,
+ flags | __GFP_NOWARN);
+ if (!new_batch)
+ return -ENOMEM;
+
+ memcpy(new_batch, iter->batch, sizeof(*iter->batch) * iter->end_sk);
+ kvfree(iter->batch);
+ iter->batch = new_batch;
+ iter->max_sk = new_batch_sz;
+
+ return 0;
+}
+
+static struct sock *bpf_iter_tcp_resume_bucket(struct sock *first_sk,
+ union bpf_tcp_iter_batch_item *cookies,
+ int n_cookies)
+{
+ struct hlist_nulls_node *node;
+ struct sock *sk;
+ int i;
+
+ for (i = 0; i < n_cookies; i++) {
+ sk = first_sk;
+ sk_nulls_for_each_from(sk, node)
+ if (cookies[i].cookie == atomic64_read(&sk->sk_cookie))
+ return sk;
+ }
+
+ return NULL;
+}
+
+static struct sock *bpf_iter_tcp_resume_listening(struct seq_file *seq)
+{
+ struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
+ struct bpf_tcp_iter_state *iter = seq->private;
+ struct tcp_iter_state *st = &iter->state;
+ unsigned int find_cookie = iter->cur_sk;
+ unsigned int end_cookie = iter->end_sk;
+ int resume_bucket = st->bucket;
+ struct sock *sk;
+
+ if (end_cookie && find_cookie == end_cookie)
+ ++st->bucket;
+
+ sk = listening_get_first(seq);
+ iter->cur_sk = 0;
+ iter->end_sk = 0;
+
+ if (sk && st->bucket == resume_bucket && end_cookie) {
+ sk = bpf_iter_tcp_resume_bucket(sk, &iter->batch[find_cookie],
+ end_cookie - find_cookie);
+ if (!sk) {
+ spin_unlock(&hinfo->lhash2[st->bucket].lock);
+ ++st->bucket;
+ sk = listening_get_first(seq);
+ }
+ }
+
+ return sk;
+}
+
+static struct sock *bpf_iter_tcp_resume_established(struct seq_file *seq)
+{
+ struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
+ struct bpf_tcp_iter_state *iter = seq->private;
+ struct tcp_iter_state *st = &iter->state;
+ unsigned int find_cookie = iter->cur_sk;
+ unsigned int end_cookie = iter->end_sk;
+ int resume_bucket = st->bucket;
+ struct sock *sk;
+
+ if (end_cookie && find_cookie == end_cookie)
+ ++st->bucket;
+
+ sk = established_get_first(seq);
+ iter->cur_sk = 0;
+ iter->end_sk = 0;
+
+ if (sk && st->bucket == resume_bucket && end_cookie) {
+ sk = bpf_iter_tcp_resume_bucket(sk, &iter->batch[find_cookie],
+ end_cookie - find_cookie);
+ if (!sk) {
+ spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
+ ++st->bucket;
+ sk = established_get_first(seq);
+ }
+ }
+
+ return sk;
+}
+
+static struct sock *bpf_iter_tcp_resume(struct seq_file *seq)
+{
+ struct bpf_tcp_iter_state *iter = seq->private;
+ struct tcp_iter_state *st = &iter->state;
+ struct sock *sk = NULL;
+
+ switch (st->state) {
+ case TCP_SEQ_STATE_LISTENING:
+ sk = bpf_iter_tcp_resume_listening(seq);
+ if (sk)
+ break;
+ st->bucket = 0;
+ st->state = TCP_SEQ_STATE_ESTABLISHED;
+ fallthrough;
+ case TCP_SEQ_STATE_ESTABLISHED:
+ sk = bpf_iter_tcp_resume_established(seq);
+ break;
+ }
+
+ return sk;
+}
+
+static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
+ struct sock **start_sk)
+{
+ struct bpf_tcp_iter_state *iter = seq->private;
+ struct hlist_nulls_node *node;
+ unsigned int expected = 1;
+ struct sock *sk;
+
+ sock_hold(*start_sk);
+ iter->batch[iter->end_sk++].sk = *start_sk;
+
+ sk = sk_nulls_next(*start_sk);
+ *start_sk = NULL;
+ sk_nulls_for_each_from(sk, node) {
+ if (seq_sk_match(seq, sk)) {
+ if (iter->end_sk < iter->max_sk) {
+ sock_hold(sk);
+ iter->batch[iter->end_sk++].sk = sk;
+ } else if (!*start_sk) {
+ /* Remember where we left off. */
+ *start_sk = sk;
+ }
+ expected++;
+ }
+ }
+
+ return expected;
+}
+
+static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq,
+ struct sock **start_sk)
+{
+ struct bpf_tcp_iter_state *iter = seq->private;
+ struct hlist_nulls_node *node;
+ struct sock *sk = *start_sk;
+ unsigned int expected = 0;
+
+ *start_sk = NULL;
+ sk_nulls_for_each_from(sk, node) {
+ if (!seq_sk_match(seq, sk))
+ continue;
+ expected++;
+ if (iter->end_sk < iter->max_sk) {
+ /* reqsk_queue_hash_req() inserts with sk_refcnt == 0
+ * and refcount_set()s it after the bucket lock drops.
+ */
+ if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))
+ continue;
+ iter->batch[iter->end_sk++].sk = sk;
+ } else if (!*start_sk) {
+ /* Remember where we left off. */
+ *start_sk = sk;
+ }
+ }
+
+ return expected;
+}
+
+static unsigned int bpf_iter_fill_batch(struct seq_file *seq,
+ struct sock **start_sk)
+{
+ struct bpf_tcp_iter_state *iter = seq->private;
+ struct tcp_iter_state *st = &iter->state;
+
+ if (st->state == TCP_SEQ_STATE_LISTENING)
+ return bpf_iter_tcp_listening_batch(seq, start_sk);
+ else
+ return bpf_iter_tcp_established_batch(seq, start_sk);
+}
+
+static void bpf_iter_tcp_unlock_bucket(struct seq_file *seq)
+{
+ struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
+ struct bpf_tcp_iter_state *iter = seq->private;
+ struct tcp_iter_state *st = &iter->state;
+
+ if (st->state == TCP_SEQ_STATE_LISTENING)
+ spin_unlock(&hinfo->lhash2[st->bucket].lock);
+ else
+ spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
+}
+
+static struct sock *bpf_iter_tcp_batch(struct seq_file *seq)
+{
+ struct bpf_tcp_iter_state *iter = seq->private;
+ unsigned int expected;
+ struct sock *sk;
+ int err;
+
+again:
+ sk = bpf_iter_tcp_resume(seq);
+ if (!sk)
+ return NULL; /* Done */
+
+ expected = bpf_iter_fill_batch(seq, &sk);
+ if (likely(!sk))
+ goto done;
+
+ /* Batch size was too small. */
+ bpf_iter_tcp_unlock_bucket(seq);
+ bpf_iter_tcp_put_batch(iter);
+ err = bpf_iter_tcp_realloc_batch(iter, expected * 3 / 2,
+ GFP_USER);
+ if (err) {
+ iter->cur_sk = 0;
+ iter->end_sk = 0;
+ return ERR_PTR(err);
+ }
+
+ sk = bpf_iter_tcp_resume(seq);
+ if (!sk)
+ return NULL; /* Done */
+
+ expected = bpf_iter_fill_batch(seq, &sk);
+ if (likely(!sk))
+ goto done;
+
+ /* Batch size was still too small. Hold onto the lock while we try
+ * again with a larger batch to make sure the current bucket's size
+ * does not change in the meantime.
+ */
+ err = bpf_iter_tcp_realloc_batch(iter, expected, GFP_NOWAIT);
+ if (err) {
+ bpf_iter_tcp_unlock_bucket(seq);
+ return ERR_PTR(err);
+ }
+
+ bpf_iter_fill_batch(seq, &sk);
+ WARN_ON_ONCE(sk);
+done:
+ bpf_iter_tcp_unlock_bucket(seq);
+ if (unlikely(!iter->end_sk)) {
+ ++iter->state.bucket;
+ goto again;
+ }
+ return iter->batch[0].sk;
+}
+
+static void *bpf_iter_tcp_seq_start(struct seq_file *seq, loff_t *pos)
+{
+ /* bpf iter does not support lseek, so it always
+ * continue from where it was stop()-ped.
+ */
+ if (*pos)
+ return bpf_iter_tcp_batch(seq);
+
+ return SEQ_START_TOKEN;
+}
+
+static void *bpf_iter_tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+ struct bpf_tcp_iter_state *iter = seq->private;
+ struct tcp_iter_state *st = &iter->state;
+ struct sock *sk;
+
+ /* Whenever seq_next() is called, the iter->cur_sk is
+ * done with seq_show(), so advance to the next sk in
+ * the batch.
+ */
+ if (iter->cur_sk < iter->end_sk) {
+ /* Keeping st->num consistent in tcp_iter_state.
+ * bpf_iter_tcp does not use st->num.
+ * meta.seq_num is used instead.
+ */
+ st->num++;
+ sock_gen_put(iter->batch[iter->cur_sk++].sk);
+ }
+
+ if (iter->cur_sk < iter->end_sk)
+ sk = iter->batch[iter->cur_sk].sk;
+ else
+ sk = bpf_iter_tcp_batch(seq);
+
+ ++*pos;
+ /* Keeping st->last_pos consistent in tcp_iter_state.
+ * bpf iter does not do lseek, so st->last_pos always equals to *pos.
+ */
+ st->last_pos = *pos;
+ return sk;
+}
+
+static int bpf_iter_tcp_seq_show(struct seq_file *seq, void *v)
+{
+ struct bpf_iter_meta meta;
+ struct bpf_prog *prog;
+ struct sock *sk = v;
+ uid_t uid;
+ int ret;
+
+ if (v == SEQ_START_TOKEN)
+ return 0;
+
+ if (sk_fullsock(sk))
+ lock_sock(sk);
+
+ if (unlikely(sk_unhashed(sk))) {
+ ret = SEQ_SKIP;
+ goto unlock;
+ }
+
+ if (sk->sk_state == TCP_TIME_WAIT) {
+ uid = 0;
+ } else if (sk->sk_state == TCP_NEW_SYN_RECV) {
+ const struct request_sock *req = v;
+
+ uid = from_kuid_munged(seq_user_ns(seq),
+ sk_uid(req->rsk_listener));
+ } else {
+ uid = from_kuid_munged(seq_user_ns(seq), sk_uid(sk));
+ }
+
+ meta.seq = seq;
+ prog = bpf_iter_get_info(&meta, false);
+ ret = tcp_prog_seq_show(prog, &meta, v, uid);
+
+unlock:
+ if (sk_fullsock(sk))
+ release_sock(sk);
+ return ret;
+}
+
+static void bpf_iter_tcp_seq_stop(struct seq_file *seq, void *v)
+{
+ struct bpf_tcp_iter_state *iter = seq->private;
+ struct bpf_iter_meta meta;
+ struct bpf_prog *prog;
+
+ if (!v) {
+ meta.seq = seq;
+ prog = bpf_iter_get_info(&meta, true);
+ if (prog)
+ (void)tcp_prog_seq_show(prog, &meta, v, 0);
+ }
+
+ if (iter->cur_sk < iter->end_sk)
+ bpf_iter_tcp_put_batch(iter);
+}
+
+static const struct seq_operations bpf_iter_tcp_seq_ops = {
+ .show = bpf_iter_tcp_seq_show,
+ .start = bpf_iter_tcp_seq_start,
+ .next = bpf_iter_tcp_seq_next,
+ .stop = bpf_iter_tcp_seq_stop,
+};
+#endif /* CONFIG_BPF_SYSCALL */
+
+static unsigned short seq_file_family(const struct seq_file *seq)
+{
+ const struct tcp_seq_afinfo *afinfo;
+
+#ifdef CONFIG_BPF_SYSCALL
+ /* Iterated from bpf_iter. Let the bpf prog to filter instead. */
+ if (seq->op == &bpf_iter_tcp_seq_ops)
+ return AF_UNSPEC;
+#endif
+
+ /* Iterated from proc fs */
+ afinfo = pde_data(file_inode(seq->file));
+ return afinfo->family;
+}
+#endif /* CONFIG_PROC_FS */
+
+#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
+DEFINE_BPF_ITER_FUNC(tcp, struct bpf_iter_meta *meta,
+ struct sock_common *sk_common, uid_t uid)
+
+#define INIT_BATCH_SZ 16
+
+static int bpf_iter_init_tcp(void *priv_data, struct bpf_iter_aux_info *aux)
+{
+ struct bpf_tcp_iter_state *iter = priv_data;
+ int err;
+
+ err = bpf_iter_init_seq_net(priv_data, aux);
+ if (err)
+ return err;
+
+ err = bpf_iter_tcp_realloc_batch(iter, INIT_BATCH_SZ, GFP_USER);
+ if (err) {
+ bpf_iter_fini_seq_net(priv_data);
+ return err;
+ }
+
+ return 0;
+}
+
+static void bpf_iter_fini_tcp(void *priv_data)
+{
+ struct bpf_tcp_iter_state *iter = priv_data;
+
+ bpf_iter_fini_seq_net(priv_data);
+ kvfree(iter->batch);
+}
+
+static const struct bpf_iter_seq_info tcp_seq_info = {
+ .seq_ops = &bpf_iter_tcp_seq_ops,
+ .init_seq_private = bpf_iter_init_tcp,
+ .fini_seq_private = bpf_iter_fini_tcp,
+ .seq_priv_size = sizeof(struct bpf_tcp_iter_state),
+};
+
+static const struct bpf_func_proto *
+bpf_iter_tcp_get_func_proto(enum bpf_func_id func_id,
+ const struct bpf_prog *prog)
+{
+ switch (func_id) {
+ case BPF_FUNC_setsockopt:
+ return &bpf_sk_setsockopt_proto;
+ case BPF_FUNC_getsockopt:
+ return &bpf_sk_getsockopt_proto;
+ default:
+ return NULL;
+ }
+}
+
+static struct bpf_iter_reg tcp_reg_info = {
+ .target = "tcp",
+ .ctx_arg_info_size = 1,
+ .ctx_arg_info = {
+ { offsetof(struct bpf_iter__tcp, sk_common),
+ PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED },
+ },
+ .get_func_proto = bpf_iter_tcp_get_func_proto,
+ .seq_info = &tcp_seq_info,
+};
+
+static void __init bpf_iter_register(void)
+{
+ tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON];
+ if (bpf_iter_reg_target(&tcp_reg_info))
+ pr_warn("Warning: could not register bpf iterator tcp\n");
+}
+
+#endif
+
+static void tcp_release_user_frags(struct sock *sk)
+{
+#ifdef CONFIG_PAGE_POOL
+ unsigned long index;
+ void *netmem;
+
+ xa_for_each(&sk->sk_user_frags, index, netmem)
+ WARN_ON_ONCE(!napi_pp_put_page((__force netmem_ref)netmem));
+#endif
+}
+
+void tcp_destroy_sock(struct sock *sk)
+{
+ struct tcp_sock *tp = tcp_sk(sk);
+
+ tcp_release_user_frags(sk);
+
+ xa_destroy(&sk->sk_user_frags);
+
+ trace_tcp_destroy_sock(sk);
+
+ tcp_clear_xmit_timers(sk);
+
+ tcp_cleanup_congestion_control(sk);
+
+ tcp_cleanup_ulp(sk);
+
+ /* Cleanup up the write buffer. */
+ tcp_write_queue_purge(sk);
+
+ /* Check if we want to disable active TFO */
+ tcp_fastopen_active_disable_ofo_check(sk);
+
+ /* Cleans up our, hopefully empty, out_of_order_queue. */
+ skb_rbtree_purge(&tp->out_of_order_queue);
+
+ /* Clean up a referenced TCP bind bucket. */
+ if (inet_csk(sk)->icsk_bind_hash)
+ inet_put_port(sk);
+
+ BUG_ON(rcu_access_pointer(tp->fastopen_rsk));
+
+ /* If socket is aborted during connect operation */
+ tcp_free_fastopen_req(tp);
+ tcp_fastopen_destroy_cipher(sk);
+ tcp_saved_syn_free(tp);
+
+ sk_sockets_allocated_dec(sk);
+}
+
+int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
+{
+ int reuse = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse);
+ const struct inet_timewait_sock *tw = inet_twsk(sktw);
+ const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
+ struct tcp_sock *tp = tcp_sk(sk);
+ int ts_recent_stamp;
+ u32 reuse_thresh;
+
+ if (READ_ONCE(tw->tw_substate) == TCP_FIN_WAIT2)
+ reuse = 0;
+
+ if (reuse == 2) {
+ /* Still does not detect *everything* that goes through
+ * lo, since we require a loopback src or dst address
+ * or direct binding to 'lo' interface.
+ */
+ bool loopback = false;
+
+ if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX)
+ loopback = true;
+#if IS_ENABLED(CONFIG_IPV6)
+ if (tw->tw_family == AF_INET6) {
+ if (ipv6_addr_loopback(&tw->tw_v6_daddr) ||
+ ipv6_addr_v4mapped_loopback(&tw->tw_v6_daddr) ||
+ ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) ||
+ ipv6_addr_v4mapped_loopback(&tw->tw_v6_rcv_saddr))
+ loopback = true;
+ } else
+#endif
+ {
+ if (ipv4_is_loopback(tw->tw_daddr) ||
+ ipv4_is_loopback(tw->tw_rcv_saddr))
+ loopback = true;
+ }
+ if (!loopback)
+ reuse = 0;
+ }
+
+ /* With PAWS, it is safe from the viewpoint
+ * of data integrity. Even without PAWS it is safe provided sequence
+ * spaces do not overlap i.e. at data rates <= 80Mbit/sec.
+ *
+ * Actually, the idea is close to VJ's one, only timestamp cache is
+ * held not per host, but per port pair and TW bucket is used as state
+ * holder.
+ *
+ * If TW bucket has been already destroyed we fall back to VJ's scheme
+ * and use initial timestamp retrieved from peer table.
+ */
+ ts_recent_stamp = READ_ONCE(tcptw->tw_ts_recent_stamp);
+ reuse_thresh = READ_ONCE(tw->tw_entry_stamp) +
+ READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse_delay);
+ if (ts_recent_stamp &&
+ (!twp || (reuse && time_after32(tcp_clock_ms(), reuse_thresh)))) {
+ /* inet_twsk_hashdance_schedule() sets sk_refcnt after putting twsk
+ * and releasing the bucket lock.
+ */
+ if (unlikely(!refcount_inc_not_zero(&sktw->sk_refcnt)))
+ return 0;
+
+ /* In case of repair and re-using TIME-WAIT sockets we still
+ * want to be sure that it is safe as above but honor the
+ * sequence numbers and time stamps set as part of the repair
+ * process.
+ *
+ * Without this check re-using a TIME-WAIT socket with TCP
+ * repair would accumulate a -1 on the repair assigned
+ * sequence number. The first time it is reused the sequence
+ * is -1, the second time -2, etc. This fixes that issue
+ * without appearing to create any others.
+ */
+ if (likely(!tp->repair)) {
+ u32 seq = tcptw->tw_snd_nxt + 65535 + 2;
+
+ if (!seq)
+ seq = 1;
+ WRITE_ONCE(tp->write_seq, seq);
+ tp->rx_opt.ts_recent = READ_ONCE(tcptw->tw_ts_recent);
+ tp->rx_opt.ts_recent_stamp = ts_recent_stamp;
+ }
+
+ return 1;
+ }
+
+ return 0;
+}
+
+enum skb_drop_reason tcp_add_backlog(struct sock *sk, struct sk_buff *skb)
+{
+ u32 tail_gso_size, tail_gso_segs;
+ struct skb_shared_info *shinfo;
+ const struct tcphdr *th;
+ struct tcphdr *thtail;
+ struct sk_buff *tail;
+ unsigned int hdrlen;
+ bool fragstolen;
+ u32 gso_segs;
+ u32 gso_size;
+ u64 limit;
+ int delta;
+ int err;
+
+ /* In case all data was pulled from skb frags (in __pskb_pull_tail()),
+ * we can fix skb->truesize to its real value to avoid future drops.
+ * This is valid because skb is not yet charged to the socket.
+ * It has been noticed pure SACK packets were sometimes dropped
+ * (if cooked by drivers without copybreak feature).
+ */
+ skb_condense(skb);
+
+ tcp_cleanup_skb(skb);
+
+ if (unlikely(tcp_checksum_complete(skb))) {
+ bh_unlock_sock(sk);
+ trace_tcp_bad_csum(skb);
+ __TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
+ __TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
+ return SKB_DROP_REASON_TCP_CSUM;
+ }
+
+ /* Attempt coalescing to last skb in backlog, even if we are
+ * above the limits.
+ * This is okay because skb capacity is limited to MAX_SKB_FRAGS.
+ */
+ th = (const struct tcphdr *)skb->data;
+ hdrlen = th->doff * 4;
+
+ tail = sk->sk_backlog.tail;
+ if (!tail)
+ goto no_coalesce;
+ thtail = (struct tcphdr *)tail->data;
+
+ if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq ||
+ TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield ||
+ ((TCP_SKB_CB(tail)->tcp_flags |
+ TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN | TCPHDR_RST | TCPHDR_URG)) ||
+ !((TCP_SKB_CB(tail)->tcp_flags &
+ TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) ||
+ ((TCP_SKB_CB(tail)->tcp_flags ^
+ TCP_SKB_CB(skb)->tcp_flags) &
+ (TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)) ||
+ !tcp_skb_can_collapse_rx(tail, skb) ||
+ thtail->doff != th->doff ||
+ memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)) ||
+ /* prior to PSP Rx policy check, retain exact PSP metadata */
+ psp_skb_coalesce_diff(tail, skb))
+ goto no_coalesce;
+
+ __skb_pull(skb, hdrlen);
+
+ shinfo = skb_shinfo(skb);
+ gso_size = shinfo->gso_size ?: skb->len;
+ gso_segs = shinfo->gso_segs ?: 1;
+
+ shinfo = skb_shinfo(tail);
+ tail_gso_size = shinfo->gso_size ?: (tail->len - hdrlen);
+ tail_gso_segs = shinfo->gso_segs ?: 1;
+
+ if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
+ TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
+
+ if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) {
+ TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
+ thtail->window = th->window;
+ }
+
+ /* We have to update both TCP_SKB_CB(tail)->tcp_flags and
+ * thtail->fin, so that the fast path in tcp_rcv_established()
+ * is not entered if we append a packet with a FIN.
+ * SYN, RST, URG are not present.
+ * ACK is set on both packets.
+ * PSH : we do not really care in TCP stack,
+ * at least for 'GRO' packets.
+ */
+ thtail->fin |= th->fin;
+ TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
+
+ if (TCP_SKB_CB(skb)->has_rxtstamp) {
+ TCP_SKB_CB(tail)->has_rxtstamp = true;
+ tail->tstamp = skb->tstamp;
+ skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
+ }
+
+ /* Not as strict as GRO. We only need to carry mss max value */
+ shinfo->gso_size = max(gso_size, tail_gso_size);
+ shinfo->gso_segs = min_t(u32, gso_segs + tail_gso_segs, 0xFFFF);
+
+ sk->sk_backlog.len += delta;
+ __NET_INC_STATS(sock_net(sk),
+ LINUX_MIB_TCPBACKLOGCOALESCE);
+ kfree_skb_partial(skb, fragstolen);
+ return SKB_NOT_DROPPED_YET;
+ }
+ __skb_push(skb, hdrlen);
+
+no_coalesce:
+ /* sk->sk_backlog.len is reset only at the end of __release_sock().
+ * Both sk->sk_backlog.len and sk->sk_rmem_alloc could reach
+ * sk_rcvbuf in normal conditions.
+ */
+ limit = ((u64)READ_ONCE(sk->sk_rcvbuf)) << 1;
+
+ limit += ((u32)READ_ONCE(sk->sk_sndbuf)) >> 1;
+
+ /* Only socket owner can try to collapse/prune rx queues
+ * to reduce memory overhead, so add a little headroom here.
+ * Few sockets backlog are possibly concurrently non empty.
+ */
+ limit += 64 * 1024;
+
+ limit = min_t(u64, limit, UINT_MAX);
+
+ err = sk_add_backlog(sk, skb, limit);
+ if (unlikely(err)) {
+ bh_unlock_sock(sk);
+ if (err == -ENOMEM) {
+ __NET_INC_STATS(sock_net(sk), LINUX_MIB_PFMEMALLOCDROP);
+ return SKB_DROP_REASON_PFMEMALLOC;
+ }
+ __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP);
+ return SKB_DROP_REASON_SOCKET_BACKLOG;
+ }
+ return SKB_NOT_DROPPED_YET;
+}
+
+/* TCP-LD (RFC 6069) logic */
+void tcp_ld_RTO_revert(struct sock *sk, u32 seq)
+{
+ struct inet_connection_sock *icsk = inet_csk(sk);
+ struct tcp_sock *tp = tcp_sk(sk);
+ struct sk_buff *skb;
+ s32 remaining;
+ u32 delta_us;
+
+ if (sock_owned_by_user(sk))
+ return;
+
+ if (seq != tp->snd_una || !icsk->icsk_retransmits ||
+ !icsk->icsk_backoff)
+ return;
+
+ skb = tcp_rtx_queue_head(sk);
+ if (WARN_ON_ONCE(!skb))
+ return;
+
+ icsk->icsk_backoff--;
+ icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) : TCP_TIMEOUT_INIT;
+ icsk->icsk_rto = inet_csk_rto_backoff(icsk, tcp_rto_max(sk));
+
+ tcp_mstamp_refresh(tp);
+ delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
+ remaining = icsk->icsk_rto - usecs_to_jiffies(delta_us);
+
+ if (remaining > 0) {
+ tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS, remaining, false);
+ } else {
+ /* RTO revert clocked out retransmission.
+ * Will retransmit now.
+ */
+ tcp_retransmit_timer(sk);
+ }
+}
+
+/* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */
+void tcp_req_err(struct sock *sk, u32 seq, bool abort)
+{
+ struct request_sock *req = inet_reqsk(sk);
+ struct net *net = sock_net(sk);
+
+ /* ICMPs are not backlogged, hence we cannot get
+ * an established socket here.
+ */
+ if (seq != tcp_rsk(req)->snt_isn) {
+ __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
+ } else if (abort) {
+ /*
+ * Still in SYN_RECV, just remove it silently.
+ * There is no good way to pass the error to the newly
+ * created socket, and POSIX does not want network
+ * errors returned from accept().
+ */
+ inet_csk_reqsk_queue_drop(req->rsk_listener, req);
+ tcp_listendrop(req->rsk_listener);
+ }
+ reqsk_put(req);
+}
+
void tcp_splice_eof(struct socket *sock)
{
struct sock *sk = sock->sk;
@@ -5291,6 +6406,182 @@ static void __init tcp_struct_check(void)
CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcvq_space);
}
+static void __net_exit tcp_sk_exit(struct net *net)
+{
+ if (net->ipv4.tcp_congestion_control)
+ bpf_module_put(net->ipv4.tcp_congestion_control,
+ net->ipv4.tcp_congestion_control->owner);
+}
+
+static void __net_init tcp_set_hashinfo(struct net *net)
+{
+ struct inet_hashinfo *hinfo;
+ unsigned int ehash_entries;
+ struct net *old_net;
+
+ if (net_eq(net, &init_net))
+ goto fallback;
+
+ old_net = current->nsproxy->net_ns;
+ ehash_entries = READ_ONCE(old_net->ipv4.sysctl_tcp_child_ehash_entries);
+ if (!ehash_entries)
+ goto fallback;
+
+ ehash_entries = roundup_pow_of_two(ehash_entries);
+ hinfo = inet_pernet_hashinfo_alloc(&tcp_hashinfo, ehash_entries);
+ if (!hinfo) {
+ pr_warn("Failed to allocate TCP ehash (entries: %u) "
+ "for a netns, fallback to the global one\n",
+ ehash_entries);
+fallback:
+ hinfo = &tcp_hashinfo;
+ ehash_entries = tcp_hashinfo.ehash_mask + 1;
+ }
+
+ net->ipv4.tcp_death_row.hashinfo = hinfo;
+ net->ipv4.tcp_death_row.sysctl_max_tw_buckets = ehash_entries / 2;
+ net->ipv4.sysctl_max_syn_backlog = max(128U, ehash_entries / 128);
+}
+
+static int __net_init tcp_sk_init(struct net *net)
+{
+ net->ipv4.sysctl_tcp_ecn = TCP_ECN_IN_ECN_OUT_NOECN;
+ net->ipv4.sysctl_tcp_ecn_option = TCP_ACCECN_OPTION_FULL;
+ net->ipv4.sysctl_tcp_ecn_option_beacon = TCP_ACCECN_OPTION_BEACON;
+ net->ipv4.sysctl_tcp_ecn_fallback = 1;
+
+ net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
+ net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS;
+ net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
+ net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
+ net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS;
+
+ net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
+ net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
+ net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;
+
+ net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
+ net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
+ net->ipv4.sysctl_tcp_syncookies = 1;
+ net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
+ net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
+ net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
+ net->ipv4.sysctl_tcp_orphan_retries = 0;
+ net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
+ net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
+ net->ipv4.sysctl_tcp_tw_reuse = 2;
+ net->ipv4.sysctl_tcp_tw_reuse_delay = 1 * MSEC_PER_SEC;
+ net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1;
+
+ refcount_set(&net->ipv4.tcp_death_row.tw_refcount, 1);
+ tcp_set_hashinfo(net);
+
+ net->ipv4.sysctl_tcp_sack = 1;
+ net->ipv4.sysctl_tcp_window_scaling = 1;
+ net->ipv4.sysctl_tcp_timestamps = 1;
+ net->ipv4.sysctl_tcp_early_retrans = 3;
+ net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION;
+ net->ipv4.sysctl_tcp_slow_start_after_idle = 1; /* By default, RFC2861 behavior. */
+ net->ipv4.sysctl_tcp_retrans_collapse = 1;
+ net->ipv4.sysctl_tcp_max_reordering = 300;
+ net->ipv4.sysctl_tcp_dsack = 1;
+ net->ipv4.sysctl_tcp_app_win = 31;
+ net->ipv4.sysctl_tcp_adv_win_scale = 1;
+ net->ipv4.sysctl_tcp_frto = 2;
+ net->ipv4.sysctl_tcp_moderate_rcvbuf = 1;
+ net->ipv4.sysctl_tcp_rcvbuf_low_rtt = USEC_PER_MSEC;
+ /* This limits the percentage of the congestion window which we
+ * will allow a single TSO frame to consume. Building TSO frames
+ * which are too large can cause TCP streams to be bursty.
+ */
+ net->ipv4.sysctl_tcp_tso_win_divisor = 3;
+ /* Default TSQ limit of 4 MB */
+ net->ipv4.sysctl_tcp_limit_output_bytes = 4 << 20;
+
+ /* rfc5961 challenge ack rate limiting, per net-ns, disabled by default. */
+ net->ipv4.sysctl_tcp_challenge_ack_limit = INT_MAX;
+
+ net->ipv4.sysctl_tcp_min_tso_segs = 2;
+ net->ipv4.sysctl_tcp_tso_rtt_log = 9; /* 2^9 = 512 usec */
+ net->ipv4.sysctl_tcp_min_rtt_wlen = 300;
+ net->ipv4.sysctl_tcp_autocorking = 1;
+ net->ipv4.sysctl_tcp_invalid_ratelimit = HZ / 2;
+ net->ipv4.sysctl_tcp_pacing_ss_ratio = 200;
+ net->ipv4.sysctl_tcp_pacing_ca_ratio = 120;
+ if (net != &init_net) {
+ memcpy(net->ipv4.sysctl_tcp_rmem,
+ init_net.ipv4.sysctl_tcp_rmem,
+ sizeof(init_net.ipv4.sysctl_tcp_rmem));
+ memcpy(net->ipv4.sysctl_tcp_wmem,
+ init_net.ipv4.sysctl_tcp_wmem,
+ sizeof(init_net.ipv4.sysctl_tcp_wmem));
+ }
+ net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
+ net->ipv4.sysctl_tcp_comp_sack_slack_ns = 10 * NSEC_PER_USEC;
+ net->ipv4.sysctl_tcp_comp_sack_nr = 44;
+ net->ipv4.sysctl_tcp_comp_sack_rtt_percent = 33;
+ net->ipv4.sysctl_tcp_backlog_ack_defer = 1;
+ net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
+ net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0;
+ atomic_set(&net->ipv4.tfo_active_disable_times, 0);
+
+ /* Set default values for PLB */
+ net->ipv4.sysctl_tcp_plb_enabled = 0; /* Disabled by default */
+ net->ipv4.sysctl_tcp_plb_idle_rehash_rounds = 3;
+ net->ipv4.sysctl_tcp_plb_rehash_rounds = 12;
+ net->ipv4.sysctl_tcp_plb_suspend_rto_sec = 60;
+ /* Default congestion threshold for PLB to mark a round is 50% */
+ net->ipv4.sysctl_tcp_plb_cong_thresh = (1 << TCP_PLB_SCALE) / 2;
+
+ /* Reno is always built in */
+ if (!net_eq(net, &init_net) &&
+ bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
+ init_net.ipv4.tcp_congestion_control->owner))
+ net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
+ else
+ net->ipv4.tcp_congestion_control = &tcp_reno;
+
+ net->ipv4.sysctl_tcp_syn_linear_timeouts = 4;
+ net->ipv4.sysctl_tcp_shrink_window = 0;
+
+ net->ipv4.sysctl_tcp_pingpong_thresh = 1;
+ net->ipv4.sysctl_tcp_rto_min_us = jiffies_to_usecs(TCP_RTO_MIN);
+ net->ipv4.sysctl_tcp_rto_max_ms = TCP_RTO_MAX_SEC * MSEC_PER_SEC;
+
+ return 0;
+}
+
+static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
+{
+ struct net *net;
+
+ /* make sure concurrent calls to tcp_sk_exit_batch from net_cleanup_work
+ * and failed setup_net error unwinding path are serialized.
+ *
+ * tcp_twsk_purge() handles twsk in any dead netns, not just those in
+ * net_exit_list, the thread that dismantles a particular twsk must
+ * do so without other thread progressing to refcount_dec_and_test() of
+ * tcp_death_row.tw_refcount.
+ */
+ mutex_lock(&tcp_exit_batch_mutex);
+
+ tcp_twsk_purge(net_exit_list);
+
+ list_for_each_entry(net, net_exit_list, exit_list) {
+ inet_pernet_hashinfo_free(net->ipv4.tcp_death_row.hashinfo);
+ WARN_ON_ONCE(!refcount_dec_and_test(&net->ipv4.tcp_death_row.tw_refcount));
+ tcp_fastopen_ctx_destroy(net);
+ }
+
+ mutex_unlock(&tcp_exit_batch_mutex);
+}
+
+static struct pernet_operations tcp_sk_ops __net_initdata = {
+ .init = tcp_sk_init,
+ .exit = tcp_sk_exit,
+ .exit_batch = tcp_sk_exit_batch,
+};
+
void __init tcp_init(void)
{
int max_rshare, max_wshare, cnt;
@@ -5385,7 +6676,14 @@ void __init tcp_init(void)
pr_info("Hash tables configured (established %u bind %u)\n",
tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
- tcp_v4_init();
+ if (IS_ENABLED(CONFIG_IPV4))
+ tcp_v4_init();
+ if (register_pernet_subsys(&tcp_sk_ops))
+ panic("Failed to create the TCP control socket.\n");
+
+#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
+ bpf_iter_register();
+#endif
tcp_metrics_init();
BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0);
tcp_tsq_work_init();
diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c
index 2e234d155b5e..23748ec9cb9c 100644
--- a/net/ipv4/tcp_bpf.c
+++ b/net/ipv4/tcp_bpf.c
@@ -671,7 +671,8 @@ static void tcp_bpf_check_v6_needs_rebuild(struct proto *ops)
static int __init tcp_bpf_v4_build_proto(void)
{
- tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV4], &tcp_prot);
+ if (IS_ENABLED(CONFIG_IPV4))
+ tcp_bpf_rebuild_protos(tcp_bpf_prots[TCP_BPF_IPV4], &tcp_prot);
return 0;
}
late_initcall(tcp_bpf_v4_build_proto);
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index c107b7e44e5b..366ed4df49d6 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -85,7 +85,6 @@
#include <linux/seq_file.h>
#include <linux/inetdevice.h>
#include <linux/btf_ids.h>
-#include <linux/skbuff_ref.h>
#include <crypto/md5.h>
#include <crypto/utils.h>
@@ -97,14 +96,10 @@ static void tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key
__be32 daddr, __be32 saddr, const struct tcphdr *th);
#endif
-struct inet_hashinfo tcp_hashinfo;
-
static DEFINE_PER_CPU(struct sock_bh_locked, ipv4_tcp_sk) = {
.bh_lock = INIT_LOCAL_LOCK(bh_lock),
};
-static DEFINE_MUTEX(tcp_exit_batch_mutex);
-
INDIRECT_CALLABLE_SCOPE union tcp_seq_and_ts_off
tcp_v4_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb)
{
@@ -115,93 +110,6 @@ tcp_v4_init_seq_and_ts_off(const struct net *net, const struct sk_buff *skb)
tcp_hdr(skb)->source);
}
-int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
-{
- int reuse = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse);
- const struct inet_timewait_sock *tw = inet_twsk(sktw);
- const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
- struct tcp_sock *tp = tcp_sk(sk);
- int ts_recent_stamp;
- u32 reuse_thresh;
-
- if (READ_ONCE(tw->tw_substate) == TCP_FIN_WAIT2)
- reuse = 0;
-
- if (reuse == 2) {
- /* Still does not detect *everything* that goes through
- * lo, since we require a loopback src or dst address
- * or direct binding to 'lo' interface.
- */
- bool loopback = false;
- if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX)
- loopback = true;
-#if IS_ENABLED(CONFIG_IPV6)
- if (tw->tw_family == AF_INET6) {
- if (ipv6_addr_loopback(&tw->tw_v6_daddr) ||
- ipv6_addr_v4mapped_loopback(&tw->tw_v6_daddr) ||
- ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) ||
- ipv6_addr_v4mapped_loopback(&tw->tw_v6_rcv_saddr))
- loopback = true;
- } else
-#endif
- {
- if (ipv4_is_loopback(tw->tw_daddr) ||
- ipv4_is_loopback(tw->tw_rcv_saddr))
- loopback = true;
- }
- if (!loopback)
- reuse = 0;
- }
-
- /* With PAWS, it is safe from the viewpoint
- of data integrity. Even without PAWS it is safe provided sequence
- spaces do not overlap i.e. at data rates <= 80Mbit/sec.
-
- Actually, the idea is close to VJ's one, only timestamp cache is
- held not per host, but per port pair and TW bucket is used as state
- holder.
-
- If TW bucket has been already destroyed we fall back to VJ's scheme
- and use initial timestamp retrieved from peer table.
- */
- ts_recent_stamp = READ_ONCE(tcptw->tw_ts_recent_stamp);
- reuse_thresh = READ_ONCE(tw->tw_entry_stamp) +
- READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse_delay);
- if (ts_recent_stamp &&
- (!twp || (reuse && time_after32(tcp_clock_ms(), reuse_thresh)))) {
- /* inet_twsk_hashdance_schedule() sets sk_refcnt after putting twsk
- * and releasing the bucket lock.
- */
- if (unlikely(!refcount_inc_not_zero(&sktw->sk_refcnt)))
- return 0;
-
- /* In case of repair and re-using TIME-WAIT sockets we still
- * want to be sure that it is safe as above but honor the
- * sequence numbers and time stamps set as part of the repair
- * process.
- *
- * Without this check re-using a TIME-WAIT socket with TCP
- * repair would accumulate a -1 on the repair assigned
- * sequence number. The first time it is reused the sequence
- * is -1, the second time -2, etc. This fixes that issue
- * without appearing to create any others.
- */
- if (likely(!tp->repair)) {
- u32 seq = tcptw->tw_snd_nxt + 65535 + 2;
-
- if (!seq)
- seq = 1;
- WRITE_ONCE(tp->write_seq, seq);
- tp->rx_opt.ts_recent = READ_ONCE(tcptw->tw_ts_recent);
- tp->rx_opt.ts_recent_stamp = ts_recent_stamp;
- }
-
- return 1;
- }
-
- return 0;
-}
-
static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
int addr_len)
{
@@ -409,69 +317,6 @@ static void do_redirect(struct sk_buff *skb, struct sock *sk)
dst->ops->redirect(dst, sk, skb);
}
-
-/* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */
-void tcp_req_err(struct sock *sk, u32 seq, bool abort)
-{
- struct request_sock *req = inet_reqsk(sk);
- struct net *net = sock_net(sk);
-
- /* ICMPs are not backlogged, hence we cannot get
- * an established socket here.
- */
- if (seq != tcp_rsk(req)->snt_isn) {
- __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
- } else if (abort) {
- /*
- * Still in SYN_RECV, just remove it silently.
- * There is no good way to pass the error to the newly
- * created socket, and POSIX does not want network
- * errors returned from accept().
- */
- inet_csk_reqsk_queue_drop(req->rsk_listener, req);
- tcp_listendrop(req->rsk_listener);
- }
- reqsk_put(req);
-}
-
-/* TCP-LD (RFC 6069) logic */
-void tcp_ld_RTO_revert(struct sock *sk, u32 seq)
-{
- struct inet_connection_sock *icsk = inet_csk(sk);
- struct tcp_sock *tp = tcp_sk(sk);
- struct sk_buff *skb;
- s32 remaining;
- u32 delta_us;
-
- if (sock_owned_by_user(sk))
- return;
-
- if (seq != tp->snd_una || !icsk->icsk_retransmits ||
- !icsk->icsk_backoff)
- return;
-
- skb = tcp_rtx_queue_head(sk);
- if (WARN_ON_ONCE(!skb))
- return;
-
- icsk->icsk_backoff--;
- icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) : TCP_TIMEOUT_INIT;
- icsk->icsk_rto = inet_csk_rto_backoff(icsk, tcp_rto_max(sk));
-
- tcp_mstamp_refresh(tp);
- delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
- remaining = icsk->icsk_rto - usecs_to_jiffies(delta_us);
-
- if (remaining > 0) {
- tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS, remaining, false);
- } else {
- /* RTO revert clocked out retransmission.
- * Will retransmit now.
- */
- tcp_retransmit_timer(sk);
- }
-}
-
/*
* This routine is called by the ICMP module when it gets some
* sort of error condition. If err < 0 then the socket should
@@ -1898,144 +1743,6 @@ int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
goto discard;
}
-enum skb_drop_reason tcp_add_backlog(struct sock *sk, struct sk_buff *skb)
-{
- u32 tail_gso_size, tail_gso_segs;
- struct skb_shared_info *shinfo;
- const struct tcphdr *th;
- struct tcphdr *thtail;
- struct sk_buff *tail;
- unsigned int hdrlen;
- bool fragstolen;
- u32 gso_segs;
- u32 gso_size;
- u64 limit;
- int delta;
- int err;
-
- /* In case all data was pulled from skb frags (in __pskb_pull_tail()),
- * we can fix skb->truesize to its real value to avoid future drops.
- * This is valid because skb is not yet charged to the socket.
- * It has been noticed pure SACK packets were sometimes dropped
- * (if cooked by drivers without copybreak feature).
- */
- skb_condense(skb);
-
- tcp_cleanup_skb(skb);
-
- if (unlikely(tcp_checksum_complete(skb))) {
- bh_unlock_sock(sk);
- trace_tcp_bad_csum(skb);
- __TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
- __TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
- return SKB_DROP_REASON_TCP_CSUM;
- }
-
- /* Attempt coalescing to last skb in backlog, even if we are
- * above the limits.
- * This is okay because skb capacity is limited to MAX_SKB_FRAGS.
- */
- th = (const struct tcphdr *)skb->data;
- hdrlen = th->doff * 4;
-
- tail = sk->sk_backlog.tail;
- if (!tail)
- goto no_coalesce;
- thtail = (struct tcphdr *)tail->data;
-
- if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq ||
- TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield ||
- ((TCP_SKB_CB(tail)->tcp_flags |
- TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN | TCPHDR_RST | TCPHDR_URG)) ||
- !((TCP_SKB_CB(tail)->tcp_flags &
- TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) ||
- ((TCP_SKB_CB(tail)->tcp_flags ^
- TCP_SKB_CB(skb)->tcp_flags) &
- (TCPHDR_ECE | TCPHDR_CWR | TCPHDR_AE)) ||
- !tcp_skb_can_collapse_rx(tail, skb) ||
- thtail->doff != th->doff ||
- memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)) ||
- /* prior to PSP Rx policy check, retain exact PSP metadata */
- psp_skb_coalesce_diff(tail, skb))
- goto no_coalesce;
-
- __skb_pull(skb, hdrlen);
-
- shinfo = skb_shinfo(skb);
- gso_size = shinfo->gso_size ?: skb->len;
- gso_segs = shinfo->gso_segs ?: 1;
-
- shinfo = skb_shinfo(tail);
- tail_gso_size = shinfo->gso_size ?: (tail->len - hdrlen);
- tail_gso_segs = shinfo->gso_segs ?: 1;
-
- if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
- TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
-
- if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) {
- TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
- thtail->window = th->window;
- }
-
- /* We have to update both TCP_SKB_CB(tail)->tcp_flags and
- * thtail->fin, so that the fast path in tcp_rcv_established()
- * is not entered if we append a packet with a FIN.
- * SYN, RST, URG are not present.
- * ACK is set on both packets.
- * PSH : we do not really care in TCP stack,
- * at least for 'GRO' packets.
- */
- thtail->fin |= th->fin;
- TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
-
- if (TCP_SKB_CB(skb)->has_rxtstamp) {
- TCP_SKB_CB(tail)->has_rxtstamp = true;
- tail->tstamp = skb->tstamp;
- skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
- }
-
- /* Not as strict as GRO. We only need to carry mss max value */
- shinfo->gso_size = max(gso_size, tail_gso_size);
- shinfo->gso_segs = min_t(u32, gso_segs + tail_gso_segs, 0xFFFF);
-
- sk->sk_backlog.len += delta;
- __NET_INC_STATS(sock_net(sk),
- LINUX_MIB_TCPBACKLOGCOALESCE);
- kfree_skb_partial(skb, fragstolen);
- return SKB_NOT_DROPPED_YET;
- }
- __skb_push(skb, hdrlen);
-
-no_coalesce:
- /* sk->sk_backlog.len is reset only at the end of __release_sock().
- * Both sk->sk_backlog.len and sk->sk_rmem_alloc could reach
- * sk_rcvbuf in normal conditions.
- */
- limit = ((u64)READ_ONCE(sk->sk_rcvbuf)) << 1;
-
- limit += ((u32)READ_ONCE(sk->sk_sndbuf)) >> 1;
-
- /* Only socket owner can try to collapse/prune rx queues
- * to reduce memory overhead, so add a little headroom here.
- * Few sockets backlog are possibly concurrently non empty.
- */
- limit += 64 * 1024;
-
- limit = min_t(u64, limit, UINT_MAX);
-
- err = sk_add_backlog(sk, skb, limit);
- if (unlikely(err)) {
- bh_unlock_sock(sk);
- if (err == -ENOMEM) {
- __NET_INC_STATS(sock_net(sk), LINUX_MIB_PFMEMALLOCDROP);
- return SKB_DROP_REASON_PFMEMALLOC;
- }
- __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP);
- return SKB_DROP_REASON_SOCKET_BACKLOG;
- }
- return SKB_NOT_DROPPED_YET;
-}
-
static void tcp_v4_restore_cb(struct sk_buff *skb)
{
memmove(IPCB(skb), &TCP_SKB_CB(skb)->header.h4,
@@ -2409,338 +2116,9 @@ static int tcp_v4_init_sock(struct sock *sk)
return 0;
}
-static void tcp_release_user_frags(struct sock *sk)
-{
-#ifdef CONFIG_PAGE_POOL
- unsigned long index;
- void *netmem;
-
- xa_for_each(&sk->sk_user_frags, index, netmem)
- WARN_ON_ONCE(!napi_pp_put_page((__force netmem_ref)netmem));
-#endif
-}
-
-void tcp_v4_destroy_sock(struct sock *sk)
-{
- struct tcp_sock *tp = tcp_sk(sk);
-
- tcp_release_user_frags(sk);
-
- xa_destroy(&sk->sk_user_frags);
-
- trace_tcp_destroy_sock(sk);
-
- tcp_clear_xmit_timers(sk);
-
- tcp_cleanup_congestion_control(sk);
-
- tcp_cleanup_ulp(sk);
-
- /* Cleanup up the write buffer. */
- tcp_write_queue_purge(sk);
-
- /* Check if we want to disable active TFO */
- tcp_fastopen_active_disable_ofo_check(sk);
-
- /* Cleans up our, hopefully empty, out_of_order_queue. */
- skb_rbtree_purge(&tp->out_of_order_queue);
-
- /* Clean up a referenced TCP bind bucket. */
- if (inet_csk(sk)->icsk_bind_hash)
- inet_put_port(sk);
-
- BUG_ON(rcu_access_pointer(tp->fastopen_rsk));
-
- /* If socket is aborted during connect operation */
- tcp_free_fastopen_req(tp);
- tcp_fastopen_destroy_cipher(sk);
- tcp_saved_syn_free(tp);
-
- sk_sockets_allocated_dec(sk);
-}
-
#ifdef CONFIG_PROC_FS
/* Proc filesystem TCP sock list dumping. */
-static unsigned short seq_file_family(const struct seq_file *seq);
-
-static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
-{
- unsigned short family = seq_file_family(seq);
-
- /* AF_UNSPEC is used as a match all */
- return ((family == AF_UNSPEC || family == sk->sk_family) &&
- net_eq(sock_net(sk), seq_file_net(seq)));
-}
-
-/* Find a non empty bucket (starting from st->bucket)
- * and return the first sk from it.
- */
-static void *listening_get_first(struct seq_file *seq)
-{
- struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
- struct tcp_iter_state *st = seq->private;
-
- st->offset = 0;
- for (; st->bucket <= hinfo->lhash2_mask; st->bucket++) {
- struct inet_listen_hashbucket *ilb2;
- struct hlist_nulls_node *node;
- struct sock *sk;
-
- ilb2 = &hinfo->lhash2[st->bucket];
- if (hlist_nulls_empty(&ilb2->nulls_head))
- continue;
-
- spin_lock(&ilb2->lock);
- sk_nulls_for_each(sk, node, &ilb2->nulls_head) {
- if (seq_sk_match(seq, sk))
- return sk;
- }
- spin_unlock(&ilb2->lock);
- }
-
- return NULL;
-}
-
-/* Find the next sk of "cur" within the same bucket (i.e. st->bucket).
- * If "cur" is the last one in the st->bucket,
- * call listening_get_first() to return the first sk of the next
- * non empty bucket.
- */
-static void *listening_get_next(struct seq_file *seq, void *cur)
-{
- struct tcp_iter_state *st = seq->private;
- struct inet_listen_hashbucket *ilb2;
- struct hlist_nulls_node *node;
- struct inet_hashinfo *hinfo;
- struct sock *sk = cur;
-
- ++st->num;
- ++st->offset;
-
- sk = sk_nulls_next(sk);
- sk_nulls_for_each_from(sk, node) {
- if (seq_sk_match(seq, sk))
- return sk;
- }
-
- hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
- ilb2 = &hinfo->lhash2[st->bucket];
- spin_unlock(&ilb2->lock);
- ++st->bucket;
- return listening_get_first(seq);
-}
-
-static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
-{
- struct tcp_iter_state *st = seq->private;
- void *rc;
-
- st->bucket = 0;
- st->offset = 0;
- rc = listening_get_first(seq);
-
- while (rc && *pos) {
- rc = listening_get_next(seq, rc);
- --*pos;
- }
- return rc;
-}
-
-static inline bool empty_bucket(struct inet_hashinfo *hinfo,
- const struct tcp_iter_state *st)
-{
- return hlist_nulls_empty(&hinfo->ehash[st->bucket].chain);
-}
-
-/*
- * Get first established socket starting from bucket given in st->bucket.
- * If st->bucket is zero, the very first socket in the hash is returned.
- */
-static void *established_get_first(struct seq_file *seq)
-{
- struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
- struct tcp_iter_state *st = seq->private;
-
- st->offset = 0;
- for (; st->bucket <= hinfo->ehash_mask; ++st->bucket) {
- struct sock *sk;
- struct hlist_nulls_node *node;
- spinlock_t *lock = inet_ehash_lockp(hinfo, st->bucket);
-
- cond_resched();
-
- /* Lockless fast path for the common case of empty buckets */
- if (empty_bucket(hinfo, st))
- continue;
-
- spin_lock_bh(lock);
- sk_nulls_for_each(sk, node, &hinfo->ehash[st->bucket].chain) {
- if (seq_sk_match(seq, sk))
- return sk;
- }
- spin_unlock_bh(lock);
- }
-
- return NULL;
-}
-
-static void *established_get_next(struct seq_file *seq, void *cur)
-{
- struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
- struct tcp_iter_state *st = seq->private;
- struct hlist_nulls_node *node;
- struct sock *sk = cur;
-
- ++st->num;
- ++st->offset;
-
- sk = sk_nulls_next(sk);
-
- sk_nulls_for_each_from(sk, node) {
- if (seq_sk_match(seq, sk))
- return sk;
- }
-
- spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
- ++st->bucket;
- return established_get_first(seq);
-}
-
-static void *established_get_idx(struct seq_file *seq, loff_t pos)
-{
- struct tcp_iter_state *st = seq->private;
- void *rc;
-
- st->bucket = 0;
- rc = established_get_first(seq);
-
- while (rc && pos) {
- rc = established_get_next(seq, rc);
- --pos;
- }
- return rc;
-}
-
-static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
-{
- void *rc;
- struct tcp_iter_state *st = seq->private;
-
- st->state = TCP_SEQ_STATE_LISTENING;
- rc = listening_get_idx(seq, &pos);
-
- if (!rc) {
- st->state = TCP_SEQ_STATE_ESTABLISHED;
- rc = established_get_idx(seq, pos);
- }
-
- return rc;
-}
-
-static void *tcp_seek_last_pos(struct seq_file *seq)
-{
- struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
- struct tcp_iter_state *st = seq->private;
- int bucket = st->bucket;
- int offset = st->offset;
- int orig_num = st->num;
- void *rc = NULL;
-
- switch (st->state) {
- case TCP_SEQ_STATE_LISTENING:
- if (st->bucket > hinfo->lhash2_mask)
- break;
- rc = listening_get_first(seq);
- while (offset-- && rc && bucket == st->bucket)
- rc = listening_get_next(seq, rc);
- if (rc)
- break;
- st->bucket = 0;
- st->state = TCP_SEQ_STATE_ESTABLISHED;
- fallthrough;
- case TCP_SEQ_STATE_ESTABLISHED:
- if (st->bucket > hinfo->ehash_mask)
- break;
- rc = established_get_first(seq);
- while (offset-- && rc && bucket == st->bucket)
- rc = established_get_next(seq, rc);
- }
-
- st->num = orig_num;
-
- return rc;
-}
-
-void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
-{
- struct tcp_iter_state *st = seq->private;
- void *rc;
-
- if (*pos && *pos == st->last_pos) {
- rc = tcp_seek_last_pos(seq);
- if (rc)
- goto out;
- }
-
- st->state = TCP_SEQ_STATE_LISTENING;
- st->num = 0;
- st->bucket = 0;
- st->offset = 0;
- rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
-
-out:
- st->last_pos = *pos;
- return rc;
-}
-
-void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
-{
- struct tcp_iter_state *st = seq->private;
- void *rc = NULL;
-
- if (v == SEQ_START_TOKEN) {
- rc = tcp_get_idx(seq, 0);
- goto out;
- }
-
- switch (st->state) {
- case TCP_SEQ_STATE_LISTENING:
- rc = listening_get_next(seq, v);
- if (!rc) {
- st->state = TCP_SEQ_STATE_ESTABLISHED;
- st->bucket = 0;
- st->offset = 0;
- rc = established_get_first(seq);
- }
- break;
- case TCP_SEQ_STATE_ESTABLISHED:
- rc = established_get_next(seq, v);
- break;
- }
-out:
- ++*pos;
- st->last_pos = *pos;
- return rc;
-}
-
-void tcp_seq_stop(struct seq_file *seq, void *v)
-{
- struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
- struct tcp_iter_state *st = seq->private;
-
- switch (st->state) {
- case TCP_SEQ_STATE_LISTENING:
- if (v != SEQ_START_TOKEN)
- spin_unlock(&hinfo->lhash2[st->bucket].lock);
- break;
- case TCP_SEQ_STATE_ESTABLISHED:
- if (v)
- spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
- break;
- }
-}
-
static void get_openreq4(const struct request_sock *req,
struct seq_file *f, int i)
{
@@ -2876,434 +2254,6 @@ static int tcp4_seq_show(struct seq_file *seq, void *v)
return 0;
}
-#ifdef CONFIG_BPF_SYSCALL
-union bpf_tcp_iter_batch_item {
- struct sock *sk;
- __u64 cookie;
-};
-
-struct bpf_tcp_iter_state {
- struct tcp_iter_state state;
- unsigned int cur_sk;
- unsigned int end_sk;
- unsigned int max_sk;
- union bpf_tcp_iter_batch_item *batch;
-};
-
-struct bpf_iter__tcp {
- __bpf_md_ptr(struct bpf_iter_meta *, meta);
- __bpf_md_ptr(struct sock_common *, sk_common);
- uid_t uid __aligned(8);
-};
-
-static int tcp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
- struct sock_common *sk_common, uid_t uid)
-{
- struct bpf_iter__tcp ctx;
-
- meta->seq_num--; /* skip SEQ_START_TOKEN */
- ctx.meta = meta;
- ctx.sk_common = sk_common;
- ctx.uid = uid;
- return bpf_iter_run_prog(prog, &ctx);
-}
-
-static void bpf_iter_tcp_put_batch(struct bpf_tcp_iter_state *iter)
-{
- union bpf_tcp_iter_batch_item *item;
- unsigned int cur_sk = iter->cur_sk;
- __u64 cookie;
-
- /* Remember the cookies of the sockets we haven't seen yet, so we can
- * pick up where we left off next time around.
- */
- while (cur_sk < iter->end_sk) {
- item = &iter->batch[cur_sk++];
- cookie = sock_gen_cookie(item->sk);
- sock_gen_put(item->sk);
- item->cookie = cookie;
- }
-}
-
-static int bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state *iter,
- unsigned int new_batch_sz, gfp_t flags)
-{
- union bpf_tcp_iter_batch_item *new_batch;
-
- new_batch = kvmalloc_objs(*new_batch, new_batch_sz,
- flags | __GFP_NOWARN);
- if (!new_batch)
- return -ENOMEM;
-
- memcpy(new_batch, iter->batch, sizeof(*iter->batch) * iter->end_sk);
- kvfree(iter->batch);
- iter->batch = new_batch;
- iter->max_sk = new_batch_sz;
-
- return 0;
-}
-
-static struct sock *bpf_iter_tcp_resume_bucket(struct sock *first_sk,
- union bpf_tcp_iter_batch_item *cookies,
- int n_cookies)
-{
- struct hlist_nulls_node *node;
- struct sock *sk;
- int i;
-
- for (i = 0; i < n_cookies; i++) {
- sk = first_sk;
- sk_nulls_for_each_from(sk, node)
- if (cookies[i].cookie == atomic64_read(&sk->sk_cookie))
- return sk;
- }
-
- return NULL;
-}
-
-static struct sock *bpf_iter_tcp_resume_listening(struct seq_file *seq)
-{
- struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
- struct bpf_tcp_iter_state *iter = seq->private;
- struct tcp_iter_state *st = &iter->state;
- unsigned int find_cookie = iter->cur_sk;
- unsigned int end_cookie = iter->end_sk;
- int resume_bucket = st->bucket;
- struct sock *sk;
-
- if (end_cookie && find_cookie == end_cookie)
- ++st->bucket;
-
- sk = listening_get_first(seq);
- iter->cur_sk = 0;
- iter->end_sk = 0;
-
- if (sk && st->bucket == resume_bucket && end_cookie) {
- sk = bpf_iter_tcp_resume_bucket(sk, &iter->batch[find_cookie],
- end_cookie - find_cookie);
- if (!sk) {
- spin_unlock(&hinfo->lhash2[st->bucket].lock);
- ++st->bucket;
- sk = listening_get_first(seq);
- }
- }
-
- return sk;
-}
-
-static struct sock *bpf_iter_tcp_resume_established(struct seq_file *seq)
-{
- struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
- struct bpf_tcp_iter_state *iter = seq->private;
- struct tcp_iter_state *st = &iter->state;
- unsigned int find_cookie = iter->cur_sk;
- unsigned int end_cookie = iter->end_sk;
- int resume_bucket = st->bucket;
- struct sock *sk;
-
- if (end_cookie && find_cookie == end_cookie)
- ++st->bucket;
-
- sk = established_get_first(seq);
- iter->cur_sk = 0;
- iter->end_sk = 0;
-
- if (sk && st->bucket == resume_bucket && end_cookie) {
- sk = bpf_iter_tcp_resume_bucket(sk, &iter->batch[find_cookie],
- end_cookie - find_cookie);
- if (!sk) {
- spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
- ++st->bucket;
- sk = established_get_first(seq);
- }
- }
-
- return sk;
-}
-
-static struct sock *bpf_iter_tcp_resume(struct seq_file *seq)
-{
- struct bpf_tcp_iter_state *iter = seq->private;
- struct tcp_iter_state *st = &iter->state;
- struct sock *sk = NULL;
-
- switch (st->state) {
- case TCP_SEQ_STATE_LISTENING:
- sk = bpf_iter_tcp_resume_listening(seq);
- if (sk)
- break;
- st->bucket = 0;
- st->state = TCP_SEQ_STATE_ESTABLISHED;
- fallthrough;
- case TCP_SEQ_STATE_ESTABLISHED:
- sk = bpf_iter_tcp_resume_established(seq);
- break;
- }
-
- return sk;
-}
-
-static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
- struct sock **start_sk)
-{
- struct bpf_tcp_iter_state *iter = seq->private;
- struct hlist_nulls_node *node;
- unsigned int expected = 1;
- struct sock *sk;
-
- sock_hold(*start_sk);
- iter->batch[iter->end_sk++].sk = *start_sk;
-
- sk = sk_nulls_next(*start_sk);
- *start_sk = NULL;
- sk_nulls_for_each_from(sk, node) {
- if (seq_sk_match(seq, sk)) {
- if (iter->end_sk < iter->max_sk) {
- sock_hold(sk);
- iter->batch[iter->end_sk++].sk = sk;
- } else if (!*start_sk) {
- /* Remember where we left off. */
- *start_sk = sk;
- }
- expected++;
- }
- }
-
- return expected;
-}
-
-static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq,
- struct sock **start_sk)
-{
- struct bpf_tcp_iter_state *iter = seq->private;
- struct hlist_nulls_node *node;
- struct sock *sk = *start_sk;
- unsigned int expected = 0;
-
- *start_sk = NULL;
- sk_nulls_for_each_from(sk, node) {
- if (!seq_sk_match(seq, sk))
- continue;
- expected++;
- if (iter->end_sk < iter->max_sk) {
- /* reqsk_queue_hash_req() inserts with sk_refcnt == 0
- * and refcount_set()s it after the bucket lock drops.
- */
- if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))
- continue;
- iter->batch[iter->end_sk++].sk = sk;
- } else if (!*start_sk) {
- /* Remember where we left off. */
- *start_sk = sk;
- }
- }
-
- return expected;
-}
-
-static unsigned int bpf_iter_fill_batch(struct seq_file *seq,
- struct sock **start_sk)
-{
- struct bpf_tcp_iter_state *iter = seq->private;
- struct tcp_iter_state *st = &iter->state;
-
- if (st->state == TCP_SEQ_STATE_LISTENING)
- return bpf_iter_tcp_listening_batch(seq, start_sk);
- else
- return bpf_iter_tcp_established_batch(seq, start_sk);
-}
-
-static void bpf_iter_tcp_unlock_bucket(struct seq_file *seq)
-{
- struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
- struct bpf_tcp_iter_state *iter = seq->private;
- struct tcp_iter_state *st = &iter->state;
-
- if (st->state == TCP_SEQ_STATE_LISTENING)
- spin_unlock(&hinfo->lhash2[st->bucket].lock);
- else
- spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
-}
-
-static struct sock *bpf_iter_tcp_batch(struct seq_file *seq)
-{
- struct bpf_tcp_iter_state *iter = seq->private;
- unsigned int expected;
- struct sock *sk;
- int err;
-
-again:
- sk = bpf_iter_tcp_resume(seq);
- if (!sk)
- return NULL; /* Done */
-
- expected = bpf_iter_fill_batch(seq, &sk);
- if (likely(!sk))
- goto done;
-
- /* Batch size was too small. */
- bpf_iter_tcp_unlock_bucket(seq);
- bpf_iter_tcp_put_batch(iter);
- err = bpf_iter_tcp_realloc_batch(iter, expected * 3 / 2,
- GFP_USER);
- if (err) {
- iter->cur_sk = 0;
- iter->end_sk = 0;
- return ERR_PTR(err);
- }
-
- sk = bpf_iter_tcp_resume(seq);
- if (!sk)
- return NULL; /* Done */
-
- expected = bpf_iter_fill_batch(seq, &sk);
- if (likely(!sk))
- goto done;
-
- /* Batch size was still too small. Hold onto the lock while we try
- * again with a larger batch to make sure the current bucket's size
- * does not change in the meantime.
- */
- err = bpf_iter_tcp_realloc_batch(iter, expected, GFP_NOWAIT);
- if (err) {
- bpf_iter_tcp_unlock_bucket(seq);
- return ERR_PTR(err);
- }
-
- bpf_iter_fill_batch(seq, &sk);
- WARN_ON_ONCE(sk);
-done:
- bpf_iter_tcp_unlock_bucket(seq);
- if (unlikely(!iter->end_sk)) {
- ++iter->state.bucket;
- goto again;
- }
- return iter->batch[0].sk;
-}
-
-static void *bpf_iter_tcp_seq_start(struct seq_file *seq, loff_t *pos)
-{
- /* bpf iter does not support lseek, so it always
- * continue from where it was stop()-ped.
- */
- if (*pos)
- return bpf_iter_tcp_batch(seq);
-
- return SEQ_START_TOKEN;
-}
-
-static void *bpf_iter_tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
-{
- struct bpf_tcp_iter_state *iter = seq->private;
- struct tcp_iter_state *st = &iter->state;
- struct sock *sk;
-
- /* Whenever seq_next() is called, the iter->cur_sk is
- * done with seq_show(), so advance to the next sk in
- * the batch.
- */
- if (iter->cur_sk < iter->end_sk) {
- /* Keeping st->num consistent in tcp_iter_state.
- * bpf_iter_tcp does not use st->num.
- * meta.seq_num is used instead.
- */
- st->num++;
- sock_gen_put(iter->batch[iter->cur_sk++].sk);
- }
-
- if (iter->cur_sk < iter->end_sk)
- sk = iter->batch[iter->cur_sk].sk;
- else
- sk = bpf_iter_tcp_batch(seq);
-
- ++*pos;
- /* Keeping st->last_pos consistent in tcp_iter_state.
- * bpf iter does not do lseek, so st->last_pos always equals to *pos.
- */
- st->last_pos = *pos;
- return sk;
-}
-
-static int bpf_iter_tcp_seq_show(struct seq_file *seq, void *v)
-{
- struct bpf_iter_meta meta;
- struct bpf_prog *prog;
- struct sock *sk = v;
- uid_t uid;
- int ret;
-
- if (v == SEQ_START_TOKEN)
- return 0;
-
- if (sk_fullsock(sk))
- lock_sock(sk);
-
- if (unlikely(sk_unhashed(sk))) {
- ret = SEQ_SKIP;
- goto unlock;
- }
-
- if (sk->sk_state == TCP_TIME_WAIT) {
- uid = 0;
- } else if (sk->sk_state == TCP_NEW_SYN_RECV) {
- const struct request_sock *req = v;
-
- uid = from_kuid_munged(seq_user_ns(seq),
- sk_uid(req->rsk_listener));
- } else {
- uid = from_kuid_munged(seq_user_ns(seq), sk_uid(sk));
- }
-
- meta.seq = seq;
- prog = bpf_iter_get_info(&meta, false);
- ret = tcp_prog_seq_show(prog, &meta, v, uid);
-
-unlock:
- if (sk_fullsock(sk))
- release_sock(sk);
- return ret;
-
-}
-
-static void bpf_iter_tcp_seq_stop(struct seq_file *seq, void *v)
-{
- struct bpf_tcp_iter_state *iter = seq->private;
- struct bpf_iter_meta meta;
- struct bpf_prog *prog;
-
- if (!v) {
- meta.seq = seq;
- prog = bpf_iter_get_info(&meta, true);
- if (prog)
- (void)tcp_prog_seq_show(prog, &meta, v, 0);
- }
-
- if (iter->cur_sk < iter->end_sk)
- bpf_iter_tcp_put_batch(iter);
-}
-
-static const struct seq_operations bpf_iter_tcp_seq_ops = {
- .show = bpf_iter_tcp_seq_show,
- .start = bpf_iter_tcp_seq_start,
- .next = bpf_iter_tcp_seq_next,
- .stop = bpf_iter_tcp_seq_stop,
-};
-#endif
-static unsigned short seq_file_family(const struct seq_file *seq)
-{
- const struct tcp_seq_afinfo *afinfo;
-
-#ifdef CONFIG_BPF_SYSCALL
- /* Iterated from bpf_iter. Let the bpf prog to filter instead. */
- if (seq->op == &bpf_iter_tcp_seq_ops)
- return AF_UNSPEC;
-#endif
-
- /* Iterated from proc fs */
- afinfo = pde_data(file_inode(seq->file));
- return afinfo->family;
-}
-
static const struct seq_operations tcp4_seq_ops = {
.show = tcp4_seq_show,
.start = tcp_seq_start,
@@ -3354,7 +2304,7 @@ struct proto tcp_prot = {
.accept = inet_csk_accept,
.ioctl = tcp_ioctl,
.init = tcp_v4_init_sock,
- .destroy = tcp_v4_destroy_sock,
+ .destroy = tcp_destroy_sock,
.shutdown = tcp_shutdown,
.setsockopt = tcp_setsockopt,
.getsockopt = tcp_getsockopt,
@@ -3397,255 +2347,6 @@ struct proto tcp_prot = {
};
EXPORT_SYMBOL(tcp_prot);
-static void __net_exit tcp_sk_exit(struct net *net)
-{
- if (net->ipv4.tcp_congestion_control)
- bpf_module_put(net->ipv4.tcp_congestion_control,
- net->ipv4.tcp_congestion_control->owner);
-}
-
-static void __net_init tcp_set_hashinfo(struct net *net)
-{
- struct inet_hashinfo *hinfo;
- unsigned int ehash_entries;
- struct net *old_net;
-
- if (net_eq(net, &init_net))
- goto fallback;
-
- old_net = current->nsproxy->net_ns;
- ehash_entries = READ_ONCE(old_net->ipv4.sysctl_tcp_child_ehash_entries);
- if (!ehash_entries)
- goto fallback;
-
- ehash_entries = roundup_pow_of_two(ehash_entries);
- hinfo = inet_pernet_hashinfo_alloc(&tcp_hashinfo, ehash_entries);
- if (!hinfo) {
- pr_warn("Failed to allocate TCP ehash (entries: %u) "
- "for a netns, fallback to the global one\n",
- ehash_entries);
-fallback:
- hinfo = &tcp_hashinfo;
- ehash_entries = tcp_hashinfo.ehash_mask + 1;
- }
-
- net->ipv4.tcp_death_row.hashinfo = hinfo;
- net->ipv4.tcp_death_row.sysctl_max_tw_buckets = ehash_entries / 2;
- net->ipv4.sysctl_max_syn_backlog = max(128U, ehash_entries / 128);
-}
-
-static int __net_init tcp_sk_init(struct net *net)
-{
- net->ipv4.sysctl_tcp_ecn = TCP_ECN_IN_ECN_OUT_NOECN;
- net->ipv4.sysctl_tcp_ecn_option = TCP_ACCECN_OPTION_FULL;
- net->ipv4.sysctl_tcp_ecn_option_beacon = TCP_ACCECN_OPTION_BEACON;
- net->ipv4.sysctl_tcp_ecn_fallback = 1;
-
- net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
- net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS;
- net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
- net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
- net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS;
-
- net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
- net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
- net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;
-
- net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
- net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
- net->ipv4.sysctl_tcp_syncookies = 1;
- net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
- net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
- net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
- net->ipv4.sysctl_tcp_orphan_retries = 0;
- net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
- net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
- net->ipv4.sysctl_tcp_tw_reuse = 2;
- net->ipv4.sysctl_tcp_tw_reuse_delay = 1 * MSEC_PER_SEC;
- net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1;
-
- refcount_set(&net->ipv4.tcp_death_row.tw_refcount, 1);
- tcp_set_hashinfo(net);
-
- net->ipv4.sysctl_tcp_sack = 1;
- net->ipv4.sysctl_tcp_window_scaling = 1;
- net->ipv4.sysctl_tcp_timestamps = 1;
- net->ipv4.sysctl_tcp_early_retrans = 3;
- net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION;
- net->ipv4.sysctl_tcp_slow_start_after_idle = 1; /* By default, RFC2861 behavior. */
- net->ipv4.sysctl_tcp_retrans_collapse = 1;
- net->ipv4.sysctl_tcp_max_reordering = 300;
- net->ipv4.sysctl_tcp_dsack = 1;
- net->ipv4.sysctl_tcp_app_win = 31;
- net->ipv4.sysctl_tcp_adv_win_scale = 1;
- net->ipv4.sysctl_tcp_frto = 2;
- net->ipv4.sysctl_tcp_moderate_rcvbuf = 1;
- net->ipv4.sysctl_tcp_rcvbuf_low_rtt = USEC_PER_MSEC;
- /* This limits the percentage of the congestion window which we
- * will allow a single TSO frame to consume. Building TSO frames
- * which are too large can cause TCP streams to be bursty.
- */
- net->ipv4.sysctl_tcp_tso_win_divisor = 3;
- /* Default TSQ limit of 4 MB */
- net->ipv4.sysctl_tcp_limit_output_bytes = 4 << 20;
-
- /* rfc5961 challenge ack rate limiting, per net-ns, disabled by default. */
- net->ipv4.sysctl_tcp_challenge_ack_limit = INT_MAX;
-
- net->ipv4.sysctl_tcp_min_tso_segs = 2;
- net->ipv4.sysctl_tcp_tso_rtt_log = 9; /* 2^9 = 512 usec */
- net->ipv4.sysctl_tcp_min_rtt_wlen = 300;
- net->ipv4.sysctl_tcp_autocorking = 1;
- net->ipv4.sysctl_tcp_invalid_ratelimit = HZ/2;
- net->ipv4.sysctl_tcp_pacing_ss_ratio = 200;
- net->ipv4.sysctl_tcp_pacing_ca_ratio = 120;
- if (net != &init_net) {
- memcpy(net->ipv4.sysctl_tcp_rmem,
- init_net.ipv4.sysctl_tcp_rmem,
- sizeof(init_net.ipv4.sysctl_tcp_rmem));
- memcpy(net->ipv4.sysctl_tcp_wmem,
- init_net.ipv4.sysctl_tcp_wmem,
- sizeof(init_net.ipv4.sysctl_tcp_wmem));
- }
- net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
- net->ipv4.sysctl_tcp_comp_sack_slack_ns = 10 * NSEC_PER_USEC;
- net->ipv4.sysctl_tcp_comp_sack_nr = 44;
- net->ipv4.sysctl_tcp_comp_sack_rtt_percent = 33;
- net->ipv4.sysctl_tcp_backlog_ack_defer = 1;
- net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
- net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0;
- atomic_set(&net->ipv4.tfo_active_disable_times, 0);
-
- /* Set default values for PLB */
- net->ipv4.sysctl_tcp_plb_enabled = 0; /* Disabled by default */
- net->ipv4.sysctl_tcp_plb_idle_rehash_rounds = 3;
- net->ipv4.sysctl_tcp_plb_rehash_rounds = 12;
- net->ipv4.sysctl_tcp_plb_suspend_rto_sec = 60;
- /* Default congestion threshold for PLB to mark a round is 50% */
- net->ipv4.sysctl_tcp_plb_cong_thresh = (1 << TCP_PLB_SCALE) / 2;
-
- /* Reno is always built in */
- if (!net_eq(net, &init_net) &&
- bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
- init_net.ipv4.tcp_congestion_control->owner))
- net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
- else
- net->ipv4.tcp_congestion_control = &tcp_reno;
-
- net->ipv4.sysctl_tcp_syn_linear_timeouts = 4;
- net->ipv4.sysctl_tcp_shrink_window = 0;
-
- net->ipv4.sysctl_tcp_pingpong_thresh = 1;
- net->ipv4.sysctl_tcp_rto_min_us = jiffies_to_usecs(TCP_RTO_MIN);
- net->ipv4.sysctl_tcp_rto_max_ms = TCP_RTO_MAX_SEC * MSEC_PER_SEC;
-
- return 0;
-}
-
-static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
-{
- struct net *net;
-
- /* make sure concurrent calls to tcp_sk_exit_batch from net_cleanup_work
- * and failed setup_net error unwinding path are serialized.
- *
- * tcp_twsk_purge() handles twsk in any dead netns, not just those in
- * net_exit_list, the thread that dismantles a particular twsk must
- * do so without other thread progressing to refcount_dec_and_test() of
- * tcp_death_row.tw_refcount.
- */
- mutex_lock(&tcp_exit_batch_mutex);
-
- tcp_twsk_purge(net_exit_list);
-
- list_for_each_entry(net, net_exit_list, exit_list) {
- inet_pernet_hashinfo_free(net->ipv4.tcp_death_row.hashinfo);
- WARN_ON_ONCE(!refcount_dec_and_test(&net->ipv4.tcp_death_row.tw_refcount));
- tcp_fastopen_ctx_destroy(net);
- }
-
- mutex_unlock(&tcp_exit_batch_mutex);
-}
-
-static struct pernet_operations __net_initdata tcp_sk_ops = {
- .init = tcp_sk_init,
- .exit = tcp_sk_exit,
- .exit_batch = tcp_sk_exit_batch,
-};
-
-#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
-DEFINE_BPF_ITER_FUNC(tcp, struct bpf_iter_meta *meta,
- struct sock_common *sk_common, uid_t uid)
-
-#define INIT_BATCH_SZ 16
-
-static int bpf_iter_init_tcp(void *priv_data, struct bpf_iter_aux_info *aux)
-{
- struct bpf_tcp_iter_state *iter = priv_data;
- int err;
-
- err = bpf_iter_init_seq_net(priv_data, aux);
- if (err)
- return err;
-
- err = bpf_iter_tcp_realloc_batch(iter, INIT_BATCH_SZ, GFP_USER);
- if (err) {
- bpf_iter_fini_seq_net(priv_data);
- return err;
- }
-
- return 0;
-}
-
-static void bpf_iter_fini_tcp(void *priv_data)
-{
- struct bpf_tcp_iter_state *iter = priv_data;
-
- bpf_iter_fini_seq_net(priv_data);
- kvfree(iter->batch);
-}
-
-static const struct bpf_iter_seq_info tcp_seq_info = {
- .seq_ops = &bpf_iter_tcp_seq_ops,
- .init_seq_private = bpf_iter_init_tcp,
- .fini_seq_private = bpf_iter_fini_tcp,
- .seq_priv_size = sizeof(struct bpf_tcp_iter_state),
-};
-
-static const struct bpf_func_proto *
-bpf_iter_tcp_get_func_proto(enum bpf_func_id func_id,
- const struct bpf_prog *prog)
-{
- switch (func_id) {
- case BPF_FUNC_setsockopt:
- return &bpf_sk_setsockopt_proto;
- case BPF_FUNC_getsockopt:
- return &bpf_sk_getsockopt_proto;
- default:
- return NULL;
- }
-}
-
-static struct bpf_iter_reg tcp_reg_info = {
- .target = "tcp",
- .ctx_arg_info_size = 1,
- .ctx_arg_info = {
- { offsetof(struct bpf_iter__tcp, sk_common),
- PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED },
- },
- .get_func_proto = bpf_iter_tcp_get_func_proto,
- .seq_info = &tcp_seq_info,
-};
-
-static void __init bpf_iter_register(void)
-{
- tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON];
- if (bpf_iter_reg_target(&tcp_reg_info))
- pr_warn("Warning: could not register bpf iterator tcp\n");
-}
-
-#endif
-
void __init tcp_v4_init(void)
{
int cpu, res;
@@ -3668,10 +2369,4 @@ void __init tcp_v4_init(void)
per_cpu(ipv4_tcp_sk.sock, cpu) = sk;
}
- if (register_pernet_subsys(&tcp_sk_ops))
- panic("Failed to create the TCP control socket.\n");
-
-#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
- bpf_iter_register();
-#endif
}
diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index afb4d8e9fa67..07a4222e1712 100644
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -220,7 +220,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
u32 exthdrlen = icsk->icsk_ext_hdr_len;
struct sockaddr_in sin;
- if (ipv6_only_sock(sk))
+ if (!IS_ENABLED(CONFIG_IPV4) || ipv6_only_sock(sk))
return -ENETUNREACH;
sin.sin_family = AF_INET;
@@ -1315,8 +1315,11 @@ u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
static int tcp_v6_conn_request(struct sock *sk, struct sk_buff *skb)
{
- if (skb->protocol == htons(ETH_P_IP))
+ if (skb->protocol == htons(ETH_P_IP)) {
+ if (!IS_ENABLED(CONFIG_IPV4))
+ goto drop;
return tcp_v4_conn_request(sk, skb);
+ }
if (!ipv6_unicast_destination(skb))
goto drop;
@@ -1401,10 +1404,14 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *
#endif
struct flowi6 fl6;
- if (skb->protocol == htons(ETH_P_IP))
+ if (skb->protocol == htons(ETH_P_IP)) {
+ if (!IS_ENABLED(CONFIG_IPV4))
+ return NULL;
return tcp_v4_syn_recv_sock(sk, skb, req, dst,
req_unhash, own_req,
tcp_v6_mapped_child_init);
+ }
+
ireq = inet_rsk(req);
if (sk_acceptq_is_full(sk))
@@ -1579,8 +1586,12 @@ int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
tcp_v6_hnd_req and tcp_v6_send_reset(). --ANK
*/
- if (skb->protocol == htons(ETH_P_IP))
- return tcp_v4_do_rcv(sk, skb);
+ if (skb->protocol == htons(ETH_P_IP)) {
+ if (IS_ENABLED(CONFIG_IPV4))
+ return tcp_v4_do_rcv(sk, skb);
+ kfree_skb(skb);
+ return 0;
+ }
reason = psp_sk_rx_policy_check(sk, skb);
if (reason)
@@ -2284,7 +2295,7 @@ struct proto tcpv6_prot = {
.accept = inet_csk_accept,
.ioctl = tcp_ioctl,
.init = tcp_v6_init_sock,
- .destroy = tcp_v4_destroy_sock,
+ .destroy = tcp_destroy_sock,
.shutdown = tcp_shutdown,
.setsockopt = tcp_setsockopt,
.getsockopt = tcp_getsockopt,
--
2.55.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 06/16 net-next v2] ipv4: udp: split IPv4 specific logic into udp_ipv4.c
[not found] <20260928193046.6698-1-fmancera@suse.de>
2026-09-28 19:29 ` [PATCH 02/16 net-next v2] net: core: add IPv4 fallback stubs and guards for CONFIG_IPV4=n Fernando Fernandez Mancera
2026-09-28 19:30 ` [PATCH 04/16 net-next v2] tcp: move protocol agnostic TCP functions out of tcp_ipv4.c Fernando Fernandez Mancera
@ 2026-09-28 19:30 ` Fernando Fernandez Mancera
2026-09-29 19:32 ` sashiko-bot
2026-09-28 19:30 ` [PATCH 08/16 net-next v2] ipv4: ping: split IPv4 specific logic into ping_ipv4.c Fernando Fernandez Mancera
2026-09-28 19:30 ` [PATCH 09/16 net-next v2] ipv4: fib: split common nexthop logic to fib_core.c Fernando Fernandez Mancera
4 siblings, 1 reply; 7+ messages in thread
From: Fernando Fernandez Mancera @ 2026-09-28 19:30 UTC (permalink / raw)
To: netdev
Cc: horms, kuba, pabeni, edumazet, davem, Fernando Fernandez Mancera,
Willem de Bruijn, Eric Dumazet, John Fastabend, Jakub Sitnicki,
Jiayuan Chen, Neal Cardwell, Florian Westphal, Julian Anastasov,
Kuniyuki Iwashima, Eric Biggers, Kees Cook, Alice Mikityanska,
Nikolay Aleksandrov, Gal Pressman, Xuanqiang Luo, linux-kernel,
bpf
To enable compiling the INET subsystem without IPv4, UDP socket IPv4
specific functions must be isolated from the generic UDP socket
infrastructure, similar to how it is done for TCP.
This patch creates udp_ipv4.c and move all the functions for packet
matching, input/output processing, multicast and the AF_INET udp_prot
from udp.c to the new file. The Makefile is updated to compile
udp_ipv4.c only when CONFIG_IPV4 is enabled.
Signed-off-by: Fernando Fernandez Mancera <fmancera@suse.de>
---
include/net/udp.h | 11 +
net/ipv4/Makefile | 2 +-
net/ipv4/af_inet.c | 10 +
net/ipv4/udp.c | 1817 +---------------------------------------
net/ipv4/udp_bpf.c | 5 +-
net/ipv4/udp_ipv4.c | 1790 +++++++++++++++++++++++++++++++++++++++
net/ipv4/udp_offload.c | 3 +
net/ipv6/udp.c | 11 +-
8 files changed, 1861 insertions(+), 1788 deletions(-)
create mode 100644 net/ipv4/udp_ipv4.c
diff --git a/include/net/udp.h b/include/net/udp.h
index 1fee17274745..9915e9b84c65 100644
--- a/include/net/udp.h
+++ b/include/net/udp.h
@@ -436,9 +436,20 @@ int udp_lib_setsockopt(struct sock *sk, int level, int optname,
int (*push_pending_frames)(struct sock *));
struct sock *udp4_lib_lookup(const struct net *net, __be32 saddr, __be16 sport,
__be32 daddr, __be16 dport, int dif);
+#if IS_ENABLED(CONFIG_IPV4)
struct sock *__udp4_lib_lookup(const struct net *net, __be32 saddr,
__be16 sport, __be32 daddr, __be16 dport,
int dif, int sdif, struct sk_buff *skb);
+#else
+static inline struct sock *__udp4_lib_lookup(const struct net *net,
+ __be32 saddr, __be16 sport,
+ __be32 daddr, __be16 dport,
+ int dif, int sdif,
+ struct sk_buff *skb)
+{
+ return NULL;
+}
+#endif
struct sock *udp4_lib_lookup_skb(const struct sk_buff *skb,
__be16 sport, __be16 dport);
struct sock *udp6_lib_lookup(const struct net *net,
diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile
index 7ac882ec5d4f..48ef3fb3b164 100644
--- a/net/ipv4/Makefile
+++ b/net/ipv4/Makefile
@@ -13,7 +13,7 @@ obj-y := inetpeer.o protocol.o inet_hashtables.o inet_timewait_sock.o \
obj-$(CONFIG_IPV4) += route.o ip_input.o ip_fragment.o ip_forward.o ip_options.o \
ip_sockglue.o tcp_ipv4.o datagram.o icmp.o arp.o devinet.o \
igmp.o fib_notifier.o ip_output.o fib_frontend.o \
- fib_semantics.o fib_trie.o raw_ipv4.o
+ fib_semantics.o fib_trie.o raw_ipv4.o udp_ipv4.o
obj-$(CONFIG_NET_IP_TUNNEL) += ip_tunnel.o
obj-$(CONFIG_SYSCTL) += sysctl_net_ipv4.o
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 6c7b0816f950..5a191d0a3bfc 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -886,8 +886,13 @@ int inet_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
return -EAGAIN;
prot = READ_ONCE(sk->sk_prot);
+#if IS_ENABLED(CONFIG_IPV4)
return INDIRECT_CALL_2(prot->sendmsg, tcp_sendmsg, udp_sendmsg,
sk, msg, size);
+#else
+ return INDIRECT_CALL_1(prot->sendmsg, tcp_sendmsg,
+ sk, msg, size);
+#endif
}
EXPORT_SYMBOL(inet_sendmsg);
@@ -914,8 +919,13 @@ int inet_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
sock_rps_record_flow(sk);
prot = READ_ONCE(sk->sk_prot);
+#if IS_ENABLED(CONFIG_IPV4)
return INDIRECT_CALL_2(prot->recvmsg, tcp_recvmsg, udp_recvmsg,
sk, msg, size, flags);
+#else
+ return INDIRECT_CALL_1(prot->recvmsg, tcp_recvmsg,
+ sk, msg, size, flags);
+#endif
}
EXPORT_SYMBOL(inet_recvmsg);
diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index b534b85bc623..0f2ff5a9d4ae 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -347,62 +347,6 @@ int udp_lib_get_port(struct sock *sk, unsigned short snum,
return error;
}
-static int udp_v4_get_port(struct sock *sk, unsigned short snum)
-{
- unsigned int hash2_nulladdr =
- ipv4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum);
- unsigned int hash2_partial =
- ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0);
-
- /* precompute partial secondary hash */
- udp_sk(sk)->udp_portaddr_hash = hash2_partial;
- return udp_lib_get_port(sk, snum, hash2_nulladdr);
-}
-
-static __always_inline int
-compute_score(struct sock *sk, const struct net *net,
- __be32 saddr, __be16 sport, __be32 daddr,
- unsigned short hnum, int dif, int sdif)
-{
- int score;
- struct inet_sock *inet;
- bool dev_match;
-
- if (!net_eq(sock_net(sk), net) ||
- udp_sk(sk)->udp_port_hash != hnum ||
- ipv6_only_sock(sk))
- return -1;
-
- if (sk->sk_rcv_saddr != daddr)
- return -1;
-
- score = (sk->sk_family == PF_INET) ? 2 : 1;
-
- inet = inet_sk(sk);
- if (inet->inet_daddr) {
- if (inet->inet_daddr != saddr)
- return -1;
- score += 4;
- }
-
- if (inet->inet_dport) {
- if (inet->inet_dport != sport)
- return -1;
- score += 4;
- }
-
- dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if,
- dif, sdif);
- if (!dev_match)
- return -1;
- if (sk->sk_bound_dev_if)
- score += 4;
-
- if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
- score++;
- return score;
-}
-
u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,
const __be32 faddr, const __be16 fport)
{
@@ -412,117 +356,7 @@ u32 udp_ehashfn(const struct net *net, const __be32 laddr, const __u16 lport,
udp_ehash_secret + net_hash_mix(net));
}
-/**
- * udp4_lib_lookup1() - Simplified lookup using primary hash (destination port)
- * @net: Network namespace
- * @saddr: Source address, network order
- * @sport: Source port, network order
- * @daddr: Destination address, network order
- * @hnum: Destination port, host order
- * @dif: Destination interface index
- * @sdif: Destination bridge port index, if relevant
- * @udptable: Set of UDP hash tables
- *
- * Simplified lookup to be used as fallback if no sockets are found due to a
- * potential race between (receive) address change, and lookup happening before
- * the rehash operation. This function ignores SO_REUSEPORT groups while scoring
- * result sockets, because if we have one, we don't need the fallback at all.
- *
- * Called under rcu_read_lock().
- *
- * Return: socket with highest matching score if any, NULL if none
- */
-static struct sock *udp4_lib_lookup1(const struct net *net,
- __be32 saddr, __be16 sport,
- __be32 daddr, unsigned int hnum,
- int dif, int sdif,
- const struct udp_table *udptable)
-{
- unsigned int slot = udp_hashfn(net, hnum, udptable->mask);
- struct udp_hslot *hslot = &udptable->hash[slot];
- struct sock *sk, *result = NULL;
- int score, badness = 0;
-
- sk_for_each_rcu(sk, &hslot->head) {
- score = compute_score(sk, net,
- saddr, sport, daddr, hnum, dif, sdif);
- if (score > badness) {
- result = sk;
- badness = score;
- }
- }
-
- return result;
-}
-
-/* called with rcu_read_lock() */
-static struct sock *udp4_lib_lookup2(const struct net *net,
- __be32 saddr, __be16 sport,
- __be32 daddr, unsigned int hnum,
- int dif, int sdif,
- struct udp_hslot *hslot2,
- struct sk_buff *skb)
-{
- struct sock *sk, *result;
- int score, badness;
- bool need_rescore;
-
- result = NULL;
- badness = 0;
- udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
- need_rescore = false;
-rescore:
- score = compute_score(need_rescore ? result : sk, net, saddr,
- sport, daddr, hnum, dif, sdif);
- if (score > badness) {
- badness = score;
-
- if (need_rescore)
- continue;
-
- if (sk->sk_state == TCP_ESTABLISHED) {
- result = sk;
- continue;
- }
-
- result = inet_lookup_reuseport(net, sk, skb, sizeof(struct udphdr),
- saddr, sport, daddr, hnum, udp_ehashfn);
- if (!result) {
- result = sk;
- continue;
- }
-
- /* Fall back to scoring if group has connections */
- if (!reuseport_has_conns(sk))
- return result;
-
- /* Reuseport logic returned an error, keep original score. */
- if (IS_ERR(result))
- continue;
-
- /* compute_score is too long of a function to be
- * inlined twice here, and calling it uninlined
- * here yields measurable overhead for some
- * workloads. Work around it by jumping
- * backwards to rescore 'result'.
- */
- need_rescore = true;
- goto rescore;
- }
- }
- return result;
-}
-
#if IS_ENABLED(CONFIG_BASE_SMALL)
-static struct sock *udp4_lib_lookup4(const struct net *net,
- __be32 saddr, __be16 sport,
- __be32 daddr, unsigned int hnum,
- int dif, int sdif,
- struct udp_table *udptable)
-{
- return NULL;
-}
-
static void udp_rehash4(struct udp_table *udptable, struct sock *sk,
u16 newhash4)
{
@@ -532,42 +366,6 @@ static void udp_unhash4(struct udp_table *udptable, struct sock *sk)
{
}
#else /* !CONFIG_BASE_SMALL */
-static struct sock *udp4_lib_lookup4(const struct net *net,
- __be32 saddr, __be16 sport,
- __be32 daddr, unsigned int hnum,
- int dif, int sdif,
- struct udp_table *udptable)
-{
- const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
- const struct hlist_nulls_node *node;
- struct udp_hslot *hslot4;
- unsigned int hash4, slot;
- struct udp_sock *up;
- struct sock *sk;
-
- hash4 = udp_ehashfn(net, daddr, hnum, saddr, sport);
- slot = hash4 & udptable->mask;
- hslot4 = &udptable->hash4[slot];
- INET_ADDR_COOKIE(acookie, saddr, daddr);
-
-begin:
- /* SLAB_TYPESAFE_BY_RCU not used, so we don't need to touch sk_refcnt */
- udp_lrpa_for_each_entry_rcu(up, node, &hslot4->nulls_head) {
- sk = (struct sock *)up;
- if (inet_match(net, sk, acookie, ports, dif, sdif))
- return sk;
- }
-
- /* if the nulls value we got at the end of this lookup is not the
- * expected one, we must restart lookup. We probably met an item that
- * was moved to another chain due to rehash.
- */
- if (get_nulls_value(node) != slot)
- goto begin;
-
- return NULL;
-}
-
/* udp_rehash4() only checks hslot4, and hash4_cnt is not processed. */
static void udp_rehash4(struct udp_table *udptable, struct sock *sk,
u16 newhash4)
@@ -654,155 +452,8 @@ void udp_lib_hash4(struct sock *sk, u16 hash)
spin_unlock_bh(&hslot->lock);
}
-
-/* call with sock lock */
-void udp4_hash4(struct sock *sk)
-{
- struct net *net = sock_net(sk);
- unsigned int hash;
-
- if (sk_unhashed(sk) || sk->sk_rcv_saddr == htonl(INADDR_ANY))
- return;
-
- hash = udp_ehashfn(net, sk->sk_rcv_saddr, sk->sk_num,
- sk->sk_daddr, sk->sk_dport);
-
- udp_lib_hash4(sk, hash);
-}
#endif /* CONFIG_BASE_SMALL */
-/* UDP is nearly always wildcards out the wazoo, it makes no sense to try
- * harder than this. -DaveM
- */
-struct sock *__udp4_lib_lookup(const struct net *net, __be32 saddr,
- __be16 sport, __be32 daddr, __be16 dport,
- int dif, int sdif, struct sk_buff *skb)
-{
- struct udp_table *udptable = net->ipv4.udp_table;
- unsigned short hnum = ntohs(dport);
- struct udp_hslot *hslot2;
- struct sock *result, *sk;
- unsigned int hash2;
-
- hash2 = ipv4_portaddr_hash(net, daddr, hnum);
- hslot2 = udp_hashslot2(udptable, hash2);
-
- if (udp_has_hash4(hslot2)) {
- result = udp4_lib_lookup4(net, saddr, sport, daddr, hnum,
- dif, sdif, udptable);
- if (result) /* udp4_lib_lookup4 return sk or NULL */
- return result;
- }
-
- /* Lookup connected or non-wildcard socket */
- result = udp4_lib_lookup2(net, saddr, sport,
- daddr, hnum, dif, sdif,
- hslot2, skb);
- if (!IS_ERR_OR_NULL(result) && result->sk_state == TCP_ESTABLISHED)
- goto done;
-
- /* Lookup redirect from BPF */
- if (static_branch_unlikely(&bpf_sk_lookup_enabled)) {
- sk = inet_lookup_run_sk_lookup(net, IPPROTO_UDP, skb, sizeof(struct udphdr),
- saddr, sport, daddr, hnum, dif,
- udp_ehashfn);
- if (sk) {
- result = sk;
- goto done;
- }
- }
-
- /* Got non-wildcard socket or error on first lookup */
- if (result)
- goto done;
-
- /* Lookup wildcard sockets */
- hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
- hslot2 = udp_hashslot2(udptable, hash2);
-
- result = udp4_lib_lookup2(net, saddr, sport,
- htonl(INADDR_ANY), hnum, dif, sdif,
- hslot2, skb);
- if (!IS_ERR_OR_NULL(result))
- goto done;
-
- /* Primary hash (destination port) lookup as fallback for this race:
- * 1. __ip4_datagram_connect() sets sk_rcv_saddr
- * 2. lookup (this function): new sk_rcv_saddr, hashes not updated yet
- * 3. rehash operation updating _secondary and four-tuple_ hashes
- * The primary hash doesn't need an update after 1., so, thanks to this
- * further step, 1. and 3. don't need to be atomic against the lookup.
- */
- result = udp4_lib_lookup1(net, saddr, sport, daddr, hnum, dif, sdif,
- udptable);
-
-done:
- if (IS_ERR(result))
- return NULL;
- return result;
-}
-EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
-
-static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
- __be16 sport, __be16 dport)
-{
- const struct iphdr *iph = ip_hdr(skb);
-
- return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
- iph->daddr, dport, inet_iif(skb),
- inet_sdif(skb), skb);
-}
-
-struct sock *udp4_lib_lookup_skb(const struct sk_buff *skb,
- __be16 sport, __be16 dport)
-{
- const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
- const struct iphdr *iph = (struct iphdr *)(skb->data + offset);
- int iif, sdif;
-
- inet_get_iif_sdif(skb, &iif, &sdif);
-
- return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
- iph->daddr, dport, iif, sdif, NULL);
-}
-
-/* Must be called under rcu_read_lock().
- * Does increment socket refcount.
- */
-#if IS_ENABLED(CONFIG_NF_TPROXY_IPV4) || IS_ENABLED(CONFIG_NF_SOCKET_IPV4)
-struct sock *udp4_lib_lookup(const struct net *net, __be32 saddr, __be16 sport,
- __be32 daddr, __be16 dport, int dif)
-{
- struct sock *sk;
-
- sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport, dif, 0, NULL);
- if (sk && !refcount_inc_not_zero(&sk->sk_refcnt))
- sk = NULL;
- return sk;
-}
-EXPORT_SYMBOL_GPL(udp4_lib_lookup);
-#endif
-
-static inline bool __udp_is_mcast_sock(struct net *net, const struct sock *sk,
- __be16 loc_port, __be32 loc_addr,
- __be16 rmt_port, __be32 rmt_addr,
- int dif, int sdif, unsigned short hnum)
-{
- const struct inet_sock *inet = inet_sk(sk);
-
- if (!net_eq(sock_net(sk), net) ||
- udp_sk(sk)->udp_port_hash != hnum ||
- (inet->inet_daddr && inet->inet_daddr != rmt_addr) ||
- (inet->inet_dport != rmt_port && inet->inet_dport) ||
- (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) ||
- ipv6_only_sock(sk) ||
- !udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))
- return false;
- if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif))
- return false;
- return true;
-}
-
DEFINE_STATIC_KEY_FALSE(udp_encap_needed_key);
#if IS_ENABLED(CONFIG_IPV6)
@@ -821,209 +472,6 @@ void udp_encap_disable(void)
}
EXPORT_SYMBOL(udp_encap_disable);
-/* Handler for tunnels with arbitrary destination ports: no socket lookup, go
- * through error handlers in encapsulations looking for a match.
- */
-static int __udp4_lib_err_encap_no_sk(struct sk_buff *skb, u32 info)
-{
- int i;
-
- for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) {
- int (*handler)(struct sk_buff *skb, u32 info);
- const struct ip_tunnel_encap_ops *encap;
-
- encap = rcu_dereference(iptun_encaps[i]);
- if (!encap)
- continue;
- handler = encap->err_handler;
- if (handler && !handler(skb, info))
- return 0;
- }
-
- return -ENOENT;
-}
-
-/* Try to match ICMP errors to UDP tunnels by looking up a socket without
- * reversing source and destination port: this will match tunnels that force the
- * same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that
- * lwtunnels might actually break this assumption by being configured with
- * different destination ports on endpoints, in this case we won't be able to
- * trace ICMP messages back to them.
- *
- * If this doesn't match any socket, probe tunnels with arbitrary destination
- * ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port
- * we've sent packets to won't necessarily match the local destination port.
- *
- * Then ask the tunnel implementation to match the error against a valid
- * association.
- *
- * Return an error if we can't find a match, the socket if we need further
- * processing, zero otherwise.
- */
-static struct sock *__udp4_lib_err_encap(struct net *net,
- const struct iphdr *iph,
- struct udphdr *uh,
- struct sock *sk,
- struct sk_buff *skb, u32 info)
-{
- int (*lookup)(struct sock *sk, struct sk_buff *skb);
- int network_offset, transport_offset;
- struct udp_sock *up;
-
- network_offset = skb_network_offset(skb);
- transport_offset = skb_transport_offset(skb);
-
- /* Network header needs to point to the outer IPv4 header inside ICMP */
- skb_reset_network_header(skb);
-
- /* Transport header needs to point to the UDP header */
- skb_set_transport_header(skb, iph->ihl << 2);
-
- if (sk) {
- up = udp_sk(sk);
-
- lookup = READ_ONCE(up->encap_err_lookup);
- if (lookup && lookup(sk, skb))
- sk = NULL;
-
- goto out;
- }
-
- sk = __udp4_lib_lookup(net, iph->daddr, uh->source,
- iph->saddr, uh->dest, skb->dev->ifindex, 0, NULL);
- if (sk) {
- up = udp_sk(sk);
-
- lookup = READ_ONCE(up->encap_err_lookup);
- if (!lookup || lookup(sk, skb))
- sk = NULL;
- }
-
-out:
- if (!sk)
- sk = ERR_PTR(__udp4_lib_err_encap_no_sk(skb, info));
-
- skb_set_transport_header(skb, transport_offset);
- skb_set_network_header(skb, network_offset);
-
- return sk;
-}
-
-static void udp_err_update_exception(struct net *net, struct sk_buff *skb,
- int type, int code, u32 info)
-{
- if (type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED)
- ipv4_update_pmtu(skb, net, info, 0, IPPROTO_UDP);
- else if (type == ICMP_REDIRECT)
- ipv4_redirect(skb, net, 0, IPPROTO_UDP);
-}
-
-/*
- * This routine is called by the ICMP module when it gets some
- * sort of error condition. If err < 0 then the socket should
- * be closed and the error returned to the user. If err > 0
- * it's just the icmp type << 8 | icmp code.
- * Header points to the ip header of the error packet. We move
- * on past this. Then (as it used to claim before adjustment)
- * header points to the first 8 bytes of the udp header. We need
- * to find the appropriate port.
- */
-int udp_err(struct sk_buff *skb, u32 info)
-{
- const struct iphdr *iph = (const struct iphdr *)skb->data;
- const int type = icmp_hdr(skb)->type;
- const int code = icmp_hdr(skb)->code;
- struct net *net = dev_net(skb->dev);
- struct inet_sock *inet;
- bool tunnel = false;
- struct udphdr *uh;
- struct sock *sk;
- int harderr;
- int err;
-
- udp_err_update_exception(net, skb, type, code, info);
-
- uh = (struct udphdr *)(skb->data + (iph->ihl << 2));
- sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
- iph->saddr, uh->source, skb->dev->ifindex,
- inet_sdif(skb), NULL);
-
- if (!sk || READ_ONCE(udp_sk(sk)->encap_type)) {
- /* No socket for error: try tunnels before discarding */
- if (static_branch_unlikely(&udp_encap_needed_key)) {
- sk = __udp4_lib_err_encap(net, iph, uh, sk, skb, info);
- if (!sk)
- return 0;
- } else
- sk = ERR_PTR(-ENOENT);
-
- if (IS_ERR(sk)) {
- __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
- return PTR_ERR(sk);
- }
-
- tunnel = true;
- }
-
- err = 0;
- harderr = 0;
- inet = inet_sk(sk);
-
- switch (type) {
- default:
- case ICMP_TIME_EXCEEDED:
- err = EHOSTUNREACH;
- break;
- case ICMP_SOURCE_QUENCH:
- goto out;
- case ICMP_PARAMETERPROB:
- err = EPROTO;
- harderr = 1;
- break;
- case ICMP_DEST_UNREACH:
- if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */
- ipv4_sk_update_pmtu(skb, sk, info);
- if (READ_ONCE(inet->pmtudisc) != IP_PMTUDISC_DONT) {
- err = EMSGSIZE;
- harderr = 1;
- break;
- }
- goto out;
- }
- err = EHOSTUNREACH;
- if (code <= NR_ICMP_UNREACH) {
- harderr = icmp_err_convert[code].fatal;
- err = icmp_err_convert[code].errno;
- }
- break;
- case ICMP_REDIRECT:
- ipv4_sk_redirect(skb, sk);
- goto out;
- }
-
- /*
- * RFC1122: OK. Passes ICMP errors back to application, as per
- * 4.1.3.3.
- */
- if (tunnel) {
- /* ...not for tunnels though: we don't have a sending socket */
- if (udp_sk(sk)->encap_err_rcv)
- udp_sk(sk)->encap_err_rcv(sk, skb, err, uh->dest, info,
- (u8 *)(uh+1));
- goto out;
- }
- if (!inet_test_bit(RECVERR, sk)) {
- if (!harderr || sk->sk_state != TCP_ESTABLISHED)
- goto out;
- } else
- ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh+1));
-
- sk->sk_err = err;
- sk_error_report(sk);
-out:
- return 0;
-}
-
/*
* Throw away all pending data and cancel the corking. Socket is locked.
*/
@@ -1038,52 +486,6 @@ void udp_flush_pending_frames(struct sock *sk)
}
}
-/**
- * udp4_hwcsum - handle outgoing HW checksumming
- * @skb: sk_buff containing the filled-in UDP header
- * (checksum field must be zeroed out)
- * @src: source IP address
- * @dst: destination IP address
- */
-void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
-{
- struct udphdr *uh = udp_hdr(skb);
- int offset = skb_transport_offset(skb);
- int len = skb->len - offset;
- int hlen = len;
- __wsum csum = 0;
-
- if (!skb_has_frag_list(skb)) {
- /*
- * Only one fragment on the socket.
- */
- skb->csum_start = skb_transport_header(skb) - skb->head;
- skb->csum_offset = offsetof(struct udphdr, check);
- uh->check = ~csum_tcpudp_magic(src, dst, len,
- IPPROTO_UDP, 0);
- } else {
- struct sk_buff *frags;
-
- /*
- * HW-checksum won't work as there are two or more
- * fragments on the socket so that all csums of sk_buffs
- * should be together
- */
- skb_walk_frags(skb, frags) {
- csum = csum_add(csum, frags->csum);
- hlen -= frags->len;
- }
-
- csum = skb_checksum(skb, offset, hlen, csum);
- skb->ip_summed = CHECKSUM_NONE;
-
- uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum);
- if (uh->check == 0)
- uh->check = CSUM_MANGLED_0;
- }
-}
-EXPORT_SYMBOL_GPL(udp4_hwcsum);
-
/* Function to set UDP checksum for an IPv4 UDP packet. This is intended
* for the simple case like when setting the checksum for a UDP tunnel.
*/
@@ -1110,112 +512,6 @@ void udp_set_csum(bool nocheck, struct sk_buff *skb,
}
EXPORT_SYMBOL(udp_set_csum);
-static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4,
- struct inet_cork *cork)
-{
- struct sock *sk = skb->sk;
- int offset, len, datalen;
- struct udphdr *uh;
- int err;
-
- offset = skb_transport_offset(skb);
- len = skb->len - offset;
- datalen = len - sizeof(*uh);
-
- /*
- * Create a UDP header
- */
- uh = udp_hdr(skb);
- uh->source = inet_sk(sk)->inet_sport;
- uh->dest = fl4->fl4_dport;
- /* Datagram length checked in udp_sendmsg. */
- udp_set_len_short(uh, len);
- uh->check = 0;
-
- if (cork->gso_size) {
- const int hlen = skb_network_header_len(skb) +
- sizeof(struct udphdr);
-
- if (hlen + min(datalen, cork->gso_size) > cork->fragsize) {
- kfree_skb(skb);
- return -EMSGSIZE;
- }
- if (datalen > cork->gso_size * UDP_MAX_SEGMENTS) {
- kfree_skb(skb);
- return -EINVAL;
- }
- if (sk->sk_no_check_tx) {
- kfree_skb(skb);
- return -EINVAL;
- }
- if (dst_xfrm(skb_dst(skb))) {
- kfree_skb(skb);
- return -EIO;
- }
-
- if (datalen > cork->gso_size) {
- skb_shinfo(skb)->gso_size = cork->gso_size;
- skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4;
- skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(datalen,
- cork->gso_size);
-
- /* Don't checksum the payload, skb will get segmented */
- goto csum_partial;
- }
- }
-
- if (sk->sk_no_check_tx) { /* UDP csum off */
- skb->ip_summed = CHECKSUM_NONE;
- goto send;
- } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
-csum_partial:
- udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
- goto send;
- }
-
- /* add protocol-dependent pseudo-header */
- uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len,
- IPPROTO_UDP, udp_csum(skb));
- if (uh->check == 0)
- uh->check = CSUM_MANGLED_0;
-
-send:
- err = ip_send_skb(sock_net(sk), skb);
- if (unlikely(err)) {
- if (err == -ENOBUFS &&
- !inet_test_bit(RECVERR, sk)) {
- UDP_INC_STATS(sock_net(sk), UDP_MIB_SNDBUFERRORS);
- err = 0;
- }
- } else {
- UDP_INC_STATS(sock_net(sk), UDP_MIB_OUTDATAGRAMS);
- }
- return err;
-}
-
-/*
- * Push out all pending data as one UDP datagram. Socket is locked.
- */
-int udp_push_pending_frames(struct sock *sk)
-{
- struct udp_sock *up = udp_sk(sk);
- struct inet_sock *inet = inet_sk(sk);
- struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
- struct sk_buff *skb;
- int err = 0;
-
- skb = ip_finish_skb(sk, fl4);
- if (!skb)
- goto out;
-
- err = udp_send_skb(skb, fl4, &inet->cork.base);
-
-out:
- up->len = 0;
- WRITE_ONCE(up->pending, 0);
- return err;
-}
-
static int __udp_cmsg_send(struct cmsghdr *cmsg, u16 *gso_size)
{
switch (cmsg->cmsg_type) {
@@ -1252,296 +548,6 @@ int udp_cmsg_send(struct sock *sk, struct msghdr *msg, u16 *gso_size)
return need_ip;
}
-int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
-{
- int corkreq = udp_test_bit(CORK, sk) || msg->msg_flags & MSG_MORE;
- DEFINE_RAW_FLEX(struct ip_options_rcu, opt_copy, opt.__data,
- IP_OPTIONS_DATA_FIXED_SIZE);
- DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
- int ulen = len, free = 0, connected = 0;
- struct inet_sock *inet = inet_sk(sk);
- struct udp_sock *up = udp_sk(sk);
- __be32 daddr, faddr, saddr;
- struct rtable *rt = NULL;
- struct flowi4 fl4_stack;
- struct ipcm_cookie ipc;
- struct sk_buff *skb;
- struct flowi4 *fl4;
- __be16 dport;
- int uc_index;
- u8 scope;
- int err;
-
- if (len > 0xFFFF)
- return -EMSGSIZE;
-
- /*
- * Check the flags.
- */
-
- if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */
- return -EOPNOTSUPP;
-
- fl4 = &inet->cork.fl.u.ip4;
- if (READ_ONCE(up->pending)) {
- /*
- * There are pending frames.
- * The socket lock must be held while it's corked.
- */
- lock_sock(sk);
- if (likely(up->pending)) {
- if (unlikely(up->pending != AF_INET)) {
- release_sock(sk);
- return -EINVAL;
- }
- goto do_append_data;
- }
- release_sock(sk);
- }
- ulen += sizeof(struct udphdr);
-
- /*
- * Get and verify the address.
- */
- if (usin) {
- if (msg->msg_namelen < sizeof(*usin))
- return -EINVAL;
- if (usin->sin_family != AF_INET) {
- if (usin->sin_family != AF_UNSPEC)
- return -EAFNOSUPPORT;
- }
-
- daddr = usin->sin_addr.s_addr;
- dport = usin->sin_port;
- if (dport == 0)
- return -EINVAL;
- } else {
- if (sk->sk_state != TCP_ESTABLISHED)
- return -EDESTADDRREQ;
- daddr = inet->inet_daddr;
- dport = inet->inet_dport;
- /* Open fast path for connected socket.
- Route will not be used, if at least one option is set.
- */
- connected = 1;
- }
-
- ipcm_init_sk(&ipc, inet);
- ipc.gso_size = READ_ONCE(up->gso_size);
-
- if (msg->msg_controllen) {
- err = udp_cmsg_send(sk, msg, &ipc.gso_size);
- if (err > 0) {
- err = ip_cmsg_send(sk, msg, &ipc,
- sk->sk_family == AF_INET6);
- connected = 0;
- }
- if (unlikely(err < 0)) {
- kfree(ipc.opt);
- return err;
- }
- if (ipc.opt)
- free = 1;
- }
- if (!ipc.opt) {
- struct ip_options_rcu *inet_opt;
-
- rcu_read_lock();
- inet_opt = rcu_dereference(inet->inet_opt);
- if (inet_opt) {
- memcpy(opt_copy, inet_opt,
- sizeof(*inet_opt) + inet_opt->opt.optlen);
- ipc.opt = opt_copy;
- }
- rcu_read_unlock();
- }
-
- if (cgroup_bpf_enabled(CGROUP_UDP4_SENDMSG) && !connected) {
- err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk,
- (struct sockaddr *)usin,
- &msg->msg_namelen,
- &ipc.addr);
- if (err)
- goto out_free;
- if (usin) {
- if (usin->sin_port == 0) {
- /* BPF program set invalid port. Reject it. */
- err = -EINVAL;
- goto out_free;
- }
- daddr = usin->sin_addr.s_addr;
- dport = usin->sin_port;
- }
- }
-
- saddr = ipc.addr;
- ipc.addr = faddr = daddr;
-
- if (ipc.opt && ipc.opt->opt.srr) {
- if (!daddr) {
- err = -EINVAL;
- goto out_free;
- }
- faddr = ipc.opt->opt.faddr;
- connected = 0;
- }
- scope = ip_sendmsg_scope(inet, &ipc, msg);
- if (scope == RT_SCOPE_LINK)
- connected = 0;
-
- uc_index = READ_ONCE(inet->uc_index);
- if (ipv4_is_multicast(daddr)) {
- if (!ipc.oif || netif_index_is_l3_master(sock_net(sk), ipc.oif))
- ipc.oif = READ_ONCE(inet->mc_index);
- if (!saddr)
- saddr = READ_ONCE(inet->mc_addr);
- connected = 0;
- } else if (!ipc.oif) {
- ipc.oif = uc_index;
- } else if (ipv4_is_lbcast(daddr) && uc_index) {
- /* oif is set, packet is to local broadcast and
- * uc_index is set. oif is most likely set
- * by sk_bound_dev_if. If uc_index != oif check if the
- * oif is an L3 master and uc_index is an L3 slave.
- * If so, we want to allow the send using the uc_index.
- */
- if (ipc.oif != uc_index &&
- ipc.oif == l3mdev_master_ifindex_by_index(sock_net(sk),
- uc_index)) {
- ipc.oif = uc_index;
- }
- }
-
- if (connected)
- rt = dst_rtable(sk_dst_check(sk, 0));
-
- if (!rt) {
- struct net *net = sock_net(sk);
- __u8 flow_flags = inet_sk_flowi_flags(sk);
-
- fl4 = &fl4_stack;
-
- flowi4_init_output(fl4, ipc.oif, ipc.sockc.mark,
- ipc.tos & INET_DSCP_MASK, scope,
- IPPROTO_UDP, flow_flags, faddr, saddr,
- dport, inet->inet_sport,
- sk_uid(sk));
-
- security_sk_classify_flow(sk, flowi4_to_flowi_common(fl4));
- rt = ip_route_output_flow(net, fl4, sk);
- if (IS_ERR(rt)) {
- err = PTR_ERR(rt);
- rt = NULL;
- if (err == -ENETUNREACH)
- IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
- goto out;
- }
-
- err = -EACCES;
- if ((rt->rt_flags & RTCF_BROADCAST) &&
- !sock_flag(sk, SOCK_BROADCAST))
- goto out;
- if (connected)
- sk_dst_set(sk, dst_clone(&rt->dst));
- }
-
- if (msg->msg_flags&MSG_CONFIRM)
- goto do_confirm;
-back_from_confirm:
-
- saddr = fl4->saddr;
- if (!ipc.addr)
- daddr = ipc.addr = fl4->daddr;
-
- /* Lockless fast path for the non-corking case. */
- if (!corkreq) {
- struct inet_cork cork;
-
- skb = ip_make_skb(sk, fl4, ip_generic_getfrag, msg, ulen,
- sizeof(struct udphdr), &ipc, &rt,
- &cork, msg->msg_flags);
- err = PTR_ERR(skb);
- if (!IS_ERR_OR_NULL(skb))
- err = udp_send_skb(skb, fl4, &cork);
- goto out;
- }
-
- lock_sock(sk);
- if (unlikely(up->pending)) {
- /* The socket is already corked while preparing it. */
- /* ... which is an evident application bug. --ANK */
- release_sock(sk);
-
- net_dbg_ratelimited("socket already corked\n");
- err = -EINVAL;
- goto out;
- }
- /*
- * Now cork the socket to pend data.
- */
- fl4 = &inet->cork.fl.u.ip4;
- fl4->daddr = daddr;
- fl4->saddr = saddr;
- fl4->fl4_dport = dport;
- fl4->fl4_sport = inet->inet_sport;
- WRITE_ONCE(up->pending, AF_INET);
-
-do_append_data:
- up->len += ulen;
- err = ip_append_data(sk, fl4, ip_generic_getfrag, msg, ulen,
- sizeof(struct udphdr), &ipc, &rt,
- corkreq ? msg->msg_flags|MSG_MORE : msg->msg_flags);
- if (err)
- udp_flush_pending_frames(sk);
- else if (!corkreq)
- err = udp_push_pending_frames(sk);
- else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
- WRITE_ONCE(up->pending, 0);
- release_sock(sk);
-
-out:
- ip_rt_put(rt);
-out_free:
- if (free)
- kfree(ipc.opt);
- if (!err)
- return len;
- /*
- * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting
- * ENOBUFS might not be good (it's not tunable per se), but otherwise
- * we don't have a good statistic (IpOutDiscards but it can be too many
- * things). We could add another new stat but at least for now that
- * seems like overkill.
- */
- if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
- UDP_INC_STATS(sock_net(sk), UDP_MIB_SNDBUFERRORS);
-
- return err;
-
-do_confirm:
- if (msg->msg_flags & MSG_PROBE)
- dst_confirm_neigh(&rt->dst, &fl4->daddr);
- if (!(msg->msg_flags&MSG_PROBE) || len)
- goto back_from_confirm;
- err = 0;
- goto out;
-}
-EXPORT_SYMBOL(udp_sendmsg);
-
-void udp_splice_eof(struct socket *sock)
-{
- struct sock *sk = sock->sk;
- struct udp_sock *up = udp_sk(sk);
-
- if (!READ_ONCE(up->pending) || udp_test_bit(CORK, sk))
- return;
-
- lock_sock(sk);
- if (up->pending && !udp_test_bit(CORK, sk))
- udp_push_pending_frames(sk);
- release_sock(sk);
-}
-
#define UDP_SKB_IS_STATELESS 0x80000000
/* all head states (dst, sk, nf conntrack) except skb extensions are
@@ -1824,21 +830,6 @@ void udp_destruct_common(struct sock *sk)
kfree(up->udp_prod_queue);
}
-static void udp_destruct_sock(struct sock *sk)
-{
- udp_destruct_common(sk);
- inet_sock_destruct(sk);
-}
-
-static int udp_init_sock(struct sock *sk)
-{
- int res = udp_lib_init_sock(sk);
-
- sk->sk_destruct = udp_destruct_sock;
- set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags);
- return res;
-}
-
void skb_consume_udp(struct sock *sk, struct sk_buff *skb, int len)
{
if (unlikely(READ_ONCE(udp_sk(sk)->peeking_with_offset)))
@@ -1998,158 +989,50 @@ struct sk_buff *__skb_recv_udp(struct sock *sk, unsigned int flags,
if (!sk_can_busy_loop(sk))
break;
- sk_busy_loop(sk, flags & MSG_DONTWAIT);
- } while (!skb_queue_empty_lockless(sk_queue));
-
- /* sk_queue is empty, reader_queue may contain peeked packets */
- } while (timeo &&
- !__skb_wait_for_more_packets(sk, &sk->sk_receive_queue,
- &error, &timeo,
- (struct sk_buff *)sk_queue));
-
- *err = error;
- return NULL;
-}
-EXPORT_SYMBOL(__skb_recv_udp);
-
-int udp_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
-{
- struct sk_buff *skb;
- int err;
-
-try_again:
- skb = skb_recv_udp(sk, MSG_DONTWAIT, &err);
- if (!skb)
- return err;
-
- if (udp_lib_checksum_complete(skb)) {
- struct net *net = sock_net(sk);
-
- __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS);
- __UDP_INC_STATS(net, UDP_MIB_INERRORS);
- udp_drops_inc(sk);
- kfree_skb_reason(skb, SKB_DROP_REASON_UDP_CSUM);
- goto try_again;
- }
-
- WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
-
- /*
- * skb->dev still aliases the UDP rx dev_scratch (its charge was freed
- * on dequeue above); a sockmap verdict program may deref it via
- * bpf_sk_lookup_*(), so clear it -> bpf_skc_lookup() uses skb->sk
- */
- skb->dev = NULL;
-
- return recv_actor(sk, skb);
-}
-
-/*
- * This should be easy, if there is something there we
- * return it, otherwise we block.
- */
-
-INDIRECT_CALLABLE_SCOPE
-int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags)
-{
- DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
- int off, err, peeking = flags & MSG_PEEK;
- struct inet_sock *inet = inet_sk(sk);
- struct net *net = sock_net(sk);
- bool checksum_valid = false;
- unsigned int ulen, copied;
- struct sk_buff *skb;
-
- if (flags & MSG_ERRQUEUE)
- return ip_recv_error(sk, msg, len);
-
-try_again:
- off = sk_peek_offset(sk, flags);
- skb = __skb_recv_udp(sk, flags, &off, &err);
- if (!skb)
- return err;
-
- ulen = udp_skb_len(skb);
- copied = len;
- if (copied > ulen - off)
- copied = ulen - off;
- else if (copied < ulen)
- msg->msg_flags |= MSG_TRUNC;
-
- /* If checksum is needed at all, try to do it while copying the
- * data. If the data is truncated, do it before the copy.
- */
- if (copied < ulen || peeking) {
- checksum_valid = udp_skb_csum_unnecessary(skb) ||
- !__udp_lib_checksum_complete(skb);
- if (!checksum_valid)
- goto csum_copy_err;
- }
-
- if (checksum_valid || udp_skb_csum_unnecessary(skb)) {
- if (udp_skb_is_linear(skb))
- err = copy_linear_skb(skb, copied, off, &msg->msg_iter);
- else
- err = skb_copy_datagram_msg(skb, off, msg, copied);
- } else {
- err = skb_copy_and_csum_datagram_msg(skb, off, msg);
-
- if (err == -EINVAL)
- goto csum_copy_err;
- }
-
- if (unlikely(err)) {
- if (!peeking) {
- udp_drops_inc(sk);
- UDP_INC_STATS(net, UDP_MIB_INERRORS);
- }
- kfree_skb(skb);
- return err;
- }
-
- if (!peeking)
- UDP_INC_STATS(net, UDP_MIB_INDATAGRAMS);
+ sk_busy_loop(sk, flags & MSG_DONTWAIT);
+ } while (!skb_queue_empty_lockless(sk_queue));
- sock_recv_cmsgs(msg, sk, skb);
+ /* sk_queue is empty, reader_queue may contain peeked packets */
+ } while (timeo &&
+ !__skb_wait_for_more_packets(sk, &sk->sk_receive_queue,
+ &error, &timeo,
+ (struct sk_buff *)sk_queue));
- /* Copy the address. */
- if (sin) {
- sin->sin_family = AF_INET;
- sin->sin_port = udp_hdr(skb)->source;
- sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
- memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
- msg->msg_namelen = sizeof(*sin);
+ *err = error;
+ return NULL;
+}
+EXPORT_SYMBOL(__skb_recv_udp);
- BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk,
- (struct sockaddr *)sin,
- &msg->msg_namelen);
- }
+int udp_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
+{
+ struct sk_buff *skb;
+ int err;
- if (udp_test_bit(GRO_ENABLED, sk))
- udp_cmsg_recv(msg, sk, skb);
+try_again:
+ skb = skb_recv_udp(sk, MSG_DONTWAIT, &err);
+ if (!skb)
+ return err;
- if (inet_cmsg_flags(inet))
- ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off);
+ if (udp_lib_checksum_complete(skb)) {
+ struct net *net = sock_net(sk);
- err = copied;
- if (flags & MSG_TRUNC)
- err = ulen;
+ __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS);
+ __UDP_INC_STATS(net, UDP_MIB_INERRORS);
+ udp_drops_inc(sk);
+ kfree_skb_reason(skb, SKB_DROP_REASON_UDP_CSUM);
+ goto try_again;
+ }
- skb_consume_udp(sk, skb, peeking ? -err : err);
- return err;
+ WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
-csum_copy_err:
- if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags,
- udp_skb_destructor)) {
- UDP_INC_STATS(net, UDP_MIB_CSUMERRORS);
- UDP_INC_STATS(net, UDP_MIB_INERRORS);
- }
- kfree_skb_reason(skb, SKB_DROP_REASON_UDP_CSUM);
+ /*
+ * skb->dev still aliases the UDP rx dev_scratch (its charge was freed
+ * on dequeue above); a sockmap verdict program may deref it via
+ * bpf_sk_lookup_*(), so clear it -> bpf_skc_lookup() uses skb->sk
+ */
+ skb->dev = NULL;
- /* starting over for a new packet, but check if we need to yield */
- cond_resched();
- msg->msg_flags &= ~MSG_TRUNC;
- goto try_again;
+ return recv_actor(sk, skb);
}
int udp_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
@@ -2165,19 +1048,6 @@ int udp_pre_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
return BPF_CGROUP_RUN_PROG_INET4_CONNECT_LOCK(sk, uaddr, &addr_len);
}
-static int udp_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
- int addr_len)
-{
- int res;
-
- lock_sock(sk);
- res = __ip4_datagram_connect(sk, uaddr, addr_len);
- if (!res)
- udp4_hash4(sk);
- release_sock(sk);
- return res;
-}
-
int __udp_disconnect(struct sock *sk, int flags)
{
struct inet_sock *inet = inet_sk(sk);
@@ -2336,155 +1206,6 @@ void udp_lib_rehash(struct sock *sk, u16 newhash, u16 newhash4)
}
}
-static void udp_v4_rehash(struct sock *sk)
-{
- u16 new_hash = ipv4_portaddr_hash(sock_net(sk),
- inet_sk(sk)->inet_rcv_saddr,
- inet_sk(sk)->inet_num);
- u16 new_hash4 = udp_ehashfn(sock_net(sk),
- sk->sk_rcv_saddr, sk->sk_num,
- sk->sk_daddr, sk->sk_dport);
-
- udp_lib_rehash(sk, new_hash, new_hash4);
-}
-
-static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
-{
- int rc;
-
- if (inet_sk(sk)->inet_daddr) {
- sock_rps_save_rxhash(sk, skb);
- sk_mark_napi_id(sk, skb);
- sk_incoming_cpu_update(sk);
- } else {
- sk_mark_napi_id_once(sk, skb);
- }
-
- rc = __udp_enqueue_schedule_skb(sk, skb);
- if (rc < 0) {
- struct net *net = sock_net(sk);
- int drop_reason;
-
- /* Note that an ENOMEM error is charged twice */
- if (rc == -ENOMEM) {
- UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS);
- drop_reason = SKB_DROP_REASON_SOCKET_RCVBUFF;
- } else {
- UDP_INC_STATS(net, UDP_MIB_MEMERRORS);
- drop_reason = SKB_DROP_REASON_PROTO_MEM;
- }
- UDP_INC_STATS(net, UDP_MIB_INERRORS);
- trace_udp_fail_queue_rcv_skb(rc, sk, skb);
- sk_skb_reason_drop(sk, skb, drop_reason);
- return -1;
- }
-
- return 0;
-}
-
-/* returns:
- * -1: error
- * 0: success
- * >0: "udp encap" protocol resubmission
- *
- * Note that in the success and error cases, the skb is assumed to
- * have either been requeued or freed.
- */
-static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
-{
- enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
- struct udp_sock *up = udp_sk(sk);
- struct net *net = sock_net(sk);
-
- /*
- * Charge it to the socket, dropping if the queue is full.
- */
- if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
- drop_reason = SKB_DROP_REASON_XFRM_POLICY;
- goto drop;
- }
- nf_reset_ct(skb);
-
- if (static_branch_unlikely(&udp_encap_needed_key) &&
- READ_ONCE(up->encap_type)) {
- int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
-
- /*
- * This is an encapsulation socket so pass the skb to
- * the socket's udp_encap_rcv() hook. Otherwise, just
- * fall through and pass this up the UDP socket.
- * up->encap_rcv() returns the following value:
- * =0 if skb was successfully passed to the encap
- * handler or was discarded by it.
- * >0 if skb should be passed on to UDP.
- * <0 if skb should be resubmitted as proto -N
- */
-
- /* if we're overly short, let UDP handle it */
- encap_rcv = READ_ONCE(up->encap_rcv);
- if (encap_rcv) {
- int ret;
-
- /* Verify checksum before giving to encap */
- if (udp_lib_checksum_complete(skb))
- goto csum_error;
-
- ret = encap_rcv(sk, skb);
- if (ret <= 0) {
- __UDP_INC_STATS(net, UDP_MIB_INDATAGRAMS);
- return -ret;
- }
- }
-
- /* FALLTHROUGH -- it's a UDP Packet */
- }
-
- prefetch(&sk->sk_rmem_alloc);
- if (rcu_access_pointer(sk->sk_filter) &&
- udp_lib_checksum_complete(skb))
- goto csum_error;
-
- drop_reason = sk_filter_trim_cap(sk, skb, sizeof(struct udphdr));
- if (drop_reason)
- goto drop;
-
- udp_csum_pull_header(skb);
-
- ipv4_pktinfo_prepare(sk, skb, true);
- return __udp_queue_rcv_skb(sk, skb);
-
-csum_error:
- drop_reason = SKB_DROP_REASON_UDP_CSUM;
- __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS);
-drop:
- __UDP_INC_STATS(net, UDP_MIB_INERRORS);
- udp_drops_inc(sk);
- sk_skb_reason_drop(sk, skb, drop_reason);
- return -1;
-}
-
-static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
-{
- struct sk_buff *next, *segs;
- int ret;
-
- if (likely(!udp_unexpected_gso(sk, skb)))
- return udp_queue_rcv_one_skb(sk, skb);
-
- BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_GSO_CB_OFFSET);
- __skb_push(skb, -skb_mac_offset(skb));
- segs = udp_rcv_segment(sk, skb, true);
- skb_list_walk_safe(segs, skb, next) {
- __skb_pull(skb, skb_transport_offset(skb));
-
- udp_post_segment_fix_csum(skb);
- ret = udp_queue_rcv_one_skb(sk, skb);
- if (ret > 0)
- ip_protocol_deliver_rcu(dev_net(skb->dev), skb, ret);
- }
- return 0;
-}
-
/* For TCP sockets, sk_rx_dst is protected by socket lock
* For UDP, we use xchg() to guard against concurrent changes.
*/
@@ -2500,401 +1221,6 @@ bool udp_sk_rx_dst_set(struct sock *sk, struct dst_entry *dst)
return false;
}
-/*
- * Multicasts and broadcasts go to each listener.
- *
- * Note: called only from the BH handler context.
- */
-static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
- struct udphdr *uh,
- __be32 saddr, __be32 daddr)
-{
- struct udp_table *udptable = net->ipv4.udp_table;
- unsigned int hash2, hash2_any, offset;
- unsigned short hnum = ntohs(uh->dest);
- struct sock *sk, *first = NULL;
- int dif = skb->dev->ifindex;
- int sdif = inet_sdif(skb);
- struct hlist_node *node;
- struct udp_hslot *hslot;
- struct sk_buff *nskb;
- bool use_hash2;
- int ret;
-
- hash2_any = 0;
- hash2 = 0;
- hslot = udp_hashslot(udptable, net, hnum);
- use_hash2 = hslot->count > 10;
- offset = offsetof(typeof(*sk), sk_node);
-
- if (use_hash2) {
- hash2_any = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &
- udptable->mask;
- hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask;
-start_lookup:
- hslot = &udptable->hash2[hash2].hslot;
- offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
- }
-
- sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
- if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr,
- uh->source, saddr, dif, sdif, hnum))
- continue;
-
- if (!first) {
- first = sk;
- continue;
- }
- nskb = skb_clone(skb, GFP_ATOMIC);
-
- if (unlikely(!nskb)) {
- udp_drops_inc(sk);
- __UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS);
- __UDP_INC_STATS(net, UDP_MIB_INERRORS);
- continue;
- }
- if (udp_queue_rcv_skb(sk, nskb) > 0)
- consume_skb(nskb);
- }
-
- /* Also lookup *:port if we are using hash2 and haven't done so yet. */
- if (use_hash2 && hash2 != hash2_any) {
- hash2 = hash2_any;
- goto start_lookup;
- }
-
- if (first) {
- ret = udp_queue_rcv_skb(first, skb);
- if (ret > 0)
- return -ret;
- } else {
- kfree_skb(skb);
- __UDP_INC_STATS(net, UDP_MIB_IGNOREDMULTI);
- }
- return 0;
-}
-
-/* Initialize UDP checksum. If exited with zero value (success),
- * CHECKSUM_UNNECESSARY means, that no more checks are required.
- * Otherwise, csum completion requires checksumming packet body,
- * including udp header and folding it to skb->csum.
- */
-static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh)
-{
- int err;
-
- /* Note, we are only interested in != 0 or == 0, thus the
- * force to int.
- */
- err = (__force int)skb_checksum_init_zero_check(skb, IPPROTO_UDP, uh->check,
- inet_compute_pseudo);
- if (err)
- return err;
-
- if (skb->ip_summed == CHECKSUM_COMPLETE && !skb->csum_valid) {
- /* If SW calculated the value, we know it's bad */
- if (skb->csum_complete_sw)
- return 1;
-
- /* HW says the value is bad. Let's validate that.
- * skb->csum is no longer the full packet checksum,
- * so don't treat it as such.
- */
- skb_checksum_complete_unset(skb);
- }
-
- return 0;
-}
-
-/* wrapper for udp_queue_rcv_skb taking care of csum conversion and
- * return code conversion for ip layer consumption
- */
-static int udp_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb,
- struct udphdr *uh)
-{
- int ret;
-
- if (inet_get_convert_csum(sk) && uh->check)
- skb_checksum_try_convert(skb, IPPROTO_UDP, inet_compute_pseudo);
-
- ret = udp_queue_rcv_skb(sk, skb);
-
- /* a return value > 0 means to resubmit the input, but
- * it wants the return to be -protocol, or 0
- */
- if (ret > 0)
- return -ret;
- return 0;
-}
-
-/*
- * All we need to do is get the socket, and then do a checksum.
- */
-
-int udp_rcv(struct sk_buff *skb)
-{
- struct rtable *rt = skb_rtable(skb);
- struct net *net = dev_net(skb->dev);
- struct sock *sk = NULL;
- __be32 saddr, daddr;
- unsigned int ulen;
- struct udphdr *uh;
- bool refcounted;
- int drop_reason;
-
- drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
-
- /*
- * Validate the packet.
- */
- if (!pskb_may_pull(skb, sizeof(struct udphdr)))
- goto drop; /* No space for header. */
-
- uh = udp_hdr(skb);
- ulen = udp_get_len(skb, uh, 0);
- saddr = ip_hdr(skb)->saddr;
- daddr = ip_hdr(skb)->daddr;
-
- if (ulen > skb->len)
- goto short_packet;
-
- if (ulen < sizeof(*uh))
- goto short_packet;
-
- if (ulen < skb->len) {
- if (pskb_trim_rcsum(skb, ulen))
- goto short_packet;
-
- uh = udp_hdr(skb);
- }
-
- if (udp4_csum_init(skb, uh))
- goto csum_error;
-
- sk = inet_steal_sock(net, skb, sizeof(struct udphdr), saddr, uh->source, daddr, uh->dest,
- &refcounted, udp_ehashfn);
- if (IS_ERR(sk))
- goto no_sk;
-
- if (sk) {
- struct dst_entry *dst = skb_dst(skb);
- int ret;
-
- if (unlikely(rcu_dereference(sk->sk_rx_dst) != dst))
- udp_sk_rx_dst_set(sk, dst);
-
- ret = udp_unicast_rcv_skb(sk, skb, uh);
- if (refcounted)
- sock_put(sk);
- return ret;
- }
-
- if (rt->rt_flags & (RTCF_BROADCAST|RTCF_MULTICAST))
- return __udp4_lib_mcast_deliver(net, skb, uh, saddr, daddr);
-
- sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest);
- if (sk)
- return udp_unicast_rcv_skb(sk, skb, uh);
-no_sk:
- if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
- goto drop;
- nf_reset_ct(skb);
-
- /* No socket. Drop packet silently, if checksum is wrong */
- if (udp_lib_checksum_complete(skb))
- goto csum_error;
-
- drop_reason = SKB_DROP_REASON_NO_SOCKET;
- __UDP_INC_STATS(net, UDP_MIB_NOPORTS);
- icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
-
- /*
- * Hmm. We got an UDP packet to a port to which we
- * don't wanna listen. Ignore it.
- */
- sk_skb_reason_drop(sk, skb, drop_reason);
- return 0;
-
-short_packet:
- drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
- net_dbg_ratelimited("UDP: short packet: From %pI4:%u %d/%d to %pI4:%u\n",
- &saddr, ntohs(uh->source),
- ulen, skb->len,
- &daddr, ntohs(uh->dest));
- goto drop;
-
-csum_error:
- /*
- * RFC1122: OK. Discards the bad packet silently (as far as
- * the network is concerned, anyway) as per 4.1.3.4 (MUST).
- */
- drop_reason = SKB_DROP_REASON_UDP_CSUM;
- net_dbg_ratelimited("UDP: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n",
- &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest),
- ulen);
- __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS);
-drop:
- __UDP_INC_STATS(net, UDP_MIB_INERRORS);
- sk_skb_reason_drop(sk, skb, drop_reason);
- return 0;
-}
-
-/* We can only early demux multicast if there is a single matching socket.
- * If more than one socket found returns NULL
- */
-static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net,
- __be16 loc_port, __be32 loc_addr,
- __be16 rmt_port, __be32 rmt_addr,
- int dif, int sdif)
-{
- struct udp_table *udptable = net->ipv4.udp_table;
- unsigned short hnum = ntohs(loc_port);
- struct sock *sk, *result;
- struct udp_hslot *hslot;
- unsigned int slot;
-
- slot = udp_hashfn(net, hnum, udptable->mask);
- hslot = &udptable->hash[slot];
-
- /* Do not bother scanning a too big list */
- if (hslot->count > 10)
- return NULL;
-
- result = NULL;
- sk_for_each_rcu(sk, &hslot->head) {
- if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr,
- rmt_port, rmt_addr, dif, sdif, hnum)) {
- if (result)
- return NULL;
- result = sk;
- }
- }
-
- return result;
-}
-
-/* For unicast we should only early demux connected sockets or we can
- * break forwarding setups. The chains here can be long so only check
- * if the first socket is an exact match and if not move on.
- */
-static struct sock *__udp4_lib_demux_lookup(struct net *net,
- __be16 loc_port, __be32 loc_addr,
- __be16 rmt_port, __be32 rmt_addr,
- int dif, int sdif)
-{
- struct udp_table *udptable = net->ipv4.udp_table;
- INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr);
- unsigned short hnum = ntohs(loc_port);
- struct udp_hslot *hslot2;
- unsigned int hash2;
- __portpair ports;
- struct sock *sk;
-
- hash2 = ipv4_portaddr_hash(net, loc_addr, hnum);
- hslot2 = udp_hashslot2(udptable, hash2);
- ports = INET_COMBINED_PORTS(rmt_port, hnum);
-
- udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
- if (inet_match(net, sk, acookie, ports, dif, sdif))
- return sk;
- /* Only check first socket in chain */
- break;
- }
- return NULL;
-}
-
-enum skb_drop_reason udp_v4_early_demux(struct sk_buff *skb)
-{
- struct net *net = dev_net(skb->dev);
- struct in_device *in_dev = NULL;
- const struct iphdr *iph;
- const struct udphdr *uh;
- struct sock *sk = NULL;
- struct dst_entry *dst;
- int dif = skb->dev->ifindex;
- int sdif = inet_sdif(skb);
- int ours;
-
- /* validate the packet */
- if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr)))
- return SKB_NOT_DROPPED_YET;
-
- iph = ip_hdr(skb);
- uh = udp_hdr(skb);
-
- if (skb->pkt_type == PACKET_MULTICAST) {
- in_dev = __in_dev_get_rcu(skb->dev);
-
- if (!in_dev)
- return SKB_NOT_DROPPED_YET;
-
- ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr,
- iph->protocol);
- if (!ours)
- return SKB_NOT_DROPPED_YET;
-
- sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr,
- uh->source, iph->saddr,
- dif, sdif);
- } else if (skb->pkt_type == PACKET_HOST) {
- sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr,
- uh->source, iph->saddr, dif, sdif);
- }
-
- if (!sk)
- return SKB_NOT_DROPPED_YET;
-
- skb->sk = sk;
- DEBUG_NET_WARN_ON_ONCE(sk_is_refcounted(sk));
- skb->destructor = sock_pfree;
- dst = rcu_dereference(sk->sk_rx_dst);
-
- if (dst)
- dst = dst_check(dst, 0);
- if (dst) {
- u32 itag = 0;
-
- /* set noref for now.
- * any place which wants to hold dst has to call
- * dst_hold_safe()
- */
- skb_dst_set_noref(skb, dst);
-
- /* for unconnected multicast sockets we need to validate
- * the source on each packet
- */
- if (!inet_sk(sk)->inet_daddr && in_dev)
- return ip_mc_validate_source(skb, iph->daddr,
- iph->saddr,
- ip4h_dscp(iph),
- skb->dev, in_dev, &itag);
- }
- return SKB_NOT_DROPPED_YET;
-}
-
-static void udp_destroy_sock(struct sock *sk)
-{
- struct udp_sock *up = udp_sk(sk);
- bool slow = lock_sock_fast(sk);
-
- /* protects from races with udp_abort() */
- sock_set_flag(sk, SOCK_DEAD);
- udp_flush_pending_frames(sk);
- unlock_sock_fast(sk, slow);
- if (static_branch_unlikely(&udp_encap_needed_key)) {
- if (up->encap_type) {
- void (*encap_destroy)(struct sock *sk);
- encap_destroy = READ_ONCE(up->encap_destroy);
- if (encap_destroy)
- encap_destroy(sk);
- }
- if (udp_test_bit(ENCAP_ENABLED, sk)) {
- static_branch_dec(&udp_encap_needed_key);
- udp_tunnel_cleanup_gro(sk);
- }
- }
-}
-
typedef struct sk_buff *(*udp_gro_receive_t)(struct sock *sk,
struct list_head *head,
struct sk_buff *skb);
@@ -3030,16 +1356,6 @@ int udp_lib_setsockopt(struct sock *sk, int level, int optname,
return err;
}
-static int udp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
- unsigned int optlen)
-{
- if (level == SOL_UDP || level == SOL_SOCKET)
- return udp_lib_setsockopt(sk, level, optname,
- optval, optlen,
- udp_push_pending_frames);
- return ip_setsockopt(sk, level, optname, optval, optlen);
-}
-
int udp_lib_getsockopt(struct sock *sk, int level, int optname,
sockopt_t *opt)
{
@@ -3088,34 +1404,6 @@ int udp_lib_getsockopt(struct sock *sk, int level, int optname,
return 0;
}
-static int udp_getsockopt(struct sock *sk, int level, int optname,
- char __user *optval, int __user *optlen)
-{
- sockopt_t opt;
- int err;
-
- /*
- * keep the old __user pointers, until ip_getsockopt() moves
- * to sockopt_t
- */
- if (level != SOL_UDP)
- return ip_getsockopt(sk, level, optname, optval, optlen);
-
- err = sockopt_init_user(&opt, optval, optlen);
- if (err)
- return err;
-
- err = udp_lib_getsockopt(sk, level, optname, &opt);
- if (err)
- return err;
-
- /* optval was written by copy_to_iter() in udp_lib_getsockopt() */
- if (put_user(opt.optlen, optlen))
- return -EFAULT;
-
- return 0;
-}
-
/**
* udp_poll - wait for a UDP event.
* @file: - file struct
@@ -3172,41 +1460,6 @@ int udp_abort(struct sock *sk, int err)
return 0;
}
-struct proto udp_prot = {
- .name = "UDP",
- .owner = THIS_MODULE,
- .close = udp_lib_close,
- .pre_connect = udp_pre_connect,
- .connect = udp_connect,
- .disconnect = udp_disconnect,
- .ioctl = udp_ioctl,
- .init = udp_init_sock,
- .destroy = udp_destroy_sock,
- .setsockopt = udp_setsockopt,
- .getsockopt = udp_getsockopt,
- .sendmsg = udp_sendmsg,
- .recvmsg = udp_recvmsg,
- .splice_eof = udp_splice_eof,
- .release_cb = ip4_datagram_release_cb,
- .hash = udp_lib_hash,
- .unhash = udp_lib_unhash,
- .rehash = udp_v4_rehash,
- .get_port = udp_v4_get_port,
- .put_port = udp_lib_unhash,
-#ifdef CONFIG_BPF_SYSCALL
- .psock_update_sk_prot = udp_bpf_update_proto,
-#endif
- .memory_allocated = &net_aligned_data.udp_memory_allocated,
- .per_cpu_fw_alloc = &udp_memory_per_cpu_fw_alloc,
-
- .sysctl_mem = sysctl_udp_mem,
- .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
- .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min),
- .obj_size = sizeof(struct udp_sock),
- .diag_destroy = udp_abort,
-};
-EXPORT_SYMBOL(udp_prot);
-
/* ------------------------------------------------------------------------ */
#ifdef CONFIG_PROC_FS
diff --git a/net/ipv4/udp_bpf.c b/net/ipv4/udp_bpf.c
index ad57c4c9eaab..a5e9e6daad3a 100644
--- a/net/ipv4/udp_bpf.c
+++ b/net/ipv4/udp_bpf.c
@@ -16,6 +16,8 @@ static int sk_udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
if (sk->sk_family == AF_INET6)
return udpv6_prot_saved->recvmsg(sk, msg, len, flags);
#endif
+ if (!IS_ENABLED(CONFIG_IPV4))
+ return -EAFNOSUPPORT;
return udp_prot.recvmsg(sk, msg, len, flags);
}
@@ -155,7 +157,8 @@ static void udp_bpf_check_v6_needs_rebuild(struct proto *ops)
static int __init udp_bpf_v4_build_proto(void)
{
- udp_bpf_rebuild_protos(&udp_bpf_prots[UDP_BPF_IPV4], &udp_prot);
+ if (IS_ENABLED(CONFIG_IPV4))
+ udp_bpf_rebuild_protos(&udp_bpf_prots[UDP_BPF_IPV4], &udp_prot);
return 0;
}
late_initcall(udp_bpf_v4_build_proto);
diff --git a/net/ipv4/udp_ipv4.c b/net/ipv4/udp_ipv4.c
new file mode 100644
index 000000000000..466008c218df
--- /dev/null
+++ b/net/ipv4/udp_ipv4.c
@@ -0,0 +1,1790 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * INET An implementation of the TCP/IP protocol suite for the LINUX
+ * operating system. INET is implemented using the BSD Socket
+ * interface as the means of communication with the user level.
+ *
+ * The User Datagram Protocol (UDP).
+ *
+ * IPv4 specific functions
+ *
+ * code split from:
+ * net/ipv4/udp.c
+ *
+ * See udp.c for author information
+ */
+
+#define pr_fmt(fmt) "UDP: " fmt
+
+#include <linux/bpf-cgroup.h>
+#include <linux/igmp.h>
+#include <linux/inetdevice.h>
+#include <linux/socket.h>
+#include <linux/types.h>
+#include <net/aligned_data.h>
+#include <net/busy_poll.h>
+#include <net/gro.h>
+#include <net/icmp.h>
+#include <net/inet_common.h>
+#include <net/inet_hashtables.h>
+#include <net/ip_tunnels.h>
+#include <net/sock_reuseport.h>
+#include <net/udp.h>
+#include <net/udp_tunnel.h>
+#include <net/xfrm.h>
+#include <trace/events/udp.h>
+
+static __always_inline int
+compute_score(struct sock *sk, const struct net *net,
+ __be32 saddr, __be16 sport, __be32 daddr,
+ unsigned short hnum, int dif, int sdif)
+{
+ int score;
+ struct inet_sock *inet;
+ bool dev_match;
+
+ if (!net_eq(sock_net(sk), net) ||
+ udp_sk(sk)->udp_port_hash != hnum ||
+ ipv6_only_sock(sk))
+ return -1;
+
+ if (sk->sk_rcv_saddr != daddr)
+ return -1;
+
+ score = (sk->sk_family == PF_INET) ? 2 : 1;
+
+ inet = inet_sk(sk);
+ if (inet->inet_daddr) {
+ if (inet->inet_daddr != saddr)
+ return -1;
+ score += 4;
+ }
+
+ if (inet->inet_dport) {
+ if (inet->inet_dport != sport)
+ return -1;
+ score += 4;
+ }
+
+ dev_match = udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if,
+ dif, sdif);
+ if (!dev_match)
+ return -1;
+ if (sk->sk_bound_dev_if)
+ score += 4;
+
+ if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
+ score++;
+ return score;
+}
+
+/**
+ * udp4_lib_lookup1() - Simplified lookup using primary hash (destination port)
+ * @net: Network namespace
+ * @saddr: Source address, network order
+ * @sport: Source port, network order
+ * @daddr: Destination address, network order
+ * @hnum: Destination port, host order
+ * @dif: Destination interface index
+ * @sdif: Destination bridge port index, if relevant
+ * @udptable: Set of UDP hash tables
+ *
+ * Simplified lookup to be used as fallback if no sockets are found due to a
+ * potential race between (receive) address change, and lookup happening before
+ * the rehash operation. This function ignores SO_REUSEPORT groups while scoring
+ * result sockets, because if we have one, we don't need the fallback at all.
+ *
+ * Called under rcu_read_lock().
+ *
+ * Return: socket with highest matching score if any, NULL if none
+ */
+static struct sock *udp4_lib_lookup1(const struct net *net,
+ __be32 saddr, __be16 sport,
+ __be32 daddr, unsigned int hnum,
+ int dif, int sdif,
+ const struct udp_table *udptable)
+{
+ unsigned int slot = udp_hashfn(net, hnum, udptable->mask);
+ struct udp_hslot *hslot = &udptable->hash[slot];
+ struct sock *sk, *result = NULL;
+ int score, badness = 0;
+
+ sk_for_each_rcu(sk, &hslot->head) {
+ score = compute_score(sk, net,
+ saddr, sport, daddr, hnum, dif, sdif);
+ if (score > badness) {
+ result = sk;
+ badness = score;
+ }
+ }
+
+ return result;
+}
+
+/* called with rcu_read_lock() */
+static struct sock *udp4_lib_lookup2(const struct net *net,
+ __be32 saddr, __be16 sport,
+ __be32 daddr, unsigned int hnum,
+ int dif, int sdif,
+ struct udp_hslot *hslot2,
+ struct sk_buff *skb)
+{
+ struct sock *sk, *result;
+ int score, badness;
+ bool need_rescore;
+
+ result = NULL;
+ badness = 0;
+ udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
+ need_rescore = false;
+rescore:
+ score = compute_score(need_rescore ? result : sk, net, saddr,
+ sport, daddr, hnum, dif, sdif);
+ if (score > badness) {
+ badness = score;
+
+ if (need_rescore)
+ continue;
+
+ if (sk->sk_state == TCP_ESTABLISHED) {
+ result = sk;
+ continue;
+ }
+
+ result = inet_lookup_reuseport(net, sk, skb, sizeof(struct udphdr),
+ saddr, sport, daddr, hnum, udp_ehashfn);
+ if (!result) {
+ result = sk;
+ continue;
+ }
+
+ /* Fall back to scoring if group has connections */
+ if (!reuseport_has_conns(sk))
+ return result;
+
+ /* Reuseport logic returned an error, keep original score. */
+ if (IS_ERR(result))
+ continue;
+
+ /* compute_score is too long of a function to be
+ * inlined twice here, and calling it uninlined
+ * here yields measurable overhead for some
+ * workloads. Work around it by jumping
+ * backwards to rescore 'result'.
+ */
+ need_rescore = true;
+ goto rescore;
+ }
+ }
+ return result;
+}
+
+#if IS_ENABLED(CONFIG_BASE_SMALL)
+static struct sock *udp4_lib_lookup4(const struct net *net,
+ __be32 saddr, __be16 sport,
+ __be32 daddr, unsigned int hnum,
+ int dif, int sdif,
+ struct udp_table *udptable)
+{
+ return NULL;
+}
+#else /* !CONFIG_BASE_SMALL */
+static struct sock *udp4_lib_lookup4(const struct net *net,
+ __be32 saddr, __be16 sport,
+ __be32 daddr, unsigned int hnum,
+ int dif, int sdif,
+ struct udp_table *udptable)
+{
+ const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
+ const struct hlist_nulls_node *node;
+ struct udp_hslot *hslot4;
+ unsigned int hash4, slot;
+ struct udp_sock *up;
+ struct sock *sk;
+
+ hash4 = udp_ehashfn(net, daddr, hnum, saddr, sport);
+ slot = hash4 & udptable->mask;
+ hslot4 = &udptable->hash4[slot];
+ INET_ADDR_COOKIE(acookie, saddr, daddr);
+
+begin:
+ /* SLAB_TYPESAFE_BY_RCU not used, so we don't need to touch sk_refcnt */
+ udp_lrpa_for_each_entry_rcu(up, node, &hslot4->nulls_head) {
+ sk = (struct sock *)up;
+ if (inet_match(net, sk, acookie, ports, dif, sdif))
+ return sk;
+ }
+
+ /* if the nulls value we got at the end of this lookup is not the
+ * expected one, we must restart lookup. We probably met an item that
+ * was moved to another chain due to rehash.
+ */
+ if (get_nulls_value(node) != slot)
+ goto begin;
+
+ return NULL;
+}
+
+/* call with sock lock */
+void udp4_hash4(struct sock *sk)
+{
+ struct net *net = sock_net(sk);
+ unsigned int hash;
+
+ if (sk_unhashed(sk) || sk->sk_rcv_saddr == htonl(INADDR_ANY))
+ return;
+
+ hash = udp_ehashfn(net, sk->sk_rcv_saddr, sk->sk_num,
+ sk->sk_daddr, sk->sk_dport);
+
+ udp_lib_hash4(sk, hash);
+}
+#endif /* CONFIG_BASE_SMALL */
+
+/* UDP is nearly always wildcards out the wazoo, it makes no sense to try
+ * harder than this. -DaveM
+ */
+struct sock *__udp4_lib_lookup(const struct net *net, __be32 saddr,
+ __be16 sport, __be32 daddr, __be16 dport,
+ int dif, int sdif, struct sk_buff *skb)
+{
+ struct udp_table *udptable = net->ipv4.udp_table;
+ unsigned short hnum = ntohs(dport);
+ struct udp_hslot *hslot2;
+ struct sock *result, *sk;
+ unsigned int hash2;
+
+ hash2 = ipv4_portaddr_hash(net, daddr, hnum);
+ hslot2 = udp_hashslot2(udptable, hash2);
+
+ if (udp_has_hash4(hslot2)) {
+ result = udp4_lib_lookup4(net, saddr, sport, daddr, hnum,
+ dif, sdif, udptable);
+ if (result) /* udp4_lib_lookup4 return sk or NULL */
+ return result;
+ }
+
+ /* Lookup connected or non-wildcard socket */
+ result = udp4_lib_lookup2(net, saddr, sport,
+ daddr, hnum, dif, sdif,
+ hslot2, skb);
+ if (!IS_ERR_OR_NULL(result) && result->sk_state == TCP_ESTABLISHED)
+ goto done;
+
+ /* Lookup redirect from BPF */
+ if (static_branch_unlikely(&bpf_sk_lookup_enabled)) {
+ sk = inet_lookup_run_sk_lookup(net, IPPROTO_UDP, skb, sizeof(struct udphdr),
+ saddr, sport, daddr, hnum, dif,
+ udp_ehashfn);
+ if (sk) {
+ result = sk;
+ goto done;
+ }
+ }
+
+ /* Got non-wildcard socket or error on first lookup */
+ if (result)
+ goto done;
+
+ /* Lookup wildcard sockets */
+ hash2 = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum);
+ hslot2 = udp_hashslot2(udptable, hash2);
+
+ result = udp4_lib_lookup2(net, saddr, sport,
+ htonl(INADDR_ANY), hnum, dif, sdif,
+ hslot2, skb);
+ if (!IS_ERR_OR_NULL(result))
+ goto done;
+
+ /* Primary hash (destination port) lookup as fallback for this race:
+ * 1. __ip4_datagram_connect() sets sk_rcv_saddr
+ * 2. lookup (this function): new sk_rcv_saddr, hashes not updated yet
+ * 3. rehash operation updating _secondary and four-tuple_ hashes
+ * The primary hash doesn't need an update after 1., so, thanks to this
+ * further step, 1. and 3. don't need to be atomic against the lookup.
+ */
+ result = udp4_lib_lookup1(net, saddr, sport, daddr, hnum, dif, sdif,
+ udptable);
+
+done:
+ if (IS_ERR(result))
+ return NULL;
+ return result;
+}
+EXPORT_SYMBOL_GPL(__udp4_lib_lookup);
+
+static inline struct sock *__udp4_lib_lookup_skb(struct sk_buff *skb,
+ __be16 sport, __be16 dport)
+{
+ const struct iphdr *iph = ip_hdr(skb);
+
+ return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
+ iph->daddr, dport, inet_iif(skb),
+ inet_sdif(skb), skb);
+}
+
+struct sock *udp4_lib_lookup_skb(const struct sk_buff *skb,
+ __be16 sport, __be16 dport)
+{
+ const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
+ const struct iphdr *iph = (struct iphdr *)(skb->data + offset);
+ int iif, sdif;
+
+ inet_get_iif_sdif(skb, &iif, &sdif);
+
+ return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,
+ iph->daddr, dport, iif, sdif, NULL);
+}
+
+/* Must be called under rcu_read_lock().
+ * Does increment socket refcount.
+ */
+#if IS_ENABLED(CONFIG_NF_TPROXY_IPV4) || IS_ENABLED(CONFIG_NF_SOCKET_IPV4)
+struct sock *udp4_lib_lookup(const struct net *net, __be32 saddr, __be16 sport,
+ __be32 daddr, __be16 dport, int dif)
+{
+ struct sock *sk;
+
+ sk = __udp4_lib_lookup(net, saddr, sport, daddr, dport, dif, 0, NULL);
+ if (sk && !refcount_inc_not_zero(&sk->sk_refcnt))
+ sk = NULL;
+ return sk;
+}
+EXPORT_SYMBOL_GPL(udp4_lib_lookup);
+#endif
+
+static int udp_v4_get_port(struct sock *sk, unsigned short snum)
+{
+ unsigned int hash2_nulladdr =
+ ipv4_portaddr_hash(sock_net(sk), htonl(INADDR_ANY), snum);
+ unsigned int hash2_partial =
+ ipv4_portaddr_hash(sock_net(sk), inet_sk(sk)->inet_rcv_saddr, 0);
+
+ /* precompute partial secondary hash */
+ udp_sk(sk)->udp_portaddr_hash = hash2_partial;
+ return udp_lib_get_port(sk, snum, hash2_nulladdr);
+}
+
+static void udp_destruct_sock(struct sock *sk)
+{
+ udp_destruct_common(sk);
+ inet_sock_destruct(sk);
+}
+
+static int udp_init_sock(struct sock *sk)
+{
+ int res = udp_lib_init_sock(sk);
+
+ sk->sk_destruct = udp_destruct_sock;
+ set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags);
+ return res;
+}
+
+/**
+ * udp4_hwcsum - handle outgoing HW checksumming
+ * @skb: sk_buff containing the filled-in UDP header
+ * (checksum field must be zeroed out)
+ * @src: source IP address
+ * @dst: destination IP address
+ */
+void udp4_hwcsum(struct sk_buff *skb, __be32 src, __be32 dst)
+{
+ struct udphdr *uh = udp_hdr(skb);
+ int offset = skb_transport_offset(skb);
+ int len = skb->len - offset;
+ int hlen = len;
+ __wsum csum = 0;
+
+ if (!skb_has_frag_list(skb)) {
+ /*
+ * Only one fragment on the socket.
+ */
+ skb->csum_start = skb_transport_header(skb) - skb->head;
+ skb->csum_offset = offsetof(struct udphdr, check);
+ uh->check = ~csum_tcpudp_magic(src, dst, len,
+ IPPROTO_UDP, 0);
+ } else {
+ struct sk_buff *frags;
+
+ /*
+ * HW-checksum won't work as there are two or more
+ * fragments on the socket so that all csums of sk_buffs
+ * should be together
+ */
+ skb_walk_frags(skb, frags) {
+ csum = csum_add(csum, frags->csum);
+ hlen -= frags->len;
+ }
+
+ csum = skb_checksum(skb, offset, hlen, csum);
+ skb->ip_summed = CHECKSUM_NONE;
+
+ uh->check = csum_tcpudp_magic(src, dst, len, IPPROTO_UDP, csum);
+ if (uh->check == 0)
+ uh->check = CSUM_MANGLED_0;
+ }
+}
+EXPORT_SYMBOL_GPL(udp4_hwcsum);
+
+static int udp_send_skb(struct sk_buff *skb, struct flowi4 *fl4,
+ struct inet_cork *cork)
+{
+ struct sock *sk = skb->sk;
+ int offset, len, datalen;
+ struct udphdr *uh;
+ int err;
+
+ offset = skb_transport_offset(skb);
+ len = skb->len - offset;
+ datalen = len - sizeof(*uh);
+
+ /*
+ * Create a UDP header
+ */
+ uh = udp_hdr(skb);
+ uh->source = inet_sk(sk)->inet_sport;
+ uh->dest = fl4->fl4_dport;
+ /* Datagram length checked in udp_sendmsg. */
+ udp_set_len_short(uh, len);
+ uh->check = 0;
+
+ if (cork->gso_size) {
+ const int hlen = skb_network_header_len(skb) +
+ sizeof(struct udphdr);
+
+ if (hlen + min(datalen, cork->gso_size) > cork->fragsize) {
+ kfree_skb(skb);
+ return -EMSGSIZE;
+ }
+ if (datalen > cork->gso_size * UDP_MAX_SEGMENTS) {
+ kfree_skb(skb);
+ return -EINVAL;
+ }
+ if (sk->sk_no_check_tx) {
+ kfree_skb(skb);
+ return -EINVAL;
+ }
+ if (dst_xfrm(skb_dst(skb))) {
+ kfree_skb(skb);
+ return -EIO;
+ }
+
+ if (datalen > cork->gso_size) {
+ skb_shinfo(skb)->gso_size = cork->gso_size;
+ skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4;
+ skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(datalen,
+ cork->gso_size);
+
+ /* Don't checksum the payload, skb will get segmented */
+ goto csum_partial;
+ }
+ }
+
+ if (sk->sk_no_check_tx) { /* UDP csum off */
+ skb->ip_summed = CHECKSUM_NONE;
+ goto send;
+ } else if (skb->ip_summed == CHECKSUM_PARTIAL) { /* UDP hardware csum */
+csum_partial:
+ udp4_hwcsum(skb, fl4->saddr, fl4->daddr);
+ goto send;
+ }
+
+ /* add protocol-dependent pseudo-header */
+ uh->check = csum_tcpudp_magic(fl4->saddr, fl4->daddr, len,
+ IPPROTO_UDP, udp_csum(skb));
+ if (uh->check == 0)
+ uh->check = CSUM_MANGLED_0;
+
+send:
+ err = ip_send_skb(sock_net(sk), skb);
+ if (unlikely(err)) {
+ if (err == -ENOBUFS &&
+ !inet_test_bit(RECVERR, sk)) {
+ UDP_INC_STATS(sock_net(sk), UDP_MIB_SNDBUFERRORS);
+ err = 0;
+ }
+ } else {
+ UDP_INC_STATS(sock_net(sk), UDP_MIB_OUTDATAGRAMS);
+ }
+ return err;
+}
+
+/*
+ * Push out all pending data as one UDP datagram. Socket is locked.
+ */
+int udp_push_pending_frames(struct sock *sk)
+{
+ struct udp_sock *up = udp_sk(sk);
+ struct inet_sock *inet = inet_sk(sk);
+ struct flowi4 *fl4 = &inet->cork.fl.u.ip4;
+ struct sk_buff *skb;
+ int err = 0;
+
+ skb = ip_finish_skb(sk, fl4);
+ if (!skb)
+ goto out;
+
+ err = udp_send_skb(skb, fl4, &inet->cork.base);
+
+out:
+ up->len = 0;
+ WRITE_ONCE(up->pending, 0);
+ return err;
+}
+
+void udp_splice_eof(struct socket *sock)
+{
+ struct sock *sk = sock->sk;
+ struct udp_sock *up = udp_sk(sk);
+
+ if (!READ_ONCE(up->pending) || udp_test_bit(CORK, sk))
+ return;
+
+ lock_sock(sk);
+ if (up->pending && !udp_test_bit(CORK, sk))
+ udp_push_pending_frames(sk);
+ release_sock(sk);
+}
+
+/* Handler for tunnels with arbitrary destination ports: no socket lookup, go
+ * through error handlers in encapsulations looking for a match.
+ */
+static int __udp4_lib_err_encap_no_sk(struct sk_buff *skb, u32 info)
+{
+ int i;
+
+ for (i = 0; i < MAX_IPTUN_ENCAP_OPS; i++) {
+ int (*handler)(struct sk_buff *skb, u32 info);
+ const struct ip_tunnel_encap_ops *encap;
+
+ encap = rcu_dereference(iptun_encaps[i]);
+ if (!encap)
+ continue;
+ handler = encap->err_handler;
+ if (handler && !handler(skb, info))
+ return 0;
+ }
+
+ return -ENOENT;
+}
+
+/* Try to match ICMP errors to UDP tunnels by looking up a socket without
+ * reversing source and destination port: this will match tunnels that force the
+ * same destination port on both endpoints (e.g. VXLAN, GENEVE). Note that
+ * lwtunnels might actually break this assumption by being configured with
+ * different destination ports on endpoints, in this case we won't be able to
+ * trace ICMP messages back to them.
+ *
+ * If this doesn't match any socket, probe tunnels with arbitrary destination
+ * ports (e.g. FoU, GUE): there, the receiving socket is useless, as the port
+ * we've sent packets to won't necessarily match the local destination port.
+ *
+ * Then ask the tunnel implementation to match the error against a valid
+ * association.
+ *
+ * Return an error if we can't find a match, the socket if we need further
+ * processing, zero otherwise.
+ */
+static struct sock *__udp4_lib_err_encap(struct net *net,
+ const struct iphdr *iph,
+ struct udphdr *uh,
+ struct sock *sk,
+ struct sk_buff *skb, u32 info)
+{
+ int (*lookup)(struct sock *sk, struct sk_buff *skb);
+ int network_offset, transport_offset;
+ struct udp_sock *up;
+
+ network_offset = skb_network_offset(skb);
+ transport_offset = skb_transport_offset(skb);
+
+ /* Network header needs to point to the outer IPv4 header inside ICMP */
+ skb_reset_network_header(skb);
+
+ /* Transport header needs to point to the UDP header */
+ skb_set_transport_header(skb, iph->ihl << 2);
+
+ if (sk) {
+ up = udp_sk(sk);
+
+ lookup = READ_ONCE(up->encap_err_lookup);
+ if (lookup && lookup(sk, skb))
+ sk = NULL;
+
+ goto out;
+ }
+
+ sk = __udp4_lib_lookup(net, iph->daddr, uh->source,
+ iph->saddr, uh->dest, skb->dev->ifindex, 0, NULL);
+ if (sk) {
+ up = udp_sk(sk);
+
+ lookup = READ_ONCE(up->encap_err_lookup);
+ if (!lookup || lookup(sk, skb))
+ sk = NULL;
+ }
+
+out:
+ if (!sk)
+ sk = ERR_PTR(__udp4_lib_err_encap_no_sk(skb, info));
+
+ skb_set_transport_header(skb, transport_offset);
+ skb_set_network_header(skb, network_offset);
+
+ return sk;
+}
+
+static void udp_err_update_exception(struct net *net, struct sk_buff *skb,
+ int type, int code, u32 info)
+{
+ if (type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED)
+ ipv4_update_pmtu(skb, net, info, 0, IPPROTO_UDP);
+ else if (type == ICMP_REDIRECT)
+ ipv4_redirect(skb, net, 0, IPPROTO_UDP);
+}
+
+/*
+ * This routine is called by the ICMP module when it gets some
+ * sort of error condition. If err < 0 then the socket should
+ * be closed and the error returned to the user. If err > 0
+ * it's just the icmp type << 8 | icmp code.
+ * Header points to the ip header of the error packet. We move
+ * on past this. Then (as it used to claim before adjustment)
+ * header points to the first 8 bytes of the udp header. We need
+ * to find the appropriate port.
+ */
+int udp_err(struct sk_buff *skb, u32 info)
+{
+ const struct iphdr *iph = (const struct iphdr *)skb->data;
+ const int type = icmp_hdr(skb)->type;
+ const int code = icmp_hdr(skb)->code;
+ struct net *net = dev_net(skb->dev);
+ struct inet_sock *inet;
+ bool tunnel = false;
+ struct udphdr *uh;
+ struct sock *sk;
+ int harderr;
+ int err;
+
+ udp_err_update_exception(net, skb, type, code, info);
+
+ uh = (struct udphdr *)(skb->data + (iph->ihl << 2));
+ sk = __udp4_lib_lookup(net, iph->daddr, uh->dest,
+ iph->saddr, uh->source, skb->dev->ifindex,
+ inet_sdif(skb), NULL);
+
+ if (!sk || READ_ONCE(udp_sk(sk)->encap_type)) {
+ /* No socket for error: try tunnels before discarding */
+ if (static_branch_unlikely(&udp_encap_needed_key)) {
+ sk = __udp4_lib_err_encap(net, iph, uh, sk, skb, info);
+ if (!sk)
+ return 0;
+ } else {
+ sk = ERR_PTR(-ENOENT);
+ }
+
+ if (IS_ERR(sk)) {
+ __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
+ return PTR_ERR(sk);
+ }
+
+ tunnel = true;
+ }
+
+ err = 0;
+ harderr = 0;
+ inet = inet_sk(sk);
+
+ switch (type) {
+ default:
+ case ICMP_TIME_EXCEEDED:
+ err = EHOSTUNREACH;
+ break;
+ case ICMP_SOURCE_QUENCH:
+ goto out;
+ case ICMP_PARAMETERPROB:
+ err = EPROTO;
+ harderr = 1;
+ break;
+ case ICMP_DEST_UNREACH:
+ if (code == ICMP_FRAG_NEEDED) { /* Path MTU discovery */
+ ipv4_sk_update_pmtu(skb, sk, info);
+ if (READ_ONCE(inet->pmtudisc) != IP_PMTUDISC_DONT) {
+ err = EMSGSIZE;
+ harderr = 1;
+ break;
+ }
+ goto out;
+ }
+ err = EHOSTUNREACH;
+ if (code <= NR_ICMP_UNREACH) {
+ harderr = icmp_err_convert[code].fatal;
+ err = icmp_err_convert[code].errno;
+ }
+ break;
+ case ICMP_REDIRECT:
+ ipv4_sk_redirect(skb, sk);
+ goto out;
+ }
+
+ /*
+ * RFC1122: OK. Passes ICMP errors back to application, as per
+ * 4.1.3.3.
+ */
+ if (tunnel) {
+ /* ...not for tunnels though: we don't have a sending socket */
+ if (udp_sk(sk)->encap_err_rcv)
+ udp_sk(sk)->encap_err_rcv(sk, skb, err, uh->dest, info,
+ (u8 *)(uh + 1));
+ goto out;
+ }
+ if (!inet_test_bit(RECVERR, sk)) {
+ if (!harderr || sk->sk_state != TCP_ESTABLISHED)
+ goto out;
+ } else {
+ ip_icmp_error(sk, skb, err, uh->dest, info, (u8 *)(uh + 1));
+ }
+
+ sk->sk_err = err;
+ sk_error_report(sk);
+out:
+ return 0;
+}
+
+static int __udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
+{
+ int rc;
+
+ if (inet_sk(sk)->inet_daddr) {
+ sock_rps_save_rxhash(sk, skb);
+ sk_mark_napi_id(sk, skb);
+ sk_incoming_cpu_update(sk);
+ } else {
+ sk_mark_napi_id_once(sk, skb);
+ }
+
+ rc = __udp_enqueue_schedule_skb(sk, skb);
+ if (rc < 0) {
+ struct net *net = sock_net(sk);
+ int drop_reason;
+
+ /* Note that an ENOMEM error is charged twice */
+ if (rc == -ENOMEM) {
+ UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS);
+ drop_reason = SKB_DROP_REASON_SOCKET_RCVBUFF;
+ } else {
+ UDP_INC_STATS(net, UDP_MIB_MEMERRORS);
+ drop_reason = SKB_DROP_REASON_PROTO_MEM;
+ }
+ UDP_INC_STATS(net, UDP_MIB_INERRORS);
+ trace_udp_fail_queue_rcv_skb(rc, sk, skb);
+ sk_skb_reason_drop(sk, skb, drop_reason);
+ return -1;
+ }
+
+ return 0;
+}
+
+/*
+ * This should be easy, if there is something there we
+ * return it, otherwise we block.
+ */
+
+INDIRECT_CALLABLE_SCOPE
+int udp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags)
+{
+ DECLARE_SOCKADDR(struct sockaddr_in *, sin, msg->msg_name);
+ int off, err, peeking = flags & MSG_PEEK;
+ struct inet_sock *inet = inet_sk(sk);
+ struct net *net = sock_net(sk);
+ bool checksum_valid = false;
+ unsigned int ulen, copied;
+ struct sk_buff *skb;
+
+ if (flags & MSG_ERRQUEUE)
+ return ip_recv_error(sk, msg, len);
+
+try_again:
+ off = sk_peek_offset(sk, flags);
+ skb = __skb_recv_udp(sk, flags, &off, &err);
+ if (!skb)
+ return err;
+
+ ulen = udp_skb_len(skb);
+ copied = len;
+ if (copied > ulen - off)
+ copied = ulen - off;
+ else if (copied < ulen)
+ msg->msg_flags |= MSG_TRUNC;
+
+ /* If checksum is needed at all, try to do it while copying the
+ * data. If the data is truncated, do it before the copy.
+ */
+ if (copied < ulen || peeking) {
+ checksum_valid = udp_skb_csum_unnecessary(skb) ||
+ !__udp_lib_checksum_complete(skb);
+ if (!checksum_valid)
+ goto csum_copy_err;
+ }
+
+ if (checksum_valid || udp_skb_csum_unnecessary(skb)) {
+ if (udp_skb_is_linear(skb))
+ err = copy_linear_skb(skb, copied, off, &msg->msg_iter);
+ else
+ err = skb_copy_datagram_msg(skb, off, msg, copied);
+ } else {
+ err = skb_copy_and_csum_datagram_msg(skb, off, msg);
+
+ if (err == -EINVAL)
+ goto csum_copy_err;
+ }
+
+ if (unlikely(err)) {
+ if (!peeking) {
+ udp_drops_inc(sk);
+ UDP_INC_STATS(net, UDP_MIB_INERRORS);
+ }
+ kfree_skb(skb);
+ return err;
+ }
+
+ if (!peeking)
+ UDP_INC_STATS(net, UDP_MIB_INDATAGRAMS);
+
+ sock_recv_cmsgs(msg, sk, skb);
+
+ /* Copy the address. */
+ if (sin) {
+ sin->sin_family = AF_INET;
+ sin->sin_port = udp_hdr(skb)->source;
+ sin->sin_addr.s_addr = ip_hdr(skb)->saddr;
+ memset(sin->sin_zero, 0, sizeof(sin->sin_zero));
+ msg->msg_namelen = sizeof(*sin);
+
+ BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk,
+ (struct sockaddr *)sin,
+ &msg->msg_namelen);
+ }
+
+ if (udp_test_bit(GRO_ENABLED, sk))
+ udp_cmsg_recv(msg, sk, skb);
+
+ if (inet_cmsg_flags(inet))
+ ip_cmsg_recv_offset(msg, sk, skb, sizeof(struct udphdr), off);
+
+ err = copied;
+ if (flags & MSG_TRUNC)
+ err = ulen;
+
+ skb_consume_udp(sk, skb, peeking ? -err : err);
+ return err;
+
+csum_copy_err:
+ if (!__sk_queue_drop_skb(sk, &udp_sk(sk)->reader_queue, skb, flags,
+ udp_skb_destructor)) {
+ UDP_INC_STATS(net, UDP_MIB_CSUMERRORS);
+ UDP_INC_STATS(net, UDP_MIB_INERRORS);
+ }
+ kfree_skb_reason(skb, SKB_DROP_REASON_UDP_CSUM);
+
+ /* starting over for a new packet, but check if we need to yield */
+ cond_resched();
+ msg->msg_flags &= ~MSG_TRUNC;
+ goto try_again;
+}
+
+/* Initialize UDP checksum. If exited with zero value (success),
+ * CHECKSUM_UNNECESSARY means, that no more checks are required.
+ * Otherwise, csum completion requires checksumming packet body,
+ * including udp header and folding it to skb->csum.
+ */
+static inline int udp4_csum_init(struct sk_buff *skb, struct udphdr *uh)
+{
+ int err;
+
+ /* Note, we are only interested in != 0 or == 0, thus the
+ * force to int.
+ */
+ err = (__force int)skb_checksum_init_zero_check(skb, IPPROTO_UDP, uh->check,
+ inet_compute_pseudo);
+ if (err)
+ return err;
+
+ if (skb->ip_summed == CHECKSUM_COMPLETE && !skb->csum_valid) {
+ /* If SW calculated the value, we know it's bad */
+ if (skb->csum_complete_sw)
+ return 1;
+
+ /* HW says the value is bad. Let's validate that.
+ * skb->csum is no longer the full packet checksum,
+ * so don't treat it as such.
+ */
+ skb_checksum_complete_unset(skb);
+ }
+
+ return 0;
+}
+
+/* returns:
+ * -1: error
+ * 0: success
+ * >0: "udp encap" protocol resubmission
+ *
+ * Note that in the success and error cases, the skb is assumed to
+ * have either been requeued or freed.
+ */
+static int udp_queue_rcv_one_skb(struct sock *sk, struct sk_buff *skb)
+{
+ enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
+ struct udp_sock *up = udp_sk(sk);
+ struct net *net = sock_net(sk);
+
+ /*
+ * Charge it to the socket, dropping if the queue is full.
+ */
+ if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
+ drop_reason = SKB_DROP_REASON_XFRM_POLICY;
+ goto drop;
+ }
+ nf_reset_ct(skb);
+
+ if (static_branch_unlikely(&udp_encap_needed_key) &&
+ READ_ONCE(up->encap_type)) {
+ int (*encap_rcv)(struct sock *sk, struct sk_buff *skb);
+
+ /*
+ * This is an encapsulation socket so pass the skb to
+ * the socket's udp_encap_rcv() hook. Otherwise, just
+ * fall through and pass this up the UDP socket.
+ * up->encap_rcv() returns the following value:
+ * =0 if skb was successfully passed to the encap
+ * handler or was discarded by it.
+ * >0 if skb should be passed on to UDP.
+ * <0 if skb should be resubmitted as proto -N
+ */
+
+ /* if we're overly short, let UDP handle it */
+ encap_rcv = READ_ONCE(up->encap_rcv);
+ if (encap_rcv) {
+ int ret;
+
+ /* Verify checksum before giving to encap */
+ if (udp_lib_checksum_complete(skb))
+ goto csum_error;
+
+ ret = encap_rcv(sk, skb);
+ if (ret <= 0) {
+ __UDP_INC_STATS(net, UDP_MIB_INDATAGRAMS);
+ return -ret;
+ }
+ }
+
+ /* FALLTHROUGH -- it's a UDP Packet */
+ }
+
+ prefetch(&sk->sk_rmem_alloc);
+ if (rcu_access_pointer(sk->sk_filter) &&
+ udp_lib_checksum_complete(skb))
+ goto csum_error;
+
+ drop_reason = sk_filter_trim_cap(sk, skb, sizeof(struct udphdr));
+ if (drop_reason)
+ goto drop;
+
+ udp_csum_pull_header(skb);
+
+ ipv4_pktinfo_prepare(sk, skb, true);
+ return __udp_queue_rcv_skb(sk, skb);
+
+csum_error:
+ drop_reason = SKB_DROP_REASON_UDP_CSUM;
+ __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS);
+drop:
+ __UDP_INC_STATS(net, UDP_MIB_INERRORS);
+ udp_drops_inc(sk);
+ sk_skb_reason_drop(sk, skb, drop_reason);
+ return -1;
+}
+
+static int udp_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
+{
+ struct sk_buff *next, *segs;
+ int ret;
+
+ if (likely(!udp_unexpected_gso(sk, skb)))
+ return udp_queue_rcv_one_skb(sk, skb);
+
+ BUILD_BUG_ON(sizeof(struct udp_skb_cb) > SKB_GSO_CB_OFFSET);
+ __skb_push(skb, -skb_mac_offset(skb));
+ segs = udp_rcv_segment(sk, skb, true);
+ skb_list_walk_safe(segs, skb, next) {
+ __skb_pull(skb, skb_transport_offset(skb));
+
+ udp_post_segment_fix_csum(skb);
+ ret = udp_queue_rcv_one_skb(sk, skb);
+ if (ret > 0)
+ ip_protocol_deliver_rcu(dev_net(skb->dev), skb, ret);
+ }
+ return 0;
+}
+
+static inline bool __udp_is_mcast_sock(struct net *net, const struct sock *sk,
+ __be16 loc_port, __be32 loc_addr,
+ __be16 rmt_port, __be32 rmt_addr,
+ int dif, int sdif, unsigned short hnum)
+{
+ const struct inet_sock *inet = inet_sk(sk);
+
+ if (!net_eq(sock_net(sk), net) ||
+ udp_sk(sk)->udp_port_hash != hnum ||
+ (inet->inet_daddr && inet->inet_daddr != rmt_addr) ||
+ (inet->inet_dport != rmt_port && inet->inet_dport) ||
+ (inet->inet_rcv_saddr && inet->inet_rcv_saddr != loc_addr) ||
+ ipv6_only_sock(sk) ||
+ !udp_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))
+ return false;
+ if (!ip_mc_sf_allow(sk, loc_addr, rmt_addr, dif, sdif))
+ return false;
+ return true;
+}
+
+/*
+ * Multicasts and broadcasts go to each listener.
+ *
+ * Note: called only from the BH handler context.
+ */
+static int __udp4_lib_mcast_deliver(struct net *net, struct sk_buff *skb,
+ struct udphdr *uh,
+ __be32 saddr, __be32 daddr)
+{
+ struct udp_table *udptable = net->ipv4.udp_table;
+ unsigned int hash2, hash2_any, offset;
+ unsigned short hnum = ntohs(uh->dest);
+ struct sock *sk, *first = NULL;
+ int dif = skb->dev->ifindex;
+ int sdif = inet_sdif(skb);
+ struct hlist_node *node;
+ struct udp_hslot *hslot;
+ struct sk_buff *nskb;
+ bool use_hash2;
+ int ret;
+
+ hash2_any = 0;
+ hash2 = 0;
+ hslot = udp_hashslot(udptable, net, hnum);
+ use_hash2 = hslot->count > 10;
+ offset = offsetof(typeof(*sk), sk_node);
+
+ if (use_hash2) {
+ hash2_any = ipv4_portaddr_hash(net, htonl(INADDR_ANY), hnum) &
+ udptable->mask;
+ hash2 = ipv4_portaddr_hash(net, daddr, hnum) & udptable->mask;
+start_lookup:
+ hslot = &udptable->hash2[hash2].hslot;
+ offset = offsetof(typeof(*sk), __sk_common.skc_portaddr_node);
+ }
+
+ sk_for_each_entry_offset_rcu(sk, node, &hslot->head, offset) {
+ if (!__udp_is_mcast_sock(net, sk, uh->dest, daddr,
+ uh->source, saddr, dif, sdif, hnum))
+ continue;
+
+ if (!first) {
+ first = sk;
+ continue;
+ }
+ nskb = skb_clone(skb, GFP_ATOMIC);
+
+ if (unlikely(!nskb)) {
+ udp_drops_inc(sk);
+ __UDP_INC_STATS(net, UDP_MIB_RCVBUFERRORS);
+ __UDP_INC_STATS(net, UDP_MIB_INERRORS);
+ continue;
+ }
+ if (udp_queue_rcv_skb(sk, nskb) > 0)
+ consume_skb(nskb);
+ }
+
+ /* Also lookup *:port if we are using hash2 and haven't done so yet. */
+ if (use_hash2 && hash2 != hash2_any) {
+ hash2 = hash2_any;
+ goto start_lookup;
+ }
+
+ if (first) {
+ ret = udp_queue_rcv_skb(first, skb);
+ if (ret > 0)
+ return -ret;
+ } else {
+ kfree_skb(skb);
+ __UDP_INC_STATS(net, UDP_MIB_IGNOREDMULTI);
+ }
+ return 0;
+}
+
+/* wrapper for udp_queue_rcv_skb taking care of csum conversion and
+ * return code conversion for ip layer consumption
+ */
+static int udp_unicast_rcv_skb(struct sock *sk, struct sk_buff *skb,
+ struct udphdr *uh)
+{
+ int ret;
+
+ if (inet_get_convert_csum(sk) && uh->check)
+ skb_checksum_try_convert(skb, IPPROTO_UDP, inet_compute_pseudo);
+
+ ret = udp_queue_rcv_skb(sk, skb);
+
+ /* a return value > 0 means to resubmit the input, but
+ * it wants the return to be -protocol, or 0
+ */
+ if (ret > 0)
+ return -ret;
+ return 0;
+}
+
+/*
+ * All we need to do is get the socket, and then do a checksum.
+ */
+
+int udp_rcv(struct sk_buff *skb)
+{
+ struct rtable *rt = skb_rtable(skb);
+ struct net *net = dev_net(skb->dev);
+ struct sock *sk = NULL;
+ unsigned short ulen;
+ __be32 saddr, daddr;
+ struct udphdr *uh;
+ bool refcounted;
+ int drop_reason;
+
+ drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
+
+ /*
+ * Validate the packet.
+ */
+ if (!pskb_may_pull(skb, sizeof(struct udphdr)))
+ goto drop; /* No space for header. */
+
+ uh = udp_hdr(skb);
+ ulen = ntohs(uh->len);
+ saddr = ip_hdr(skb)->saddr;
+ daddr = ip_hdr(skb)->daddr;
+
+ if (ulen > skb->len)
+ goto short_packet;
+
+ if (ulen < sizeof(*uh))
+ goto short_packet;
+
+ if (ulen < skb->len) {
+ if (pskb_trim_rcsum(skb, ulen))
+ goto short_packet;
+
+ uh = udp_hdr(skb);
+ }
+
+ if (udp4_csum_init(skb, uh))
+ goto csum_error;
+
+ sk = inet_steal_sock(net, skb, sizeof(struct udphdr), saddr, uh->source, daddr, uh->dest,
+ &refcounted, udp_ehashfn);
+ if (IS_ERR(sk))
+ goto no_sk;
+
+ if (sk) {
+ struct dst_entry *dst = skb_dst(skb);
+ int ret;
+
+ if (unlikely(rcu_dereference(sk->sk_rx_dst) != dst))
+ udp_sk_rx_dst_set(sk, dst);
+
+ ret = udp_unicast_rcv_skb(sk, skb, uh);
+ if (refcounted)
+ sock_put(sk);
+ return ret;
+ }
+
+ if (rt->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
+ return __udp4_lib_mcast_deliver(net, skb, uh, saddr, daddr);
+
+ sk = __udp4_lib_lookup_skb(skb, uh->source, uh->dest);
+ if (sk)
+ return udp_unicast_rcv_skb(sk, skb, uh);
+no_sk:
+ if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
+ goto drop;
+ nf_reset_ct(skb);
+
+ /* No socket. Drop packet silently, if checksum is wrong */
+ if (udp_lib_checksum_complete(skb))
+ goto csum_error;
+
+ drop_reason = SKB_DROP_REASON_NO_SOCKET;
+ __UDP_INC_STATS(net, UDP_MIB_NOPORTS);
+ icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
+
+ /*
+ * Hmm. We got an UDP packet to a port to which we
+ * don't wanna listen. Ignore it.
+ */
+ sk_skb_reason_drop(sk, skb, drop_reason);
+ return 0;
+
+short_packet:
+ drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
+ net_dbg_ratelimited("UDP: short packet: From %pI4:%u %d/%d to %pI4:%u\n",
+ &saddr, ntohs(uh->source),
+ ulen, skb->len,
+ &daddr, ntohs(uh->dest));
+ goto drop;
+
+csum_error:
+ /*
+ * RFC1122: OK. Discards the bad packet silently (as far as
+ * the network is concerned, anyway) as per 4.1.3.4 (MUST).
+ */
+ drop_reason = SKB_DROP_REASON_UDP_CSUM;
+ net_dbg_ratelimited("UDP: bad checksum. From %pI4:%u to %pI4:%u ulen %d\n",
+ &saddr, ntohs(uh->source), &daddr, ntohs(uh->dest),
+ ulen);
+ __UDP_INC_STATS(net, UDP_MIB_CSUMERRORS);
+drop:
+ __UDP_INC_STATS(net, UDP_MIB_INERRORS);
+ sk_skb_reason_drop(sk, skb, drop_reason);
+ return 0;
+}
+
+int udp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
+{
+ int corkreq = udp_test_bit(CORK, sk) || msg->msg_flags & MSG_MORE;
+ DEFINE_RAW_FLEX(struct ip_options_rcu, opt_copy, opt.__data,
+ IP_OPTIONS_DATA_FIXED_SIZE);
+ DECLARE_SOCKADDR(struct sockaddr_in *, usin, msg->msg_name);
+ int ulen = len, free = 0, connected = 0;
+ struct inet_sock *inet = inet_sk(sk);
+ struct udp_sock *up = udp_sk(sk);
+ __be32 daddr, faddr, saddr;
+ struct rtable *rt = NULL;
+ struct flowi4 fl4_stack;
+ struct ipcm_cookie ipc;
+ struct sk_buff *skb;
+ struct flowi4 *fl4;
+ __be16 dport;
+ int uc_index;
+ u8 scope;
+ int err;
+
+ if (len > 0xFFFF)
+ return -EMSGSIZE;
+
+ /*
+ * Check the flags.
+ */
+
+ if (msg->msg_flags & MSG_OOB) /* Mirror BSD error message compatibility */
+ return -EOPNOTSUPP;
+
+ fl4 = &inet->cork.fl.u.ip4;
+ if (READ_ONCE(up->pending)) {
+ /*
+ * There are pending frames.
+ * The socket lock must be held while it's corked.
+ */
+ lock_sock(sk);
+ if (likely(up->pending)) {
+ if (unlikely(up->pending != AF_INET)) {
+ release_sock(sk);
+ return -EINVAL;
+ }
+ goto do_append_data;
+ }
+ release_sock(sk);
+ }
+ ulen += sizeof(struct udphdr);
+
+ /*
+ * Get and verify the address.
+ */
+ if (usin) {
+ if (msg->msg_namelen < sizeof(*usin))
+ return -EINVAL;
+ if (usin->sin_family != AF_INET) {
+ if (usin->sin_family != AF_UNSPEC)
+ return -EAFNOSUPPORT;
+ }
+
+ daddr = usin->sin_addr.s_addr;
+ dport = usin->sin_port;
+ if (dport == 0)
+ return -EINVAL;
+ } else {
+ if (sk->sk_state != TCP_ESTABLISHED)
+ return -EDESTADDRREQ;
+ daddr = inet->inet_daddr;
+ dport = inet->inet_dport;
+ /* Open fast path for connected socket.
+ * Route will not be used, if at least one option is set.
+ */
+ connected = 1;
+ }
+
+ ipcm_init_sk(&ipc, inet);
+ ipc.gso_size = READ_ONCE(up->gso_size);
+
+ if (msg->msg_controllen) {
+ err = udp_cmsg_send(sk, msg, &ipc.gso_size);
+ if (err > 0) {
+ err = ip_cmsg_send(sk, msg, &ipc,
+ sk->sk_family == AF_INET6);
+ connected = 0;
+ }
+ if (unlikely(err < 0)) {
+ kfree(ipc.opt);
+ return err;
+ }
+ if (ipc.opt)
+ free = 1;
+ }
+ if (!ipc.opt) {
+ struct ip_options_rcu *inet_opt;
+
+ rcu_read_lock();
+ inet_opt = rcu_dereference(inet->inet_opt);
+ if (inet_opt) {
+ memcpy(opt_copy, inet_opt,
+ sizeof(*inet_opt) + inet_opt->opt.optlen);
+ ipc.opt = opt_copy;
+ }
+ rcu_read_unlock();
+ }
+
+ if (cgroup_bpf_enabled(CGROUP_UDP4_SENDMSG) && !connected) {
+ err = BPF_CGROUP_RUN_PROG_UDP4_SENDMSG_LOCK(sk,
+ (struct sockaddr *)usin,
+ &msg->msg_namelen,
+ &ipc.addr);
+ if (err)
+ goto out_free;
+ if (usin) {
+ if (usin->sin_port == 0) {
+ /* BPF program set invalid port. Reject it. */
+ err = -EINVAL;
+ goto out_free;
+ }
+ daddr = usin->sin_addr.s_addr;
+ dport = usin->sin_port;
+ }
+ }
+
+ saddr = ipc.addr;
+ ipc.addr = daddr;
+ faddr = daddr;
+
+ if (ipc.opt && ipc.opt->opt.srr) {
+ if (!daddr) {
+ err = -EINVAL;
+ goto out_free;
+ }
+ faddr = ipc.opt->opt.faddr;
+ connected = 0;
+ }
+ scope = ip_sendmsg_scope(inet, &ipc, msg);
+ if (scope == RT_SCOPE_LINK)
+ connected = 0;
+
+ uc_index = READ_ONCE(inet->uc_index);
+ if (ipv4_is_multicast(daddr)) {
+ if (!ipc.oif || netif_index_is_l3_master(sock_net(sk), ipc.oif))
+ ipc.oif = READ_ONCE(inet->mc_index);
+ if (!saddr)
+ saddr = READ_ONCE(inet->mc_addr);
+ connected = 0;
+ } else if (!ipc.oif) {
+ ipc.oif = uc_index;
+ } else if (ipv4_is_lbcast(daddr) && uc_index) {
+ /* oif is set, packet is to local broadcast and
+ * uc_index is set. oif is most likely set
+ * by sk_bound_dev_if. If uc_index != oif check if the
+ * oif is an L3 master and uc_index is an L3 slave.
+ * If so, we want to allow the send using the uc_index.
+ */
+ if (ipc.oif != uc_index &&
+ ipc.oif == l3mdev_master_ifindex_by_index(sock_net(sk),
+ uc_index)) {
+ ipc.oif = uc_index;
+ }
+ }
+
+ if (connected)
+ rt = dst_rtable(sk_dst_check(sk, 0));
+
+ if (!rt) {
+ struct net *net = sock_net(sk);
+ __u8 flow_flags = inet_sk_flowi_flags(sk);
+
+ fl4 = &fl4_stack;
+
+ flowi4_init_output(fl4, ipc.oif, ipc.sockc.mark,
+ ipc.tos & INET_DSCP_MASK, scope,
+ IPPROTO_UDP, flow_flags, faddr, saddr,
+ dport, inet->inet_sport,
+ sk_uid(sk));
+
+ security_sk_classify_flow(sk, flowi4_to_flowi_common(fl4));
+ rt = ip_route_output_flow(net, fl4, sk);
+ if (IS_ERR(rt)) {
+ err = PTR_ERR(rt);
+ rt = NULL;
+ if (err == -ENETUNREACH)
+ IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
+ goto out;
+ }
+
+ err = -EACCES;
+ if ((rt->rt_flags & RTCF_BROADCAST) &&
+ !sock_flag(sk, SOCK_BROADCAST))
+ goto out;
+ if (connected)
+ sk_dst_set(sk, dst_clone(&rt->dst));
+ }
+
+ if (msg->msg_flags & MSG_CONFIRM)
+ goto do_confirm;
+back_from_confirm:
+
+ saddr = fl4->saddr;
+ if (!ipc.addr) {
+ daddr = fl4->daddr;
+ ipc.addr = fl4->daddr;
+ }
+
+ /* Lockless fast path for the non-corking case. */
+ if (!corkreq) {
+ struct inet_cork cork;
+
+ skb = ip_make_skb(sk, fl4, ip_generic_getfrag, msg, ulen,
+ sizeof(struct udphdr), &ipc, &rt,
+ &cork, msg->msg_flags);
+ err = PTR_ERR(skb);
+ if (!IS_ERR_OR_NULL(skb))
+ err = udp_send_skb(skb, fl4, &cork);
+ goto out;
+ }
+
+ lock_sock(sk);
+ if (unlikely(up->pending)) {
+ /* The socket is already corked while preparing it. */
+ /* ... which is an evident application bug. --ANK */
+ release_sock(sk);
+
+ net_dbg_ratelimited("socket already corked\n");
+ err = -EINVAL;
+ goto out;
+ }
+ /*
+ * Now cork the socket to pend data.
+ */
+ fl4 = &inet->cork.fl.u.ip4;
+ fl4->daddr = daddr;
+ fl4->saddr = saddr;
+ fl4->fl4_dport = dport;
+ fl4->fl4_sport = inet->inet_sport;
+ WRITE_ONCE(up->pending, AF_INET);
+
+do_append_data:
+ up->len += ulen;
+ err = ip_append_data(sk, fl4, ip_generic_getfrag, msg, ulen,
+ sizeof(struct udphdr), &ipc, &rt,
+ corkreq ? msg->msg_flags | MSG_MORE : msg->msg_flags);
+ if (err)
+ udp_flush_pending_frames(sk);
+ else if (!corkreq)
+ err = udp_push_pending_frames(sk);
+ else if (unlikely(skb_queue_empty(&sk->sk_write_queue)))
+ WRITE_ONCE(up->pending, 0);
+ release_sock(sk);
+
+out:
+ ip_rt_put(rt);
+out_free:
+ if (free)
+ kfree(ipc.opt);
+ if (!err)
+ return len;
+ /*
+ * ENOBUFS = no kernel mem, SOCK_NOSPACE = no sndbuf space. Reporting
+ * ENOBUFS might not be good (it's not tunable per se), but otherwise
+ * we don't have a good statistic (IpOutDiscards but it can be too many
+ * things). We could add another new stat but at least for now that
+ * seems like overkill.
+ */
+ if (err == -ENOBUFS || test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
+ UDP_INC_STATS(sock_net(sk), UDP_MIB_SNDBUFERRORS);
+
+ return err;
+
+do_confirm:
+ if (msg->msg_flags & MSG_PROBE)
+ dst_confirm_neigh(&rt->dst, &fl4->daddr);
+ if (!(msg->msg_flags & MSG_PROBE) || len)
+ goto back_from_confirm;
+ err = 0;
+ goto out;
+}
+EXPORT_SYMBOL(udp_sendmsg);
+
+static int udp_connect(struct sock *sk, struct sockaddr_unsized *uaddr,
+ int addr_len)
+{
+ int res;
+
+ lock_sock(sk);
+ res = __ip4_datagram_connect(sk, uaddr, addr_len);
+ if (!res)
+ udp4_hash4(sk);
+ release_sock(sk);
+ return res;
+}
+
+static void udp_v4_rehash(struct sock *sk)
+{
+ u16 new_hash = ipv4_portaddr_hash(sock_net(sk),
+ inet_sk(sk)->inet_rcv_saddr,
+ inet_sk(sk)->inet_num);
+ u16 new_hash4 = udp_ehashfn(sock_net(sk),
+ sk->sk_rcv_saddr, sk->sk_num,
+ sk->sk_daddr, sk->sk_dport);
+
+ udp_lib_rehash(sk, new_hash, new_hash4);
+}
+
+static void udp_destroy_sock(struct sock *sk)
+{
+ struct udp_sock *up = udp_sk(sk);
+ bool slow = lock_sock_fast(sk);
+
+ /* protects from races with udp_abort() */
+ sock_set_flag(sk, SOCK_DEAD);
+ udp_flush_pending_frames(sk);
+ unlock_sock_fast(sk, slow);
+ if (static_branch_unlikely(&udp_encap_needed_key)) {
+ if (up->encap_type) {
+ void (*encap_destroy)(struct sock *sk);
+
+ encap_destroy = READ_ONCE(up->encap_destroy);
+ if (encap_destroy)
+ encap_destroy(sk);
+ }
+ if (udp_test_bit(ENCAP_ENABLED, sk)) {
+ static_branch_dec(&udp_encap_needed_key);
+ udp_tunnel_cleanup_gro(sk);
+ }
+ }
+}
+
+/* We can only early demux multicast if there is a single matching socket.
+ * If more than one socket found returns NULL
+ */
+static struct sock *__udp4_lib_mcast_demux_lookup(struct net *net,
+ __be16 loc_port, __be32 loc_addr,
+ __be16 rmt_port, __be32 rmt_addr,
+ int dif, int sdif)
+{
+ struct udp_table *udptable = net->ipv4.udp_table;
+ unsigned short hnum = ntohs(loc_port);
+ struct sock *sk, *result;
+ struct udp_hslot *hslot;
+ unsigned int slot;
+
+ slot = udp_hashfn(net, hnum, udptable->mask);
+ hslot = &udptable->hash[slot];
+
+ /* Do not bother scanning a too big list */
+ if (hslot->count > 10)
+ return NULL;
+
+ result = NULL;
+ sk_for_each_rcu(sk, &hslot->head) {
+ if (__udp_is_mcast_sock(net, sk, loc_port, loc_addr,
+ rmt_port, rmt_addr, dif, sdif, hnum)) {
+ if (result)
+ return NULL;
+ result = sk;
+ }
+ }
+
+ return result;
+}
+
+/* For unicast we should only early demux connected sockets or we can
+ * break forwarding setups. The chains here can be long so only check
+ * if the first socket is an exact match and if not move on.
+ */
+static struct sock *__udp4_lib_demux_lookup(struct net *net,
+ __be16 loc_port, __be32 loc_addr,
+ __be16 rmt_port, __be32 rmt_addr,
+ int dif, int sdif)
+{
+ struct udp_table *udptable = net->ipv4.udp_table;
+ INET_ADDR_COOKIE(acookie, rmt_addr, loc_addr);
+ unsigned short hnum = ntohs(loc_port);
+ struct udp_hslot *hslot2;
+ unsigned int hash2;
+ __portpair ports;
+ struct sock *sk;
+
+ hash2 = ipv4_portaddr_hash(net, loc_addr, hnum);
+ hslot2 = udp_hashslot2(udptable, hash2);
+ ports = INET_COMBINED_PORTS(rmt_port, hnum);
+
+ udp_portaddr_for_each_entry_rcu(sk, &hslot2->head) {
+ if (inet_match(net, sk, acookie, ports, dif, sdif))
+ return sk;
+ /* Only check first socket in chain */
+ break;
+ }
+ return NULL;
+}
+
+enum skb_drop_reason udp_v4_early_demux(struct sk_buff *skb)
+{
+ struct net *net = dev_net(skb->dev);
+ struct in_device *in_dev = NULL;
+ const struct iphdr *iph;
+ const struct udphdr *uh;
+ struct sock *sk = NULL;
+ struct dst_entry *dst;
+ int dif = skb->dev->ifindex;
+ int sdif = inet_sdif(skb);
+ int ours;
+
+ /* validate the packet */
+ if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct udphdr)))
+ return SKB_NOT_DROPPED_YET;
+
+ iph = ip_hdr(skb);
+ uh = udp_hdr(skb);
+
+ if (skb->pkt_type == PACKET_MULTICAST) {
+ in_dev = __in_dev_get_rcu(skb->dev);
+
+ if (!in_dev)
+ return SKB_NOT_DROPPED_YET;
+
+ ours = ip_check_mc_rcu(in_dev, iph->daddr, iph->saddr,
+ iph->protocol);
+ if (!ours)
+ return SKB_NOT_DROPPED_YET;
+
+ sk = __udp4_lib_mcast_demux_lookup(net, uh->dest, iph->daddr,
+ uh->source, iph->saddr,
+ dif, sdif);
+ } else if (skb->pkt_type == PACKET_HOST) {
+ sk = __udp4_lib_demux_lookup(net, uh->dest, iph->daddr,
+ uh->source, iph->saddr, dif, sdif);
+ }
+
+ if (!sk)
+ return SKB_NOT_DROPPED_YET;
+
+ skb->sk = sk;
+ DEBUG_NET_WARN_ON_ONCE(sk_is_refcounted(sk));
+ skb->destructor = sock_pfree;
+ dst = rcu_dereference(sk->sk_rx_dst);
+
+ if (dst)
+ dst = dst_check(dst, 0);
+ if (dst) {
+ u32 itag = 0;
+
+ /* set noref for now.
+ * any place which wants to hold dst has to call
+ * dst_hold_safe()
+ */
+ skb_dst_set_noref(skb, dst);
+
+ /* for unconnected multicast sockets we need to validate
+ * the source on each packet
+ */
+ if (!inet_sk(sk)->inet_daddr && in_dev)
+ return ip_mc_validate_source(skb, iph->daddr,
+ iph->saddr,
+ ip4h_dscp(iph),
+ skb->dev, in_dev, &itag);
+ }
+ return SKB_NOT_DROPPED_YET;
+}
+
+static int udp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
+ unsigned int optlen)
+{
+ if (level == SOL_UDP || level == SOL_SOCKET)
+ return udp_lib_setsockopt(sk, level, optname,
+ optval, optlen,
+ udp_push_pending_frames);
+ return ip_setsockopt(sk, level, optname, optval, optlen);
+}
+
+static int udp_getsockopt(struct sock *sk, int level, int optname,
+ char __user *optval, int __user *optlen)
+{
+ sockopt_t opt;
+ int err;
+
+ /*
+ * keep the old __user pointers, until ip_getsockopt() moves
+ * to sockopt_t
+ */
+ if (level != SOL_UDP)
+ return ip_getsockopt(sk, level, optname, optval, optlen);
+
+ err = sockopt_init_user(&opt, optval, optlen);
+ if (err)
+ return err;
+
+ err = udp_lib_getsockopt(sk, level, optname, &opt);
+ if (err)
+ return err;
+
+ /* optval was written by copy_to_iter() in udp_lib_getsockopt() */
+ if (put_user(opt.optlen, optlen))
+ return -EFAULT;
+
+ return 0;
+}
+
+struct proto udp_prot = {
+ .name = "UDP",
+ .owner = THIS_MODULE,
+ .close = udp_lib_close,
+ .pre_connect = udp_pre_connect,
+ .connect = udp_connect,
+ .disconnect = udp_disconnect,
+ .ioctl = udp_ioctl,
+ .init = udp_init_sock,
+ .destroy = udp_destroy_sock,
+ .setsockopt = udp_setsockopt,
+ .getsockopt = udp_getsockopt,
+ .sendmsg = udp_sendmsg,
+ .recvmsg = udp_recvmsg,
+ .splice_eof = udp_splice_eof,
+ .release_cb = ip4_datagram_release_cb,
+ .hash = udp_lib_hash,
+ .unhash = udp_lib_unhash,
+ .rehash = udp_v4_rehash,
+ .get_port = udp_v4_get_port,
+ .put_port = udp_lib_unhash,
+#ifdef CONFIG_BPF_SYSCALL
+ .psock_update_sk_prot = udp_bpf_update_proto,
+#endif
+ .memory_allocated = &net_aligned_data.udp_memory_allocated,
+ .per_cpu_fw_alloc = &udp_memory_per_cpu_fw_alloc,
+
+ .sysctl_mem = sysctl_udp_mem,
+ .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_udp_wmem_min),
+ .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_udp_rmem_min),
+ .obj_size = sizeof(struct udp_sock),
+ .diag_destroy = udp_abort,
+};
+EXPORT_SYMBOL(udp_prot);
diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c
index cf07c3c6611a..bea037d6b4eb 100644
--- a/net/ipv4/udp_offload.c
+++ b/net/ipv4/udp_offload.c
@@ -958,6 +958,9 @@ INDIRECT_CALLABLE_SCOPE int udp4_gro_complete(struct sk_buff *skb, int nhoff)
const struct iphdr *iph = (struct iphdr *)(skb->data + offset);
struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
+ if (!IS_ENABLED(CONFIG_IPV4))
+ return -EAFNOSUPPORT;
+
/* do fraglist only if there is no outer UDP encap (or we already processed it) */
if (NAPI_GRO_CB(skb)->is_flist && !NAPI_GRO_CB(skb)->encap_mark) {
udp_set_len(uh, skb->len - nhoff);
diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c
index 59f9872c3552..8cdc24ed996a 100644
--- a/net/ipv6/udp.c
+++ b/net/ipv6/udp.c
@@ -326,7 +326,8 @@ static void udp6_hash4(struct sock *sk)
struct net *net = sock_net(sk);
unsigned int hash;
- if (ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) {
+ if (IS_ENABLED(CONFIG_IPV4) &&
+ ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr)) {
udp4_hash4(sk);
return;
}
@@ -1449,7 +1450,8 @@ static int udp_v6_push_pending_frames(struct sock *sk)
int err = 0;
if (up->pending == AF_INET)
- return udp_push_pending_frames(sk);
+ return IS_ENABLED(CONFIG_IPV4) ? udp_push_pending_frames(sk) :
+ -EAFNOSUPPORT;
skb = ip6_finish_skb(sk);
if (!skb)
@@ -1529,7 +1531,7 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
msg->msg_name = &sin;
msg->msg_namelen = sizeof(sin);
do_udp_sendmsg:
- err = ipv6_only_sock(sk) ?
+ err = (!IS_ENABLED(CONFIG_IPV4) || ipv6_only_sock(sk)) ?
-ENETUNREACH : udp_sendmsg(sk, msg, len);
msg->msg_name = sin6;
msg->msg_namelen = addr_len;
@@ -1545,7 +1547,8 @@ int udpv6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
if (READ_ONCE(up->pending)) {
if (READ_ONCE(up->pending) == AF_INET)
- return udp_sendmsg(sk, msg, len);
+ return IS_ENABLED(CONFIG_IPV4) ?
+ udp_sendmsg(sk, msg, len) : -EAFNOSUPPORT;
/*
* There are pending frames.
* The socket lock must be held while it's corked.
--
2.55.0
^ permalink raw reply related [flat|nested] 7+ messages in thread