[PATCH net-next] net/packet: remove po->xmit

[PATCH net-next] net/packet: remove po->xmit

[Eric Dumazet]

Use PACKET_SOCK_QDISC_BYPASS atomic bit instead of a pointer.

This removes one indirect call in fast path,
and READ_ONCE()/WRITE_ONCE() annotations as well.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Suggested-by: Willem de Bruijn <willemb@google.com>
Cc: Daniel Borkmann <daniel@iogearbox.net>
---
 net/packet/af_packet.c | 24 +++++++++---------------
 net/packet/internal.h  |  2 +-
 2 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c
index 7b9367b233d34d1a8338233f2c1bd96e9a28e14c..497193f73030c385a2d33b71dfbc299fbf9b763d 100644
--- a/net/packet/af_packet.c
+++ b/net/packet/af_packet.c
@@ -270,8 +270,11 @@ static noinline struct sk_buff *nf_hook_direct_egress(struct sk_buff *skb)
 }
 #endif
 
-static int packet_direct_xmit(struct sk_buff *skb)
+static int packet_xmit(const struct packet_sock *po, struct sk_buff *skb)
 {
+	if (!packet_sock_flag(po, PACKET_SOCK_QDISC_BYPASS))
+		return dev_queue_xmit(skb);
+
 #ifdef CONFIG_NETFILTER_EGRESS
 	if (nf_hook_egress_active()) {
 		skb = nf_hook_direct_egress(skb);
@@ -305,12 +308,6 @@ static void packet_cached_dev_reset(struct packet_sock *po)
 	RCU_INIT_POINTER(po->cached_dev, NULL);
 }
 
-static bool packet_use_direct_xmit(const struct packet_sock *po)
-{
-	/* Paired with WRITE_ONCE() in packet_setsockopt() */
-	return READ_ONCE(po->xmit) == packet_direct_xmit;
-}
-
 static u16 packet_pick_tx_queue(struct sk_buff *skb)
 {
 	struct net_device *dev = skb->dev;
@@ -2872,8 +2869,7 @@ static int tpacket_snd(struct packet_sock *po, struct msghdr *msg)
 		packet_inc_pending(&po->tx_ring);
 
 		status = TP_STATUS_SEND_REQUEST;
-		/* Paired with WRITE_ONCE() in packet_setsockopt() */
-		err = READ_ONCE(po->xmit)(skb);
+		err = packet_xmit(po, skb);
 		if (unlikely(err != 0)) {
 			if (err > 0)
 				err = net_xmit_errno(err);
@@ -3076,8 +3072,8 @@ static int packet_snd(struct socket *sock, struct msghdr *msg, size_t len)
 		virtio_net_hdr_set_proto(skb, &vnet_hdr);
 	}
 
-	/* Paired with WRITE_ONCE() in packet_setsockopt() */
-	err = READ_ONCE(po->xmit)(skb);
+	err = packet_xmit(po, skb);
+
 	if (unlikely(err != 0)) {
 		if (err > 0)
 			err = net_xmit_errno(err);
@@ -3359,7 +3355,6 @@ static int packet_create(struct net *net, struct socket *sock, int protocol,
 	init_completion(&po->skb_completion);
 	sk->sk_family = PF_PACKET;
 	po->num = proto;
-	po->xmit = dev_queue_xmit;
 
 	err = packet_alloc_pending(po);
 	if (err)
@@ -4010,8 +4005,7 @@ packet_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval,
 		if (copy_from_sockptr(&val, optval, sizeof(val)))
 			return -EFAULT;
 
-		/* Paired with all lockless reads of po->xmit */
-		WRITE_ONCE(po->xmit, val ? packet_direct_xmit : dev_queue_xmit);
+		packet_sock_flag_set(po, PACKET_SOCK_QDISC_BYPASS, val);
 		return 0;
 	}
 	default:
@@ -4126,7 +4120,7 @@ static int packet_getsockopt(struct socket *sock, int level, int optname,
 		val = packet_sock_flag(po, PACKET_SOCK_TX_HAS_OFF);
 		break;
 	case PACKET_QDISC_BYPASS:
-		val = packet_use_direct_xmit(po);
+		val = packet_sock_flag(po, PACKET_SOCK_QDISC_BYPASS);
 		break;
 	default:
 		return -ENOPROTOOPT;
diff --git a/net/packet/internal.h b/net/packet/internal.h
index 680703dbce5e04fc26d0fdeab1c1c911b71a8729..62e9c3eefba26638557facfd291456da3dfb6119 100644
--- a/net/packet/internal.h
+++ b/net/packet/internal.h
@@ -128,7 +128,6 @@ struct packet_sock {
 	unsigned int		tp_tstamp;
 	struct completion	skb_completion;
 	struct net_device __rcu	*cached_dev;
-	int			(*xmit)(struct sk_buff *skb);
 	struct packet_type	prot_hook ____cacheline_aligned_in_smp;
 	atomic_t		tp_drops ____cacheline_aligned_in_smp;
 };
@@ -146,6 +145,7 @@ enum packet_sock_flags {
 	PACKET_SOCK_HAS_VNET_HDR,
 	PACKET_SOCK_RUNNING,
 	PACKET_SOCK_PRESSURE,
+	PACKET_SOCK_QDISC_BYPASS,
 };
 
 static inline void packet_sock_flag_set(struct packet_sock *po,
-- 
2.40.0.rc2.332.ga46443480c-goog

RE: [PATCH net-next] net/packet: remove po->xmit

[Willem de Bruijn]

Eric Dumazet wrote:
> Use PACKET_SOCK_QDISC_BYPASS atomic bit instead of a pointer.
> 
> This removes one indirect call in fast path,
> and READ_ONCE()/WRITE_ONCE() annotations as well.
> 
> Signed-off-by: Eric Dumazet <edumazet@google.com>
> Suggested-by: Willem de Bruijn <willemb@google.com>
> Cc: Daniel Borkmann <daniel@iogearbox.net>

Reviewed-by: Willem de Bruijn <willemb@google.com>

Re: [PATCH net-next] net/packet: remove po->xmit

[patchwork-bot+netdevbpf]

Hello:

This patch was applied to netdev/net-next.git (main)
by David S. Miller <davem@davemloft.net>:

On Fri, 17 Mar 2023 16:20:02 +0000 you wrote:
> Use PACKET_SOCK_QDISC_BYPASS atomic bit instead of a pointer.
> 
> This removes one indirect call in fast path,
> and READ_ONCE()/WRITE_ONCE() annotations as well.
> 
> Signed-off-by: Eric Dumazet <edumazet@google.com>
> Suggested-by: Willem de Bruijn <willemb@google.com>
> Cc: Daniel Borkmann <daniel@iogearbox.net>
> 
> [...]

Here is the summary with links:
  - [net-next] net/packet: remove po->xmit
    https://git.kernel.org/netdev/net-next/c/105a201ebf33

You are awesome, thank you!
-- 
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html