[PATCH RFC bpf-next 41/52] net, xdp: replace net_device::xdp_prog pointer with &xdp_attachment_info

Alexander Lobakin
2022-06-28T19:53:52Z

To have access and store not only BPF prog pointer, but also
auxiliary params on Generic (skb) XDP path, replace it with
an &xdp_attachment_info struct and use xdp_attachment_setup_rcu()
(since Generic XDP code RCU-protects the pointer already).
This slightly changes the struct &net_device cacheline layout, but
nothing performance-critical.

Signed-off-by: Alexander Lobakin <alexandr.lobakin@intel.com>
---
include/linux/netdevice.h | 7 +++----
net/bpf/dev.c | 11 ++++-------
net/core/dev.c | 4 +++-
net/core/rtnetlink.c | 2 +-
4 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 60df42b3f116..1c033c164257 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -2168,7 +2168,7 @@ struct net_device {
unsigned int num_rx_queues;
unsigned int real_num_rx_queues;

- struct bpf_prog __rcu *xdp_prog;
+ struct xdp_attachment_info xdp_info;
unsigned long gro_flush_timeout;
int napi_defer_hard_irqs;
#define GRO_LEGACY_MAX_SIZE 65536u
@@ -2343,9 +2343,8 @@ struct net_device {

static inline bool netif_elide_gro(const struct net_device *dev)
{
- if (!(dev->features & NETIF_F_GRO) || dev->xdp_prog)
- return true;
- return false;
+ return !(dev->features & NETIF_F_GRO) ||
+ rcu_access_pointer(dev->xdp_info.prog_rcu);
}

#define NETDEV_ALIGN 32
diff --git a/net/bpf/dev.c b/net/bpf/dev.c
index 82948d0536c8..cc43f73929f3 100644
--- a/net/bpf/dev.c
+++ b/net/bpf/dev.c
@@ -242,19 +242,16 @@ static void dev_disable_gro_hw(struct net_device *dev)

static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
{
- struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
- struct bpf_prog *new = xdp->prog;
+ bool old = !!rtnl_dereference(dev->xdp_info.prog_rcu);
int ret = 0;

switch (xdp->command) {
case XDP_SETUP_PROG:
- rcu_assign_pointer(dev->xdp_prog, new);
- if (old)
- bpf_prog_put(old);
+ xdp_attachment_setup_rcu(&dev->xdp_info, xdp);

- if (old && !new) {
+ if (old && !xdp->prog) {
static_branch_dec(&generic_xdp_needed_key);
- } else if (new && !old) {
+ } else if (xdp->prog && !old) {
static_branch_inc(&generic_xdp_needed_key);
dev_disable_lro(dev);
dev_disable_gro_hw(dev);
diff --git a/net/core/dev.c b/net/core/dev.c
index 62bf6ee00741..e57ae87d619e 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -5055,10 +5055,12 @@ static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
__this_cpu_inc(softnet_data.processed);

if (static_branch_unlikely(&generic_xdp_needed_key)) {
+ struct bpf_prog *prog;
int ret2;

migrate_disable();
- ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), skb);
+ prog = rcu_dereference(skb->dev->xdp_info.prog_rcu);
+ ret2 = do_xdp_generic(prog, skb);
migrate_enable();

if (ret2 != XDP_PASS) {
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 500420d5017c..72f696b12df2 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -1451,7 +1451,7 @@ static u32 rtnl_xdp_prog_skb(struct net_device *dev)

ASSERT_RTNL();

- generic_xdp_prog = rtnl_dereference(dev->xdp_prog);
+ generic_xdp_prog = rtnl_dereference(dev->xdp_info.prog_rcu);
if (!generic_xdp_prog)
return 0;
return generic_xdp_prog->aux->id;
--
2.36.1