[PATCH bpf-next] net: Add a warning if NAPI cb missed xdp_do_flush().

[PATCH bpf-next] net: Add a warning if NAPI cb missed xdp_do_flush().

[Sebastian Andrzej Siewior]

A few drivers were missing a xdp_do_flush() invocation after
XDP_REDIRECT.

Add three helper functions each for one of the per-CPU lists. Return
true if the per-CPU list is non-empty and flush the list.
Add xdp_do_check_flushed() which invokes each helper functions and
creats a warning if one of the functions had a non-empty list.
Hide everything behind CONFIG_DEBUG_NET.

Suggested-by: Jesper Dangaard Brouer <hawk@kernel.org>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---

This is follow-up to
	https://lore.kernel.org/all/cb2f7931-5ae5-8583-acff-4a186fed6632@kernel.org

It has been compile tested.

 include/linux/bpf.h    | 16 ++++++++++++++++
 include/linux/filter.h |  8 ++++++++
 include/net/xdp_sock.h | 10 ++++++++++
 kernel/bpf/cpumap.c    | 10 ++++++++++
 kernel/bpf/devmap.c    | 10 ++++++++++
 net/core/dev.c         |  2 ++
 net/core/filter.c      | 14 ++++++++++++++
 net/xdp/xsk.c          | 10 ++++++++++
 8 files changed, 80 insertions(+)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 30063a760b5af..a4eb8f23d35e6 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2730,6 +2730,22 @@ static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr)
 }
 #endif /* CONFIG_BPF_SYSCALL */
 
+#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_DEBUG_NET)
+bool __dev_check_flush(void);
+bool __cpu_map_check_flush(void);
+
+#else
+static inline bool __dev_check_flush(void)
+{
+	return false;
+}
+
+static inline bool __cpu_map_check_flush(void)
+{
+	return false;
+}
+#endif
+
 static __always_inline int
 bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr)
 {
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 27406aee2d402..db095d731813e 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1025,6 +1025,14 @@ int xdp_do_redirect_frame(struct net_device *dev,
 			  struct bpf_prog *prog);
 void xdp_do_flush(void);
 
+#ifdef CONFIG_DEBUG_NET
+void xdp_do_check_flushed(struct napi_struct *napi);
+
+#else
+static inline void xdp_do_check_flushed(struct napi_struct *napi) { }
+
+#endif
+
 /* The xdp_do_flush_map() helper has been renamed to drop the _map suffix, as
  * it is no longer only flushing maps. Keep this define for compatibility
  * until all drivers are updated - do not use xdp_do_flush_map() in new code!
diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
index 69b472604b86f..c250b78712771 100644
--- a/include/net/xdp_sock.h
+++ b/include/net/xdp_sock.h
@@ -109,4 +109,14 @@ static inline void __xsk_map_flush(void)
 
 #endif /* CONFIG_XDP_SOCKETS */
 
+#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)
+bool __xsk_map_check_flush(void);
+
+#else
+static inline bool __xsk_map_check_flush(void)
+{
+	return false;
+}
+#endif
+
 #endif /* _LINUX_XDP_SOCK_H */
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index e42a1bdb7f536..2cded02d83815 100644
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -764,6 +764,16 @@ void __cpu_map_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool __cpu_map_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&cpu_map_flush_list)))
+		return false;
+	__cpu_map_flush();
+	return true;
+}
+#endif
+
 static int __init cpu_map_init(void)
 {
 	int cpu;
diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c
index 4d42f6ed6c11a..8619ac1b879ed 100644
--- a/kernel/bpf/devmap.c
+++ b/kernel/bpf/devmap.c
@@ -418,6 +418,16 @@ void __dev_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool __dev_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&dev_flush_list)))
+		return false;
+	__dev_flush();
+	return true;
+}
+#endif
+
 /* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
  * by local_bh_disable() (from XDP calls inside NAPI). The
  * rcu_read_lock_bh_held() below makes lockdep accept both.
diff --git a/net/core/dev.c b/net/core/dev.c
index 606a366cc2095..9273b12ecf6fa 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6526,6 +6526,8 @@ static int __napi_poll(struct napi_struct *n, bool *repoll)
 	if (test_bit(NAPI_STATE_SCHED, &n->state)) {
 		work = n->poll(n, weight);
 		trace_napi_poll(n, work, weight);
+
+		xdp_do_check_flushed(n);
 	}
 
 	if (unlikely(work > weight))
diff --git a/net/core/filter.c b/net/core/filter.c
index a094694899c99..9841d0e32cb94 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -4207,6 +4207,20 @@ void xdp_do_flush(void)
 }
 EXPORT_SYMBOL_GPL(xdp_do_flush);
 
+#ifdef CONFIG_DEBUG_NET
+void xdp_do_check_flushed(struct napi_struct *napi)
+{
+	bool ret;
+
+	ret = __dev_check_flush();
+	ret |= __cpu_map_check_flush();
+	ret |= __xsk_map_check_flush();
+
+	WARN_ONCE(ret, "Missing xdp_do_flush() invocation after NAPI by %ps\n",
+		  napi->poll);
+}
+#endif
+
 void bpf_clear_redirect_map(struct bpf_map *map)
 {
 	struct bpf_redirect_info *ri;
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 7482d0aca5046..4eae53478db07 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -391,6 +391,16 @@ void __xsk_map_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool __xsk_map_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&xskmap_flush_list)))
+		return false;
+	__xsk_map_flush();
+	return true;
+}
+#endif
+
 void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
 {
 	xskq_prod_submit_n(pool->cq, nb_entries);
-- 
2.42.0

Re: [PATCH bpf-next] net: Add a warning if NAPI cb missed xdp_do_flush().

[Toke Høiland-Jørgensen]

Sebastian Andrzej Siewior <bigeasy@linutronix.de> writes:

> A few drivers were missing a xdp_do_flush() invocation after
> XDP_REDIRECT.
>
> Add three helper functions each for one of the per-CPU lists. Return
> true if the per-CPU list is non-empty and flush the list.
> Add xdp_do_check_flushed() which invokes each helper functions and
> creats a warning if one of the functions had a non-empty list.
> Hide everything behind CONFIG_DEBUG_NET.
>
> Suggested-by: Jesper Dangaard Brouer <hawk@kernel.org>
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> ---
>
> This is follow-up to
> 	https://lore.kernel.org/all/cb2f7931-5ae5-8583-acff-4a186fed6632@kernel.org
>
> It has been compile tested.

Both with and without CONFIG_DEBUG_NET enabled, I trust? ;)

Anyway, LGTM!

Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com>

Re: [PATCH bpf-next] net: Add a warning if NAPI cb missed xdp_do_flush().

[Jakub Kicinski]

On Fri, 29 Sep 2023 18:58:25 +0200 Sebastian Andrzej Siewior wrote:
> +#if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_DEBUG_NET)
> +bool __dev_check_flush(void);
> +bool __cpu_map_check_flush(void);
> +
> +#else
> +static inline bool __dev_check_flush(void)
> +{
> +	return false;
> +}
> +
> +static inline bool __cpu_map_check_flush(void)
> +{
> +	return false;
> +}
> +#endif

I think you're going too hard with the ifdefs.
These functions are only called if DEBUG_NET, add if BPF on the call
site and spare us the static inlines for all the __ helpers.

>  static __always_inline int
>  bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr)
>  {
> diff --git a/include/linux/filter.h b/include/linux/filter.h
> index 27406aee2d402..db095d731813e 100644
> --- a/include/linux/filter.h
> +++ b/include/linux/filter.h
> @@ -1025,6 +1025,14 @@ int xdp_do_redirect_frame(struct net_device *dev,
>  			  struct bpf_prog *prog);
>  void xdp_do_flush(void);
>  
> +#ifdef CONFIG_DEBUG_NET
> +void xdp_do_check_flushed(struct napi_struct *napi);
> +
> +#else
> +static inline void xdp_do_check_flushed(struct napi_struct *napi) { }
> +
> +#endif

Can you move this to net/core/dev.h? Or a new header under net/core
if you prefer? This looks internal to the stack.
Also nit: drop the empty lines inside the #ifdef?

[PATCH bpf-next v2] net: Add a warning if NAPI cb missed xdp_do_flush().

[Sebastian Andrzej Siewior]

A few drivers were missing a xdp_do_flush() invocation after
XDP_REDIRECT.

Add three helper functions each for one of the per-CPU lists. Return
true if the per-CPU list is non-empty and flush the list.
Add xdp_do_check_flushed() which invokes each helper functions and
creats a warning if one of the functions had a non-empty list.
Hide everything behind CONFIG_DEBUG_NET.

Suggested-by: Jesper Dangaard Brouer <hawk@kernel.org>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
v1…v2:
  - Moved xdp_do_check_flushed() to net/core/dev.h.
  - Stripped __ from function names.
  - Removed empty lines within an ifdef block.
  - xdp_do_check_flushed() is now behind CONFIG_DEBUG_NET &&
    CONFIG_BPF_SYSCALL. dev_check_flush and cpu_map_check_flush are now
    only behind CONFIG_DEBUG_NET. They have no empty inline function for
    the !CONFIG_DEBUG_NET case since they are only called in
    CONFIG_DEBUG_NET case.


 include/linux/bpf.h    |  3 +++
 include/net/xdp_sock.h |  9 +++++++++
 kernel/bpf/cpumap.c    | 10 ++++++++++
 kernel/bpf/devmap.c    | 10 ++++++++++
 net/core/dev.c         |  2 ++
 net/core/dev.h         |  6 ++++++
 net/core/filter.c      | 14 ++++++++++++++
 net/xdp/xsk.c          | 10 ++++++++++
 8 files changed, 64 insertions(+)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index a34ac7f00c86c..584adabd411fc 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2478,6 +2478,9 @@ void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
 		     enum bpf_dynptr_type type, u32 offset, u32 size);
 void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
 void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
+
+bool dev_check_flush(void);
+bool cpu_map_check_flush(void);
 #else /* !CONFIG_BPF_SYSCALL */
 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
 {
diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
index 69b472604b86f..7dd0df2f6f8e6 100644
--- a/include/net/xdp_sock.h
+++ b/include/net/xdp_sock.h
@@ -109,4 +109,13 @@ static inline void __xsk_map_flush(void)
 
 #endif /* CONFIG_XDP_SOCKETS */
 
+#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)
+bool xsk_map_check_flush(void);
+#else
+static inline bool xsk_map_check_flush(void)
+{
+	return false;
+}
+#endif
+
 #endif /* _LINUX_XDP_SOCK_H */
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index e42a1bdb7f536..8a0bb80fe48a3 100644
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -764,6 +764,16 @@ void __cpu_map_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool cpu_map_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&cpu_map_flush_list)))
+		return false;
+	__cpu_map_flush();
+	return true;
+}
+#endif
+
 static int __init cpu_map_init(void)
 {
 	int cpu;
diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c
index 4d42f6ed6c11a..a936c704d4e77 100644
--- a/kernel/bpf/devmap.c
+++ b/kernel/bpf/devmap.c
@@ -418,6 +418,16 @@ void __dev_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool dev_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&dev_flush_list)))
+		return false;
+	__dev_flush();
+	return true;
+}
+#endif
+
 /* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
  * by local_bh_disable() (from XDP calls inside NAPI). The
  * rcu_read_lock_bh_held() below makes lockdep accept both.
diff --git a/net/core/dev.c b/net/core/dev.c
index 606a366cc2095..9273b12ecf6fa 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6526,6 +6526,8 @@ static int __napi_poll(struct napi_struct *n, bool *repoll)
 	if (test_bit(NAPI_STATE_SCHED, &n->state)) {
 		work = n->poll(n, weight);
 		trace_napi_poll(n, work, weight);
+
+		xdp_do_check_flushed(n);
 	}
 
 	if (unlikely(work > weight))
diff --git a/net/core/dev.h b/net/core/dev.h
index e075e198092cc..f66125857af77 100644
--- a/net/core/dev.h
+++ b/net/core/dev.h
@@ -136,4 +136,10 @@ static inline void netif_set_gro_ipv4_max_size(struct net_device *dev,
 }
 
 int rps_cpumask_housekeeping(struct cpumask *mask);
+
+#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
+void xdp_do_check_flushed(struct napi_struct *napi);
+#else
+static inline void xdp_do_check_flushed(struct napi_struct *napi) { }
+#endif
 #endif
diff --git a/net/core/filter.c b/net/core/filter.c
index a094694899c99..5d5921c0ab4a3 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -4207,6 +4207,20 @@ void xdp_do_flush(void)
 }
 EXPORT_SYMBOL_GPL(xdp_do_flush);
 
+#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
+void xdp_do_check_flushed(struct napi_struct *napi)
+{
+	bool ret;
+
+	ret = dev_check_flush();
+	ret |= cpu_map_check_flush();
+	ret |= xsk_map_check_flush();
+
+	WARN_ONCE(ret, "Missing xdp_do_flush() invocation after NAPI by %ps\n",
+		  napi->poll);
+}
+#endif
+
 void bpf_clear_redirect_map(struct bpf_map *map)
 {
 	struct bpf_redirect_info *ri;
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index f5e96e0d6e01d..ba070fd37d244 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -391,6 +391,16 @@ void __xsk_map_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool xsk_map_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&xskmap_flush_list)))
+		return false;
+	__xsk_map_flush();
+	return true;
+}
+#endif
+
 void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
 {
 	xskq_prod_submit_n(pool->cq, nb_entries);
-- 
2.42.0

Re: [PATCH bpf-next v2] net: Add a warning if NAPI cb missed xdp_do_flush().

[kernel test robot]

Hi Sebastian,

kernel test robot noticed the following build warnings:

[auto build test WARNING on bpf-next/master]

url:    https://github.com/intel-lab-lkp/linux/commits/Sebastian-Andrzej-Siewior/net-Add-a-warning-if-NAPI-cb-missed-xdp_do_flush/20231006-235117
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
patch link:    https://lore.kernel.org/r/20231006154933.mQgxQHHt%40linutronix.de
patch subject: [PATCH bpf-next v2] net: Add a warning if NAPI cb missed xdp_do_flush().
config: m68k-allyesconfig (https://download.01.org/0day-ci/archive/20231007/202310070134.JX5try68-lkp@intel.com/config)
compiler: m68k-linux-gcc (GCC) 13.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20231007/202310070134.JX5try68-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202310070134.JX5try68-lkp@intel.com/

All warnings (new ones prefixed by >>):

>> net/core/filter.c:4211:6: warning: no previous prototype for 'xdp_do_check_flushed' [-Wmissing-prototypes]
    4211 | void xdp_do_check_flushed(struct napi_struct *napi)
         |      ^~~~~~~~~~~~~~~~~~~~


vim +/xdp_do_check_flushed +4211 net/core/filter.c

  4209	
  4210	#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
> 4211	void xdp_do_check_flushed(struct napi_struct *napi)
  4212	{
  4213		bool ret;
  4214	
  4215		ret = dev_check_flush();
  4216		ret |= cpu_map_check_flush();
  4217		ret |= xsk_map_check_flush();
  4218	
  4219		WARN_ONCE(ret, "Missing xdp_do_flush() invocation after NAPI by %ps\n",
  4220			  napi->poll);
  4221	}
  4222	#endif
  4223	

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki

Re: [PATCH bpf-next v2] net: Add a warning if NAPI cb missed xdp_do_flush().

[Jakub Kicinski]

On Fri, 6 Oct 2023 17:49:33 +0200 Sebastian Andrzej Siewior wrote:
>   - Moved xdp_do_check_flushed() to net/core/dev.h.
>   - Stripped __ from function names.
>   - Removed empty lines within an ifdef block.
>   - xdp_do_check_flushed() is now behind CONFIG_DEBUG_NET &&
>     CONFIG_BPF_SYSCALL. dev_check_flush and cpu_map_check_flush are now
>     only behind CONFIG_DEBUG_NET. They have no empty inline function for
>     the !CONFIG_DEBUG_NET case since they are only called in
>     CONFIG_DEBUG_NET case.


Other than the minor nit from the build bot - LGTM now thanks!

Acked-by: Jakub Kicinski <kuba@kernel.org>

[PATCH bpf-next v3] net: Add a warning if NAPI cb missed xdp_do_flush().

[Sebastian Andrzej Siewior]

A few drivers were missing a xdp_do_flush() invocation after
XDP_REDIRECT.

Add three helper functions each for one of the per-CPU lists. Return
true if the per-CPU list is non-empty and flush the list.
Add xdp_do_check_flushed() which invokes each helper functions and
creats a warning if one of the functions had a non-empty list.
Hide everything behind CONFIG_DEBUG_NET.

Suggested-by: Jesper Dangaard Brouer <hawk@kernel.org>
Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com>
Acked-by: Jakub Kicinski <kuba@kernel.org>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
v2…v3:
  - Collected Reviewed/Acked from the list.
  - Added an include dev.h to filter.c, the robot pointed out a missing
    prototype.

v1…v2:
  - Moved xdp_do_check_flushed() to net/core/dev.h.
  - Stripped __ from function names.
  - Removed empty lines within an ifdef block.
  - xdp_do_check_flushed() is now behind CONFIG_DEBUG_NET &&
    CONFIG_BPF_SYSCALL. dev_check_flush and cpu_map_check_flush are now
    only behind CONFIG_DEBUG_NET. They have no empty inline function for
    the !CONFIG_DEBUG_NET case since they are only called in
    CONFIG_DEBUG_NET case.

 include/linux/bpf.h    |  3 +++
 include/net/xdp_sock.h |  9 +++++++++
 kernel/bpf/cpumap.c    | 10 ++++++++++
 kernel/bpf/devmap.c    | 10 ++++++++++
 net/core/dev.c         |  2 ++
 net/core/dev.h         |  6 ++++++
 net/core/filter.c      | 16 ++++++++++++++++
 net/xdp/xsk.c          | 10 ++++++++++
 8 files changed, 66 insertions(+)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index a34ac7f00c86c..584adabd411fc 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2478,6 +2478,9 @@ void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
 		     enum bpf_dynptr_type type, u32 offset, u32 size);
 void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
 void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
+
+bool dev_check_flush(void);
+bool cpu_map_check_flush(void);
 #else /* !CONFIG_BPF_SYSCALL */
 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
 {
diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
index 69b472604b86f..7dd0df2f6f8e6 100644
--- a/include/net/xdp_sock.h
+++ b/include/net/xdp_sock.h
@@ -109,4 +109,13 @@ static inline void __xsk_map_flush(void)
 
 #endif /* CONFIG_XDP_SOCKETS */
 
+#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)
+bool xsk_map_check_flush(void);
+#else
+static inline bool xsk_map_check_flush(void)
+{
+	return false;
+}
+#endif
+
 #endif /* _LINUX_XDP_SOCK_H */
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index e42a1bdb7f536..8a0bb80fe48a3 100644
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -764,6 +764,16 @@ void __cpu_map_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool cpu_map_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&cpu_map_flush_list)))
+		return false;
+	__cpu_map_flush();
+	return true;
+}
+#endif
+
 static int __init cpu_map_init(void)
 {
 	int cpu;
diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c
index 4d42f6ed6c11a..a936c704d4e77 100644
--- a/kernel/bpf/devmap.c
+++ b/kernel/bpf/devmap.c
@@ -418,6 +418,16 @@ void __dev_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool dev_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&dev_flush_list)))
+		return false;
+	__dev_flush();
+	return true;
+}
+#endif
+
 /* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
  * by local_bh_disable() (from XDP calls inside NAPI). The
  * rcu_read_lock_bh_held() below makes lockdep accept both.
diff --git a/net/core/dev.c b/net/core/dev.c
index 606a366cc2095..9273b12ecf6fa 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6526,6 +6526,8 @@ static int __napi_poll(struct napi_struct *n, bool *repoll)
 	if (test_bit(NAPI_STATE_SCHED, &n->state)) {
 		work = n->poll(n, weight);
 		trace_napi_poll(n, work, weight);
+
+		xdp_do_check_flushed(n);
 	}
 
 	if (unlikely(work > weight))
diff --git a/net/core/dev.h b/net/core/dev.h
index e075e198092cc..f66125857af77 100644
--- a/net/core/dev.h
+++ b/net/core/dev.h
@@ -136,4 +136,10 @@ static inline void netif_set_gro_ipv4_max_size(struct net_device *dev,
 }
 
 int rps_cpumask_housekeeping(struct cpumask *mask);
+
+#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
+void xdp_do_check_flushed(struct napi_struct *napi);
+#else
+static inline void xdp_do_check_flushed(struct napi_struct *napi) { }
+#endif
 #endif
diff --git a/net/core/filter.c b/net/core/filter.c
index a094694899c99..af2d34d5e1815 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -82,6 +82,8 @@
 #include <net/mptcp.h>
 #include <net/netfilter/nf_conntrack_bpf.h>
 
+#include "dev.h"
+
 static const struct bpf_func_proto *
 bpf_sk_base_func_proto(enum bpf_func_id func_id);
 
@@ -4207,6 +4209,20 @@ void xdp_do_flush(void)
 }
 EXPORT_SYMBOL_GPL(xdp_do_flush);
 
+#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
+void xdp_do_check_flushed(struct napi_struct *napi)
+{
+	bool ret;
+
+	ret = dev_check_flush();
+	ret |= cpu_map_check_flush();
+	ret |= xsk_map_check_flush();
+
+	WARN_ONCE(ret, "Missing xdp_do_flush() invocation after NAPI by %ps\n",
+		  napi->poll);
+}
+#endif
+
 void bpf_clear_redirect_map(struct bpf_map *map)
 {
 	struct bpf_redirect_info *ri;
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index f5e96e0d6e01d..ba070fd37d244 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -391,6 +391,16 @@ void __xsk_map_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool xsk_map_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&xskmap_flush_list)))
+		return false;
+	__xsk_map_flush();
+	return true;
+}
+#endif
+
 void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
 {
 	xskq_prod_submit_n(pool->cq, nb_entries);
-- 
2.42.0

[PATCH bpf-next -v4] net: Add a warning if NAPI cb missed xdp_do_flush().

[Sebastian Andrzej Siewior]

A few drivers were missing a xdp_do_flush() invocation after
XDP_REDIRECT.

Add three helper functions each for one of the per-CPU lists. Return
true if the per-CPU list is non-empty and flush the list.
Add xdp_do_check_flushed() which invokes each helper functions and
creates a warning if one of the functions had a non-empty list.
Hide everything behind CONFIG_DEBUG_NET.

Suggested-by: Jesper Dangaard Brouer <hawk@kernel.org>
Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com>
Acked-by: Jakub Kicinski <kuba@kernel.org>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
v3…v4:
  - s/creats/creates as per Simon Horman.

v2…v3:
  - Collected Reviewed/Acked from the list.
  - Added an include dev.h to filter.c, the robot pointed out a missing
    prototype.

v1…v2:
  - Moved xdp_do_check_flushed() to net/core/dev.h.
  - Stripped __ from function names.
  - Removed empty lines within an ifdef block.
  - xdp_do_check_flushed() is now behind CONFIG_DEBUG_NET &&
    CONFIG_BPF_SYSCALL. dev_check_flush and cpu_map_check_flush are now
    only behind CONFIG_DEBUG_NET. They have no empty inline function for
    the !CONFIG_DEBUG_NET case since they are only called in
    CONFIG_DEBUG_NET case.

 include/linux/bpf.h    |  3 +++
 include/net/xdp_sock.h |  9 +++++++++
 kernel/bpf/cpumap.c    | 10 ++++++++++
 kernel/bpf/devmap.c    | 10 ++++++++++
 net/core/dev.c         |  2 ++
 net/core/dev.h         |  6 ++++++
 net/core/filter.c      | 16 ++++++++++++++++
 net/xdp/xsk.c          | 10 ++++++++++
 8 files changed, 66 insertions(+)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index a34ac7f00c86c..584adabd411fc 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2478,6 +2478,9 @@ void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
 		     enum bpf_dynptr_type type, u32 offset, u32 size);
 void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
 void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
+
+bool dev_check_flush(void);
+bool cpu_map_check_flush(void);
 #else /* !CONFIG_BPF_SYSCALL */
 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
 {
diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
index 69b472604b86f..7dd0df2f6f8e6 100644
--- a/include/net/xdp_sock.h
+++ b/include/net/xdp_sock.h
@@ -109,4 +109,13 @@ static inline void __xsk_map_flush(void)
 
 #endif /* CONFIG_XDP_SOCKETS */
 
+#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)
+bool xsk_map_check_flush(void);
+#else
+static inline bool xsk_map_check_flush(void)
+{
+	return false;
+}
+#endif
+
 #endif /* _LINUX_XDP_SOCK_H */
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index e42a1bdb7f536..8a0bb80fe48a3 100644
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -764,6 +764,16 @@ void __cpu_map_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool cpu_map_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&cpu_map_flush_list)))
+		return false;
+	__cpu_map_flush();
+	return true;
+}
+#endif
+
 static int __init cpu_map_init(void)
 {
 	int cpu;
diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c
index 4d42f6ed6c11a..a936c704d4e77 100644
--- a/kernel/bpf/devmap.c
+++ b/kernel/bpf/devmap.c
@@ -418,6 +418,16 @@ void __dev_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool dev_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&dev_flush_list)))
+		return false;
+	__dev_flush();
+	return true;
+}
+#endif
+
 /* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
  * by local_bh_disable() (from XDP calls inside NAPI). The
  * rcu_read_lock_bh_held() below makes lockdep accept both.
diff --git a/net/core/dev.c b/net/core/dev.c
index 606a366cc2095..9273b12ecf6fa 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6526,6 +6526,8 @@ static int __napi_poll(struct napi_struct *n, bool *repoll)
 	if (test_bit(NAPI_STATE_SCHED, &n->state)) {
 		work = n->poll(n, weight);
 		trace_napi_poll(n, work, weight);
+
+		xdp_do_check_flushed(n);
 	}
 
 	if (unlikely(work > weight))
diff --git a/net/core/dev.h b/net/core/dev.h
index e075e198092cc..f66125857af77 100644
--- a/net/core/dev.h
+++ b/net/core/dev.h
@@ -136,4 +136,10 @@ static inline void netif_set_gro_ipv4_max_size(struct net_device *dev,
 }
 
 int rps_cpumask_housekeeping(struct cpumask *mask);
+
+#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
+void xdp_do_check_flushed(struct napi_struct *napi);
+#else
+static inline void xdp_do_check_flushed(struct napi_struct *napi) { }
+#endif
 #endif
diff --git a/net/core/filter.c b/net/core/filter.c
index a094694899c99..af2d34d5e1815 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -82,6 +82,8 @@
 #include <net/mptcp.h>
 #include <net/netfilter/nf_conntrack_bpf.h>
 
+#include "dev.h"
+
 static const struct bpf_func_proto *
 bpf_sk_base_func_proto(enum bpf_func_id func_id);
 
@@ -4207,6 +4209,20 @@ void xdp_do_flush(void)
 }
 EXPORT_SYMBOL_GPL(xdp_do_flush);
 
+#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
+void xdp_do_check_flushed(struct napi_struct *napi)
+{
+	bool ret;
+
+	ret = dev_check_flush();
+	ret |= cpu_map_check_flush();
+	ret |= xsk_map_check_flush();
+
+	WARN_ONCE(ret, "Missing xdp_do_flush() invocation after NAPI by %ps\n",
+		  napi->poll);
+}
+#endif
+
 void bpf_clear_redirect_map(struct bpf_map *map)
 {
 	struct bpf_redirect_info *ri;
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index f5e96e0d6e01d..ba070fd37d244 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -391,6 +391,16 @@ void __xsk_map_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool xsk_map_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&xskmap_flush_list)))
+		return false;
+	__xsk_map_flush();
+	return true;
+}
+#endif
+
 void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
 {
 	xskq_prod_submit_n(pool->cq, nb_entries);
-- 
2.42.0

RE: [PATCH bpf-next -v4] net: Add a warning if NAPI cb missed xdp_do_flush().

[John Fastabend]

Sebastian Andrzej Siewior wrote:
> A few drivers were missing a xdp_do_flush() invocation after
> XDP_REDIRECT.
> 
> Add three helper functions each for one of the per-CPU lists. Return
> true if the per-CPU list is non-empty and flush the list.
> Add xdp_do_check_flushed() which invokes each helper functions and
> creates a warning if one of the functions had a non-empty list.
> Hide everything behind CONFIG_DEBUG_NET.
> 
> Suggested-by: Jesper Dangaard Brouer <hawk@kernel.org>
> Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com>
> Acked-by: Jakub Kicinski <kuba@kernel.org>
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> ---

LGTM.

Acked-by: John Fastabend <john.fastabend@gmail.com>

Re: [PATCH bpf-next -v4] net: Add a warning if NAPI cb missed xdp_do_flush().

[Daniel Borkmann]

Hi Sebastian,

On 10/11/23 6:42 AM, John Fastabend wrote:
> Sebastian Andrzej Siewior wrote:
>> A few drivers were missing a xdp_do_flush() invocation after
>> XDP_REDIRECT.
>>
>> Add three helper functions each for one of the per-CPU lists. Return
>> true if the per-CPU list is non-empty and flush the list.
>> Add xdp_do_check_flushed() which invokes each helper functions and
>> creates a warning if one of the functions had a non-empty list.
>> Hide everything behind CONFIG_DEBUG_NET.
>>
>> Suggested-by: Jesper Dangaard Brouer <hawk@kernel.org>
>> Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com>
>> Acked-by: Jakub Kicinski <kuba@kernel.org>
>> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
> 
> LGTM.
> 
> Acked-by: John Fastabend <john.fastabend@gmail.com>

Do you have a chance to send a v5 rebase? It does not apply to bpf-next.

Other than that, the patch lgtm.

Thanks,
Daniel

[PATCH bpf-next -v5] net: Add a warning if NAPI cb missed xdp_do_flush().

[Sebastian Andrzej Siewior]

A few drivers were missing a xdp_do_flush() invocation after
XDP_REDIRECT.

Add three helper functions each for one of the per-CPU lists. Return
true if the per-CPU list is non-empty and flush the list.
Add xdp_do_check_flushed() which invokes each helper functions and
creates a warning if one of the functions had a non-empty list.
Hide everything behind CONFIG_DEBUG_NET.

Suggested-by: Jesper Dangaard Brouer <hawk@kernel.org>
Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com>
Acked-by: Jakub Kicinski <kuba@kernel.org>
Acked-by: John Fastabend <john.fastabend@gmail.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
---
v4…v5:
  - rebase on top of bpf-next
  - Collected Reviewed/Acked from the list.

v3…v4:
  - s/creats/creates as per Simon Horman.

v2…v3:
  - Collected Reviewed/Acked from the list.
  - Added an include dev.h to filter.c, the robot pointed out a missing
    prototype.

v1…v2:
  - Moved xdp_do_check_flushed() to net/core/dev.h.
  - Stripped __ from function names.
  - Removed empty lines within an ifdef block.
  - xdp_do_check_flushed() is now behind CONFIG_DEBUG_NET &&
    CONFIG_BPF_SYSCALL. dev_check_flush and cpu_map_check_flush are now
    only behind CONFIG_DEBUG_NET. They have no empty inline function for
    the !CONFIG_DEBUG_NET case since they are only called in
    CONFIG_DEBUG_NET case.

 include/linux/bpf.h    |  3 +++
 include/net/xdp_sock.h |  9 +++++++++
 kernel/bpf/cpumap.c    | 10 ++++++++++
 kernel/bpf/devmap.c    | 10 ++++++++++
 net/core/dev.c         |  2 ++
 net/core/dev.h         |  6 ++++++
 net/core/filter.c      | 16 ++++++++++++++++
 net/xdp/xsk.c          | 10 ++++++++++
 8 files changed, 66 insertions(+)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index f0891ba24cb1c..92b76360ff58a 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -2478,6 +2478,9 @@ void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data,
 		     enum bpf_dynptr_type type, u32 offset, u32 size);
 void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
 void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
+
+bool dev_check_flush(void);
+bool cpu_map_check_flush(void);
 #else /* !CONFIG_BPF_SYSCALL */
 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
 {
diff --git a/include/net/xdp_sock.h b/include/net/xdp_sock.h
index 69b472604b86f..7dd0df2f6f8e6 100644
--- a/include/net/xdp_sock.h
+++ b/include/net/xdp_sock.h
@@ -109,4 +109,13 @@ static inline void __xsk_map_flush(void)
 
 #endif /* CONFIG_XDP_SOCKETS */
 
+#if defined(CONFIG_XDP_SOCKETS) && defined(CONFIG_DEBUG_NET)
+bool xsk_map_check_flush(void);
+#else
+static inline bool xsk_map_check_flush(void)
+{
+	return false;
+}
+#endif
+
 #endif /* _LINUX_XDP_SOCK_H */
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index e42a1bdb7f536..8a0bb80fe48a3 100644
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -764,6 +764,16 @@ void __cpu_map_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool cpu_map_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&cpu_map_flush_list)))
+		return false;
+	__cpu_map_flush();
+	return true;
+}
+#endif
+
 static int __init cpu_map_init(void)
 {
 	int cpu;
diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c
index 4d42f6ed6c11a..a936c704d4e77 100644
--- a/kernel/bpf/devmap.c
+++ b/kernel/bpf/devmap.c
@@ -418,6 +418,16 @@ void __dev_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool dev_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&dev_flush_list)))
+		return false;
+	__dev_flush();
+	return true;
+}
+#endif
+
 /* Elements are kept alive by RCU; either by rcu_read_lock() (from syscall) or
  * by local_bh_disable() (from XDP calls inside NAPI). The
  * rcu_read_lock_bh_held() below makes lockdep accept both.
diff --git a/net/core/dev.c b/net/core/dev.c
index 606a366cc2095..9273b12ecf6fa 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -6526,6 +6526,8 @@ static int __napi_poll(struct napi_struct *n, bool *repoll)
 	if (test_bit(NAPI_STATE_SCHED, &n->state)) {
 		work = n->poll(n, weight);
 		trace_napi_poll(n, work, weight);
+
+		xdp_do_check_flushed(n);
 	}
 
 	if (unlikely(work > weight))
diff --git a/net/core/dev.h b/net/core/dev.h
index e075e198092cc..f66125857af77 100644
--- a/net/core/dev.h
+++ b/net/core/dev.h
@@ -136,4 +136,10 @@ static inline void netif_set_gro_ipv4_max_size(struct net_device *dev,
 }
 
 int rps_cpumask_housekeeping(struct cpumask *mask);
+
+#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
+void xdp_do_check_flushed(struct napi_struct *napi);
+#else
+static inline void xdp_do_check_flushed(struct napi_struct *napi) { }
+#endif
 #endif
diff --git a/net/core/filter.c b/net/core/filter.c
index cc2e4babc85fb..21d75108c2e94 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -83,6 +83,8 @@
 #include <net/netfilter/nf_conntrack_bpf.h>
 #include <linux/un.h>
 
+#include "dev.h"
+
 static const struct bpf_func_proto *
 bpf_sk_base_func_proto(enum bpf_func_id func_id);
 
@@ -4208,6 +4210,20 @@ void xdp_do_flush(void)
 }
 EXPORT_SYMBOL_GPL(xdp_do_flush);
 
+#if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL)
+void xdp_do_check_flushed(struct napi_struct *napi)
+{
+	bool ret;
+
+	ret = dev_check_flush();
+	ret |= cpu_map_check_flush();
+	ret |= xsk_map_check_flush();
+
+	WARN_ONCE(ret, "Missing xdp_do_flush() invocation after NAPI by %ps\n",
+		  napi->poll);
+}
+#endif
+
 void bpf_clear_redirect_map(struct bpf_map *map)
 {
 	struct bpf_redirect_info *ri;
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 7482d0aca5046..e23689b82914f 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -391,6 +391,16 @@ void __xsk_map_flush(void)
 	}
 }
 
+#ifdef CONFIG_DEBUG_NET
+bool xsk_map_check_flush(void)
+{
+	if (list_empty(this_cpu_ptr(&xskmap_flush_list)))
+		return false;
+	__xsk_map_flush();
+	return true;
+}
+#endif
+
 void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
 {
 	xskq_prod_submit_n(pool->cq, nb_entries);
-- 
2.42.0

Re: [PATCH bpf-next -v5] net: Add a warning if NAPI cb missed xdp_do_flush().

[patchwork-bot+netdevbpf]

Hello:

This patch was applied to bpf/bpf-next.git (master)
by Daniel Borkmann <daniel@iogearbox.net>:

On Mon, 16 Oct 2023 14:57:38 +0200 you wrote:
> A few drivers were missing a xdp_do_flush() invocation after
> XDP_REDIRECT.
> 
> Add three helper functions each for one of the per-CPU lists. Return
> true if the per-CPU list is non-empty and flush the list.
> Add xdp_do_check_flushed() which invokes each helper functions and
> creates a warning if one of the functions had a non-empty list.
> Hide everything behind CONFIG_DEBUG_NET.
> 
> [...]

Here is the summary with links:
  - [bpf-next,-v5] net: Add a warning if NAPI cb missed xdp_do_flush().
    https://git.kernel.org/bpf/bpf-next/c/9a675ba55a96

You are awesome, thank you!
-- 
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html

Re: [PATCH bpf-next v2] net: Add a warning if NAPI cb missed xdp_do_flush().

[Simon Horman]

On Fri, Oct 06, 2023 at 05:49:33PM +0200, Sebastian Andrzej Siewior wrote:
> A few drivers were missing a xdp_do_flush() invocation after
> XDP_REDIRECT.
> 
> Add three helper functions each for one of the per-CPU lists. Return
> true if the per-CPU list is non-empty and flush the list.
> Add xdp_do_check_flushed() which invokes each helper functions and
> creats a warning if one of the functions had a non-empty list.

nit: creates

> Hide everything behind CONFIG_DEBUG_NET.
> 
> Suggested-by: Jesper Dangaard Brouer <hawk@kernel.org>
> Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>

...