[PATCH net v2 1/2] net: napi: add completion with an ordered IRQ mask callback

[Congjie Zhou <zcjie0802@qq.com>]

napi_complete_done() can clear SCHED, arm the watchdog and return false
when deferring hard IRQs. A false return therefore does not guarantee
that the caller still owns NAPI. Masking interrupts after this return
can overwrite the interrupt state established by a successor poll.

Add napi_complete_done_with_irq_mask() with a mask callback that runs on
false-return paths before ownership can be handed off. The callback
must be non-sleeping, idempotent and leave NAPI state unchanged, since
a state-update retry may invoke it more than once.

Keep napi_complete_done() as a wrapper without a callback, and inline
the shared implementation to eliminate the callback path for existing
callers.

Signed-off-by: Congjie Zhou <zcjie0802@qq.com>
---
Changes in v2:
 - Add the Fixes tag to patch 2/2; no code changes in either patch.

v1: https://lore.kernel.org/netdev/tencent_74EA7A2E8BF5758E9C5EF18C9B1CED401D06@qq.com/
Bug report: https://lore.kernel.org/netdev/tencent_AA4F39DA7ADE330C721C03AAA832B92DD608@qq.com/
Discovery and validation: https://lore.kernel.org/netdev/tencent_4DBDF69909B9FD811F54F2DC0E6BAC1C0507@qq.com/

 include/linux/netdevice.h | 18 ++++++++++++++++++
 net/core/dev.c            | 29 +++++++++++++++++++++++++++--
 2 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 435be7880..810527cd8 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -601,6 +601,24 @@ static inline void napi_schedule_irqoff(struct napi_struct *n)
  */
 bool napi_complete_done(struct napi_struct *n, int work_done);
 
+/**
+ * napi_complete_done_with_irq_mask - complete with an ordered IRQ mask callback
+ * @n: NAPI context
+ * @work_done: number of packets processed
+ * @mask: non-sleeping, idempotent device interrupt mask callback
+ * @data: callback argument
+ *
+ * Like napi_complete_done(), but call @mask on false-return paths before
+ * releasing NAPI ownership. Drivers which prepare interrupt rearming before
+ * completion must not mask interrupts after ownership has been released.
+ * The callback must not modify NAPI state and may run more than once if the
+ * state update is retried.
+ *
+ * Return: false if device interrupts have been masked by the callback.
+ */
+bool napi_complete_done_with_irq_mask(struct napi_struct *n, int work_done,
+				      void (*mask)(void *data), void *data);
+
 static inline bool napi_complete(struct napi_struct *n)
 {
 	return napi_complete_done(n, 0);
diff --git a/net/core/dev.c b/net/core/dev.c
index df1a840b0..4ba9e4c1b 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -7001,7 +7001,10 @@ void __napi_schedule_irqoff(struct napi_struct *n)
 }
 EXPORT_SYMBOL(__napi_schedule_irqoff);
 
-bool napi_complete_done(struct napi_struct *n, int work_done)
+/* Inline so the legacy API eliminates the optional mask callback path. */
+static __always_inline bool
+__napi_complete_done(struct napi_struct *n, int work_done,
+		     void (*mask)(void *data), void *data)
 {
 	unsigned long flags, val, new, timeout = 0;
 	bool ret = true;
@@ -7013,8 +7016,11 @@ bool napi_complete_done(struct napi_struct *n, int work_done)
 	 *    the guarantee we will be called later.
 	 */
 	if (unlikely(n->state & (NAPIF_STATE_NPSVC |
-				 NAPIF_STATE_IN_BUSY_POLL)))
+				 NAPIF_STATE_IN_BUSY_POLL))) {
+		if (mask)
+			mask(data);
 		return false;
+	}
 
 	if (work_done) {
 		if (n->gro.bitmask)
@@ -7047,6 +7053,13 @@ bool napi_complete_done(struct napi_struct *n, int work_done)
 	do {
 		WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
 
+		/*
+		 * A false result is not proof of retained ownership. Mask
+		 * before the state transition lets another CPU enter poll.
+		 */
+		if (mask && (!ret || (val & NAPIF_STATE_MISSED)))
+			mask(data);
+
 		new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED |
 			      NAPIF_STATE_SCHED_THREADED |
 			      NAPIF_STATE_PREFER_BUSY_POLL);
@@ -7069,8 +7082,20 @@ bool napi_complete_done(struct napi_struct *n, int work_done)
 			      HRTIMER_MODE_REL_PINNED);
 	return ret;
 }
+
+bool napi_complete_done(struct napi_struct *n, int work_done)
+{
+	return __napi_complete_done(n, work_done, NULL, NULL);
+}
 EXPORT_SYMBOL(napi_complete_done);
 
+bool napi_complete_done_with_irq_mask(struct napi_struct *n, int work_done,
+				      void (*mask)(void *data), void *data)
+{
+	return __napi_complete_done(n, work_done, mask, data);
+}
+EXPORT_SYMBOL(napi_complete_done_with_irq_mask);
+
 static void __skb_defer_free_flush(struct skb_defer_node *sdn, int budget)
 {
 	struct llist_node *free_list;

base-commit: b3dfbde62cf1981c4638939c159dfd087b7a72e9
-- 
2.43.0