[PATCH bpf-next v1 2/6] bpf: Defer stream file notifications from NMI context

[Kumar Kartikeya Dwivedi <memxor@gmail.com>]

bpf_stream_vprintk() and staged stream writers can run in NMI context.
The stream file interface currently wakes its wait queue directly after
publishing data. Wait queue wakeups take a spin lock and can invoke epoll
callbacks that take further locks, so calling them from NMI context can
deadlock.

Give each stream an irq_work item and queue it after publishing data. The
irq_work callback reports readable data to poll waiters outside NMI
context, while naturally coalescing concurrent notifications. This matches
poll and epoll readiness semantics: notifications do not count records,
but prompt waiters to re-evaluate persistent readable state. Publications
made before a coalesced queue attempt are ordered before the pending
callback, and the work can be queued again once that callback begins.
EPOLLET consumers drain until EAGAIN, so one wakeup may safely represent a
batch.

Synchronize pending work before releasing the final stream reference so
the callback cannot outlive the stream.

Signed-off-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
---
 include/linux/bpf.h |  2 ++
 kernel/bpf/stream.c | 18 ++++++++++++++++--
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 1745686331be..0af3c79f5d03 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -17,6 +17,7 @@
 #include <linux/numa.h>
 #include <linux/mm_types.h>
 #include <linux/wait.h>
+#include <linux/irq_work_types.h>
 #include <linux/refcount.h>
 #include <linux/mutex.h>
 #include <linux/module.h>
@@ -1719,6 +1720,7 @@ struct bpf_stream {
 	struct llist_node *backlog_head; /* list of in-flight stream elements in FIFO order */
 	struct llist_node *backlog_tail; /* tail of the list above */
 	wait_queue_head_t waitq;
+	struct irq_work notify_work;
 	bool dead;
 };
 
diff --git a/kernel/bpf/stream.c b/kernel/bpf/stream.c
index d3dbb1aca792..99a89533eaef 100644
--- a/kernel/bpf/stream.c
+++ b/kernel/bpf/stream.c
@@ -6,6 +6,7 @@
 #include <linux/filter.h>
 #include <linux/bpf_mem_alloc.h>
 #include <linux/gfp.h>
+#include <linux/irq_work.h>
 #include <linux/memory.h>
 #include <linux/mutex.h>
 #include <linux/poll.h>
@@ -76,6 +77,17 @@ static void bpf_stream_release_capacity(struct bpf_stream *stream, int len)
 	atomic_sub(len, &stream->capacity);
 }
 
+static void bpf_stream_notify(struct irq_work *work)
+{
+	struct bpf_stream *stream = container_of(work, struct bpf_stream, notify_work);
+
+	/*
+	 * Stream writers can run in NMI context, while wait queue callbacks may
+	 * acquire locks. Defer those callbacks to irq_work context.
+	 */
+	wake_up_interruptible_poll(&stream->waitq, EPOLLIN | EPOLLRDNORM);
+}
+
 static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int len)
 {
 	int ret = bpf_stream_consume_capacity(stream, len);
@@ -87,7 +99,7 @@ static int bpf_stream_push_str(struct bpf_stream *stream, const char *str, int l
 	if (ret)
 		bpf_stream_release_capacity(stream, len);
 	else if (len)
-		wake_up_interruptible_poll(&stream->waitq, EPOLLIN | EPOLLRDNORM);
+		irq_work_queue(&stream->notify_work);
 
 	return ret;
 }
@@ -230,6 +242,7 @@ static void bpf_stream_put(struct bpf_stream *stream)
 	if (refcount_dec_and_test(&stream->refcnt)) {
 		struct llist_node *list;
 
+		irq_work_sync(&stream->notify_work);
 		list = llist_del_all(&stream->log);
 		bpf_stream_free_list(list);
 		bpf_stream_free_list(stream->backlog_head);
@@ -402,6 +415,7 @@ int bpf_prog_stream_init(struct bpf_prog *prog, gfp_t gfp_extra_flags)
 		init_llist_head(&stream->log);
 		mutex_init(&stream->lock);
 		init_waitqueue_head(&stream->waitq);
+		init_irq_work(&stream->notify_work, bpf_stream_notify);
 		prog->aux->stream[i] = stream;
 	}
 	return 0;
@@ -482,7 +496,7 @@ int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog,
 		list = tail;
 	}
 	llist_add_batch(head, tail, &stream->log);
-	wake_up_interruptible_poll(&stream->waitq, EPOLLIN | EPOLLRDNORM);
+	irq_work_queue(&stream->notify_work);
 	return 0;
 }
 
-- 
2.53.0