[RFC 9/9] io_uring/zcrx: add rq_lock cache of "user" niov refs

[Pavel Begunkov <asml.silence@gmail.com>]

Now the "user" refs are acquired and released by NAPI/page pool for the
optimised path, cache them on the NAPI side and protect it by rq.lock.
Store it in net_iov as we'd be touching the cache line by refilling soon
anyway.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
 io_uring/zcrx.c | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/io_uring/zcrx.c b/io_uring/zcrx.c
index 23669471a8f0..04a80d1a2b3a 100644
--- a/io_uring/zcrx.c
+++ b/io_uring/zcrx.c
@@ -393,9 +393,20 @@ static inline atomic_t *io_get_user_counter(struct net_iov *niov)
 
 static bool io_zcrx_put_niov_uref(struct net_iov *niov, unsigned refs)
 {
-	atomic_t *uref = io_get_user_counter(niov);
+	unsigned *cached_ref = &niov->mp_private;
+	atomic_t *uref;
 	int old;
 
+	lockdep_assert_held(&io_zcrx_iov_to_area(niov)->ifq->rq.lock);
+
+	if (likely(*cached_ref >= refs)) {
+		*cached_ref -= refs;
+		return true;
+	}
+	refs -= *cached_ref;
+	*cached_ref = 0;
+
+	uref = io_get_user_counter(niov);
 	old = atomic_read(uref);
 	do {
 		if (unlikely(old < refs))
@@ -744,6 +755,16 @@ static void io_zcrx_scrub_area(struct io_zcrx_ifq *ifq, struct io_zcrx_area *are
 {
 	int i;
 
+	scoped_guard(spinlock_bh, &ifq->rq.lock) {
+		for (i = 0; i < area->nia.num_niovs; i++) {
+			struct net_iov *niov = &area->nia.niovs[i];
+			unsigned *ref = &niov->mp_private;
+
+			atomic_add(*ref, &area->user_refs[i]);
+			*ref = 0;
+		}
+	}
+
 	/* Reclaim back all buffers given to the user space. */
 	for (i = 0; i < area->nia.num_niovs; i++) {
 		struct net_iov *niov = &area->nia.niovs[i];
@@ -1327,9 +1348,9 @@ static void zcrx_release_skbs(struct io_zcrx_ifq *ifq)
 		for (i = 0; i < shi->nr_frags; i++) {
 			const skb_frag_t *frag = &shi->frags[i];
 			struct net_iov *niov = netmem_to_net_iov(frag->netmem);
+			unsigned *ref = &niov->mp_private;
 
-			/* Take niov references the skb holds */
-			io_zcrx_get_niov_uref(niov);
+			*ref += 1;
 		}
 		shi->nr_frags = 0;
 
-- 
2.54.0