[PATCH 4/3] io_uring: add __io_cqring_add_overflow() helper

[Jens Axboe <axboe@kernel.dk>]

This splits the overflow addition into the typical two helpers, one that
already holds ->completion_lock, if it needs it, and one that grabs it.

Also mark both of them as __cold, as this should be mostly out of line
invocations, and ditto for the ocqe allocation helper.

Signed-off-by: Jens Axboe <axboe@kernel.dk>

---

diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index b50c2d434e74..d14a9b5eeb59 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -697,8 +697,8 @@ static __cold void io_uring_drop_tctx_refs(struct task_struct *task)
 	}
 }
 
-static bool io_cqring_add_overflow(struct io_ring_ctx *ctx,
-				   struct io_overflow_cqe *ocqe)
+static __cold bool __io_cqring_add_overflow(struct io_ring_ctx *ctx,
+					    struct io_overflow_cqe *ocqe)
 {
 	lockdep_assert_held(&ctx->completion_lock);
 
@@ -723,9 +723,19 @@ static bool io_cqring_add_overflow(struct io_ring_ctx *ctx,
 	return true;
 }
 
-static struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx,
-					     struct io_cqe *cqe, u64 extra1,
-					     u64 extra2, gfp_t gfp)
+
+static __cold void io_cqring_add_overflow(struct io_ring_ctx *ctx,
+					  struct io_overflow_cqe *ocqe)
+{
+	spin_lock(&ctx->completion_lock);
+	__io_cqring_add_overflow(ctx, ocqe);
+	spin_unlock(&ctx->completion_lock);
+}
+
+static __cold struct io_overflow_cqe *io_alloc_ocqe(struct io_ring_ctx *ctx,
+						    struct io_cqe *cqe,
+						    u64 extra1, u64 extra2,
+						    gfp_t gfp)
 {
 	struct io_overflow_cqe *ocqe;
 	size_t ocq_size = sizeof(struct io_overflow_cqe);
@@ -820,7 +830,7 @@ bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags
 		struct io_cqe cqe = io_init_cqe(user_data, res, cflags);
 
 		ocqe = io_alloc_ocqe(ctx, &cqe, 0, 0, GFP_ATOMIC);
-		filled = io_cqring_add_overflow(ctx, ocqe);
+		filled = __io_cqring_add_overflow(ctx, ocqe);
 	}
 	io_cq_unlock_post(ctx);
 	return filled;
@@ -840,9 +850,7 @@ void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags)
 		struct io_cqe cqe = io_init_cqe(user_data, res, cflags);
 
 		ocqe = io_alloc_ocqe(ctx, &cqe, 0, 0, GFP_KERNEL);
-		spin_lock(&ctx->completion_lock);
 		io_cqring_add_overflow(ctx, ocqe);
-		spin_unlock(&ctx->completion_lock);
 	}
 	ctx->submit_state.cq_flush = true;
 }
@@ -1451,13 +1459,10 @@ void __io_submit_flush_completions(struct io_ring_ctx *ctx)
 
 			ocqe = io_alloc_ocqe(ctx, &req->cqe, req->big_cqe.extra1,
 					     req->big_cqe.extra2, gfp);
-			if (ctx->lockless_cq) {
-				spin_lock(&ctx->completion_lock);
+			if (ctx->lockless_cq)
 				io_cqring_add_overflow(ctx, ocqe);
-				spin_unlock(&ctx->completion_lock);
-			} else {
-				io_cqring_add_overflow(ctx, ocqe);
-			}
+			else
+				__io_cqring_add_overflow(ctx, ocqe);
 			memset(&req->big_cqe, 0, sizeof(req->big_cqe));
 		}
 	}