* [PATCH 5.15.y 2/6] block: use flags instead of bit fields for blkdev_dio
2026-09-04 15:53 ` [PATCH 5.15.y 1/6] block: don't try to poll multi-bio I/Os in __blkdev_direct_IO Sasha Levin
@ 2026-09-04 15:53 ` Sasha Levin
2026-09-04 15:53 ` [PATCH 5.15.y 3/6] usb: remove res2 argument from gadget code completions Sasha Levin
` (3 subsequent siblings)
4 siblings, 0 replies; 7+ messages in thread
From: Sasha Levin @ 2026-09-04 15:53 UTC (permalink / raw)
To: stable; +Cc: Jens Axboe, Christoph Hellwig, Sasha Levin
From: Jens Axboe <axboe@kernel.dk>
[ Upstream commit 09ce8744253a038eb658c14f9dc3b77fa021fc9f ]
This generates a lot better code for me, and bumps performance from
7650K IOPS to 7750K IOPS. Looking at profiles for the run and running
perf diff, it confirms that we're now sending a lot less time there:
6.38% -2.80% [kernel.vmlinux] [k] blkdev_direct_IO
Taking it from the 2nd most cycle consumer to only the 9th most at
3.35% of the CPU time.
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Stable-dep-of: e78dcb1f7ec2 ("usb: gadget: f_fs: Fix Use-After-Free in AIO error path")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
block/fops.c | 34 ++++++++++++++++++++--------------
1 file changed, 20 insertions(+), 14 deletions(-)
diff --git a/block/fops.c b/block/fops.c
index c23529d08929d..9359368ca7cf4 100644
--- a/block/fops.c
+++ b/block/fops.c
@@ -126,6 +126,12 @@ static ssize_t __blkdev_direct_IO_simple(struct kiocb *iocb,
return ret;
}
+enum {
+ DIO_MULTI_BIO = 1,
+ DIO_SHOULD_DIRTY = 2,
+ DIO_IS_SYNC = 4,
+};
+
struct blkdev_dio {
union {
struct kiocb *iocb;
@@ -133,9 +139,7 @@ struct blkdev_dio {
};
size_t size;
atomic_t ref;
- bool multi_bio : 1;
- bool should_dirty : 1;
- bool is_sync : 1;
+ unsigned int flags;
struct bio bio;
};
@@ -152,13 +156,13 @@ static int blkdev_iopoll(struct kiocb *kiocb, bool wait)
static void blkdev_bio_end_io(struct bio *bio)
{
struct blkdev_dio *dio = bio->bi_private;
- bool should_dirty = dio->should_dirty;
+ bool should_dirty = dio->flags & DIO_SHOULD_DIRTY;
if (bio->bi_status && !dio->bio.bi_status)
dio->bio.bi_status = bio->bi_status;
- if (!dio->multi_bio || atomic_dec_and_test(&dio->ref)) {
- if (!dio->is_sync) {
+ if (!(dio->flags & DIO_MULTI_BIO) || atomic_dec_and_test(&dio->ref)) {
+ if (!(dio->flags & DIO_IS_SYNC)) {
struct kiocb *iocb = dio->iocb;
ssize_t ret;
@@ -170,7 +174,7 @@ static void blkdev_bio_end_io(struct bio *bio)
}
dio->iocb->ki_complete(iocb, ret, 0);
- if (dio->multi_bio)
+ if (dio->flags & DIO_MULTI_BIO)
bio_put(&dio->bio);
} else {
struct task_struct *waiter = dio->waiter;
@@ -210,17 +214,19 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
bio = bio_alloc_kiocb(iocb, nr_pages, &blkdev_dio_pool);
dio = container_of(bio, struct blkdev_dio, bio);
- dio->is_sync = is_sync = is_sync_kiocb(iocb);
- if (dio->is_sync) {
+ is_sync = is_sync_kiocb(iocb);
+ if (is_sync) {
+ dio->flags = DIO_IS_SYNC;
dio->waiter = current;
bio_get(bio);
} else {
+ dio->flags = 0;
dio->iocb = iocb;
}
dio->size = 0;
- dio->multi_bio = false;
- dio->should_dirty = is_read && iter_is_iovec(iter);
+ if (is_read && iter_is_iovec(iter))
+ dio->flags |= DIO_SHOULD_DIRTY;
/*
* Don't plug for HIPRI/polled IO, as those should go straight
@@ -263,7 +269,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
if (is_read) {
bio->bi_opf = REQ_OP_READ;
- if (dio->should_dirty)
+ if (dio->flags & DIO_SHOULD_DIRTY)
bio_set_pages_dirty(bio);
} else {
bio->bi_opf = dio_bio_write_op(iocb);
@@ -285,7 +291,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
WRITE_ONCE(iocb->ki_cookie, qc);
break;
}
- if (!dio->multi_bio) {
+ if (!(dio->flags & DIO_MULTI_BIO)) {
/*
* AIO needs an extra reference to ensure the dio
* structure which is embedded into the first bio
@@ -293,7 +299,7 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter,
*/
if (!is_sync)
bio_get(bio);
- dio->multi_bio = true;
+ dio->flags |= DIO_MULTI_BIO;
atomic_set(&dio->ref, 2);
do_poll = false;
} else {
--
2.53.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 5.15.y 4/6] fs: get rid of the res2 iocb->ki_complete argument
2026-09-04 15:53 ` [PATCH 5.15.y 1/6] block: don't try to poll multi-bio I/Os in __blkdev_direct_IO Sasha Levin
2026-09-04 15:53 ` [PATCH 5.15.y 2/6] block: use flags instead of bit fields for blkdev_dio Sasha Levin
2026-09-04 15:53 ` [PATCH 5.15.y 3/6] usb: remove res2 argument from gadget code completions Sasha Levin
@ 2026-09-04 15:53 ` Sasha Levin
2026-09-04 15:53 ` [PATCH 5.15.y 5/6] USB: gadget: ffs: fix mm lifetime handling Sasha Levin
2026-09-04 15:53 ` [PATCH 5.15.y 6/6] usb: gadget: f_fs: Fix Use-After-Free in AIO error path Sasha Levin
4 siblings, 0 replies; 7+ messages in thread
From: Sasha Levin @ 2026-09-04 15:53 UTC (permalink / raw)
To: stable; +Cc: Jens Axboe, Darrick J. Wong, Sasha Levin
From: Jens Axboe <axboe@kernel.dk>
[ Upstream commit 6b19b766e8f077f29cdb47da5003469a85bbfb9c ]
The second argument was only used by the USB gadget code, yet everyone
pays the overhead of passing a zero to be passed into aio, where it
ends up being part of the aio res2 value.
Now that everybody is passing in zero, kill off the extra argument.
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Stable-dep-of: e78dcb1f7ec2 ("usb: gadget: f_fs: Fix Use-After-Free in AIO error path")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
block/fops.c | 2 +-
crypto/af_alg.c | 2 +-
drivers/block/loop.c | 4 ++--
drivers/nvme/target/io-cmd-file.c | 4 ++--
drivers/target/target_core_file.c | 4 ++--
drivers/usb/gadget/function/f_fs.c | 2 +-
drivers/usb/gadget/legacy/inode.c | 5 ++---
fs/aio.c | 6 +++---
fs/cachefiles/io.c | 12 ++++++------
fs/ceph/file.c | 2 +-
fs/cifs/file.c | 4 ++--
fs/direct-io.c | 2 +-
fs/fuse/file.c | 2 +-
fs/iomap/direct-io.c | 2 +-
fs/nfs/direct.c | 2 +-
fs/overlayfs/file.c | 4 ++--
include/linux/fs.h | 2 +-
io_uring/io_uring.c | 6 +++---
18 files changed, 33 insertions(+), 34 deletions(-)
diff --git a/block/fops.c b/block/fops.c
index 9359368ca7cf4..45f57060c9296 100644
--- a/block/fops.c
+++ b/block/fops.c
@@ -173,7 +173,7 @@ static void blkdev_bio_end_io(struct bio *bio)
ret = blk_status_to_errno(dio->bio.bi_status);
}
- dio->iocb->ki_complete(iocb, ret, 0);
+ dio->iocb->ki_complete(iocb, ret);
if (dio->flags & DIO_MULTI_BIO)
bio_put(&dio->bio);
} else {
diff --git a/crypto/af_alg.c b/crypto/af_alg.c
index 8dfb94aa99285..cbc6e3f8db8ff 100644
--- a/crypto/af_alg.c
+++ b/crypto/af_alg.c
@@ -1030,7 +1030,7 @@ void af_alg_async_cb(struct crypto_async_request *_req, int err)
af_alg_free_resources(areq);
sock_put(sk);
- iocb->ki_complete(iocb, err ? err : (int)resultlen, 0);
+ iocb->ki_complete(iocb, err ? err : (int)resultlen);
}
EXPORT_SYMBOL_GPL(af_alg_async_cb);
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index 45684136e8376..3fad77cd16bb5 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -542,7 +542,7 @@ static void lo_rw_aio_do_completion(struct loop_cmd *cmd)
blk_mq_complete_request(rq);
}
-static void lo_rw_aio_complete(struct kiocb *iocb, long ret, long ret2)
+static void lo_rw_aio_complete(struct kiocb *iocb, long ret)
{
struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb);
@@ -615,7 +615,7 @@ static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
lo_rw_aio_do_completion(cmd);
if (ret != -EIOCBQUEUED)
- cmd->iocb.ki_complete(&cmd->iocb, ret, 0);
+ lo_rw_aio_complete(&cmd->iocb, ret);
return 0;
}
diff --git a/drivers/nvme/target/io-cmd-file.c b/drivers/nvme/target/io-cmd-file.c
index f576a959e36c9..682d64b95d3a5 100644
--- a/drivers/nvme/target/io-cmd-file.c
+++ b/drivers/nvme/target/io-cmd-file.c
@@ -107,7 +107,7 @@ static ssize_t nvmet_file_submit_bvec(struct nvmet_req *req, loff_t pos,
return call_iter(iocb, &iter);
}
-static void nvmet_file_io_done(struct kiocb *iocb, long ret, long ret2)
+static void nvmet_file_io_done(struct kiocb *iocb, long ret)
{
struct nvmet_req *req = container_of(iocb, struct nvmet_req, f.iocb);
u16 status = NVME_SC_SUCCESS;
@@ -204,7 +204,7 @@ static bool nvmet_file_execute_io(struct nvmet_req *req, int ki_flags)
}
complete:
- nvmet_file_io_done(&req->f.iocb, ret, 0);
+ nvmet_file_io_done(&req->f.iocb, ret);
return true;
}
diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c
index e11d9897a5e43..c94d83e089f09 100644
--- a/drivers/target/target_core_file.c
+++ b/drivers/target/target_core_file.c
@@ -244,7 +244,7 @@ struct target_core_file_cmd {
struct bio_vec bvecs[];
};
-static void cmd_rw_aio_complete(struct kiocb *iocb, long ret, long ret2)
+static void cmd_rw_aio_complete(struct kiocb *iocb, long ret)
{
struct target_core_file_cmd *cmd;
@@ -302,7 +302,7 @@ fd_execute_rw_aio(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents,
ret = call_read_iter(file, &aio_cmd->iocb, &iter);
if (ret != -EIOCBQUEUED)
- cmd_rw_aio_complete(&aio_cmd->iocb, ret, 0);
+ cmd_rw_aio_complete(&aio_cmd->iocb, ret);
return 0;
}
diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c
index bb1fe2f37289e..0588be1109022 100644
--- a/drivers/usb/gadget/function/f_fs.c
+++ b/drivers/usb/gadget/function/f_fs.c
@@ -841,7 +841,7 @@ static void ffs_user_copy_worker(struct work_struct *work)
kthread_unuse_mm(io_data->mm);
}
- io_data->kiocb->ki_complete(io_data->kiocb, ret, 0);
+ io_data->kiocb->ki_complete(io_data->kiocb, ret);
if (io_data->ffs->ffs_eventfd && !kiocb_has_eventfd)
eventfd_signal(io_data->ffs->ffs_eventfd, 1);
diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c
index e6ffbc82d48c4..33b427c1b9cb8 100644
--- a/drivers/usb/gadget/legacy/inode.c
+++ b/drivers/usb/gadget/legacy/inode.c
@@ -473,7 +473,7 @@ static void ep_user_copy_worker(struct work_struct *work)
ret = -EFAULT;
/* completing the iocb can drop the ctx and mm, don't touch mm after */
- iocb->ki_complete(iocb, ret, 0);
+ iocb->ki_complete(iocb, ret);
kfree(priv->buf);
kfree(priv->to_free);
@@ -501,8 +501,7 @@ static void ep_aio_complete(struct usb_ep *ep, struct usb_request *req)
kfree(priv);
iocb->private = NULL;
iocb->ki_complete(iocb,
- req->actual ? req->actual : (long)req->status,
- 0);
+ req->actual ? req->actual : (long)req->status);
} else {
/* ep_copy_to_user() won't report both; we hide some faults */
if (unlikely(0 != req->status))
diff --git a/fs/aio.c b/fs/aio.c
index f502a7c841909..cdb1030f4abc0 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -1433,7 +1433,7 @@ static void aio_remove_iocb(struct aio_kiocb *iocb)
spin_unlock_irqrestore(&ctx->ctx_lock, flags);
}
-static void aio_complete_rw(struct kiocb *kiocb, long res, long res2)
+static void aio_complete_rw(struct kiocb *kiocb, long res)
{
struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, rw);
@@ -1453,7 +1453,7 @@ static void aio_complete_rw(struct kiocb *kiocb, long res, long res2)
}
iocb->ki_res.res = res;
- iocb->ki_res.res2 = res2;
+ iocb->ki_res.res2 = 0;
iocb_put(iocb);
}
@@ -1524,7 +1524,7 @@ static inline void aio_rw_done(struct kiocb *req, ssize_t ret)
ret = -EINTR;
fallthrough;
default:
- req->ki_complete(req, ret, 0);
+ req->ki_complete(req, ret);
}
}
diff --git a/fs/cachefiles/io.c b/fs/cachefiles/io.c
index fac2e8e7b533e..effe37ef86291 100644
--- a/fs/cachefiles/io.c
+++ b/fs/cachefiles/io.c
@@ -37,11 +37,11 @@ static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki)
/*
* Handle completion of a read from the cache.
*/
-static void cachefiles_read_complete(struct kiocb *iocb, long ret, long ret2)
+static void cachefiles_read_complete(struct kiocb *iocb, long ret)
{
struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
- _enter("%ld,%ld", ret, ret2);
+ _enter("%ld", ret);
if (ki->term_func) {
if (ret >= 0)
@@ -139,7 +139,7 @@ static int cachefiles_read(struct netfs_cache_resources *cres,
fallthrough;
default:
ki->was_async = false;
- cachefiles_read_complete(&ki->iocb, ret, 0);
+ cachefiles_read_complete(&ki->iocb, ret);
if (ret > 0)
ret = 0;
break;
@@ -159,12 +159,12 @@ static int cachefiles_read(struct netfs_cache_resources *cres,
/*
* Handle completion of a write to the cache.
*/
-static void cachefiles_write_complete(struct kiocb *iocb, long ret, long ret2)
+static void cachefiles_write_complete(struct kiocb *iocb, long ret)
{
struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);
struct inode *inode = file_inode(ki->iocb.ki_filp);
- _enter("%ld,%ld", ret, ret2);
+ _enter("%ld", ret);
/* Tell lockdep we inherited freeze protection from submission thread */
__sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE);
@@ -244,7 +244,7 @@ static int cachefiles_write(struct netfs_cache_resources *cres,
fallthrough;
default:
ki->was_async = false;
- cachefiles_write_complete(&ki->iocb, ret, 0);
+ cachefiles_write_complete(&ki->iocb, ret);
if (ret > 0)
ret = 0;
break;
diff --git a/fs/ceph/file.c b/fs/ceph/file.c
index 7ecb51384497b..cc8044d0471d4 100644
--- a/fs/ceph/file.c
+++ b/fs/ceph/file.c
@@ -1061,7 +1061,7 @@ static void ceph_aio_complete(struct inode *inode,
ceph_put_cap_refs(ci, (aio_req->write ? CEPH_CAP_FILE_WR :
CEPH_CAP_FILE_RD));
- aio_req->iocb->ki_complete(aio_req->iocb, ret, 0);
+ aio_req->iocb->ki_complete(aio_req->iocb, ret);
ceph_free_cap_flush(aio_req->prealloc_cf);
kfree(aio_req);
diff --git a/fs/cifs/file.c b/fs/cifs/file.c
index 71f585a41d456..11f0c4ee690ae 100644
--- a/fs/cifs/file.c
+++ b/fs/cifs/file.c
@@ -3210,7 +3210,7 @@ static void collect_uncached_write_data(struct cifs_aio_ctx *ctx)
mutex_unlock(&ctx->aio_mutex);
if (ctx->iocb && ctx->iocb->ki_complete)
- ctx->iocb->ki_complete(ctx->iocb, ctx->rc, 0);
+ ctx->iocb->ki_complete(ctx->iocb, ctx->rc);
else
complete(&ctx->done);
}
@@ -3951,7 +3951,7 @@ collect_uncached_read_data(struct cifs_aio_ctx *ctx)
mutex_unlock(&ctx->aio_mutex);
if (ctx->iocb && ctx->iocb->ki_complete)
- ctx->iocb->ki_complete(ctx->iocb, ctx->rc, 0);
+ ctx->iocb->ki_complete(ctx->iocb, ctx->rc);
else
complete(&ctx->done);
}
diff --git a/fs/direct-io.c b/fs/direct-io.c
index b2e86e739d7a1..072c4b5eeef9c 100644
--- a/fs/direct-io.c
+++ b/fs/direct-io.c
@@ -308,7 +308,7 @@ static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags)
if (ret > 0 && dio->op == REQ_OP_WRITE)
ret = generic_write_sync(dio->iocb, ret);
- dio->iocb->ki_complete(dio->iocb, ret, 0);
+ dio->iocb->ki_complete(dio->iocb, ret);
}
kmem_cache_free(dio_cache, dio);
diff --git a/fs/fuse/file.c b/fs/fuse/file.c
index db9211464f1e0..ad3ab7c86b0e2 100644
--- a/fs/fuse/file.c
+++ b/fs/fuse/file.c
@@ -725,7 +725,7 @@ static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
spin_unlock(&fi->lock);
}
- io->iocb->ki_complete(io->iocb, res, 0);
+ io->iocb->ki_complete(io->iocb, res);
}
kref_put(&io->refcnt, fuse_io_release);
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
index f5b36732b89b3..71f50ae0996df 100644
--- a/fs/iomap/direct-io.c
+++ b/fs/iomap/direct-io.c
@@ -139,7 +139,7 @@ static void iomap_dio_complete_work(struct work_struct *work)
struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);
struct kiocb *iocb = dio->iocb;
- iocb->ki_complete(iocb, iomap_dio_complete(dio), 0);
+ iocb->ki_complete(iocb, iomap_dio_complete(dio));
}
/*
diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c
index 3185899676adf..cb08d8d57945c 100644
--- a/fs/nfs/direct.c
+++ b/fs/nfs/direct.c
@@ -275,7 +275,7 @@ static void nfs_direct_complete(struct nfs_direct_req *dreq)
res = (long) dreq->count;
WARN_ON_ONCE(dreq->count < 0);
}
- dreq->iocb->ki_complete(dreq->iocb, res, 0);
+ dreq->iocb->ki_complete(dreq->iocb, res);
}
complete(&dreq->completion);
diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c
index 49d3feded593f..a99a73afa4862 100644
--- a/fs/overlayfs/file.c
+++ b/fs/overlayfs/file.c
@@ -279,14 +279,14 @@ static void ovl_aio_cleanup_handler(struct ovl_aio_req *aio_req)
ovl_aio_put(aio_req);
}
-static void ovl_aio_rw_complete(struct kiocb *iocb, long res, long res2)
+static void ovl_aio_rw_complete(struct kiocb *iocb, long res)
{
struct ovl_aio_req *aio_req = container_of(iocb,
struct ovl_aio_req, iocb);
struct kiocb *orig_iocb = aio_req->orig_iocb;
ovl_aio_cleanup_handler(aio_req);
- orig_iocb->ki_complete(orig_iocb, res, res2);
+ orig_iocb->ki_complete(orig_iocb, res);
}
static ssize_t ovl_read_iter(struct kiocb *iocb, struct iov_iter *iter)
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 41b3ebe4e8e71..326598e3cba87 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -336,7 +336,7 @@ struct kiocb {
randomized_struct_fields_start
loff_t ki_pos;
- void (*ki_complete)(struct kiocb *iocb, long ret, long ret2);
+ void (*ki_complete)(struct kiocb *iocb, long ret);
void *private;
int ki_flags;
u16 ki_hint;
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 8747c585a0770..1cacfe63d7544 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2801,7 +2801,7 @@ static void io_req_rw_complete(struct io_kiocb *req, bool *locked)
io_req_task_complete(req, locked);
}
-static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
+static void io_complete_rw(struct kiocb *kiocb, long res)
{
struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
@@ -2812,7 +2812,7 @@ static void io_complete_rw(struct kiocb *kiocb, long res, long res2)
io_req_task_work_add(req);
}
-static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2)
+static void io_complete_rw_iopoll(struct kiocb *kiocb, long res)
{
struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);
@@ -3037,7 +3037,7 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
ret = -EINTR;
fallthrough;
default:
- kiocb->ki_complete(kiocb, ret, 0);
+ kiocb->ki_complete(kiocb, ret);
}
}
--
2.53.0
^ permalink raw reply related [flat|nested] 7+ messages in thread