* [PATCH v2 0/2] block: enable per-cpu bio cache by default @ 2025-11-07 2:05 Fengnan Chang 2025-11-07 2:05 ` [PATCH v2 1/2] block: use bio_alloc_bioset for passthru IO " Fengnan Chang 2025-11-07 2:05 ` [PATCH v2 2/2] block: enable per-cpu bio cache " Fengnan Chang 0 siblings, 2 replies; 5+ messages in thread From: Fengnan Chang @ 2025-11-07 2:05 UTC (permalink / raw) To: axboe, viro, brauner, jack, asml.silence, willy, djwong, hch, ritesh.list, linux-fsdevel, io-uring, linux-xfs, linux-ext4, linux-block, ming.lei, linux-nvme Cc: Fengnan Chang For now, per-cpu bio cache was only used in the io_uring + raw block device, filesystem also can use this to improve performance. After discussion in [1], we think it's better to enable per-cpu bio cache by default. v2: enable per-cpu bio cache for passthru IO by default. v1: https://lore.kernel.org/linux-fsdevel/CAPFOzZs5mJ9Ts+TYkhioO8aAYfzevcgw7O3hjexFNb_tM+kEZA@mail.gmail.com/ [1] https://lore.kernel.org/linux-fsdevel/c4bc7c33-b1e1-47d1-9d22-b189c86c6c7d@gmail.com/ Fengnan Chang (2): block: use bio_alloc_bioset for passthru IO by default block: enable per-cpu bio cache by default block/bio.c | 26 ++++++------ block/blk-map.c | 89 +++++++++++++++------------------------ block/fops.c | 4 -- drivers/nvme/host/ioctl.c | 2 +- include/linux/fs.h | 3 -- io_uring/rw.c | 1 - 6 files changed, 48 insertions(+), 77 deletions(-) base-commit: 4a0c9b3391999818e2c5b93719699b255be1f682 -- 2.39.5 (Apple Git-154) ^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH v2 1/2] block: use bio_alloc_bioset for passthru IO by default 2025-11-07 2:05 [PATCH v2 0/2] block: enable per-cpu bio cache by default Fengnan Chang @ 2025-11-07 2:05 ` Fengnan Chang 2025-11-07 3:17 ` kernel test robot 2025-11-08 1:15 ` kernel test robot 2025-11-07 2:05 ` [PATCH v2 2/2] block: enable per-cpu bio cache " Fengnan Chang 1 sibling, 2 replies; 5+ messages in thread From: Fengnan Chang @ 2025-11-07 2:05 UTC (permalink / raw) To: axboe, viro, brauner, jack, asml.silence, willy, djwong, hch, ritesh.list, linux-fsdevel, io-uring, linux-xfs, linux-ext4, linux-block, ming.lei, linux-nvme Cc: Fengnan Chang Use bio_alloc_bioset for passthru IO by default, so that we can enable bio cache for irq and polled passthru IO in later. Signed-off-by: Fengnan Chang <changfengnan@bytedance.com> --- block/blk-map.c | 89 +++++++++++++++------------------------ drivers/nvme/host/ioctl.c | 2 +- 2 files changed, 36 insertions(+), 55 deletions(-) diff --git a/block/blk-map.c b/block/blk-map.c index 60faf036fb6e..272a23d8ef8e 100644 --- a/block/blk-map.c +++ b/block/blk-map.c @@ -37,6 +37,25 @@ static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, return bmd; } +static inline void blk_mq_map_bio_put(struct bio *bio) +{ + bio_put(bio); +} + +static struct bio *blk_rq_map_bio_alloc(struct request *rq, + unsigned int nr_vecs, gfp_t gfp_mask) +{ + struct block_device *bdev = rq->q->disk ? rq->q->disk->part0 : NULL; + struct bio *bio; + + bio = bio_alloc_bioset(bdev, nr_vecs, rq->cmd_flags, gfp_mask, + &fs_bio_set); + if (!bio) + return NULL; + + return bio; +} + /** * bio_copy_from_iter - copy all pages from iov_iter to bio * @bio: The &struct bio which describes the I/O as destination @@ -154,10 +173,9 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE)); ret = -ENOMEM; - bio = bio_kmalloc(nr_pages, gfp_mask); + bio = blk_rq_map_bio_alloc(rq, nr_pages, gfp_mask); if (!bio) goto out_bmd; - bio_init_inline(bio, NULL, nr_pages, req_op(rq)); if (map_data) { nr_pages = 1U << map_data->page_order; @@ -233,43 +251,12 @@ static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, cleanup: if (!map_data) bio_free_pages(bio); - bio_uninit(bio); - kfree(bio); + blk_mq_map_bio_put(bio); out_bmd: kfree(bmd); return ret; } -static void blk_mq_map_bio_put(struct bio *bio) -{ - if (bio->bi_opf & REQ_ALLOC_CACHE) { - bio_put(bio); - } else { - bio_uninit(bio); - kfree(bio); - } -} - -static struct bio *blk_rq_map_bio_alloc(struct request *rq, - unsigned int nr_vecs, gfp_t gfp_mask) -{ - struct block_device *bdev = rq->q->disk ? rq->q->disk->part0 : NULL; - struct bio *bio; - - if (rq->cmd_flags & REQ_ALLOC_CACHE && (nr_vecs <= BIO_INLINE_VECS)) { - bio = bio_alloc_bioset(bdev, nr_vecs, rq->cmd_flags, gfp_mask, - &fs_bio_set); - if (!bio) - return NULL; - } else { - bio = bio_kmalloc(nr_vecs, gfp_mask); - if (!bio) - return NULL; - bio_init_inline(bio, bdev, nr_vecs, req_op(rq)); - } - return bio; -} - static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, gfp_t gfp_mask) { @@ -318,25 +305,23 @@ static void bio_invalidate_vmalloc_pages(struct bio *bio) static void bio_map_kern_endio(struct bio *bio) { bio_invalidate_vmalloc_pages(bio); - bio_uninit(bio); - kfree(bio); + blk_mq_map_bio_put(bio); } -static struct bio *bio_map_kern(void *data, unsigned int len, enum req_op op, +static struct bio *bio_map_kern(struct request *rq, void *data, unsigned int len, gfp_t gfp_mask) { unsigned int nr_vecs = bio_add_max_vecs(data, len); struct bio *bio; - bio = bio_kmalloc(nr_vecs, gfp_mask); + bio = blk_rq_map_bio_alloc(rq, nr_vecs, gfp_mask); if (!bio) return ERR_PTR(-ENOMEM); - bio_init_inline(bio, NULL, nr_vecs, op); + if (is_vmalloc_addr(data)) { bio->bi_private = data; if (!bio_add_vmalloc(bio, data, len)) { - bio_uninit(bio); - kfree(bio); + blk_mq_map_bio_put(bio); return ERR_PTR(-EINVAL); } } else { @@ -349,8 +334,7 @@ static struct bio *bio_map_kern(void *data, unsigned int len, enum req_op op, static void bio_copy_kern_endio(struct bio *bio) { bio_free_pages(bio); - bio_uninit(bio); - kfree(bio); + blk_mq_map_bio_put(bio); } static void bio_copy_kern_endio_read(struct bio *bio) @@ -377,9 +361,10 @@ static void bio_copy_kern_endio_read(struct bio *bio) * copy the kernel address into a bio suitable for io to a block * device. Returns an error pointer in case of error. */ -static struct bio *bio_copy_kern(void *data, unsigned int len, enum req_op op, +static struct bio *bio_copy_kern(struct request *rq, void *data, unsigned int len, gfp_t gfp_mask) { + enum req_op op = req_op(rq); unsigned long kaddr = (unsigned long)data; unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; unsigned long start = kaddr >> PAGE_SHIFT; @@ -394,10 +379,9 @@ static struct bio *bio_copy_kern(void *data, unsigned int len, enum req_op op, return ERR_PTR(-EINVAL); nr_pages = end - start; - bio = bio_kmalloc(nr_pages, gfp_mask); + bio = blk_rq_map_bio_alloc(rq, nr_pages, gfp_mask); if (!bio) return ERR_PTR(-ENOMEM); - bio_init_inline(bio, NULL, nr_pages, op); while (len) { struct page *page; @@ -431,8 +415,7 @@ static struct bio *bio_copy_kern(void *data, unsigned int len, enum req_op op, cleanup: bio_free_pages(bio); - bio_uninit(bio); - kfree(bio); + blk_mq_map_bio_put(bio); return ERR_PTR(-ENOMEM); } @@ -676,18 +659,16 @@ int blk_rq_map_kern(struct request *rq, void *kbuf, unsigned int len, return -EINVAL; if (!blk_rq_aligned(rq->q, addr, len) || object_is_on_stack(kbuf)) - bio = bio_copy_kern(kbuf, len, req_op(rq), gfp_mask); + bio = bio_copy_kern(rq, kbuf, len, gfp_mask); else - bio = bio_map_kern(kbuf, len, req_op(rq), gfp_mask); + bio = bio_map_kern(rq, kbuf, len, gfp_mask); if (IS_ERR(bio)) return PTR_ERR(bio); ret = blk_rq_append_bio(rq, bio); - if (unlikely(ret)) { - bio_uninit(bio); - kfree(bio); - } + if (unlikely(ret)) + blk_mq_map_bio_put(bio); return ret; } EXPORT_SYMBOL(blk_rq_map_kern); diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index c212fa952c0f..cd6bca8a5233 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -446,7 +446,7 @@ static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, struct iov_iter iter; struct iov_iter *map_iter = NULL; struct request *req; - blk_opf_t rq_flags = REQ_ALLOC_CACHE; + blk_opf_t rq_flags; blk_mq_req_flags_t blk_flags = 0; int ret; -- 2.39.5 (Apple Git-154) ^ permalink raw reply related [flat|nested] 5+ messages in thread
* Re: [PATCH v2 1/2] block: use bio_alloc_bioset for passthru IO by default 2025-11-07 2:05 ` [PATCH v2 1/2] block: use bio_alloc_bioset for passthru IO " Fengnan Chang @ 2025-11-07 3:17 ` kernel test robot 2025-11-08 1:15 ` kernel test robot 1 sibling, 0 replies; 5+ messages in thread From: kernel test robot @ 2025-11-07 3:17 UTC (permalink / raw) To: Fengnan Chang, axboe, viro, brauner, jack, asml.silence, willy, djwong, hch, ritesh.list, linux-fsdevel, io-uring, linux-xfs, linux-ext4, linux-block, ming.lei, linux-nvme Cc: oe-kbuild-all, Fengnan Chang Hi Fengnan, kernel test robot noticed the following build warnings: [auto build test WARNING on 4a0c9b3391999818e2c5b93719699b255be1f682] url: https://github.com/intel-lab-lkp/linux/commits/Fengnan-Chang/block-use-bio_alloc_bioset-for-passthru-IO-by-default/20251107-100851 base: 4a0c9b3391999818e2c5b93719699b255be1f682 patch link: https://lore.kernel.org/r/20251107020557.10097-2-changfengnan%40bytedance.com patch subject: [PATCH v2 1/2] block: use bio_alloc_bioset for passthru IO by default config: m68k-allnoconfig (https://download.01.org/0day-ci/archive/20251107/202511071021.3YIvZw6u-lkp@intel.com/config) compiler: m68k-linux-gcc (GCC) 15.1.0 reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251107/202511071021.3YIvZw6u-lkp@intel.com/reproduce) If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags | Reported-by: kernel test robot <lkp@intel.com> | Closes: https://lore.kernel.org/oe-kbuild-all/202511071021.3YIvZw6u-lkp@intel.com/ All warnings (new ones prefixed by >>): >> Warning: block/blk-map.c:365 function parameter 'rq' not described in 'bio_copy_kern' -- 0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki ^ permalink raw reply [flat|nested] 5+ messages in thread
* Re: [PATCH v2 1/2] block: use bio_alloc_bioset for passthru IO by default 2025-11-07 2:05 ` [PATCH v2 1/2] block: use bio_alloc_bioset for passthru IO " Fengnan Chang 2025-11-07 3:17 ` kernel test robot @ 2025-11-08 1:15 ` kernel test robot 1 sibling, 0 replies; 5+ messages in thread From: kernel test robot @ 2025-11-08 1:15 UTC (permalink / raw) To: Fengnan Chang, axboe, viro, brauner, jack, asml.silence, willy, djwong, hch, ritesh.list, linux-fsdevel, io-uring, linux-xfs, linux-ext4, linux-block, ming.lei, linux-nvme Cc: llvm, oe-kbuild-all, Fengnan Chang Hi Fengnan, kernel test robot noticed the following build warnings: [auto build test WARNING on 4a0c9b3391999818e2c5b93719699b255be1f682] url: https://github.com/intel-lab-lkp/linux/commits/Fengnan-Chang/block-use-bio_alloc_bioset-for-passthru-IO-by-default/20251107-100851 base: 4a0c9b3391999818e2c5b93719699b255be1f682 patch link: https://lore.kernel.org/r/20251107020557.10097-2-changfengnan%40bytedance.com patch subject: [PATCH v2 1/2] block: use bio_alloc_bioset for passthru IO by default config: x86_64-kexec (https://download.01.org/0day-ci/archive/20251108/202511080837.qd2MmgFS-lkp@intel.com/config) compiler: clang version 20.1.8 (https://github.com/llvm/llvm-project 87f0227cb60147a26a1eeb4fb06e3b505e9c7261) reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20251108/202511080837.qd2MmgFS-lkp@intel.com/reproduce) If you fix the issue in a separate patch/commit (i.e. not just a new version of the same patch/commit), kindly add following tags | Reported-by: kernel test robot <lkp@intel.com> | Closes: https://lore.kernel.org/oe-kbuild-all/202511080837.qd2MmgFS-lkp@intel.com/ All warnings (new ones prefixed by >>): >> drivers/nvme/host/ioctl.c:500:3: warning: variable 'rq_flags' is uninitialized when used here [-Wuninitialized] 500 | rq_flags |= REQ_NOWAIT; | ^~~~~~~~ drivers/nvme/host/ioctl.c:449:20: note: initialize the variable 'rq_flags' to silence this warning 449 | blk_opf_t rq_flags; | ^ | = 0 1 warning generated. vim +/rq_flags +500 drivers/nvme/host/ioctl.c c0a7ba77e81b84 Jens Axboe 2022-09-21 437 456cba386e94f2 Kanchan Joshi 2022-05-11 438 static int nvme_uring_cmd_io(struct nvme_ctrl *ctrl, struct nvme_ns *ns, f569add47119fa Anuj Gupta 2022-05-11 439 struct io_uring_cmd *ioucmd, unsigned int issue_flags, bool vec) 456cba386e94f2 Kanchan Joshi 2022-05-11 440 { 456cba386e94f2 Kanchan Joshi 2022-05-11 441 struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); fd9b8547bc5c34 Breno Leitao 2023-05-04 442 const struct nvme_uring_cmd *cmd = io_uring_sqe_cmd(ioucmd->sqe); 456cba386e94f2 Kanchan Joshi 2022-05-11 443 struct request_queue *q = ns ? ns->queue : ctrl->admin_q; 456cba386e94f2 Kanchan Joshi 2022-05-11 444 struct nvme_uring_data d; 456cba386e94f2 Kanchan Joshi 2022-05-11 445 struct nvme_command c; 38808af53c6e72 Caleb Sander Mateos 2025-03-28 446 struct iov_iter iter; 38808af53c6e72 Caleb Sander Mateos 2025-03-28 447 struct iov_iter *map_iter = NULL; 456cba386e94f2 Kanchan Joshi 2022-05-11 448 struct request *req; 070157fe67aee9 Fengnan Chang 2025-11-07 449 blk_opf_t rq_flags; 456cba386e94f2 Kanchan Joshi 2022-05-11 450 blk_mq_req_flags_t blk_flags = 0; 470e900c8036ff Kanchan Joshi 2022-09-30 451 int ret; 456cba386e94f2 Kanchan Joshi 2022-05-11 452 456cba386e94f2 Kanchan Joshi 2022-05-11 453 c.common.opcode = READ_ONCE(cmd->opcode); 456cba386e94f2 Kanchan Joshi 2022-05-11 454 c.common.flags = READ_ONCE(cmd->flags); 456cba386e94f2 Kanchan Joshi 2022-05-11 455 if (c.common.flags) 456cba386e94f2 Kanchan Joshi 2022-05-11 456 return -EINVAL; 456cba386e94f2 Kanchan Joshi 2022-05-11 457 456cba386e94f2 Kanchan Joshi 2022-05-11 458 c.common.command_id = 0; 456cba386e94f2 Kanchan Joshi 2022-05-11 459 c.common.nsid = cpu_to_le32(cmd->nsid); 456cba386e94f2 Kanchan Joshi 2022-05-11 460 if (!nvme_validate_passthru_nsid(ctrl, ns, le32_to_cpu(c.common.nsid))) 456cba386e94f2 Kanchan Joshi 2022-05-11 461 return -EINVAL; 456cba386e94f2 Kanchan Joshi 2022-05-11 462 456cba386e94f2 Kanchan Joshi 2022-05-11 463 c.common.cdw2[0] = cpu_to_le32(READ_ONCE(cmd->cdw2)); 456cba386e94f2 Kanchan Joshi 2022-05-11 464 c.common.cdw2[1] = cpu_to_le32(READ_ONCE(cmd->cdw3)); 456cba386e94f2 Kanchan Joshi 2022-05-11 465 c.common.metadata = 0; 456cba386e94f2 Kanchan Joshi 2022-05-11 466 c.common.dptr.prp1 = c.common.dptr.prp2 = 0; 456cba386e94f2 Kanchan Joshi 2022-05-11 467 c.common.cdw10 = cpu_to_le32(READ_ONCE(cmd->cdw10)); 456cba386e94f2 Kanchan Joshi 2022-05-11 468 c.common.cdw11 = cpu_to_le32(READ_ONCE(cmd->cdw11)); 456cba386e94f2 Kanchan Joshi 2022-05-11 469 c.common.cdw12 = cpu_to_le32(READ_ONCE(cmd->cdw12)); 456cba386e94f2 Kanchan Joshi 2022-05-11 470 c.common.cdw13 = cpu_to_le32(READ_ONCE(cmd->cdw13)); 456cba386e94f2 Kanchan Joshi 2022-05-11 471 c.common.cdw14 = cpu_to_le32(READ_ONCE(cmd->cdw14)); 456cba386e94f2 Kanchan Joshi 2022-05-11 472 c.common.cdw15 = cpu_to_le32(READ_ONCE(cmd->cdw15)); 456cba386e94f2 Kanchan Joshi 2022-05-11 473 7d9d7d59d44b7e Christoph Hellwig 2023-06-08 474 if (!nvme_cmd_allowed(ns, &c, 0, ioucmd->file->f_mode & FMODE_WRITE)) 855b7717f44b13 Kanchan Joshi 2022-10-31 475 return -EACCES; 855b7717f44b13 Kanchan Joshi 2022-10-31 476 456cba386e94f2 Kanchan Joshi 2022-05-11 477 d.metadata = READ_ONCE(cmd->metadata); 456cba386e94f2 Kanchan Joshi 2022-05-11 478 d.addr = READ_ONCE(cmd->addr); 456cba386e94f2 Kanchan Joshi 2022-05-11 479 d.data_len = READ_ONCE(cmd->data_len); 456cba386e94f2 Kanchan Joshi 2022-05-11 480 d.metadata_len = READ_ONCE(cmd->metadata_len); 456cba386e94f2 Kanchan Joshi 2022-05-11 481 d.timeout_ms = READ_ONCE(cmd->timeout_ms); 456cba386e94f2 Kanchan Joshi 2022-05-11 482 38808af53c6e72 Caleb Sander Mateos 2025-03-28 483 if (d.data_len && (ioucmd->flags & IORING_URING_CMD_FIXED)) { 3c12a8939e0474 Pavel Begunkov 2025-05-20 484 int ddir = nvme_is_write(&c) ? WRITE : READ; 38808af53c6e72 Caleb Sander Mateos 2025-03-28 485 3c12a8939e0474 Pavel Begunkov 2025-05-20 486 if (vec) 3c12a8939e0474 Pavel Begunkov 2025-05-20 487 ret = io_uring_cmd_import_fixed_vec(ioucmd, 3c12a8939e0474 Pavel Begunkov 2025-05-20 488 u64_to_user_ptr(d.addr), d.data_len, 3c12a8939e0474 Pavel Begunkov 2025-05-20 489 ddir, &iter, issue_flags); 3c12a8939e0474 Pavel Begunkov 2025-05-20 490 else 38808af53c6e72 Caleb Sander Mateos 2025-03-28 491 ret = io_uring_cmd_import_fixed(d.addr, d.data_len, 3c12a8939e0474 Pavel Begunkov 2025-05-20 492 ddir, &iter, ioucmd, issue_flags); 38808af53c6e72 Caleb Sander Mateos 2025-03-28 493 if (ret < 0) 38808af53c6e72 Caleb Sander Mateos 2025-03-28 494 return ret; 38808af53c6e72 Caleb Sander Mateos 2025-03-28 495 38808af53c6e72 Caleb Sander Mateos 2025-03-28 496 map_iter = &iter; 38808af53c6e72 Caleb Sander Mateos 2025-03-28 497 } 38808af53c6e72 Caleb Sander Mateos 2025-03-28 498 456cba386e94f2 Kanchan Joshi 2022-05-11 499 if (issue_flags & IO_URING_F_NONBLOCK) { 888545cb43d763 Anuj Gupta 2023-01-17 @500 rq_flags |= REQ_NOWAIT; 456cba386e94f2 Kanchan Joshi 2022-05-11 501 blk_flags = BLK_MQ_REQ_NOWAIT; 456cba386e94f2 Kanchan Joshi 2022-05-11 502 } 585079b6e42538 Kanchan Joshi 2022-08-23 503 if (issue_flags & IO_URING_F_IOPOLL) 585079b6e42538 Kanchan Joshi 2022-08-23 504 rq_flags |= REQ_POLLED; 456cba386e94f2 Kanchan Joshi 2022-05-11 505 470e900c8036ff Kanchan Joshi 2022-09-30 506 req = nvme_alloc_user_request(q, &c, rq_flags, blk_flags); 456cba386e94f2 Kanchan Joshi 2022-05-11 507 if (IS_ERR(req)) 456cba386e94f2 Kanchan Joshi 2022-05-11 508 return PTR_ERR(req); 470e900c8036ff Kanchan Joshi 2022-09-30 509 req->timeout = d.timeout_ms ? msecs_to_jiffies(d.timeout_ms) : 0; 470e900c8036ff Kanchan Joshi 2022-09-30 510 99fde895ff56ac Xinyu Zhang 2025-02-27 511 if (d.data_len) { 38808af53c6e72 Caleb Sander Mateos 2025-03-28 512 ret = nvme_map_user_request(req, d.addr, d.data_len, 38808af53c6e72 Caleb Sander Mateos 2025-03-28 513 nvme_to_user_ptr(d.metadata), d.metadata_len, c4b680ac286382 Pavel Begunkov 2025-05-20 514 map_iter, vec ? NVME_IOCTL_VEC : 0); 470e900c8036ff Kanchan Joshi 2022-09-30 515 if (ret) cd683de63e1d7c Caleb Sander Mateos 2025-03-28 516 goto out_free_req; 470e900c8036ff Kanchan Joshi 2022-09-30 517 } 456cba386e94f2 Kanchan Joshi 2022-05-11 518 456cba386e94f2 Kanchan Joshi 2022-05-11 519 /* to free bio on completion, as req->bio will be null at that time */ 456cba386e94f2 Kanchan Joshi 2022-05-11 520 pdu->bio = req->bio; d6aacee9255e7f Keith Busch 2023-11-30 521 pdu->req = req; c0a7ba77e81b84 Jens Axboe 2022-09-21 522 req->end_io_data = ioucmd; c0a7ba77e81b84 Jens Axboe 2022-09-21 523 req->end_io = nvme_uring_cmd_end_io; e2e530867245d0 Christoph Hellwig 2022-05-24 524 blk_execute_rq_nowait(req, false); 456cba386e94f2 Kanchan Joshi 2022-05-11 525 return -EIOCBQUEUED; cd683de63e1d7c Caleb Sander Mateos 2025-03-28 526 cd683de63e1d7c Caleb Sander Mateos 2025-03-28 527 out_free_req: cd683de63e1d7c Caleb Sander Mateos 2025-03-28 528 blk_mq_free_request(req); cd683de63e1d7c Caleb Sander Mateos 2025-03-28 529 return ret; 456cba386e94f2 Kanchan Joshi 2022-05-11 530 } 456cba386e94f2 Kanchan Joshi 2022-05-11 531 -- 0-DAY CI Kernel Test Service https://github.com/intel/lkp-tests/wiki ^ permalink raw reply [flat|nested] 5+ messages in thread
* [PATCH v2 2/2] block: enable per-cpu bio cache by default 2025-11-07 2:05 [PATCH v2 0/2] block: enable per-cpu bio cache by default Fengnan Chang 2025-11-07 2:05 ` [PATCH v2 1/2] block: use bio_alloc_bioset for passthru IO " Fengnan Chang @ 2025-11-07 2:05 ` Fengnan Chang 1 sibling, 0 replies; 5+ messages in thread From: Fengnan Chang @ 2025-11-07 2:05 UTC (permalink / raw) To: axboe, viro, brauner, jack, asml.silence, willy, djwong, hch, ritesh.list, linux-fsdevel, io-uring, linux-xfs, linux-ext4, linux-block, ming.lei, linux-nvme Cc: Fengnan Chang Since after commit 12e4e8c7ab59 ("io_uring/rw: enable bio caches for IRQ rw"), bio_put is safe for task and irq context, bio_alloc_bioset is safe for task context and no one calls in irq context, so we can enable per cpu bio cache by default. Benchmarked with t/io_uring and ext4+nvme: taskset -c 6 /root/fio/t/io_uring -p0 -d128 -b4096 -s1 -c1 -F1 -B1 -R1 -X1 -n1 -P1 /mnt/testfile base IOPS is 562K, patch IOPS is 574K. The CPU usage of bio_alloc_bioset decrease from 1.42% to 1.22%. The worst case is allocate bio in CPU A but free in CPU B, still use t/io_uring and ext4+nvme: base IOPS is 648K, patch IOPS is 647K. Also use fio test ext4/xfs with libaio/sync/io_uring on null_blk and nvme, no obvious performance regression. Signed-off-by: Fengnan Chang <changfengnan@bytedance.com> --- block/bio.c | 26 ++++++++++++-------------- block/fops.c | 4 ---- include/linux/fs.h | 3 --- io_uring/rw.c | 1 - 4 files changed, 12 insertions(+), 22 deletions(-) diff --git a/block/bio.c b/block/bio.c index b3a79285c278..64a1599a5930 100644 --- a/block/bio.c +++ b/block/bio.c @@ -516,20 +516,18 @@ struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs, if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_vecs > 0)) return NULL; - if (opf & REQ_ALLOC_CACHE) { - if (bs->cache && nr_vecs <= BIO_INLINE_VECS) { - bio = bio_alloc_percpu_cache(bdev, nr_vecs, opf, - gfp_mask, bs); - if (bio) - return bio; - /* - * No cached bio available, bio returned below marked with - * REQ_ALLOC_CACHE to particpate in per-cpu alloc cache. - */ - } else { - opf &= ~REQ_ALLOC_CACHE; - } - } + if (bs->cache && nr_vecs <= BIO_INLINE_VECS) { + opf |= REQ_ALLOC_CACHE; + bio = bio_alloc_percpu_cache(bdev, nr_vecs, opf, + gfp_mask, bs); + if (bio) + return bio; + /* + * No cached bio available, bio returned below marked with + * REQ_ALLOC_CACHE to participate in per-cpu alloc cache. + */ + } else + opf &= ~REQ_ALLOC_CACHE; /* * submit_bio_noacct() converts recursion to iteration; this means if diff --git a/block/fops.c b/block/fops.c index 5e3db9fead77..7ef2848244b1 100644 --- a/block/fops.c +++ b/block/fops.c @@ -184,8 +184,6 @@ static ssize_t __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, loff_t pos = iocb->ki_pos; int ret = 0; - if (iocb->ki_flags & IOCB_ALLOC_CACHE) - opf |= REQ_ALLOC_CACHE; bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL, &blkdev_dio_pool); dio = container_of(bio, struct blkdev_dio, bio); @@ -333,8 +331,6 @@ static ssize_t __blkdev_direct_IO_async(struct kiocb *iocb, loff_t pos = iocb->ki_pos; int ret = 0; - if (iocb->ki_flags & IOCB_ALLOC_CACHE) - opf |= REQ_ALLOC_CACHE; bio = bio_alloc_bioset(bdev, nr_pages, opf, GFP_KERNEL, &blkdev_dio_pool); dio = container_of(bio, struct blkdev_dio, bio); diff --git a/include/linux/fs.h b/include/linux/fs.h index c895146c1444..1be899ac8b5a 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -365,8 +365,6 @@ struct readahead_control; /* iocb->ki_waitq is valid */ #define IOCB_WAITQ (1 << 19) #define IOCB_NOIO (1 << 20) -/* can use bio alloc cache */ -#define IOCB_ALLOC_CACHE (1 << 21) /* * IOCB_DIO_CALLER_COMP can be set by the iocb owner, to indicate that the * iocb completion can be passed back to the owner for execution from a safe @@ -399,7 +397,6 @@ struct readahead_control; { IOCB_WRITE, "WRITE" }, \ { IOCB_WAITQ, "WAITQ" }, \ { IOCB_NOIO, "NOIO" }, \ - { IOCB_ALLOC_CACHE, "ALLOC_CACHE" }, \ { IOCB_DIO_CALLER_COMP, "CALLER_COMP" }, \ { IOCB_AIO_RW, "AIO_RW" }, \ { IOCB_HAS_METADATA, "AIO_HAS_METADATA" } diff --git a/io_uring/rw.c b/io_uring/rw.c index 5b2241a5813c..c0c59eb358a8 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -862,7 +862,6 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode, int rw_type) ret = kiocb_set_rw_flags(kiocb, rw->flags, rw_type); if (unlikely(ret)) return ret; - kiocb->ki_flags |= IOCB_ALLOC_CACHE; /* * If the file is marked O_NONBLOCK, still allow retry for it if it -- 2.39.5 (Apple Git-154) ^ permalink raw reply related [flat|nested] 5+ messages in thread
end of thread, other threads:[~2025-11-08 1:16 UTC | newest] Thread overview: 5+ messages (download: mbox.gz follow: Atom feed -- links below jump to the message on this page -- 2025-11-07 2:05 [PATCH v2 0/2] block: enable per-cpu bio cache by default Fengnan Chang 2025-11-07 2:05 ` [PATCH v2 1/2] block: use bio_alloc_bioset for passthru IO " Fengnan Chang 2025-11-07 3:17 ` kernel test robot 2025-11-08 1:15 ` kernel test robot 2025-11-07 2:05 ` [PATCH v2 2/2] block: enable per-cpu bio cache " Fengnan Chang
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox; as well as URLs for NNTP newsgroup(s).