* [PATCH 6.18.y 2/6] block: add helpers to bounce buffer an iov_iter into bios
2026-09-08 12:19 ` [PATCH 6.18.y 1/6] iov_iter: extract a iov_iter_extract_bvecs helper from bio code Sasha Levin
@ 2026-09-08 12:19 ` Sasha Levin
2026-09-08 12:19 ` [PATCH 6.18.y 3/6] iomap: simplify iomap_dio_bio_iter Sasha Levin
` (3 subsequent siblings)
4 siblings, 0 replies; 7+ messages in thread
From: Sasha Levin @ 2026-09-08 12:19 UTC (permalink / raw)
To: stable
Cc: Christoph Hellwig, Anuj Gupta, Damien Le Moal, Johannes Thumshirn,
Martin K. Petersen, Darrick J. Wong, Jens Axboe, Sasha Levin
From: Christoph Hellwig <hch@lst.de>
[ Upstream commit 8dd5e7c75d7bb2635c7efd219ff20693fc24096a ]
Add helpers to implement bounce buffering of data into a bio to implement
direct I/O for cases where direct user access is not possible because
stable in-flight data is required. These are intended to be used as
easily as bio_iov_iter_get_pages for the zero-copy path.
The write side is trivial and just copies data into the bounce buffer.
The read side is a lot more complex because it needs to perform the copy
from the completion context, and without preserving the iov_iter through
the call chain. It steals a trick from the integrity data user interface
and uses the first vector in the bio for the bounce buffer data that is
fed to the block I/O stack, and uses the others to record the user
buffer fragments.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Anuj Gupta <anuj20.g@samsung.com>
Reviewed-by: Damien Le Moal <dlemoal@kernel.org>
Reviewed-by: Johannes Thumshirn <johannes.thumshirn@wdc.com>
Tested-by: Anuj Gupta <anuj20.g@samsung.com>
Reviewed-by: Martin K. Petersen <martin.petersen@oracle.com>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Stable-dep-of: 14b007e17881 ("block: validate user space vectors during extraction")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
block/bio.c | 179 ++++++++++++++++++++++++++++++++++++++++++++
include/linux/bio.h | 26 +++++++
2 files changed, 205 insertions(+)
diff --git a/block/bio.c b/block/bio.c
index 63594139efe05..600f110e5c0dc 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -1258,6 +1258,185 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
return bio_iov_iter_align_down(bio, iter, len_align_mask);
}
+static struct folio *folio_alloc_greedy(gfp_t gfp, size_t *size)
+{
+ struct folio *folio;
+
+ while (*size > PAGE_SIZE) {
+ folio = folio_alloc(gfp | __GFP_NORETRY, get_order(*size));
+ if (folio)
+ return folio;
+ *size = rounddown_pow_of_two(*size - 1);
+ }
+
+ return folio_alloc(gfp, get_order(*size));
+}
+
+static void bio_free_folios(struct bio *bio)
+{
+ struct bio_vec *bv;
+ int i;
+
+ bio_for_each_bvec_all(bv, bio, i) {
+ struct folio *folio = page_folio(bv->bv_page);
+
+ if (!is_zero_folio(folio))
+ folio_put(folio);
+ }
+}
+
+static int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter)
+{
+ size_t total_len = iov_iter_count(iter);
+
+ if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)))
+ return -EINVAL;
+ if (WARN_ON_ONCE(bio->bi_iter.bi_size))
+ return -EINVAL;
+ if (WARN_ON_ONCE(bio->bi_vcnt >= bio->bi_max_vecs))
+ return -EINVAL;
+
+ do {
+ size_t this_len = min(total_len, SZ_1M);
+ struct folio *folio;
+
+ if (this_len > PAGE_SIZE * 2)
+ this_len = rounddown_pow_of_two(this_len);
+
+ if (bio->bi_iter.bi_size > BIO_MAX_SIZE - this_len)
+ break;
+
+ folio = folio_alloc_greedy(GFP_KERNEL, &this_len);
+ if (!folio)
+ break;
+ bio_add_folio_nofail(bio, folio, this_len, 0);
+
+ if (copy_from_iter(folio_address(folio), this_len, iter) !=
+ this_len) {
+ bio_free_folios(bio);
+ return -EFAULT;
+ }
+
+ total_len -= this_len;
+ } while (total_len && bio->bi_vcnt < bio->bi_max_vecs);
+
+ if (!bio->bi_iter.bi_size)
+ return -ENOMEM;
+ return 0;
+}
+
+static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter)
+{
+ size_t len = min(iov_iter_count(iter), SZ_1M);
+ struct folio *folio;
+
+ folio = folio_alloc_greedy(GFP_KERNEL, &len);
+ if (!folio)
+ return -ENOMEM;
+
+ do {
+ ssize_t ret;
+
+ ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec + 1, len,
+ &bio->bi_vcnt, bio->bi_max_vecs - 1, 0);
+ if (ret <= 0) {
+ if (!bio->bi_vcnt)
+ return ret;
+ break;
+ }
+ len -= ret;
+ bio->bi_iter.bi_size += ret;
+ } while (len && bio->bi_vcnt < bio->bi_max_vecs - 1);
+
+ /*
+ * Set the folio directly here. The above loop has already calculated
+ * the correct bi_size, and we use bi_vcnt for the user buffers. That
+ * is safe as bi_vcnt is only used by the submitter and not the actual
+ * I/O path.
+ */
+ bvec_set_folio(&bio->bi_io_vec[0], folio, bio->bi_iter.bi_size, 0);
+ if (iov_iter_extract_will_pin(iter))
+ bio_set_flag(bio, BIO_PAGE_PINNED);
+ return 0;
+}
+
+/**
+ * bio_iov_iter_bounce - bounce buffer data from an iter into a bio
+ * @bio: bio to send
+ * @iter: iter to read from / write into
+ *
+ * Helper for direct I/O implementations that need to bounce buffer because
+ * we need to checksum the data or perform other operations that require
+ * consistency. Allocates folios to back the bounce buffer, and for writes
+ * copies the data into it. Needs to be paired with bio_iov_iter_unbounce()
+ * called on completion.
+ */
+int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter)
+{
+ if (op_is_write(bio_op(bio)))
+ return bio_iov_iter_bounce_write(bio, iter);
+ return bio_iov_iter_bounce_read(bio, iter);
+}
+
+static void bvec_unpin(struct bio_vec *bv, bool mark_dirty)
+{
+ struct folio *folio = page_folio(bv->bv_page);
+ size_t nr_pages = (bv->bv_offset + bv->bv_len - 1) / PAGE_SIZE -
+ bv->bv_offset / PAGE_SIZE + 1;
+
+ if (mark_dirty)
+ folio_mark_dirty_lock(folio);
+ unpin_user_folio(folio, nr_pages);
+}
+
+static void bio_iov_iter_unbounce_read(struct bio *bio, bool is_error,
+ bool mark_dirty)
+{
+ unsigned int len = bio->bi_io_vec[0].bv_len;
+
+ if (likely(!is_error)) {
+ void *buf = bvec_virt(&bio->bi_io_vec[0]);
+ struct iov_iter to;
+
+ iov_iter_bvec(&to, ITER_DEST, bio->bi_io_vec + 1, bio->bi_vcnt,
+ len);
+ /* copying to pinned pages should always work */
+ WARN_ON_ONCE(copy_to_iter(buf, len, &to) != len);
+ } else {
+ /* No need to mark folios dirty if never copied to them */
+ mark_dirty = false;
+ }
+
+ if (bio_flagged(bio, BIO_PAGE_PINNED)) {
+ int i;
+
+ for (i = 0; i < bio->bi_vcnt; i++)
+ bvec_unpin(&bio->bi_io_vec[1 + i], mark_dirty);
+ }
+
+ folio_put(page_folio(bio->bi_io_vec[0].bv_page));
+}
+
+/**
+ * bio_iov_iter_unbounce - finish a bounce buffer operation
+ * @bio: completed bio
+ * @is_error: %true if an I/O error occurred and data should not be copied
+ * @mark_dirty: If %true, folios will be marked dirty.
+ *
+ * Helper for direct I/O implementations that need to bounce buffer because
+ * we need to checksum the data or perform other operations that require
+ * consistency. Called to complete a bio set up by bio_iov_iter_bounce().
+ * Copies data back for reads, and marks the original folios dirty if
+ * requested and then frees the bounce buffer.
+ */
+void bio_iov_iter_unbounce(struct bio *bio, bool is_error, bool mark_dirty)
+{
+ if (op_is_write(bio_op(bio)))
+ bio_free_folios(bio);
+ else
+ bio_iov_iter_unbounce_read(bio, is_error, mark_dirty);
+}
+
static void submit_bio_wait_endio(struct bio *bio)
{
complete(bio->bi_private);
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 16c1c85613b76..627973d7ed3a7 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -401,6 +401,29 @@ static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs)
return iov_iter_npages(iter, max_segs);
}
+/**
+ * bio_iov_bounce_nr_vecs - calculate number of bvecs for a bounce bio
+ * @iter: iter to bounce from
+ * @op: REQ_OP_* for the bio
+ *
+ * Calculates how many bvecs are needed for the next bio to bounce from/to
+ * @iter.
+ */
+static inline unsigned short
+bio_iov_bounce_nr_vecs(struct iov_iter *iter, blk_opf_t op)
+{
+ /*
+ * We still need to bounce bvec iters, so don't special case them
+ * here unlike in bio_iov_vecs_to_alloc.
+ *
+ * For reads we need to use a vector for the bounce buffer, account
+ * for that here.
+ */
+ if (op_is_write(op))
+ return iov_iter_npages(iter, BIO_MAX_VECS);
+ return iov_iter_npages(iter, BIO_MAX_VECS - 1) + 1;
+}
+
struct request_queue;
void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table,
@@ -454,6 +477,9 @@ void __bio_release_pages(struct bio *bio, bool mark_dirty);
extern void bio_set_pages_dirty(struct bio *bio);
extern void bio_check_pages_dirty(struct bio *bio);
+int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter);
+void bio_iov_iter_unbounce(struct bio *bio, bool is_error, bool mark_dirty);
+
extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
struct bio *src, struct bvec_iter *src_iter);
extern void bio_copy_data(struct bio *dst, struct bio *src);
--
2.53.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 6.18.y 3/6] iomap: simplify iomap_dio_bio_iter
2026-09-08 12:19 ` [PATCH 6.18.y 1/6] iov_iter: extract a iov_iter_extract_bvecs helper from bio code Sasha Levin
2026-09-08 12:19 ` [PATCH 6.18.y 2/6] block: add helpers to bounce buffer an iov_iter into bios Sasha Levin
@ 2026-09-08 12:19 ` Sasha Levin
2026-09-08 12:19 ` [PATCH 6.18.y 4/6] iomap: split out the per-bio logic from iomap_dio_bio_iter Sasha Levin
` (2 subsequent siblings)
4 siblings, 0 replies; 7+ messages in thread
From: Sasha Levin @ 2026-09-08 12:19 UTC (permalink / raw)
To: stable
Cc: Christoph Hellwig, Damien Le Moal, Darrick J. Wong, Anuj Gupta,
Jens Axboe, Sasha Levin
From: Christoph Hellwig <hch@lst.de>
[ Upstream commit 6e7a6c80198ead08b11aa6cdc92e60a42fc5895f ]
Use iov_iter_count to check if we need to continue as that just reads
a field in the iov_iter, and only use bio_iov_vecs_to_alloc to calculate
the actual number of vectors to allocate for the bio.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Damien Le Moal <dlemoal@kernel.org>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Tested-by: Anuj Gupta <anuj20.g@samsung.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Stable-dep-of: 14b007e17881 ("block: validate user space vectors during extraction")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
fs/iomap/direct-io.c | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
index e73c71f39bd45..6ea14d07dc618 100644
--- a/fs/iomap/direct-io.c
+++ b/fs/iomap/direct-io.c
@@ -344,7 +344,7 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
blk_opf_t bio_opf = REQ_SYNC | REQ_IDLE;
struct bio *bio;
bool need_zeroout = false;
- int nr_pages, ret = 0;
+ int ret = 0;
u64 copied = 0;
size_t orig_count;
@@ -426,7 +426,6 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
goto out;
}
- nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS);
do {
size_t n;
@@ -439,7 +438,9 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
goto out;
}
- bio = iomap_dio_alloc_bio(iter, dio, nr_pages, bio_opf);
+ bio = iomap_dio_alloc_bio(iter, dio,
+ bio_iov_vecs_to_alloc(dio->submit.iter,
+ BIO_MAX_VECS), bio_opf);
fscrypt_set_bio_crypt_ctx(bio, inode, pos >> inode->i_blkbits,
GFP_KERNEL);
bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
@@ -481,16 +482,14 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
dio->size += n;
copied += n;
- nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter,
- BIO_MAX_VECS);
/*
* We can only poll for single bio I/Os.
*/
- if (nr_pages)
+ if (iov_iter_count(dio->submit.iter))
dio->iocb->ki_flags &= ~IOCB_HIPRI;
iomap_dio_submit_bio(iter, dio, bio, pos);
pos += n;
- } while (nr_pages);
+ } while (iov_iter_count(dio->submit.iter));
/*
* We need to zeroout the tail of a sub-block write if the extent type
--
2.53.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 6.18.y 4/6] iomap: split out the per-bio logic from iomap_dio_bio_iter
2026-09-08 12:19 ` [PATCH 6.18.y 1/6] iov_iter: extract a iov_iter_extract_bvecs helper from bio code Sasha Levin
2026-09-08 12:19 ` [PATCH 6.18.y 2/6] block: add helpers to bounce buffer an iov_iter into bios Sasha Levin
2026-09-08 12:19 ` [PATCH 6.18.y 3/6] iomap: simplify iomap_dio_bio_iter Sasha Levin
@ 2026-09-08 12:19 ` Sasha Levin
2026-09-08 12:19 ` [PATCH 6.18.y 5/6] iomap: rename IOMAP_DIO_DIRTY to IOMAP_DIO_USER_BACKED Sasha Levin
2026-09-08 12:19 ` [PATCH 6.18.y 6/6] block: validate user space vectors during extraction Sasha Levin
4 siblings, 0 replies; 7+ messages in thread
From: Sasha Levin @ 2026-09-08 12:19 UTC (permalink / raw)
To: stable
Cc: Christoph Hellwig, Damien Le Moal, Darrick J. Wong, Anuj Gupta,
Jens Axboe, Sasha Levin
From: Christoph Hellwig <hch@lst.de>
[ Upstream commit 2631c94602297090febd8f93d6f96d9d2045466d ]
Factor out a separate helper that builds and submits a single bio.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Damien Le Moal <dlemoal@kernel.org>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Tested-by: Anuj Gupta <anuj20.g@samsung.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Stable-dep-of: 14b007e17881 ("block: validate user space vectors during extraction")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
block/bio.c | 2 +-
fs/iomap/direct-io.c | 157 ++++++++++++++++++++++----------------
include/linux/blk_types.h | 3 +-
include/linux/iomap.h | 9 +++
4 files changed, 104 insertions(+), 67 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index 600f110e5c0dc..56eae6ef6ec07 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -1243,7 +1243,7 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
ssize_t ret;
ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec,
- UINT_MAX - bio->bi_iter.bi_size,
+ BIO_MAX_SIZE - bio->bi_iter.bi_size,
&bio->bi_vcnt, bio->bi_max_vecs, flags);
if (ret <= 0) {
if (!bio->bi_vcnt)
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
index 6ea14d07dc618..2dc3c26cf870e 100644
--- a/fs/iomap/direct-io.c
+++ b/fs/iomap/direct-io.c
@@ -233,32 +233,48 @@ static void iomap_dio_done(struct iomap_dio *dio)
void iomap_dio_bio_end_io(struct bio *bio)
{
struct iomap_dio *dio = bio->bi_private;
+ bool should_bounce = (dio->flags & IOMAP_DIO_BOUNCE);
bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
if (bio->bi_status)
iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));
- if (atomic_dec_and_test(&dio->ref))
- iomap_dio_done(dio);
-
- if (should_dirty) {
+ if (should_bounce) {
+ bio_iov_iter_unbounce(bio, !!dio->error, should_dirty);
+ bio_put(bio);
+ } else if (should_dirty) {
bio_check_pages_dirty(bio);
} else {
bio_release_pages(bio, false);
bio_put(bio);
}
+
+ if (atomic_dec_and_test(&dio->ref))
+ iomap_dio_done(dio);
}
EXPORT_SYMBOL_GPL(iomap_dio_bio_end_io);
u32 iomap_finish_ioend_direct(struct iomap_ioend *ioend)
{
struct iomap_dio *dio = ioend->io_bio.bi_private;
+ bool should_bounce = (dio->flags & IOMAP_DIO_BOUNCE);
bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
u32 vec_count = ioend->io_bio.bi_vcnt;
if (ioend->io_error)
iomap_dio_set_error(dio, ioend->io_error);
+ if (should_bounce) {
+ bio_iov_iter_unbounce(&ioend->io_bio, !!dio->error,
+ should_dirty);
+ bio_put(&ioend->io_bio);
+ } else if (should_dirty) {
+ bio_check_pages_dirty(&ioend->io_bio);
+ } else {
+ bio_release_pages(&ioend->io_bio, false);
+ bio_put(&ioend->io_bio);
+ }
+
if (atomic_dec_and_test(&dio->ref)) {
/*
* Try to avoid another context switch for the completion given
@@ -276,13 +292,6 @@ u32 iomap_finish_ioend_direct(struct iomap_ioend *ioend)
iomap_dio_done(dio);
}
- if (should_dirty) {
- bio_check_pages_dirty(&ioend->io_bio);
- } else {
- bio_release_pages(&ioend->io_bio, false);
- bio_put(&ioend->io_bio);
- }
-
/*
* Return the number of bvecs completed as even direct I/O completions
* do significant per-folio work and we'll still want to give up the
@@ -334,6 +343,65 @@ static inline bool iomap_dio_can_use_fua(const struct iomap *iomap,
return !bdev_write_cache(iomap->bdev) || bdev_fua(iomap->bdev);
}
+static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter,
+ struct iomap_dio *dio, loff_t pos, unsigned int alignment,
+ blk_opf_t op)
+{
+ unsigned int nr_vecs;
+ struct bio *bio;
+ ssize_t ret;
+
+ if (dio->flags & IOMAP_DIO_BOUNCE)
+ nr_vecs = bio_iov_bounce_nr_vecs(dio->submit.iter, op);
+ else
+ nr_vecs = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS);
+
+ bio = iomap_dio_alloc_bio(iter, dio, nr_vecs, op);
+ fscrypt_set_bio_crypt_ctx(bio, iter->inode,
+ pos >> iter->inode->i_blkbits, GFP_KERNEL);
+ bio->bi_iter.bi_sector = iomap_sector(&iter->iomap, pos);
+ bio->bi_write_hint = iter->inode->i_write_hint;
+ bio->bi_ioprio = dio->iocb->ki_ioprio;
+ bio->bi_private = dio;
+ bio->bi_end_io = iomap_dio_bio_end_io;
+
+ if (dio->flags & IOMAP_DIO_BOUNCE)
+ ret = bio_iov_iter_bounce(bio, dio->submit.iter);
+ else
+ ret = bio_iov_iter_get_pages(bio, dio->submit.iter,
+ alignment - 1);
+ if (unlikely(ret))
+ goto out_put_bio;
+ ret = bio->bi_iter.bi_size;
+
+ /*
+ * An atomic write bio must cover the complete length. If it doesn't,
+ * error out.
+ */
+ if ((op & REQ_ATOMIC) && WARN_ON_ONCE(ret != iomap_length(iter))) {
+ ret = -EINVAL;
+ goto out_put_bio;
+ }
+
+ if (dio->flags & IOMAP_DIO_WRITE)
+ task_io_account_write(ret);
+ else if ((dio->flags & IOMAP_DIO_DIRTY) &&
+ !(dio->flags & IOMAP_DIO_BOUNCE))
+ bio_set_pages_dirty(bio);
+
+ /*
+ * We can only poll for single bio I/Os.
+ */
+ if (iov_iter_count(dio->submit.iter))
+ dio->iocb->ki_flags &= ~IOCB_HIPRI;
+ iomap_dio_submit_bio(iter, dio, bio, pos);
+ return ret;
+
+out_put_bio:
+ bio_put(bio);
+ return ret;
+}
+
static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
{
const struct iomap *iomap = &iter->iomap;
@@ -342,13 +410,14 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
const loff_t length = iomap_length(iter);
loff_t pos = iter->pos;
blk_opf_t bio_opf = REQ_SYNC | REQ_IDLE;
- struct bio *bio;
bool need_zeroout = false;
- int ret = 0;
u64 copied = 0;
size_t orig_count;
+ unsigned int alignment;
+ ssize_t ret = 0;
- if ((pos | length) & (bdev_logical_block_size(iomap->bdev) - 1))
+ alignment = bdev_logical_block_size(iomap->bdev);
+ if ((pos | length) & (alignment - 1))
return -EINVAL;
if (dio->flags & IOMAP_DIO_WRITE) {
@@ -427,68 +496,27 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
}
do {
- size_t n;
-
/*
* If completions already occurred and reported errors, give up now and
* don't bother submitting more bios.
*/
- if (unlikely(data_race(dio->error))) {
- ret = 0;
+ if (unlikely(data_race(dio->error)))
goto out;
- }
- bio = iomap_dio_alloc_bio(iter, dio,
- bio_iov_vecs_to_alloc(dio->submit.iter,
- BIO_MAX_VECS), bio_opf);
- fscrypt_set_bio_crypt_ctx(bio, inode, pos >> inode->i_blkbits,
- GFP_KERNEL);
- bio->bi_iter.bi_sector = iomap_sector(iomap, pos);
- bio->bi_write_hint = inode->i_write_hint;
- bio->bi_ioprio = dio->iocb->ki_ioprio;
- bio->bi_private = dio;
- bio->bi_end_io = iomap_dio_bio_end_io;
-
- ret = bio_iov_iter_get_pages(bio, dio->submit.iter,
- bdev_logical_block_size(iomap->bdev) - 1);
- if (unlikely(ret)) {
+ ret = iomap_dio_bio_iter_one(iter, dio, pos, alignment, bio_opf);
+ if (unlikely(ret < 0)) {
/*
* We have to stop part way through an IO. We must fall
* through to the sub-block tail zeroing here, otherwise
* this short IO may expose stale data in the tail of
* the block we haven't written data to.
*/
- bio_put(bio);
- goto zero_tail;
- }
-
- n = bio->bi_iter.bi_size;
- if (WARN_ON_ONCE((bio_opf & REQ_ATOMIC) && n != length)) {
- /*
- * An atomic write bio must cover the complete length,
- * which it doesn't, so error. We may need to zero out
- * the tail (complete FS block), similar to when
- * bio_iov_iter_get_pages() returns an error, above.
- */
- ret = -EINVAL;
- bio_put(bio);
- goto zero_tail;
+ break;
}
- if (dio->flags & IOMAP_DIO_WRITE)
- task_io_account_write(n);
- else if (dio->flags & IOMAP_DIO_DIRTY)
- bio_set_pages_dirty(bio);
-
- dio->size += n;
- copied += n;
-
- /*
- * We can only poll for single bio I/Os.
- */
- if (iov_iter_count(dio->submit.iter))
- dio->iocb->ki_flags &= ~IOCB_HIPRI;
- iomap_dio_submit_bio(iter, dio, bio, pos);
- pos += n;
+ dio->size += ret;
+ copied += ret;
+ pos += ret;
+ ret = 0;
} while (iov_iter_count(dio->submit.iter));
/*
@@ -497,7 +525,6 @@ static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)
* the block tail in the latter case, we can expose stale data via mmap
* reads of the EOF block.
*/
-zero_tail:
if (need_zeroout ||
((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {
/* zero out from the end of the write to the end of the block */
@@ -644,7 +671,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
dio->i_size = i_size_read(inode);
dio->dops = dops;
dio->error = 0;
- dio->flags = 0;
+ dio->flags = dio_flags & IOMAP_DIO_BOUNCE;
dio->done_before = done_before;
dio->submit.iter = iter;
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 4e2e3aed32f5f..fca0dfcd5a522 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -269,7 +269,8 @@ struct bio {
};
#define BIO_RESET_BYTES offsetof(struct bio, bi_max_vecs)
-#define BIO_MAX_SECTORS (UINT_MAX >> SECTOR_SHIFT)
+#define BIO_MAX_SIZE UINT_MAX
+#define BIO_MAX_SECTORS (BIO_MAX_SIZE >> SECTOR_SHIFT)
static inline struct bio_vec *bio_inline_vecs(struct bio *bio)
{
diff --git a/include/linux/iomap.h b/include/linux/iomap.h
index 73dceabc21c8c..c536e659970b2 100644
--- a/include/linux/iomap.h
+++ b/include/linux/iomap.h
@@ -518,6 +518,15 @@ struct iomap_dio_ops {
*/
#define IOMAP_DIO_PARTIAL (1 << 2)
+/*
+ * Bounce buffer instead of using zero copy access.
+ *
+ * This is needed if the device needs stable data to checksum or generate
+ * parity. The file system must hook into the I/O submission and offload
+ * completions to user context for reads when this is set.
+ */
+#define IOMAP_DIO_BOUNCE (1 << 4)
+
ssize_t iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
const struct iomap_ops *ops, const struct iomap_dio_ops *dops,
unsigned int dio_flags, void *private, size_t done_before);
--
2.53.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 6.18.y 5/6] iomap: rename IOMAP_DIO_DIRTY to IOMAP_DIO_USER_BACKED
2026-09-08 12:19 ` [PATCH 6.18.y 1/6] iov_iter: extract a iov_iter_extract_bvecs helper from bio code Sasha Levin
` (2 preceding siblings ...)
2026-09-08 12:19 ` [PATCH 6.18.y 4/6] iomap: split out the per-bio logic from iomap_dio_bio_iter Sasha Levin
@ 2026-09-08 12:19 ` Sasha Levin
2026-09-08 12:19 ` [PATCH 6.18.y 6/6] block: validate user space vectors during extraction Sasha Levin
4 siblings, 0 replies; 7+ messages in thread
From: Sasha Levin @ 2026-09-08 12:19 UTC (permalink / raw)
To: stable
Cc: Christoph Hellwig, Damien Le Moal, Darrick J. Wong, Anuj Gupta,
Jens Axboe, Sasha Levin
From: Christoph Hellwig <hch@lst.de>
[ Upstream commit c96b8b220271024c04289d6d9779dc2ccbd12be2 ]
Match the more descriptive iov_iter terminology instead of encoding
what we do with them for reads only.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Damien Le Moal <dlemoal@kernel.org>
Reviewed-by: Darrick J. Wong <djwong@kernel.org>
Tested-by: Anuj Gupta <anuj20.g@samsung.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Stable-dep-of: 14b007e17881 ("block: validate user space vectors during extraction")
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
fs/iomap/direct-io.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
index 2dc3c26cf870e..6c7293c3ba592 100644
--- a/fs/iomap/direct-io.c
+++ b/fs/iomap/direct-io.c
@@ -22,7 +22,7 @@
#define IOMAP_DIO_WRITE_THROUGH (1U << 28)
#define IOMAP_DIO_NEED_SYNC (1U << 29)
#define IOMAP_DIO_WRITE (1U << 30)
-#define IOMAP_DIO_DIRTY (1U << 31)
+#define IOMAP_DIO_USER_BACKED (1U << 31)
/*
* Used for sub block zeroing in iomap_dio_zero()
@@ -234,7 +234,7 @@ void iomap_dio_bio_end_io(struct bio *bio)
{
struct iomap_dio *dio = bio->bi_private;
bool should_bounce = (dio->flags & IOMAP_DIO_BOUNCE);
- bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
+ bool should_dirty = (dio->flags & IOMAP_DIO_USER_BACKED);
if (bio->bi_status)
iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));
@@ -258,7 +258,7 @@ u32 iomap_finish_ioend_direct(struct iomap_ioend *ioend)
{
struct iomap_dio *dio = ioend->io_bio.bi_private;
bool should_bounce = (dio->flags & IOMAP_DIO_BOUNCE);
- bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
+ bool should_dirty = (dio->flags & IOMAP_DIO_USER_BACKED);
u32 vec_count = ioend->io_bio.bi_vcnt;
if (ioend->io_error)
@@ -385,7 +385,7 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter,
if (dio->flags & IOMAP_DIO_WRITE)
task_io_account_write(ret);
- else if ((dio->flags & IOMAP_DIO_DIRTY) &&
+ else if ((dio->flags & IOMAP_DIO_USER_BACKED) &&
!(dio->flags & IOMAP_DIO_BOUNCE))
bio_set_pages_dirty(bio);
@@ -688,7 +688,7 @@ __iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter,
goto out_free_dio;
if (user_backed_iter(iter))
- dio->flags |= IOMAP_DIO_DIRTY;
+ dio->flags |= IOMAP_DIO_USER_BACKED;
ret = kiocb_write_and_wait(iocb, iomi.len);
if (ret)
--
2.53.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 6.18.y 6/6] block: validate user space vectors during extraction
2026-09-08 12:19 ` [PATCH 6.18.y 1/6] iov_iter: extract a iov_iter_extract_bvecs helper from bio code Sasha Levin
` (3 preceding siblings ...)
2026-09-08 12:19 ` [PATCH 6.18.y 5/6] iomap: rename IOMAP_DIO_DIRTY to IOMAP_DIO_USER_BACKED Sasha Levin
@ 2026-09-08 12:19 ` Sasha Levin
4 siblings, 0 replies; 7+ messages in thread
From: Sasha Levin @ 2026-09-08 12:19 UTC (permalink / raw)
To: stable
Cc: Keith Busch, Hannes Reinecke, Christoph Hellwig, Jens Axboe,
Sasha Levin
From: Keith Busch <kbusch@kernel.org>
[ Upstream commit 14b007e178811db72fbb1ebb3535160db6ec1e6a ]
The bio-based drivers don't necessarily check the alignment split, and
stacking block drivers don't always handle a misalignment detected after
submitting the bio. Validate user vectors against the device's
dma_alignment as the bio is built from the iov_iter, rejecting
misaligned early with -EINVAL.
Cc: stable@vger.kernel.org
Fixes: 5ff3f74e145a ("block: simplify direct io validity check")
Fixes: 7eac33186957 ("iomap: simplify direct io validity check")
Reviewed-by: Hannes Reinecke <hare@kernel.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Keith Busch <kbusch@kernel.org>
Link: https://patch.msgid.link/20260720201057.1862857-6-kbusch@meta.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
block/bio.c | 56 +++++++++++++++++++++++++++++++++++++++++---
block/blk-map.c | 2 +-
block/fops.c | 2 +-
fs/iomap/direct-io.c | 1 +
include/linux/bio.h | 2 +-
include/linux/uio.h | 10 +++++++-
lib/iov_iter.c | 9 ++++++-
7 files changed, 74 insertions(+), 8 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index 56eae6ef6ec07..9d5cc882ca90e 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -1199,10 +1199,45 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter,
return 0;
}
+#ifdef CONFIG_DEBUG_KERNEL
+static inline bool bio_iov_bvec_aligned(const struct bio *bio,
+ unsigned mem_align_mask)
+{
+ struct bvec_iter iter;
+ struct bio_vec bv;
+
+ /*
+ * Correct callers never break the alignment requirements, so this
+ * exhaustive check is only paid for in debug builds.
+ */
+ for_each_mp_bvec(bv, bio->bi_io_vec, iter, bio->bi_iter)
+ if ((bv.bv_offset | bv.bv_len) & mem_align_mask)
+ return false;
+ return true;
+}
+#else
+static inline bool bio_iov_bvec_aligned(const struct bio *bio,
+ unsigned mem_align_mask)
+{
+ /*
+ * We forward the bio_vec as-is, so ITER_BVEC callers must provide
+ * segments already aligned to the device's DMA alignment. The only
+ * unchecked user-controllable offset that reaches here is an io_uring
+ * registered buffer where just the first segment can be unaligned
+ * (the rest is virtually contiguous), so checking only that one is
+ * sufficient to know if the entire vector is valid.
+ */
+ return !(mp_bvec_iter_offset(bio->bi_io_vec, bio->bi_iter) &
+ mem_align_mask);
+}
+#endif
+
/**
* bio_iov_iter_get_pages - add user or kernel pages to a bio
* @bio: bio to add pages to
* @iter: iov iterator describing the region to be added
+ * @mem_align_mask: the mask the source address and length must be aligned to,
+ * 0 for no requirement
* @len_align_mask: the mask to align the total size to, 0 for any length
*
* This takes either an iterator pointing to user memory, or one pointing to
@@ -1221,7 +1256,7 @@ static int bio_iov_iter_align_down(struct bio *bio, struct iov_iter *iter,
* is returned only if 0 pages could be pinned.
*/
int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
- unsigned len_align_mask)
+ unsigned mem_align_mask, unsigned len_align_mask)
{
iov_iter_extraction_t flags = 0;
@@ -1230,6 +1265,10 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
if (iov_iter_is_bvec(iter)) {
bio_iov_bvec_set(bio, iter);
+
+ if (!bio_iov_bvec_aligned(bio, mem_align_mask))
+ return -EINVAL;
+
iov_iter_advance(iter, bio->bi_iter.bi_size);
return 0;
}
@@ -1244,8 +1283,19 @@ int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec,
BIO_MAX_SIZE - bio->bi_iter.bi_size,
- &bio->bi_vcnt, bio->bi_max_vecs, flags);
+ &bio->bi_vcnt, bio->bi_max_vecs,
+ mem_align_mask, flags);
if (ret <= 0) {
+ /*
+ * A misaligned vector fails the whole I/O. Release any
+ * pages pinned by earlier iterations before returning
+ * since this bio won't be submitted to release them.
+ */
+ if (ret == -EINVAL) {
+ bio_release_pages(bio, false);
+ bio_clear_flag(bio, BIO_PAGE_PINNED);
+ bio->bi_vcnt = 0;
+ }
if (!bio->bi_vcnt)
return ret;
break;
@@ -1338,7 +1388,7 @@ static int bio_iov_iter_bounce_read(struct bio *bio, struct iov_iter *iter)
ssize_t ret;
ret = iov_iter_extract_bvecs(iter, bio->bi_io_vec + 1, len,
- &bio->bi_vcnt, bio->bi_max_vecs - 1, 0);
+ &bio->bi_vcnt, bio->bi_max_vecs - 1, 0, 0);
if (ret <= 0) {
if (!bio->bi_vcnt)
return ret;
diff --git a/block/blk-map.c b/block/blk-map.c
index 60faf036fb6e4..1cf7165d49149 100644
--- a/block/blk-map.c
+++ b/block/blk-map.c
@@ -287,7 +287,7 @@ static int bio_map_user_iov(struct request *rq, struct iov_iter *iter,
* No alignment requirements on our part to support arbitrary
* passthrough commands.
*/
- ret = bio_iov_iter_get_pages(bio, iter, 0);
+ ret = bio_iov_iter_get_pages(bio, iter, 0, 0);
if (ret)
goto out_put;
ret = blk_rq_append_bio(rq, bio);
diff --git a/block/fops.c b/block/fops.c
index 5e3db9fead77c..ba9ac3372fd90 100644
--- a/block/fops.c
+++ b/block/fops.c
@@ -46,7 +46,7 @@ static bool blkdev_dio_invalid(struct block_device *bdev, struct kiocb *iocb,
static inline int blkdev_iov_iter_get_pages(struct bio *bio,
struct iov_iter *iter, struct block_device *bdev)
{
- return bio_iov_iter_get_pages(bio, iter,
+ return bio_iov_iter_get_pages(bio, iter, bdev_dma_alignment(bdev),
bdev_logical_block_size(bdev) - 1);
}
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
index 6c7293c3ba592..2f10b15d27a5d 100644
--- a/fs/iomap/direct-io.c
+++ b/fs/iomap/direct-io.c
@@ -369,6 +369,7 @@ static ssize_t iomap_dio_bio_iter_one(struct iomap_iter *iter,
ret = bio_iov_iter_bounce(bio, dio->submit.iter);
else
ret = bio_iov_iter_get_pages(bio, dio->submit.iter,
+ bdev_dma_alignment(bio->bi_bdev),
alignment - 1);
if (unlikely(ret))
goto out_put_bio;
diff --git a/include/linux/bio.h b/include/linux/bio.h
index 627973d7ed3a7..fd75ceba60777 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -470,7 +470,7 @@ int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data,
size_t len, enum req_op op);
int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter,
- unsigned len_align_mask);
+ unsigned mem_align_mask, unsigned len_align_mask);
void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter);
void __bio_release_pages(struct bio *bio, bool mark_dirty);
diff --git a/include/linux/uio.h b/include/linux/uio.h
index a9bc5b3067e32..fe2e985d74d24 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -389,9 +389,17 @@ ssize_t iov_iter_extract_pages(struct iov_iter *i, struct page ***pages,
size_t maxsize, unsigned int maxpages,
iov_iter_extraction_t extraction_flags,
size_t *offset0);
+/*
+ * Block-layer consumers (e.g. bio_iov_iter_get_pages()) require that the
+ * segments of an ITER_BVEC iterator are already aligned to the target device's
+ * DMA alignment, and forward them as-is. In-kernel users that build their own
+ * bvecs must not create sub-aligned segments; iov_iter_extract_bvecs() enforces
+ * the same for the segments it extracts via @mem_align_mask.
+ */
ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv,
size_t max_size, unsigned short *nr_vecs,
- unsigned short max_vecs, iov_iter_extraction_t extraction_flags);
+ unsigned short max_vecs, unsigned mem_align_mask,
+ iov_iter_extraction_t extraction_flags);
/**
* iov_iter_extract_will_pin - Indicate how pages from the iterator will be retained
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 53dcd9649d59a..b8782f5e7dbbf 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -1874,6 +1874,8 @@ static unsigned int get_contig_folio_len(struct page **pages,
* @max_size: maximum size to extract from @iter
* @nr_vecs: number of vectors in @bv (on in and output)
* @max_vecs: maximum vectors in @bv, including those filled before calling
+ * @mem_align_mask: reject with -EINVAL if the source address or
+ * length is not aligned to this mask
* @extraction_flags: flags to qualify request
*
* Like iov_iter_extract_pages(), but returns physically contiguous ranges
@@ -1885,14 +1887,19 @@ static unsigned int get_contig_folio_len(struct page **pages,
*/
ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv,
size_t max_size, unsigned short *nr_vecs,
- unsigned short max_vecs, iov_iter_extraction_t extraction_flags)
+ unsigned short max_vecs, unsigned mem_align_mask,
+ iov_iter_extraction_t extraction_flags)
{
+ unsigned long start = (unsigned long)iter_iov_addr(iter);
unsigned short entries_left = max_vecs - *nr_vecs;
unsigned short nr_pages, i = 0;
size_t left, offset, len;
struct page **pages;
ssize_t size;
+ if ((start | iter_iov_len(iter)) & mem_align_mask)
+ return -EINVAL;
+
/*
* Move page array up in the allocated memory for the bio vecs as far as
* possible so that we can start filling biovecs from the beginning
--
2.53.0
^ permalink raw reply related [flat|nested] 7+ messages in thread