* [PATCHv5 2/3] dm-crypt: handle memory allocation failures
2026-07-09 16:15 [PATCHv5 0/3] dm-crypt: finer grained memory alignment Keith Busch
2026-07-09 16:15 ` [PATCHv5 1/3] dm: initialize dma_alignment to 0 Keith Busch
@ 2026-07-09 16:15 ` Keith Busch
2026-07-09 16:15 ` [PATCHv5 3/3] dm-crypt: allow unaligned bio_vecs for direct io Keith Busch
2 siblings, 0 replies; 8+ messages in thread
From: Keith Busch @ 2026-07-09 16:15 UTC (permalink / raw)
To: dm-devel, mpatocka; +Cc: snitzer, bmarzins, Keith Busch
From: Keith Busch <kbusch@kernel.org>
There are a few paths that can fail with -ENOMEM, like skcipher_copy_iv,
which had been treated as a generic failure. This is a transient host
resource issue that can be retried later.
Signed-off-by: Keith Busch <kbusch@kernel.org>
---
drivers/md/dm-crypt.c | 24 +++++++++++++++++++-----
1 file changed, 19 insertions(+), 5 deletions(-)
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 608b617fb817f..fc9c875cccfd2 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -1338,6 +1338,8 @@ static int crypt_convert_block_aead(struct crypt_config *cc,
dm_audit_log_bio(DM_MSG_PREFIX, "integrity-aead",
ctx->bio_in, s, 0);
}
+ } else if (unlikely(r == -ENOMEM)) {
+ return r;
}
if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
@@ -1417,6 +1419,9 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
else
r = crypto_skcipher_decrypt(req);
+ if (unlikely(r == -ENOMEM))
+ return r;
+
if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
cc->iv_gen_ops->post(cc, org_iv, dmreq);
@@ -1593,6 +1598,13 @@ static blk_status_t crypt_convert(struct crypt_config *cc,
case -EBADMSG:
atomic_dec(&ctx->cc_pending);
return BLK_STS_PROTECTION;
+ /*
+ * There was a host resource issue.
+ */
+ case -ENOMEM:
+ atomic_dec(&ctx->cc_pending);
+ complete(&ctx->restart);
+ return BLK_STS_RESOURCE;
/*
* There was an error while processing the request.
*/
@@ -2072,10 +2084,11 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags), true);
/*
* Crypto API backlogged the request, because its queue was full
- * and we're in softirq context, so continue from a workqueue
- * (TODO: is it actually possible to be in softirq in the write path?)
+ * and we're in softirq context, or a host memory allocation
+ * temporarily failed, so continue from a workqueue (TODO: is it
+ * actually possible to be in softirq in the write path?)
*/
- if (r == BLK_STS_DEV_RESOURCE) {
+ if (r == BLK_STS_DEV_RESOURCE || r == BLK_STS_RESOURCE) {
INIT_WORK(&io->work, kcryptd_crypt_write_continue);
queue_work(cc->crypt_queue, &io->work);
return;
@@ -2148,9 +2161,10 @@ static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)
}
/*
* Crypto API backlogged the request, because its queue was full
- * and we're in softirq context, so continue from a workqueue
+ * and we're in softirq context, or a host memory allocation
+ * temporarily failed, so continue from a workqueue
*/
- if (r == BLK_STS_DEV_RESOURCE) {
+ if (r == BLK_STS_DEV_RESOURCE || r == BLK_STS_RESOURCE) {
INIT_WORK(&io->work, kcryptd_crypt_read_continue);
queue_work(cc->crypt_queue, &io->work);
return;
--
2.52.0
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCHv5 3/3] dm-crypt: allow unaligned bio_vecs for direct io
2026-07-09 16:15 [PATCHv5 0/3] dm-crypt: finer grained memory alignment Keith Busch
2026-07-09 16:15 ` [PATCHv5 1/3] dm: initialize dma_alignment to 0 Keith Busch
2026-07-09 16:15 ` [PATCHv5 2/3] dm-crypt: handle memory allocation failures Keith Busch
@ 2026-07-09 16:15 ` Keith Busch
2026-07-11 0:54 ` Eric Biggers
2 siblings, 1 reply; 8+ messages in thread
From: Keith Busch @ 2026-07-09 16:15 UTC (permalink / raw)
To: dm-devel, mpatocka; +Cc: snitzer, bmarzins, Keith Busch
From: Keith Busch <kbusch@kernel.org>
Many storage devices can handle DMA for data that is not aligned to the
logical block size. The block and filesystem layers have introduced
updates to allow that kind of memory alignment flexibility when
possible.
dm-crypt, however, currently constrains itself to aligned memory because
it sends a single scatterlist element for the in/out list to the encrypt
and decrypt algorithms. This forces applications that have unaligned
data to copy through a bounce buffer, increasing CPU and memory
utilization.
Use multiple scatterlist elements as needed to relax the memory
alignment requirement. To keep this simple, this more flexible
constraint is enabled only for certain encryption and initialization
vector types, specifically the ones that don't have additional use for
the request scatterlist elements beyond pointing to user data.
In the unlikely case where the incoming bio uses a highly fragmented
vector, the four inline scatterlist elements may not be enough, so
allocate a temporary scatterlist when needed, falling back to a mempool
for the in and out buffers to guarantee forward progress if the initial
allocation fails.
Signed-off-by: Keith Busch <kbusch@kernel.org>
---
drivers/md/dm-crypt.c | 161 ++++++++++++++++++++++++++++++++++++------
1 file changed, 140 insertions(+), 21 deletions(-)
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index fc9c875cccfd2..66f666d9812f0 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -101,6 +101,10 @@ struct dm_crypt_request {
struct scatterlist sg_in[4];
struct scatterlist sg_out[4];
u64 iv_sector;
+ struct scatterlist *__sg_in;
+ struct scatterlist *__sg_out;
+ bool sg_in_pooled;
+ bool sg_out_pooled;
};
struct crypt_config;
@@ -216,6 +220,9 @@ struct crypt_config {
unsigned int key_extra_size; /* additional keys length */
unsigned int key_mac_size; /* MAC key size for authenc(...) */
+ unsigned int io_alignment;
+ mempool_t sg_in_pool;
+ mempool_t sg_out_pool;
unsigned int integrity_tag_size;
unsigned int integrity_iv_size;
unsigned int used_tag_size;
@@ -1351,22 +1358,92 @@ static int crypt_convert_block_aead(struct crypt_config *cc,
return r;
}
+static void crypt_free_sg(struct scatterlist *sg, struct scatterlist *inline_sg,
+ mempool_t *pool, bool from_pool)
+{
+ if (sg == inline_sg)
+ return;
+ if (from_pool)
+ mempool_free(sg, pool);
+ else
+ kfree(sg);
+}
+
+static void crypt_free_sgls(struct crypt_config *cc,
+ struct dm_crypt_request *dmreq)
+{
+ crypt_free_sg(dmreq->__sg_in, dmreq->sg_in,
+ &cc->sg_in_pool, dmreq->sg_in_pooled);
+ crypt_free_sg(dmreq->__sg_out, dmreq->sg_out,
+ &cc->sg_out_pool, dmreq->sg_out_pooled);
+ dmreq->__sg_in = NULL;
+ dmreq->__sg_out = NULL;
+}
+
+static int crypt_build_sgl(struct crypt_config *cc, struct scatterlist **psg,
+ struct bvec_iter iter, struct bio *bio,
+ int max_segs, mempool_t *pool, bool *pooled)
+{
+ unsigned int bytes = cc->sector_size;
+ struct scatterlist *sg = *psg;
+ struct bvec_iter tmp = iter;
+ int segs, i = 0;
+
+ *pooled = false;
+ bio_advance_iter(bio, &tmp, bytes);
+ segs = tmp.bi_idx - iter.bi_idx + !!tmp.bi_bvec_done;
+ if (segs > max_segs) {
+ if (unlikely(segs > BIO_MAX_VECS))
+ return -EIO;
+ sg = kmalloc_array(segs, sizeof(struct scatterlist),
+ GFP_NOWAIT | __GFP_NOMEMALLOC);
+ if (!sg) {
+ sg = mempool_alloc(pool, in_interrupt() ? GFP_ATOMIC :
+ GFP_NOIO);
+ if (!sg)
+ return -ENOMEM;
+ *pooled = true;
+ }
+ }
+
+ sg_init_table(sg, segs);
+ do {
+ struct bio_vec bv = mp_bvec_iter_bvec(bio->bi_io_vec, iter);
+ int len = min(bytes, bv.bv_len);
+
+ /* Reject unexpected unaligned bio. */
+ if (unlikely((len | bv.bv_offset) & cc->io_alignment))
+ goto error;
+
+ sg_set_page(&sg[i++], bv.bv_page, len, bv.bv_offset);
+ bio_advance_iter_single(bio, &iter, len);
+ bytes -= len;
+ } while (bytes);
+
+ if (WARN_ON_ONCE(i != segs))
+ goto error;
+ *psg = sg;
+ return 0;
+error:
+ if (sg != *psg) {
+ if (*pooled)
+ mempool_free(sg, pool);
+ else
+ kfree(sg);
+ }
+ return -EIO;
+}
+
static int crypt_convert_block_skcipher(struct crypt_config *cc,
struct convert_context *ctx,
struct skcipher_request *req,
unsigned int tag_offset)
{
- struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
- struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
struct scatterlist *sg_in, *sg_out;
struct dm_crypt_request *dmreq;
u8 *iv, *org_iv, *tag_iv;
__le64 *sector;
- int r = 0;
-
- /* Reject unexpected unaligned bio. */
- if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
- return -EIO;
+ int r;
dmreq = dmreq_of_req(cc, req);
dmreq->iv_sector = ctx->cc_sector;
@@ -1383,15 +1460,23 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
sector = org_sector_of_dmreq(cc, dmreq);
*sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
- /* For skcipher we use only the first sg item */
- sg_in = &dmreq->sg_in[0];
- sg_out = &dmreq->sg_out[0];
+ dmreq->__sg_in = &dmreq->sg_in[0];
+ dmreq->__sg_out = &dmreq->sg_out[0];
+
+ r = crypt_build_sgl(cc, &dmreq->__sg_in, ctx->iter_in, ctx->bio_in,
+ ARRAY_SIZE(dmreq->sg_in), &cc->sg_in_pool,
+ &dmreq->sg_in_pooled);
+ if (r < 0)
+ goto out;
- sg_init_table(sg_in, 1);
- sg_set_page(sg_in, bv_in.bv_page, cc->sector_size, bv_in.bv_offset);
+ r = crypt_build_sgl(cc, &dmreq->__sg_out, ctx->iter_out, ctx->bio_out,
+ ARRAY_SIZE(dmreq->sg_out), &cc->sg_out_pool,
+ &dmreq->sg_out_pooled);
+ if (r < 0)
+ goto out;
- sg_init_table(sg_out, 1);
- sg_set_page(sg_out, bv_out.bv_page, cc->sector_size, bv_out.bv_offset);
+ sg_in = dmreq->__sg_in;
+ sg_out = dmreq->__sg_out;
if (cc->iv_gen_ops) {
/* For READs use IV stored in integrity metadata */
@@ -1400,7 +1485,7 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
} else {
r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
if (r < 0)
- return r;
+ goto out;
/* Data can be already preprocessed in generator */
if (test_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags))
sg_in = sg_out;
@@ -1420,13 +1505,16 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
r = crypto_skcipher_decrypt(req);
if (unlikely(r == -ENOMEM))
- return r;
+ goto out;
if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
cc->iv_gen_ops->post(cc, org_iv, dmreq);
bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size);
bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size);
+out:
+ if (r != -EINPROGRESS && r != -EBUSY)
+ crypt_free_sgls(cc, dmreq);
return r;
}
@@ -1492,7 +1580,9 @@ static void crypt_free_req_skcipher(struct crypt_config *cc,
struct skcipher_request *req, struct bio *base_bio)
{
struct dm_crypt_io *io = dm_per_bio_data(base_bio, cc->per_bio_data_size);
+ struct dm_crypt_request *dmreq = dmreq_of_req(cc, req);
+ crypt_free_sgls(cc, dmreq);
if ((struct skcipher_request *)(io + 1) != req)
mempool_free(req, &cc->req_pool);
}
@@ -2731,6 +2821,8 @@ static void crypt_dtr(struct dm_target *ti)
mempool_exit(&cc->page_pool);
mempool_exit(&cc->req_pool);
+ mempool_exit(&cc->sg_in_pool);
+ mempool_exit(&cc->sg_out_pool);
mempool_exit(&cc->tag_pool);
WARN_ON(percpu_counter_sum(&cc->n_allocated_pages) != 0);
@@ -2765,10 +2857,12 @@ static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
{
struct crypt_config *cc = ti->private;
- if (crypt_integrity_aead(cc))
+ if (crypt_integrity_aead(cc)) {
cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc));
- else
+ cc->io_alignment = cc->sector_size - 1;
+ } else {
cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));
+ }
if (cc->iv_size)
/* at least a 64 bit sector number should fit in our buffer */
@@ -2803,6 +2897,7 @@ static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
if (cc->key_extra_size > ELEPHANT_MAX_KEY_SIZE)
return -EINVAL;
set_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags);
+ cc->io_alignment = cc->sector_size - 1;
} else if (strcmp(ivmode, "lmk") == 0) {
cc->iv_gen_ops = &crypt_iv_lmk_ops;
/*
@@ -2815,10 +2910,12 @@ static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
cc->key_parts++;
cc->key_extra_size = cc->key_size / cc->key_parts;
}
+ cc->io_alignment = cc->sector_size - 1;
} else if (strcmp(ivmode, "tcw") == 0) {
cc->iv_gen_ops = &crypt_iv_tcw_ops;
cc->key_parts += 2; /* IV + whitening */
cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE;
+ cc->io_alignment = cc->sector_size - 1;
} else if (strcmp(ivmode, "random") == 0) {
cc->iv_gen_ops = &crypt_iv_random_ops;
/* Need storage space in integrity fields. */
@@ -3285,6 +3382,20 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start + additional_req_size,
ARCH_DMA_MINALIGN);
+ ret = mempool_init_kmalloc_pool(&cc->sg_in_pool, MIN_IOS,
+ BIO_MAX_VECS * sizeof(struct scatterlist));
+ if (ret) {
+ ti->error = "Cannot allocate crypt scatterlist mempool";
+ goto bad;
+ }
+
+ ret = mempool_init_kmalloc_pool(&cc->sg_out_pool, MIN_IOS,
+ BIO_MAX_VECS * sizeof(struct scatterlist));
+ if (ret) {
+ ti->error = "Cannot allocate crypt scatterlist mempool";
+ goto bad;
+ }
+
ret = mempool_init(&cc->page_pool, BIO_MAX_VECS, crypt_page_alloc, crypt_page_free, cc);
if (ret) {
ti->error = "Cannot allocate page mempool";
@@ -3495,10 +3606,16 @@ static int crypt_map(struct dm_target *ti, struct bio *bio)
}
}
- if (crypt_integrity_aead(cc))
+ if (crypt_integrity_aead(cc)) {
io->ctx.r.req_aead = (struct aead_request *)(io + 1);
- else
+ } else {
+ struct dm_crypt_request *dmreq;
+
io->ctx.r.req = (struct skcipher_request *)(io + 1);
+ dmreq = dmreq_of_req(cc, io->ctx.r.req);
+ dmreq->__sg_in = &dmreq->sg_in[0];
+ dmreq->__sg_out = &dmreq->sg_out[0];
+ }
if (bio_data_dir(io->base_bio) == READ) {
if (kcryptd_io_read(io, CRYPT_MAP_READ_GFP))
@@ -3694,7 +3811,9 @@ static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
struct crypt_config *cc = ti->private;
dm_stack_bs_limits(limits, cc->sector_size);
- limits->dma_alignment = limits->logical_block_size - 1;
+ limits->dma_alignment = max(bdev_dma_alignment(cc->dev->bdev),
+ cc->io_alignment);
+ cc->io_alignment = limits->dma_alignment;
/*
* For zoned dm-crypt targets, there will be no internal splitting of
--
2.52.0
^ permalink raw reply related [flat|nested] 8+ messages in thread