[PATCH v5 4/5] dm crypt: batch a bio segment's sectors via dun()

[Leonid Ravich <lravich@amazon.com>]

Submit one skcipher request per contiguous bio segment (a single
bio_vec) with data_unit_size = cc->sector_size, instead of one request
per sector.  E.g. the default 512-byte sector with a 4 KiB bio_vec
becomes one request of 8 data units; the crypto layer (the dun()
template, or a native driver) walks the per-sector IV as a data-unit
counter.  Because a bio_vec is one contiguous segment, the request uses
only the existing inline dmreq->sg_in[0]/sg_out[0] entry -- no per-bio
scatterlist allocation, and no regression on small random I/O.

crypt_alloc_tfms() wraps the skcipher in dun(<cipher>,<endian>) when
crypt_can_batch_dun() holds: an IV mode that is a data-unit counter (its
crypt_iv_operations sets dun_endian to the counter endianness -- "le" for
plain64, "be" for plain64be; non-counter modes such as lmk/tcw/eboiv
leave it NULL and are excluded), single-tfm, non-aead, and sector_size
512 or iv_large_sectors so the per-unit IV step is exactly one.  This is
the same kind of name rewrite as essiv(), done in the one alloc helper so
callers are unchanged.

DM_CRYPT selects CRYPTO_DUN and dun() resolves against a sync inner
cipher, so wrapping has no acceptable failure that the bare cipher would
survive -- there is no fallback; any error propagates.  (A config whose
only xts provider is async with no generic CRYPTO_XTS would now fail to
activate rather than silently run per-sector; generic xts is selected by
the dependency chain, so this does not arise in practice.)

crypt_convert_block_skcipher() handles both cases in one function: the
length is crypt_skcipher_len() -- a whole contiguous segment when
batching, else a single sector -- and data_unit_size is set
unconditionally (a dun() tfm reads it; a plain skcipher ignores it).  It
advances the bio iterators itself (as the aead path already does) and
reports the bytes processed, so crypt_convert() advances cc_sector /
tag_offset uniformly via one helper, no per-case duplication.

Verified byte-equivalent to the per-sector path: plain64 and plain64be
dm-crypt with dun() produce ciphertext bit-identical to an unpatched
kernel over a 256 MB device (xts-aes driving the split).

Signed-off-by: Leonid Ravich <lravich@amazon.com>
---
 drivers/md/Kconfig    |   1 +
 drivers/md/dm-crypt.c | 208 +++++++++++++++++++++++++++++++++---------
 2 files changed, 166 insertions(+), 43 deletions(-)

diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index a3fcdca7e6db..e8e299566374 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -299,6 +299,7 @@ config DM_CRYPT
 	select CRC32
 	select CRYPTO
 	select CRYPTO_CBC
+	select CRYPTO_DUN # multi-data-unit batching of contiguous sectors
 	select CRYPTO_ESSIV
 	select CRYPTO_LIB_AES
 	select CRYPTO_LIB_MD5 # needed by lmk IV mode
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 608b617fb817..44938223ad3e 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -115,6 +115,13 @@ struct crypt_iv_operations {
 			 struct dm_crypt_request *dmreq);
 	void (*post)(struct crypt_config *cc, u8 *iv,
 		     struct dm_crypt_request *dmreq);
+
+	/*
+	 * Counter endianness ("le"/"be") for IV modes whose per-sector IV is a
+	 * data-unit-number counter (IV(s+i) == IV(s)+i), batchable via
+	 * dun(<cipher>,<dun_endian>).  NULL for non-counter modes (lmk, tcw, ...).
+	 */
+	const char *dun_endian;
 };
 
 struct iv_benbi_private {
@@ -151,6 +158,7 @@ enum cipher_flags {
 	CRYPT_IV_LARGE_SECTORS,		/* Calculate IV from sector_size, not 512B sectors */
 	CRYPT_ENCRYPT_PREPROCESS,	/* Must preprocess data for encryption (elephant) */
 	CRYPT_KEY_MAC_SIZE_SET,		/* The integrity_key_size option was used */
+	CRYPT_MULTI_DATA_UNIT,		/* Batch a bio segment's sectors per crypto request */
 };
 
 /*
@@ -1018,15 +1026,19 @@ static const struct crypt_iv_operations crypt_iv_plain_ops = {
 };
 
 static const struct crypt_iv_operations crypt_iv_plain64_ops = {
-	.generator = crypt_iv_plain64_gen
+	.generator = crypt_iv_plain64_gen,
+	.dun_endian = "le",
 };
 
 static const struct crypt_iv_operations crypt_iv_plain64be_ops = {
-	.generator = crypt_iv_plain64be_gen
+	.generator = crypt_iv_plain64be_gen,
+	.dun_endian = "be",
 };
 
 static const struct crypt_iv_operations crypt_iv_essiv_ops = {
-	.generator = crypt_iv_essiv_gen
+	.generator = crypt_iv_essiv_gen,
+	/* IV input is le64(sector); the salt-encrypt lives in essiv(). */
+	.dun_endian = "le",
 };
 
 static const struct crypt_iv_operations crypt_iv_benbi_ops = {
@@ -1349,21 +1361,51 @@ static int crypt_convert_block_aead(struct crypt_config *cc,
 	return r;
 }
 
+/*
+ * Bytes to process in one skcipher request: a whole contiguous segment when
+ * batching (multi-data-unit), else one sector.  0 means an unusable
+ * (sub-sector / misaligned) segment.
+ */
+static unsigned int crypt_skcipher_len(struct crypt_config *cc,
+				       const struct bio_vec *bv_in,
+				       const struct bio_vec *bv_out)
+{
+	const unsigned int sector_size = cc->sector_size;
+
+	if (test_bit(CRYPT_MULTI_DATA_UNIT, &cc->cipher_flags))
+		return round_down(min(bv_in->bv_len, bv_out->bv_len),
+				  sector_size);
+
+	/* Reject unexpected unaligned bio. */
+	if (unlikely(bv_in->bv_len & (sector_size - 1)))
+		return 0;
+	return sector_size;
+}
+
+/*
+ * Encrypt/decrypt one bio segment (one sector, or a whole segment when
+ * batching) and report the bytes done in *out_processed.  The integrity /
+ * preprocess / post handling is inert when batching (crypt_can_batch_dun()
+ * excludes those configs).
+ */
 static int crypt_convert_block_skcipher(struct crypt_config *cc,
 					struct convert_context *ctx,
 					struct skcipher_request *req,
-					unsigned int tag_offset)
+					unsigned int tag_offset,
+					unsigned int *out_processed)
 {
 	struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
 	struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
+	const unsigned int sector_size = cc->sector_size;
 	struct scatterlist *sg_in, *sg_out;
 	struct dm_crypt_request *dmreq;
 	u8 *iv, *org_iv, *tag_iv;
 	__le64 *sector;
+	unsigned int len;
 	int r = 0;
 
-	/* Reject unexpected unaligned bio. */
-	if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
+	len = crypt_skcipher_len(cc, &bv_in, &bv_out);
+	if (unlikely(!len))
 		return -EIO;
 
 	dmreq = dmreq_of_req(cc, req);
@@ -1386,10 +1428,10 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
 	sg_out = &dmreq->sg_out[0];
 
 	sg_init_table(sg_in, 1);
-	sg_set_page(sg_in, bv_in.bv_page, cc->sector_size, bv_in.bv_offset);
+	sg_set_page(sg_in, bv_in.bv_page, len, bv_in.bv_offset);
 
 	sg_init_table(sg_out, 1);
-	sg_set_page(sg_out, bv_out.bv_page, cc->sector_size, bv_out.bv_offset);
+	sg_set_page(sg_out, bv_out.bv_page, len, bv_out.bv_offset);
 
 	if (cc->iv_gen_ops) {
 		/* For READs use IV stored in integrity metadata */
@@ -1410,7 +1452,9 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
 		memcpy(iv, org_iv, cc->iv_size);
 	}
 
-	skcipher_request_set_crypt(req, sg_in, sg_out, cc->sector_size, iv);
+	skcipher_request_set_crypt(req, sg_in, sg_out, len, iv);
+	/* A dun() tfm reads this; a plain skcipher ignores it (len is one sector). */
+	skcipher_request_set_data_unit_size(req, sector_size);
 
 	if (bio_data_dir(ctx->bio_in) == WRITE)
 		r = crypto_skcipher_encrypt(req);
@@ -1420,9 +1464,10 @@ static int crypt_convert_block_skcipher(struct crypt_config *cc,
 	if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
 		cc->iv_gen_ops->post(cc, org_iv, dmreq);
 
-	bio_advance_iter(ctx->bio_in, &ctx->iter_in, cc->sector_size);
-	bio_advance_iter(ctx->bio_out, &ctx->iter_out, cc->sector_size);
+	bio_advance_iter(ctx->bio_in, &ctx->iter_in, len);
+	bio_advance_iter(ctx->bio_out, &ctx->iter_out, len);
 
+	*out_processed = len;
 	return r;
 }
 
@@ -1509,13 +1554,25 @@ static void crypt_free_req(struct crypt_config *cc, void *req, struct bio *base_
 		crypt_free_req_skcipher(cc, req, base_bio);
 }
 
+/*
+ * Advance the IV-sector and integrity-tag cursors by @processed bytes; the
+ * bio iterators are advanced by the per-block helpers themselves.
+ */
+static void crypt_convert_advance(struct crypt_config *cc,
+				  struct convert_context *ctx,
+				  unsigned int processed)
+{
+	ctx->cc_sector += processed >> SECTOR_SHIFT;
+	ctx->tag_offset += processed / cc->sector_size;
+}
+
 /*
  * Encrypt / decrypt data from one bio to another one (can be the same one)
  */
 static blk_status_t crypt_convert(struct crypt_config *cc,
 			 struct convert_context *ctx, bool atomic, bool reset_pending)
 {
-	unsigned int sector_step = cc->sector_size >> SECTOR_SHIFT;
+	unsigned int processed;
 	int r;
 
 	/*
@@ -1536,10 +1593,12 @@ static blk_status_t crypt_convert(struct crypt_config *cc,
 
 		atomic_inc(&ctx->cc_pending);
 
+		processed = cc->sector_size;
 		if (crypt_integrity_aead(cc))
 			r = crypt_convert_block_aead(cc, ctx, ctx->r.req_aead, ctx->tag_offset);
 		else
-			r = crypt_convert_block_skcipher(cc, ctx, ctx->r.req, ctx->tag_offset);
+			r = crypt_convert_block_skcipher(cc, ctx, ctx->r.req,
+							 ctx->tag_offset, &processed);
 
 		switch (r) {
 		/*
@@ -1559,8 +1618,7 @@ static blk_status_t crypt_convert(struct crypt_config *cc,
 					 * exit and continue processing in a workqueue
 					 */
 					ctx->r.req = NULL;
-					ctx->tag_offset++;
-					ctx->cc_sector += sector_step;
+					crypt_convert_advance(cc, ctx, processed);
 					return BLK_STS_DEV_RESOURCE;
 				}
 			} else {
@@ -1574,16 +1632,14 @@ static blk_status_t crypt_convert(struct crypt_config *cc,
 		 */
 		case -EINPROGRESS:
 			ctx->r.req = NULL;
-			ctx->tag_offset++;
-			ctx->cc_sector += sector_step;
+			crypt_convert_advance(cc, ctx, processed);
 			continue;
 		/*
 		 * The request was already processed (synchronously).
 		 */
 		case 0:
 			atomic_dec(&ctx->cc_pending);
-			ctx->cc_sector += sector_step;
-			ctx->tag_offset++;
+			crypt_convert_advance(cc, ctx, processed);
 			if (!atomic)
 				cond_resched();
 			continue;
@@ -2345,12 +2401,37 @@ static int crypt_alloc_tfms_aead(struct crypt_config *cc, char *ciphermode)
 	return 0;
 }
 
+/*
+ * Whether to wrap the cipher in dun() for multi-data-unit batching: a counter
+ * IV mode (dun_endian set: plain64 "le", plain64be "be", essiv "le"), single-
+ * tfm, non-aead, and a per-unit IV step of exactly one (512B sectors or
+ * iv_large_sectors).  Integrity is configured
+ * after alloc, so it is re-checked post-alloc in crypt_ctr_cipher(); an
+ * integrity config keeps an inert dun() wrapper but never sets the batch flag.
+ */
+static bool crypt_can_batch_dun(struct crypt_config *cc)
+{
+	return !crypt_integrity_aead(cc) && cc->tfms_count == 1 &&
+		cc->iv_gen_ops && cc->iv_gen_ops->dun_endian &&
+		(cc->sector_size == (1 << SECTOR_SHIFT) ||
+		 test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags));
+}
+
 static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)
 {
+	char dun_api[CRYPTO_MAX_ALG_NAME];
+
 	if (crypt_integrity_aead(cc))
 		return crypt_alloc_tfms_aead(cc, ciphermode);
-	else
-		return crypt_alloc_tfms_skcipher(cc, ciphermode);
+
+	/* Wrap in dun() for batching when eligible (like the essiv() rewrite). */
+	if (crypt_can_batch_dun(cc)) {
+		if (snprintf(dun_api, sizeof(dun_api), "dun(%s,%s)", ciphermode,
+			     cc->iv_gen_ops->dun_endian) >= (int)sizeof(dun_api))
+			return -ENAMETOOLONG;
+		ciphermode = dun_api;
+	}
+	return crypt_alloc_tfms_skcipher(cc, ciphermode);
 }
 
 static unsigned int crypt_subkey_size(struct crypt_config *cc)
@@ -2747,25 +2828,15 @@ static void crypt_dtr(struct dm_target *ti)
 	dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1);
 }
 
-static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
+/*
+ * Select cc->iv_gen_ops from the IV mode string -- pure parsing, no tfm
+ * dependency, so it runs before alloc and lets crypt_can_batch_dun() see the
+ * mode.  The tfm-dependent IV sizing is finished later by crypt_ctr_ivmode().
+ */
+static int crypt_select_ivmode(struct dm_target *ti, const char *ivmode)
 {
 	struct crypt_config *cc = ti->private;
 
-	if (crypt_integrity_aead(cc))
-		cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc));
-	else
-		cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));
-
-	if (cc->iv_size)
-		/* at least a 64 bit sector number should fit in our buffer */
-		cc->iv_size = max(cc->iv_size,
-				  (unsigned int)(sizeof(u64) / sizeof(u8)));
-	else if (ivmode) {
-		DMWARN("Selected cipher does not support IVs");
-		ivmode = NULL;
-	}
-
-	/* Choose ivmode, see comments at iv code. */
 	if (ivmode == NULL)
 		cc->iv_gen_ops = NULL;
 	else if (strcmp(ivmode, "plain") == 0)
@@ -2803,12 +2874,8 @@ static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
 		}
 	} else if (strcmp(ivmode, "tcw") == 0) {
 		cc->iv_gen_ops = &crypt_iv_tcw_ops;
-		cc->key_parts += 2; /* IV + whitening */
-		cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE;
 	} else if (strcmp(ivmode, "random") == 0) {
 		cc->iv_gen_ops = &crypt_iv_random_ops;
-		/* Need storage space in integrity fields. */
-		cc->integrity_iv_size = cc->iv_size;
 	} else {
 		ti->error = "Invalid IV mode";
 		return -EINVAL;
@@ -2817,6 +2884,37 @@ static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
 	return 0;
 }
 
+static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
+{
+	struct crypt_config *cc = ti->private;
+
+	if (crypt_integrity_aead(cc))
+		cc->iv_size = crypto_aead_ivsize(any_tfm_aead(cc));
+	else
+		cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));
+
+	if (cc->iv_size)
+		/* at least a 64 bit sector number should fit in our buffer */
+		cc->iv_size = max(cc->iv_size,
+				  (unsigned int)(sizeof(u64) / sizeof(u8)));
+	else if (ivmode) {
+		DMWARN("Selected cipher does not support IVs");
+		ivmode = NULL;
+		cc->iv_gen_ops = NULL;
+	}
+
+	/* Finish the tfm-dependent IV sizing; modes are already selected. */
+	if (cc->iv_gen_ops == &crypt_iv_tcw_ops) {
+		cc->key_parts += 2; /* IV + whitening */
+		cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE;
+	} else if (cc->iv_gen_ops == &crypt_iv_random_ops) {
+		/* Need storage space in integrity fields. */
+		cc->integrity_iv_size = cc->iv_size;
+	}
+
+	return 0;
+}
+
 /*
  * Workaround to parse HMAC algorithm from AEAD crypto API spec.
  * The HMAC is needed to calculate tag size (HMAC digest size).
@@ -2914,7 +3012,12 @@ static int crypt_ctr_cipher_new(struct dm_target *ti, char *cipher_in, char *key
 
 	cc->key_parts = cc->tfms_count;
 
-	/* Allocate cipher */
+	/* Select IV mode before alloc so dun() wrapping can be decided. */
+	ret = crypt_select_ivmode(ti, *ivmode);
+	if (ret < 0)
+		return ret;
+
+	/* Allocate cipher (skcipher may be wrapped in dun()). */
 	ret = crypt_alloc_tfms(cc, cipher_api);
 	if (ret < 0) {
 		ti->error = "Error allocating crypto tfm";
@@ -2999,7 +3102,13 @@ static int crypt_ctr_cipher_old(struct dm_target *ti, char *cipher_in, char *key
 		goto bad_mem;
 	}
 
-	/* Allocate cipher */
+	/* Select IV mode before alloc so dun() wrapping can be decided. */
+	ret = crypt_select_ivmode(ti, *ivmode);
+	if (ret < 0) {
+		kfree(cipher_api);
+		return ret;
+	}
+
 	ret = crypt_alloc_tfms(cc, cipher_api);
 	if (ret < 0) {
 		ti->error = "Error allocating crypto tfm";
@@ -3063,6 +3172,19 @@ static int crypt_ctr_cipher(struct dm_target *ti, char *cipher_in, char *key)
 		}
 	}
 
+	/*
+	 * Enable batching only if the cipher was dun()-wrapped at alloc time and
+	 * no integrity was configured (integrity is set up after cipher alloc).
+	 */
+	if (!crypt_integrity_aead(cc) && !cc->integrity_tag_size &&
+	    !cc->integrity_iv_size &&
+	    !strncmp(crypto_skcipher_alg(any_tfm(cc))->base.cra_name,
+		     "dun(", 4)) {
+		set_bit(CRYPT_MULTI_DATA_UNIT, &cc->cipher_flags);
+		DMINFO("Using multi-data-unit crypto offload (du=%u)",
+		       cc->sector_size);
+	}
+
 	/* wipe the kernel key payload copy */
 	if (cc->key_string)
 		memset(cc->key, 0, cc->key_size * sizeof(u8));
-- 
2.47.3