[PATCH v5 5/5] blk-crypto: fallback - batch a segment's data units via dun()

[Leonid Ravich <lravich@amazon.com>]

blk-crypto-fallback open-codes a per-data-unit loop, issuing one
skcipher request per data unit with the IV walked as a DUN counter.
Allocate dun(<cipher>,le) instead of the bare cipher so a contiguous bio
segment is encrypted/decrypted as one multi-data-unit request, the
crypto layer walking the per-unit IV.  Every blk-crypto mode feeds the
DUN as a little-endian counter, and dun() handles any counter width up
to 32 bytes, so all modes -- including Adiantum (32-byte IV) -- are
wrapped and the open-coded inner per-unit loop is removed from both the
encrypt and decrypt paths.  This makes blk-crypto-fallback a second
consumer of the template (after dm-crypt) and lets a higher-priority
hardware dun(...) driver, if present, handle the request in one pass.

Output is unchanged: the template's little-endian per-unit counter is
exactly blk_crypto_dun_to_iv()/bio_crypt_dun_increment().

Signed-off-by: Leonid Ravich <lravich@amazon.com>
---
 block/Kconfig               |  1 +
 block/blk-crypto-fallback.c | 74 ++++++++++++++++++-------------------
 2 files changed, 36 insertions(+), 39 deletions(-)

diff --git a/block/Kconfig b/block/Kconfig
index 15027963472d..0c9025f9b0f6 100644
--- a/block/Kconfig
+++ b/block/Kconfig
@@ -204,6 +204,7 @@ config BLK_INLINE_ENCRYPTION_FALLBACK
 	depends on BLK_INLINE_ENCRYPTION
 	select CRYPTO
 	select CRYPTO_SKCIPHER
+	select CRYPTO_DUN # batches a segment's data units per crypto request
 	help
 	  Enabling this lets the block layer handle inline encryption
 	  by falling back to the kernel crypto API when inline
diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c
index 61f595410832..8337d56ba1dc 100644
--- a/block/blk-crypto-fallback.c
+++ b/block/blk-crypto-fallback.c
@@ -250,7 +250,6 @@ static void __blk_crypto_fallback_encrypt_bio(struct bio *src_bio,
 	unsigned int nr_enc_pages, enc_idx;
 	struct page **enc_pages;
 	struct bio *enc_bio;
-	unsigned int i;
 
 	skcipher_request_set_callback(ciph_req,
 			CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
@@ -260,9 +259,6 @@ static void __blk_crypto_fallback_encrypt_bio(struct bio *src_bio,
 	sg_init_table(&src, 1);
 	sg_init_table(&dst, 1);
 
-	skcipher_request_set_crypt(ciph_req, &src, &dst, data_unit_size,
-				   iv.bytes);
-
 	/*
 	 * Encrypt each page in the source bio.  Because the source bio could
 	 * have bio_vecs that span more than a single page, but the encrypted
@@ -287,29 +283,26 @@ static void __blk_crypto_fallback_encrypt_bio(struct bio *src_bio,
 		__bio_add_page(enc_bio, enc_page, src_bv.bv_len,
 				src_bv.bv_offset);
 
-		sg_set_page(&src, src_bv.bv_page, data_unit_size,
-			    src_bv.bv_offset);
-		sg_set_page(&dst, enc_page, data_unit_size, src_bv.bv_offset);
-
 		/*
 		 * Increment the index now that the encrypted page is added to
 		 * the bio.  This is important for the error unwind path.
 		 */
 		enc_idx++;
 
-		/*
-		 * Encrypt each data unit in this page.
-		 */
-		for (i = 0; i < src_bv.bv_len; i += data_unit_size) {
-			blk_crypto_dun_to_iv(curr_dun, &iv);
-			if (crypto_skcipher_encrypt(ciph_req)) {
-				enc_bio->bi_status = BLK_STS_IOERR;
-				goto out_free_enc_bio;
-			}
-			bio_crypt_dun_increment(curr_dun, 1);
-			src.offset += data_unit_size;
-			dst.offset += data_unit_size;
+		/* Encrypt the whole segment as one multi-data-unit request. */
+		blk_crypto_dun_to_iv(curr_dun, &iv);
+		sg_set_page(&src, src_bv.bv_page, src_bv.bv_len,
+			    src_bv.bv_offset);
+		sg_set_page(&dst, enc_page, src_bv.bv_len, src_bv.bv_offset);
+		skcipher_request_set_crypt(ciph_req, &src, &dst, src_bv.bv_len,
+					   iv.bytes);
+		skcipher_request_set_data_unit_size(ciph_req, data_unit_size);
+		if (crypto_skcipher_encrypt(ciph_req)) {
+			enc_bio->bi_status = BLK_STS_IOERR;
+			goto out_free_enc_bio;
 		}
+		bio_crypt_dun_increment(curr_dun,
+					src_bv.bv_len / data_unit_size);
 
 		bio_advance_iter_single(src_bio, &src_bio->bi_iter,
 				src_bv.bv_len);
@@ -380,7 +373,6 @@ static blk_status_t __blk_crypto_fallback_decrypt_bio(struct bio *bio,
 	struct scatterlist sg;
 	struct bio_vec bv;
 	const int data_unit_size = bc->bc_key->crypto_cfg.data_unit_size;
-	unsigned int i;
 
 	skcipher_request_set_callback(ciph_req,
 			CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
@@ -388,26 +380,20 @@ static blk_status_t __blk_crypto_fallback_decrypt_bio(struct bio *bio,
 
 	memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun));
 	sg_init_table(&sg, 1);
-	skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size,
-				   iv.bytes);
 
-	/* Decrypt each segment in the bio */
+	/* One dun() request per segment; the crypto layer walks the per-unit DUN. */
 	__bio_for_each_segment(bv, bio, iter, iter) {
-		struct page *page = bv.bv_page;
-
 		if (!IS_ALIGNED(bv.bv_len | bv.bv_offset, data_unit_size))
 			return BLK_STS_INVAL;
 
-		sg_set_page(&sg, page, data_unit_size, bv.bv_offset);
-
-		/* Decrypt each data unit in the segment */
-		for (i = 0; i < bv.bv_len; i += data_unit_size) {
-			blk_crypto_dun_to_iv(curr_dun, &iv);
-			if (crypto_skcipher_decrypt(ciph_req))
-				return BLK_STS_IOERR;
-			bio_crypt_dun_increment(curr_dun, 1);
-			sg.offset += data_unit_size;
-		}
+		blk_crypto_dun_to_iv(curr_dun, &iv);
+		sg_set_page(&sg, bv.bv_page, bv.bv_len, bv.bv_offset);
+		skcipher_request_set_crypt(ciph_req, &sg, &sg, bv.bv_len,
+					   iv.bytes);
+		skcipher_request_set_data_unit_size(ciph_req, data_unit_size);
+		if (crypto_skcipher_decrypt(ciph_req))
+			return BLK_STS_IOERR;
+		bio_crypt_dun_increment(curr_dun, bv.bv_len / data_unit_size);
 	}
 
 	return BLK_STS_OK;
@@ -621,6 +607,7 @@ static int blk_crypto_fallback_init(void)
 int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num)
 {
 	const char *cipher_str = blk_crypto_modes[mode_num].cipher_str;
+	char dun_str[CRYPTO_MAX_ALG_NAME];
 	struct blk_crypto_fallback_keyslot *slotp;
 	unsigned int i;
 	int err = 0;
@@ -641,15 +628,24 @@ int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num)
 	if (err)
 		goto out;
 
+	/*
+	 * Wrap in dun() to handle a whole segment per request (a higher-priority
+	 * hardware dun() wins if present).  The blk-crypto DUN is little-endian.
+	 */
+	if (snprintf(dun_str, sizeof(dun_str), "dun(%s,le)", cipher_str) >=
+	    (int)sizeof(dun_str)) {
+		err = -EINVAL;
+		goto out;
+	}
+
 	for (i = 0; i < blk_crypto_num_keyslots; i++) {
 		slotp = &blk_crypto_keyslots[i];
-		slotp->tfms[mode_num] = crypto_alloc_sync_skcipher(cipher_str,
-				0, 0);
+		slotp->tfms[mode_num] = crypto_alloc_sync_skcipher(dun_str, 0, 0);
 		if (IS_ERR(slotp->tfms[mode_num])) {
 			err = PTR_ERR(slotp->tfms[mode_num]);
 			if (err == -ENOENT) {
 				pr_warn_once("Missing crypto API support for \"%s\"\n",
-					     cipher_str);
+					     dun_str);
 				err = -ENOPKG;
 			}
 			slotp->tfms[mode_num] = NULL;
-- 
2.47.3