* [PATCH] crypto: arm64/aes-blk - honour iv_out requirement in CBC and CTR modes
From: Ard Biesheuvel @ 2017-01-17 13:46 UTC (permalink / raw)
To: herbert, linux-crypto; +Cc: linux-arm-kernel, Ard Biesheuvel
Update the ARMv8 Crypto Extensions and the plain NEON AES implementations
in CBC and CTR modes to return the next IV back to the skcipher API client.
This is necessary for chaining to work correctly.
Note that for CTR, this is only done if the request is a round multiple of
the block size, since otherwise, chaining is impossible anyway.
Cc: <stable@vger.kernel.org> # v3.16+
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/aes-modes.S | 88 ++++++++++----------
1 file changed, 42 insertions(+), 46 deletions(-)
diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S
index c53dbeae79f2..838dad5c209f 100644
--- a/arch/arm64/crypto/aes-modes.S
+++ b/arch/arm64/crypto/aes-modes.S
@@ -193,15 +193,16 @@ AES_ENTRY(aes_cbc_encrypt)
cbz w6, .Lcbcencloop
ld1 {v0.16b}, [x5] /* get iv */
- enc_prepare w3, x2, x5
+ enc_prepare w3, x2, x6
.Lcbcencloop:
ld1 {v1.16b}, [x1], #16 /* get next pt block */
eor v0.16b, v0.16b, v1.16b /* ..and xor with iv */
- encrypt_block v0, w3, x2, x5, w6
+ encrypt_block v0, w3, x2, x6, w7
st1 {v0.16b}, [x0], #16
subs w4, w4, #1
bne .Lcbcencloop
+ st1 {v0.16b}, [x5] /* return iv */
ret
AES_ENDPROC(aes_cbc_encrypt)
@@ -211,7 +212,7 @@ AES_ENTRY(aes_cbc_decrypt)
cbz w6, .LcbcdecloopNx
ld1 {v7.16b}, [x5] /* get iv */
- dec_prepare w3, x2, x5
+ dec_prepare w3, x2, x6
.LcbcdecloopNx:
#if INTERLEAVE >= 2
@@ -248,7 +249,7 @@ AES_ENTRY(aes_cbc_decrypt)
.Lcbcdecloop:
ld1 {v1.16b}, [x1], #16 /* get next ct block */
mov v0.16b, v1.16b /* ...and copy to v0 */
- decrypt_block v0, w3, x2, x5, w6
+ decrypt_block v0, w3, x2, x6, w7
eor v0.16b, v0.16b, v7.16b /* xor with iv => pt */
mov v7.16b, v1.16b /* ct is next iv */
st1 {v0.16b}, [x0], #16
@@ -256,6 +257,7 @@ AES_ENTRY(aes_cbc_decrypt)
bne .Lcbcdecloop
.Lcbcdecout:
FRAME_POP
+ st1 {v7.16b}, [x5] /* return iv */
ret
AES_ENDPROC(aes_cbc_decrypt)
@@ -267,24 +269,15 @@ AES_ENDPROC(aes_cbc_decrypt)
AES_ENTRY(aes_ctr_encrypt)
FRAME_PUSH
- cbnz w6, .Lctrfirst /* 1st time around? */
- umov x5, v4.d[1] /* keep swabbed ctr in reg */
- rev x5, x5
-#if INTERLEAVE >= 2
- cmn w5, w4 /* 32 bit overflow? */
- bcs .Lctrinc
- add x5, x5, #1 /* increment BE ctr */
- b .LctrincNx
-#else
- b .Lctrinc
-#endif
-.Lctrfirst:
+ cbz w6, .Lctrnotfirst /* 1st time around? */
enc_prepare w3, x2, x6
ld1 {v4.16b}, [x5]
- umov x5, v4.d[1] /* keep swabbed ctr in reg */
- rev x5, x5
+
+.Lctrnotfirst:
+ umov x8, v4.d[1] /* keep swabbed ctr in reg */
+ rev x8, x8
#if INTERLEAVE >= 2
- cmn w5, w4 /* 32 bit overflow? */
+ cmn w8, w4 /* 32 bit overflow? */
bcs .Lctrloop
.LctrloopNx:
subs w4, w4, #INTERLEAVE
@@ -292,11 +285,11 @@ AES_ENTRY(aes_ctr_encrypt)
#if INTERLEAVE == 2
mov v0.8b, v4.8b
mov v1.8b, v4.8b
- rev x7, x5
- add x5, x5, #1
+ rev x7, x8
+ add x8, x8, #1
ins v0.d[1], x7
- rev x7, x5
- add x5, x5, #1
+ rev x7, x8
+ add x8, x8, #1
ins v1.d[1], x7
ld1 {v2.16b-v3.16b}, [x1], #32 /* get 2 input blocks */
do_encrypt_block2x
@@ -305,7 +298,7 @@ AES_ENTRY(aes_ctr_encrypt)
st1 {v0.16b-v1.16b}, [x0], #32
#else
ldr q8, =0x30000000200000001 /* addends 1,2,3[,0] */
- dup v7.4s, w5
+ dup v7.4s, w8
mov v0.16b, v4.16b
add v7.4s, v7.4s, v8.4s
mov v1.16b, v4.16b
@@ -323,18 +316,12 @@ AES_ENTRY(aes_ctr_encrypt)
eor v2.16b, v7.16b, v2.16b
eor v3.16b, v5.16b, v3.16b
st1 {v0.16b-v3.16b}, [x0], #64
- add x5, x5, #INTERLEAVE
+ add x8, x8, #INTERLEAVE
#endif
- cbz w4, .LctroutNx
-.LctrincNx:
- rev x7, x5
+ rev x7, x8
ins v4.d[1], x7
+ cbz w4, .Lctrout
b .LctrloopNx
-.LctroutNx:
- sub x5, x5, #1
- rev x7, x5
- ins v4.d[1], x7
- b .Lctrout
.Lctr1x:
adds w4, w4, #INTERLEAVE
beq .Lctrout
@@ -342,30 +329,39 @@ AES_ENTRY(aes_ctr_encrypt)
.Lctrloop:
mov v0.16b, v4.16b
encrypt_block v0, w3, x2, x6, w7
+
+ adds x8, x8, #1 /* increment BE ctr */
+ rev x7, x8
+ ins v4.d[1], x7
+ bcs .Lctrcarry /* overflow? */
+
+.Lctrcarrydone:
subs w4, w4, #1
bmi .Lctrhalfblock /* blocks < 0 means 1/2 block */
ld1 {v3.16b}, [x1], #16
eor v3.16b, v0.16b, v3.16b
st1 {v3.16b}, [x0], #16
- beq .Lctrout
-.Lctrinc:
- adds x5, x5, #1 /* increment BE ctr */
- rev x7, x5
- ins v4.d[1], x7
- bcc .Lctrloop /* no overflow? */
- umov x7, v4.d[0] /* load upper word of ctr */
- rev x7, x7 /* ... to handle the carry */
- add x7, x7, #1
- rev x7, x7
- ins v4.d[0], x7
- b .Lctrloop
+ bne .Lctrloop
+
+.Lctrout:
+ st1 {v4.16b}, [x5] /* return next CTR value */
+ FRAME_POP
+ ret
+
.Lctrhalfblock:
ld1 {v3.8b}, [x1]
eor v3.8b, v0.8b, v3.8b
st1 {v3.8b}, [x0]
-.Lctrout:
FRAME_POP
ret
+
+.Lctrcarry:
+ umov x7, v4.d[0] /* load upper word of ctr */
+ rev x7, x7 /* ... to handle the carry */
+ add x7, x7, #1
+ rev x7, x7
+ ins v4.d[0], x7
+ b .Lctrcarrydone
AES_ENDPROC(aes_ctr_encrypt)
.ltorg
--
2.7.4
^ permalink raw reply related
* [PATCH 00/10] crypto - AES for ARM/arm64 updates for v4.11 (round #2)
From: Ard Biesheuvel @ 2017-01-17 15:22 UTC (permalink / raw)
To: linux-crypto, herbert; +Cc: linux-arm-kernel, Ard Biesheuvel
Patch #1 is a fix for the CBC chaining issue that was discussed on the
mailing list. The driver itself is queued for v4.11, so this fix can go
right on top.
Patches #2 - #6 clear the cra_alignmasks of various drivers: all NEON
capable CPUs can perform unaligned accesses, and the advantage of using
the slightly faster aligned accessors (which only exist on ARM not arm64)
is certainly outweighed by the cost of copying data to suitably aligned
buffers.
NOTE: patch #5 won't apply unless 'crypto: arm64/aes-blk - honour iv_out
requirement in CBC and CTR modes' is applied first, which was sent out
separately as a bugfix for v3.16 - v4.9. If this is a problem, this patch
can wait.
Patch #7 and #8 are minor tweaks to the new scalar AES code.
Patch #9 improves the performance of the plain NEON AES code, to make it
more suitable as a fallback for the new bitsliced NEON code, which can
only operate on 8 blocks in parallel, and needs another driver to perform
CBC encryption or XTS tweak generation.
Patch #10 updates the new bitsliced AES NEON code to switch to the plain
NEON driver as a fallback.
Patches #9 and #10 improve the performance of CBC encryption by ~35% on
low end cores such as the Cortex-A53 that can be found in the Raspberry Pi3
Ard Biesheuvel (10):
crypto: arm64/aes-neon-bs - honour iv_out requirement in CTR mode
crypto: arm/aes-ce - remove cra_alignmask
crypto: arm/chacha20 - remove cra_alignmask
crypto: arm64/aes-ce-ccm - remove cra_alignmask
crypto: arm64/aes-blk - remove cra_alignmask
crypto: arm64/chacha20 - remove cra_alignmask
crypto: arm64/aes - avoid literals for cross-module symbol references
crypto: arm64/aes - performance tweak
crypto: arm64/aes-neon-blk - tweak performance for low end cores
crypto: arm64/aes - replace scalar fallback with plain NEON fallback
arch/arm/crypto/aes-ce-core.S | 84 ++++-----
arch/arm/crypto/aes-ce-glue.c | 15 +-
arch/arm/crypto/chacha20-neon-glue.c | 1 -
arch/arm64/crypto/Kconfig | 2 +-
arch/arm64/crypto/aes-ce-ccm-glue.c | 1 -
arch/arm64/crypto/aes-cipher-core.S | 59 +++---
arch/arm64/crypto/aes-glue.c | 18 +-
arch/arm64/crypto/aes-modes.S | 8 +-
arch/arm64/crypto/aes-neon.S | 199 ++++++++------------
arch/arm64/crypto/aes-neonbs-core.S | 25 ++-
arch/arm64/crypto/aes-neonbs-glue.c | 38 +++-
arch/arm64/crypto/chacha20-neon-glue.c | 1 -
12 files changed, 199 insertions(+), 252 deletions(-)
--
2.7.4
^ permalink raw reply
* [PATCH 01/10] crypto: arm64/aes-neon-bs - honour iv_out requirement in CTR mode
From: Ard Biesheuvel @ 2017-01-17 15:22 UTC (permalink / raw)
To: linux-crypto, herbert; +Cc: linux-arm-kernel, Ard Biesheuvel
In-Reply-To: <1484666557-31458-1-git-send-email-ard.biesheuvel@linaro.org>
Update the new bitsliced NEON AES implementation in CTR mode to return
the next IV back to the skcipher API client. This is necessary for
chaining to work correctly.
Note that this is only done if the request is a round multiple of the
block size, since otherwise, chaining is impossible anyway.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/aes-neonbs-core.S | 25 +++++++++++++-------
1 file changed, 16 insertions(+), 9 deletions(-)
diff --git a/arch/arm64/crypto/aes-neonbs-core.S b/arch/arm64/crypto/aes-neonbs-core.S
index 8d0cdaa2768d..2ada12dd768e 100644
--- a/arch/arm64/crypto/aes-neonbs-core.S
+++ b/arch/arm64/crypto/aes-neonbs-core.S
@@ -874,12 +874,19 @@ CPU_LE( rev x8, x8 )
csel x4, x4, xzr, pl
csel x9, x9, xzr, le
+ tbnz x9, #1, 0f
next_ctr v1
+ tbnz x9, #2, 0f
next_ctr v2
+ tbnz x9, #3, 0f
next_ctr v3
+ tbnz x9, #4, 0f
next_ctr v4
+ tbnz x9, #5, 0f
next_ctr v5
+ tbnz x9, #6, 0f
next_ctr v6
+ tbnz x9, #7, 0f
next_ctr v7
0: mov bskey, x2
@@ -928,11 +935,11 @@ CPU_LE( rev x8, x8 )
eor v5.16b, v5.16b, v15.16b
st1 {v5.16b}, [x0], #16
- next_ctr v0
+8: next_ctr v0
cbnz x4, 99b
0: st1 {v0.16b}, [x5]
-8: ldp x29, x30, [sp], #16
+9: ldp x29, x30, [sp], #16
ret
/*
@@ -941,23 +948,23 @@ CPU_LE( rev x8, x8 )
*/
1: cbz x6, 8b
st1 {v1.16b}, [x5]
- b 8b
+ b 9b
2: cbz x6, 8b
st1 {v4.16b}, [x5]
- b 8b
+ b 9b
3: cbz x6, 8b
st1 {v6.16b}, [x5]
- b 8b
+ b 9b
4: cbz x6, 8b
st1 {v3.16b}, [x5]
- b 8b
+ b 9b
5: cbz x6, 8b
st1 {v7.16b}, [x5]
- b 8b
+ b 9b
6: cbz x6, 8b
st1 {v2.16b}, [x5]
- b 8b
+ b 9b
7: cbz x6, 8b
st1 {v5.16b}, [x5]
- b 8b
+ b 9b
ENDPROC(aesbs_ctr_encrypt)
--
2.7.4
^ permalink raw reply related
* [PATCH 03/10] crypto: arm/chacha20 - remove cra_alignmask
From: Ard Biesheuvel @ 2017-01-17 15:22 UTC (permalink / raw)
To: linux-crypto, herbert; +Cc: linux-arm-kernel, Ard Biesheuvel
In-Reply-To: <1484666557-31458-1-git-send-email-ard.biesheuvel@linaro.org>
Remove the unnecessary alignmask: it is much more efficient to deal with
the misalignment in the core algorithm than relying on the crypto API to
copy the data to a suitably aligned buffer.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm/crypto/chacha20-neon-glue.c | 1 -
1 file changed, 1 deletion(-)
diff --git a/arch/arm/crypto/chacha20-neon-glue.c b/arch/arm/crypto/chacha20-neon-glue.c
index 592f75ae4fa1..59a7be08e80c 100644
--- a/arch/arm/crypto/chacha20-neon-glue.c
+++ b/arch/arm/crypto/chacha20-neon-glue.c
@@ -94,7 +94,6 @@ static struct skcipher_alg alg = {
.base.cra_priority = 300,
.base.cra_blocksize = 1,
.base.cra_ctxsize = sizeof(struct chacha20_ctx),
- .base.cra_alignmask = 1,
.base.cra_module = THIS_MODULE,
.min_keysize = CHACHA20_KEY_SIZE,
--
2.7.4
^ permalink raw reply related
* [PATCH 05/10] crypto: arm64/aes-blk - remove cra_alignmask
From: Ard Biesheuvel @ 2017-01-17 15:22 UTC (permalink / raw)
To: linux-crypto, herbert; +Cc: linux-arm-kernel, Ard Biesheuvel
In-Reply-To: <1484666557-31458-1-git-send-email-ard.biesheuvel@linaro.org>
Remove the unnecessary alignmask: it is much more efficient to deal with
the misalignment in the core algorithm than relying on the crypto API to
copy the data to a suitably aligned buffer.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
NOTE: this won't apply unless 'crypto: arm64/aes-blk - honour iv_out
requirement in CBC and CTR modes' is applied first, which was sent out
separately as a bugfix for v3.16 - v4.9
arch/arm64/crypto/aes-glue.c | 16 ++++++----------
arch/arm64/crypto/aes-modes.S | 8 +++-----
2 files changed, 9 insertions(+), 15 deletions(-)
diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index 5164aaf82c6a..8ee1fb7aaa4f 100644
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -215,14 +215,15 @@ static int ctr_encrypt(struct skcipher_request *req)
u8 *tsrc = walk.src.virt.addr;
/*
- * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need
- * to tell aes_ctr_encrypt() to only read half a block.
+ * Tell aes_ctr_encrypt() to process a tail block.
*/
- blocks = (nbytes <= 8) ? -1 : 1;
+ blocks = -1;
- aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc, rounds,
+ aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc, rounds,
blocks, walk.iv, first);
- memcpy(tdst, tail, nbytes);
+ if (tdst != tsrc)
+ memcpy(tdst, tsrc, nbytes);
+ crypto_xor(tdst, tail, nbytes);
err = skcipher_walk_done(&walk, 0);
}
kernel_neon_end();
@@ -282,7 +283,6 @@ static struct skcipher_alg aes_algs[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_alignmask = 7,
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
@@ -298,7 +298,6 @@ static struct skcipher_alg aes_algs[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_alignmask = 7,
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
@@ -315,7 +314,6 @@ static struct skcipher_alg aes_algs[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_alignmask = 7,
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
@@ -332,7 +330,6 @@ static struct skcipher_alg aes_algs[] = { {
.cra_priority = PRIO - 1,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_alignmask = 7,
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
@@ -350,7 +347,6 @@ static struct skcipher_alg aes_algs[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_aes_xts_ctx),
- .cra_alignmask = 7,
.cra_module = THIS_MODULE,
},
.min_keysize = 2 * AES_MIN_KEY_SIZE,
diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S
index 838dad5c209f..92b982a8b112 100644
--- a/arch/arm64/crypto/aes-modes.S
+++ b/arch/arm64/crypto/aes-modes.S
@@ -337,7 +337,7 @@ AES_ENTRY(aes_ctr_encrypt)
.Lctrcarrydone:
subs w4, w4, #1
- bmi .Lctrhalfblock /* blocks < 0 means 1/2 block */
+ bmi .Lctrtailblock /* blocks <0 means tail block */
ld1 {v3.16b}, [x1], #16
eor v3.16b, v0.16b, v3.16b
st1 {v3.16b}, [x0], #16
@@ -348,10 +348,8 @@ AES_ENTRY(aes_ctr_encrypt)
FRAME_POP
ret
-.Lctrhalfblock:
- ld1 {v3.8b}, [x1]
- eor v3.8b, v0.8b, v3.8b
- st1 {v3.8b}, [x0]
+.Lctrtailblock:
+ st1 {v0.16b}, [x0]
FRAME_POP
ret
--
2.7.4
^ permalink raw reply related
* [PATCH 06/10] crypto: arm64/chacha20 - remove cra_alignmask
From: Ard Biesheuvel @ 2017-01-17 15:22 UTC (permalink / raw)
To: linux-crypto, herbert; +Cc: linux-arm-kernel, Ard Biesheuvel
In-Reply-To: <1484666557-31458-1-git-send-email-ard.biesheuvel@linaro.org>
Remove the unnecessary alignmask: it is much more efficient to deal with
the misalignment in the core algorithm than relying on the crypto API to
copy the data to a suitably aligned buffer.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/chacha20-neon-glue.c | 1 -
1 file changed, 1 deletion(-)
diff --git a/arch/arm64/crypto/chacha20-neon-glue.c b/arch/arm64/crypto/chacha20-neon-glue.c
index a7f2337d46cf..a7cd575ea223 100644
--- a/arch/arm64/crypto/chacha20-neon-glue.c
+++ b/arch/arm64/crypto/chacha20-neon-glue.c
@@ -93,7 +93,6 @@ static struct skcipher_alg alg = {
.base.cra_priority = 300,
.base.cra_blocksize = 1,
.base.cra_ctxsize = sizeof(struct chacha20_ctx),
- .base.cra_alignmask = 1,
.base.cra_module = THIS_MODULE,
.min_keysize = CHACHA20_KEY_SIZE,
--
2.7.4
^ permalink raw reply related
* [PATCH 04/10] crypto: arm64/aes-ce-ccm - remove cra_alignmask
From: Ard Biesheuvel @ 2017-01-17 15:22 UTC (permalink / raw)
To: linux-crypto, herbert; +Cc: linux-arm-kernel, Ard Biesheuvel
In-Reply-To: <1484666557-31458-1-git-send-email-ard.biesheuvel@linaro.org>
Remove the unnecessary alignmask: it is much more efficient to deal with
the misalignment in the core algorithm than relying on the crypto API to
copy the data to a suitably aligned buffer.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/aes-ce-ccm-glue.c | 1 -
1 file changed, 1 deletion(-)
diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c
index cc5515dac74a..6a7dbc7c83a6 100644
--- a/arch/arm64/crypto/aes-ce-ccm-glue.c
+++ b/arch/arm64/crypto/aes-ce-ccm-glue.c
@@ -258,7 +258,6 @@ static struct aead_alg ccm_aes_alg = {
.cra_priority = 300,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_alignmask = 7,
.cra_module = THIS_MODULE,
},
.ivsize = AES_BLOCK_SIZE,
--
2.7.4
^ permalink raw reply related
* [PATCH 08/10] crypto: arm64/aes - performance tweak
From: Ard Biesheuvel @ 2017-01-17 15:22 UTC (permalink / raw)
To: linux-crypto, herbert; +Cc: linux-arm-kernel, Ard Biesheuvel
In-Reply-To: <1484666557-31458-1-git-send-email-ard.biesheuvel@linaro.org>
Shuffle some instructions around in the __hround macro to shave off
0.1 cycles per byte on Cortex-A57.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/aes-cipher-core.S | 52 +++++++-------------
1 file changed, 19 insertions(+), 33 deletions(-)
diff --git a/arch/arm64/crypto/aes-cipher-core.S b/arch/arm64/crypto/aes-cipher-core.S
index cd58c61e6677..f2f9cc519309 100644
--- a/arch/arm64/crypto/aes-cipher-core.S
+++ b/arch/arm64/crypto/aes-cipher-core.S
@@ -20,46 +20,32 @@
tt .req x4
lt .req x2
- .macro __hround, out0, out1, in0, in1, in2, in3, t0, t1, enc
- ldp \out0, \out1, [rk], #8
-
- ubfx w13, \in0, #0, #8
- ubfx w14, \in1, #8, #8
- ldr w13, [tt, w13, uxtw #2]
- ldr w14, [tt, w14, uxtw #2]
-
+ .macro __pair, enc, reg0, reg1, in0, in1e, in1d, shift
+ ubfx \reg0, \in0, #\shift, #8
.if \enc
- ubfx w17, \in1, #0, #8
- ubfx w18, \in2, #8, #8
+ ubfx \reg1, \in1e, #\shift, #8
.else
- ubfx w17, \in3, #0, #8
- ubfx w18, \in0, #8, #8
+ ubfx \reg1, \in1d, #\shift, #8
.endif
- ldr w17, [tt, w17, uxtw #2]
- ldr w18, [tt, w18, uxtw #2]
+ ldr \reg0, [tt, \reg0, uxtw #2]
+ ldr \reg1, [tt, \reg1, uxtw #2]
+ .endm
- ubfx w15, \in2, #16, #8
- ubfx w16, \in3, #24, #8
- ldr w15, [tt, w15, uxtw #2]
- ldr w16, [tt, w16, uxtw #2]
+ .macro __hround, out0, out1, in0, in1, in2, in3, t0, t1, enc
+ ldp \out0, \out1, [rk], #8
- .if \enc
- ubfx \t0, \in3, #16, #8
- ubfx \t1, \in0, #24, #8
- .else
- ubfx \t0, \in1, #16, #8
- ubfx \t1, \in2, #24, #8
- .endif
- ldr \t0, [tt, \t0, uxtw #2]
- ldr \t1, [tt, \t1, uxtw #2]
+ __pair \enc, w13, w14, \in0, \in1, \in3, 0
+ __pair \enc, w15, w16, \in1, \in2, \in0, 8
+ __pair \enc, w17, w18, \in2, \in3, \in1, 16
+ __pair \enc, \t0, \t1, \in3, \in0, \in2, 24
eor \out0, \out0, w13
- eor \out1, \out1, w17
- eor \out0, \out0, w14, ror #24
- eor \out1, \out1, w18, ror #24
- eor \out0, \out0, w15, ror #16
- eor \out1, \out1, \t0, ror #16
- eor \out0, \out0, w16, ror #8
+ eor \out1, \out1, w14
+ eor \out0, \out0, w15, ror #24
+ eor \out1, \out1, w16, ror #24
+ eor \out0, \out0, w17, ror #16
+ eor \out1, \out1, w18, ror #16
+ eor \out0, \out0, \t0, ror #8
eor \out1, \out1, \t1, ror #8
.endm
--
2.7.4
^ permalink raw reply related
* [PATCH 07/10] crypto: arm64/aes - avoid literals for cross-module symbol references
From: Ard Biesheuvel @ 2017-01-17 15:22 UTC (permalink / raw)
To: linux-crypto, herbert; +Cc: linux-arm-kernel, Ard Biesheuvel
In-Reply-To: <1484666557-31458-1-git-send-email-ard.biesheuvel@linaro.org>
Using simple adrp/add pairs to refer to the AES lookup tables exposed by
the generic AES driver (which could be loaded far away from this driver
when KASLR is in effect) was unreliable at module load time before commit
41c066f2c4d4 ("arm64: assembler: make adr_l work in modules under KASLR"),
which is why the AES code used literals instead.
So now we can get rid of the literals, and switch to the adr_l macro.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/aes-cipher-core.S | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/arch/arm64/crypto/aes-cipher-core.S b/arch/arm64/crypto/aes-cipher-core.S
index 37590ab8121a..cd58c61e6677 100644
--- a/arch/arm64/crypto/aes-cipher-core.S
+++ b/arch/arm64/crypto/aes-cipher-core.S
@@ -89,8 +89,8 @@ CPU_BE( rev w8, w8 )
eor w7, w7, w11
eor w8, w8, w12
- ldr tt, =\ttab
- ldr lt, =\ltab
+ adr_l tt, \ttab
+ adr_l lt, \ltab
tbnz rounds, #1, 1f
@@ -111,9 +111,6 @@ CPU_BE( rev w8, w8 )
stp w5, w6, [out]
stp w7, w8, [out, #8]
ret
-
- .align 4
- .ltorg
.endm
.align 5
--
2.7.4
^ permalink raw reply related
* [PATCH 10/10] crypto: arm64/aes - replace scalar fallback with plain NEON fallback
From: Ard Biesheuvel @ 2017-01-17 15:22 UTC (permalink / raw)
To: linux-crypto, herbert; +Cc: linux-arm-kernel, Ard Biesheuvel
In-Reply-To: <1484666557-31458-1-git-send-email-ard.biesheuvel@linaro.org>
The new bitsliced NEON implementation of AES uses a fallback in two
places: CBC encryption (which is strictly sequential, whereas this
driver can only operate efficiently on 8 blocks at a time), and the
XTS tweak generation, which involves encrypting a single AES block
with a different key schedule.
The plain (i.e., non-bitsliced) NEON code is more suitable as a fallback,
given that it is faster than scalar on low end cores (which is what
the NEON implementations target, since high end cores have dedicated
instructions for AES), and shows similar behavior in terms of D-cache
footprint and sensitivity to cache timing attacks. So switch the fallback
handling to the plain NEON driver.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/Kconfig | 2 +-
arch/arm64/crypto/aes-neonbs-glue.c | 38 ++++++++++++++------
2 files changed, 29 insertions(+), 11 deletions(-)
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 5de75c3dcbd4..bed7feddfeed 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -86,7 +86,7 @@ config CRYPTO_AES_ARM64_BS
tristate "AES in ECB/CBC/CTR/XTS modes using bit-sliced NEON algorithm"
depends on KERNEL_MODE_NEON
select CRYPTO_BLKCIPHER
- select CRYPTO_AES_ARM64
+ select CRYPTO_AES_ARM64_NEON_BLK
select CRYPTO_SIMD
endif
diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c
index 323dd76ae5f0..863e436ecf89 100644
--- a/arch/arm64/crypto/aes-neonbs-glue.c
+++ b/arch/arm64/crypto/aes-neonbs-glue.c
@@ -10,7 +10,6 @@
#include <asm/neon.h>
#include <crypto/aes.h>
-#include <crypto/cbc.h>
#include <crypto/internal/simd.h>
#include <crypto/internal/skcipher.h>
#include <crypto/xts.h>
@@ -42,7 +41,12 @@ asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[],
asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[],
int rounds, int blocks, u8 iv[]);
-asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
+/* borrowed from aes-neon-blk.ko */
+asmlinkage void neon_aes_ecb_encrypt(u8 out[], u8 const in[], u32 const rk[],
+ int rounds, int blocks, int first);
+asmlinkage void neon_aes_cbc_encrypt(u8 out[], u8 const in[], u32 const rk[],
+ int rounds, int blocks, u8 iv[],
+ int first);
struct aesbs_ctx {
u8 rk[13 * (8 * AES_BLOCK_SIZE) + 32];
@@ -140,16 +144,28 @@ static int aesbs_cbc_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
return 0;
}
-static void cbc_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst)
+static int cbc_encrypt(struct skcipher_request *req)
{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ int err, first = 1;
- __aes_arm64_encrypt(ctx->enc, dst, src, ctx->key.rounds);
-}
+ err = skcipher_walk_virt(&walk, req, true);
-static int cbc_encrypt(struct skcipher_request *req)
-{
- return crypto_cbc_encrypt_walk(req, cbc_encrypt_one);
+ kernel_neon_begin();
+ while (walk.nbytes >= AES_BLOCK_SIZE) {
+ unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
+
+ /* fall back to the non-bitsliced NEON implementation */
+ neon_aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
+ ctx->enc, ctx->key.rounds, blocks, walk.iv,
+ first);
+ err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
+ first = 0;
+ }
+ kernel_neon_end();
+ return err;
}
static int cbc_decrypt(struct skcipher_request *req)
@@ -254,9 +270,11 @@ static int __xts_crypt(struct skcipher_request *req,
err = skcipher_walk_virt(&walk, req, true);
- __aes_arm64_encrypt(ctx->twkey, walk.iv, walk.iv, ctx->key.rounds);
-
kernel_neon_begin();
+
+ neon_aes_ecb_encrypt(walk.iv, walk.iv, ctx->twkey,
+ ctx->key.rounds, 1, 1);
+
while (walk.nbytes >= AES_BLOCK_SIZE) {
unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
--
2.7.4
^ permalink raw reply related
* [PATCH 09/10] crypto: arm64/aes-neon-blk - tweak performance for low end cores
From: Ard Biesheuvel @ 2017-01-17 15:22 UTC (permalink / raw)
To: linux-crypto, herbert; +Cc: linux-arm-kernel, Ard Biesheuvel
In-Reply-To: <1484666557-31458-1-git-send-email-ard.biesheuvel@linaro.org>
The non-bitsliced AES implementation using the NEON is highly sensitive
to micro-architectural details, and, as it turns out, the Cortex-A53 on
the Raspberry Pi 3 is a core that can benefit from this code, given that
its scalar AES performance is abysmal (32.9 cycles per byte).
The new bitsliced AES code manages 19.8 cycles per byte on this core,
but can only operate on 8 blocks at a time, which is not supported by
all chaining modes. With a bit of tweaking, we can get the plain NEON
code to run at 24.0 cycles per byte, making it useful for sequential
modes like CBC encryption. (Like bitsliced NEON, the plain NEON
implementation does not use any lookup tables, which makes it easy on
the D-cache, and invulnerable to cache timing attacks)
So tweak the plain NEON AES code to use tbl instructions rather than
shl/sri pairs, and to avoid the need to reload permutation vectors or
other constants from memory in every round.
To allow the ECB and CBC encrypt routines to be reused by the bitsliced
NEON code in a subsequent patch, export them from the module.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/aes-glue.c | 2 +
arch/arm64/crypto/aes-neon.S | 199 ++++++++------------
2 files changed, 77 insertions(+), 124 deletions(-)
diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index 8ee1fb7aaa4f..055bc3f61138 100644
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -409,5 +409,7 @@ static int __init aes_init(void)
module_cpu_feature_match(AES, aes_init);
#else
module_init(aes_init);
+EXPORT_SYMBOL(neon_aes_ecb_encrypt);
+EXPORT_SYMBOL(neon_aes_cbc_encrypt);
#endif
module_exit(aes_exit);
diff --git a/arch/arm64/crypto/aes-neon.S b/arch/arm64/crypto/aes-neon.S
index 85f07ead7c5c..67c68462bc20 100644
--- a/arch/arm64/crypto/aes-neon.S
+++ b/arch/arm64/crypto/aes-neon.S
@@ -1,7 +1,7 @@
/*
* linux/arch/arm64/crypto/aes-neon.S - AES cipher for ARMv8 NEON
*
- * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
+ * Copyright (C) 2013 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
@@ -25,9 +25,9 @@
/* preload the entire Sbox */
.macro prepare, sbox, shiftrows, temp
adr \temp, \sbox
- movi v12.16b, #0x40
+ movi v12.16b, #0x1b
ldr q13, \shiftrows
- movi v14.16b, #0x1b
+ ldr q14, .Lror32by8
ld1 {v16.16b-v19.16b}, [\temp], #64
ld1 {v20.16b-v23.16b}, [\temp], #64
ld1 {v24.16b-v27.16b}, [\temp], #64
@@ -50,37 +50,33 @@
/* apply SubBytes transformation using the the preloaded Sbox */
.macro sub_bytes, in
- sub v9.16b, \in\().16b, v12.16b
+ sub v9.16b, \in\().16b, v15.16b
tbl \in\().16b, {v16.16b-v19.16b}, \in\().16b
- sub v10.16b, v9.16b, v12.16b
+ sub v10.16b, v9.16b, v15.16b
tbx \in\().16b, {v20.16b-v23.16b}, v9.16b
- sub v11.16b, v10.16b, v12.16b
+ sub v11.16b, v10.16b, v15.16b
tbx \in\().16b, {v24.16b-v27.16b}, v10.16b
tbx \in\().16b, {v28.16b-v31.16b}, v11.16b
.endm
/* apply MixColumns transformation */
- .macro mix_columns, in
- mul_by_x v10.16b, \in\().16b, v9.16b, v14.16b
- rev32 v8.8h, \in\().8h
- eor \in\().16b, v10.16b, \in\().16b
- shl v9.4s, v8.4s, #24
- shl v11.4s, \in\().4s, #24
- sri v9.4s, v8.4s, #8
- sri v11.4s, \in\().4s, #8
- eor v9.16b, v9.16b, v8.16b
- eor v10.16b, v10.16b, v9.16b
- eor \in\().16b, v10.16b, v11.16b
- .endm
-
+ .macro mix_columns, in, enc
+ .if \enc == 0
/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
- .macro inv_mix_columns, in
- mul_by_x v11.16b, \in\().16b, v10.16b, v14.16b
- mul_by_x v11.16b, v11.16b, v10.16b, v14.16b
+ mul_by_x v11.16b, \in\().16b, v10.16b, v12.16b
+ mul_by_x v11.16b, v11.16b, v10.16b, v12.16b
eor \in\().16b, \in\().16b, v11.16b
rev32 v11.8h, v11.8h
eor \in\().16b, \in\().16b, v11.16b
- mix_columns \in
+ .endif
+
+ mul_by_x v10.16b, \in\().16b, v9.16b, v12.16b
+ rev32 v8.8h, \in\().8h
+ eor \in\().16b, \in\().16b, v10.16b
+ eor v10.16b, v10.16b, v8.16b
+ eor v11.16b, \in\().16b, v8.16b
+ tbl v11.16b, {v11.16b}, v14.16b
+ eor \in\().16b, v10.16b, v11.16b
.endm
.macro do_block, enc, in, rounds, rk, rkp, i
@@ -88,16 +84,13 @@
add \rkp, \rk, #16
mov \i, \rounds
1111: eor \in\().16b, \in\().16b, v15.16b /* ^round key */
+ movi v15.16b, #0x40
tbl \in\().16b, {\in\().16b}, v13.16b /* ShiftRows */
sub_bytes \in
- ld1 {v15.4s}, [\rkp], #16
subs \i, \i, #1
+ ld1 {v15.4s}, [\rkp], #16
beq 2222f
- .if \enc == 1
- mix_columns \in
- .else
- inv_mix_columns \in
- .endif
+ mix_columns \in, \enc
b 1111b
2222: eor \in\().16b, \in\().16b, v15.16b /* ^round key */
.endm
@@ -116,48 +109,48 @@
*/
.macro sub_bytes_2x, in0, in1
- sub v8.16b, \in0\().16b, v12.16b
- sub v9.16b, \in1\().16b, v12.16b
+ sub v8.16b, \in0\().16b, v15.16b
tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b
+ sub v9.16b, \in1\().16b, v15.16b
tbl \in1\().16b, {v16.16b-v19.16b}, \in1\().16b
- sub v10.16b, v8.16b, v12.16b
- sub v11.16b, v9.16b, v12.16b
+ sub v10.16b, v8.16b, v15.16b
tbx \in0\().16b, {v20.16b-v23.16b}, v8.16b
+ sub v11.16b, v9.16b, v15.16b
tbx \in1\().16b, {v20.16b-v23.16b}, v9.16b
- sub v8.16b, v10.16b, v12.16b
- sub v9.16b, v11.16b, v12.16b
+ sub v8.16b, v10.16b, v15.16b
tbx \in0\().16b, {v24.16b-v27.16b}, v10.16b
+ sub v9.16b, v11.16b, v15.16b
tbx \in1\().16b, {v24.16b-v27.16b}, v11.16b
tbx \in0\().16b, {v28.16b-v31.16b}, v8.16b
tbx \in1\().16b, {v28.16b-v31.16b}, v9.16b
.endm
.macro sub_bytes_4x, in0, in1, in2, in3
- sub v8.16b, \in0\().16b, v12.16b
+ sub v8.16b, \in0\().16b, v15.16b
tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b
- sub v9.16b, \in1\().16b, v12.16b
+ sub v9.16b, \in1\().16b, v15.16b
tbl \in1\().16b, {v16.16b-v19.16b}, \in1\().16b
- sub v10.16b, \in2\().16b, v12.16b
+ sub v10.16b, \in2\().16b, v15.16b
tbl \in2\().16b, {v16.16b-v19.16b}, \in2\().16b
- sub v11.16b, \in3\().16b, v12.16b
+ sub v11.16b, \in3\().16b, v15.16b
tbl \in3\().16b, {v16.16b-v19.16b}, \in3\().16b
tbx \in0\().16b, {v20.16b-v23.16b}, v8.16b
tbx \in1\().16b, {v20.16b-v23.16b}, v9.16b
- sub v8.16b, v8.16b, v12.16b
+ sub v8.16b, v8.16b, v15.16b
tbx \in2\().16b, {v20.16b-v23.16b}, v10.16b
- sub v9.16b, v9.16b, v12.16b
+ sub v9.16b, v9.16b, v15.16b
tbx \in3\().16b, {v20.16b-v23.16b}, v11.16b
- sub v10.16b, v10.16b, v12.16b
+ sub v10.16b, v10.16b, v15.16b
tbx \in0\().16b, {v24.16b-v27.16b}, v8.16b
- sub v11.16b, v11.16b, v12.16b
+ sub v11.16b, v11.16b, v15.16b
tbx \in1\().16b, {v24.16b-v27.16b}, v9.16b
- sub v8.16b, v8.16b, v12.16b
+ sub v8.16b, v8.16b, v15.16b
tbx \in2\().16b, {v24.16b-v27.16b}, v10.16b
- sub v9.16b, v9.16b, v12.16b
+ sub v9.16b, v9.16b, v15.16b
tbx \in3\().16b, {v24.16b-v27.16b}, v11.16b
- sub v10.16b, v10.16b, v12.16b
+ sub v10.16b, v10.16b, v15.16b
tbx \in0\().16b, {v28.16b-v31.16b}, v8.16b
- sub v11.16b, v11.16b, v12.16b
+ sub v11.16b, v11.16b, v15.16b
tbx \in1\().16b, {v28.16b-v31.16b}, v9.16b
tbx \in2\().16b, {v28.16b-v31.16b}, v10.16b
tbx \in3\().16b, {v28.16b-v31.16b}, v11.16b
@@ -174,59 +167,32 @@
eor \out1\().16b, \out1\().16b, \tmp1\().16b
.endm
- .macro mix_columns_2x, in0, in1
- mul_by_x_2x v8, v9, \in0, \in1, v10, v11, v14
- rev32 v10.8h, \in0\().8h
- rev32 v11.8h, \in1\().8h
- eor \in0\().16b, v8.16b, \in0\().16b
- eor \in1\().16b, v9.16b, \in1\().16b
- shl v12.4s, v10.4s, #24
- shl v13.4s, v11.4s, #24
- eor v8.16b, v8.16b, v10.16b
- sri v12.4s, v10.4s, #8
- shl v10.4s, \in0\().4s, #24
- eor v9.16b, v9.16b, v11.16b
- sri v13.4s, v11.4s, #8
- shl v11.4s, \in1\().4s, #24
- sri v10.4s, \in0\().4s, #8
- eor \in0\().16b, v8.16b, v12.16b
- sri v11.4s, \in1\().4s, #8
- eor \in1\().16b, v9.16b, v13.16b
- eor \in0\().16b, v10.16b, \in0\().16b
- eor \in1\().16b, v11.16b, \in1\().16b
- .endm
-
- .macro inv_mix_cols_2x, in0, in1
- mul_by_x_2x v8, v9, \in0, \in1, v10, v11, v14
- mul_by_x_2x v8, v9, v8, v9, v10, v11, v14
+ .macro mix_columns_2x, in0, in1, enc
+ .if \enc == 0
+ /* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
+ mul_by_x_2x v8, v9, \in0, \in1, v10, v11, v12
+ mul_by_x_2x v8, v9, v8, v9, v10, v11, v12
eor \in0\().16b, \in0\().16b, v8.16b
- eor \in1\().16b, \in1\().16b, v9.16b
rev32 v8.8h, v8.8h
+ eor \in1\().16b, \in1\().16b, v9.16b
rev32 v9.8h, v9.8h
eor \in0\().16b, \in0\().16b, v8.16b
eor \in1\().16b, \in1\().16b, v9.16b
- mix_columns_2x \in0, \in1
- .endm
+ .endif
- .macro inv_mix_cols_4x, in0, in1, in2, in3
- mul_by_x_2x v8, v9, \in0, \in1, v10, v11, v14
- mul_by_x_2x v10, v11, \in2, \in3, v12, v13, v14
- mul_by_x_2x v8, v9, v8, v9, v12, v13, v14
- mul_by_x_2x v10, v11, v10, v11, v12, v13, v14
+ mul_by_x_2x v8, v9, \in0, \in1, v10, v11, v12
+ rev32 v10.8h, \in0\().8h
eor \in0\().16b, \in0\().16b, v8.16b
+ rev32 v11.8h, \in1\().8h
eor \in1\().16b, \in1\().16b, v9.16b
- eor \in2\().16b, \in2\().16b, v10.16b
- eor \in3\().16b, \in3\().16b, v11.16b
- rev32 v8.8h, v8.8h
- rev32 v9.8h, v9.8h
- rev32 v10.8h, v10.8h
- rev32 v11.8h, v11.8h
+ eor v8.16b, v8.16b, v10.16b
+ eor \in0\().16b, \in0\().16b, v10.16b
+ eor v9.16b, v9.16b, v11.16b
+ eor \in1\().16b, \in1\().16b, v11.16b
+ tbl \in0\().16b, {\in0\().16b}, v14.16b
+ tbl \in1\().16b, {\in1\().16b}, v14.16b
eor \in0\().16b, \in0\().16b, v8.16b
eor \in1\().16b, \in1\().16b, v9.16b
- eor \in2\().16b, \in2\().16b, v10.16b
- eor \in3\().16b, \in3\().16b, v11.16b
- mix_columns_2x \in0, \in1
- mix_columns_2x \in2, \in3
.endm
.macro do_block_2x, enc, in0, in1 rounds, rk, rkp, i
@@ -235,20 +201,14 @@
mov \i, \rounds
1111: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */
eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */
- sub_bytes_2x \in0, \in1
+ movi v15.16b, #0x40
tbl \in0\().16b, {\in0\().16b}, v13.16b /* ShiftRows */
tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */
- ld1 {v15.4s}, [\rkp], #16
+ sub_bytes_2x \in0, \in1
subs \i, \i, #1
+ ld1 {v15.4s}, [\rkp], #16
beq 2222f
- .if \enc == 1
- mix_columns_2x \in0, \in1
- ldr q13, .LForward_ShiftRows
- .else
- inv_mix_cols_2x \in0, \in1
- ldr q13, .LReverse_ShiftRows
- .endif
- movi v12.16b, #0x40
+ mix_columns_2x \in0, \in1, \enc
b 1111b
2222: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */
eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */
@@ -262,23 +222,17 @@
eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */
eor \in2\().16b, \in2\().16b, v15.16b /* ^round key */
eor \in3\().16b, \in3\().16b, v15.16b /* ^round key */
- sub_bytes_4x \in0, \in1, \in2, \in3
+ movi v15.16b, #0x40
tbl \in0\().16b, {\in0\().16b}, v13.16b /* ShiftRows */
tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */
tbl \in2\().16b, {\in2\().16b}, v13.16b /* ShiftRows */
tbl \in3\().16b, {\in3\().16b}, v13.16b /* ShiftRows */
- ld1 {v15.4s}, [\rkp], #16
+ sub_bytes_4x \in0, \in1, \in2, \in3
subs \i, \i, #1
+ ld1 {v15.4s}, [\rkp], #16
beq 2222f
- .if \enc == 1
- mix_columns_2x \in0, \in1
- mix_columns_2x \in2, \in3
- ldr q13, .LForward_ShiftRows
- .else
- inv_mix_cols_4x \in0, \in1, \in2, \in3
- ldr q13, .LReverse_ShiftRows
- .endif
- movi v12.16b, #0x40
+ mix_columns_2x \in0, \in1, \enc
+ mix_columns_2x \in2, \in3, \enc
b 1111b
2222: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */
eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */
@@ -305,19 +259,7 @@
#include "aes-modes.S"
.text
- .align 4
-.LForward_ShiftRows:
-CPU_LE( .byte 0x0, 0x5, 0xa, 0xf, 0x4, 0x9, 0xe, 0x3 )
-CPU_LE( .byte 0x8, 0xd, 0x2, 0x7, 0xc, 0x1, 0x6, 0xb )
-CPU_BE( .byte 0xb, 0x6, 0x1, 0xc, 0x7, 0x2, 0xd, 0x8 )
-CPU_BE( .byte 0x3, 0xe, 0x9, 0x4, 0xf, 0xa, 0x5, 0x0 )
-
-.LReverse_ShiftRows:
-CPU_LE( .byte 0x0, 0xd, 0xa, 0x7, 0x4, 0x1, 0xe, 0xb )
-CPU_LE( .byte 0x8, 0x5, 0x2, 0xf, 0xc, 0x9, 0x6, 0x3 )
-CPU_BE( .byte 0x3, 0x6, 0x9, 0xc, 0xf, 0x2, 0x5, 0x8 )
-CPU_BE( .byte 0xb, 0xe, 0x1, 0x4, 0x7, 0xa, 0xd, 0x0 )
-
+ .align 6
.LForward_Sbox:
.byte 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5
.byte 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76
@@ -385,3 +327,12 @@ CPU_BE( .byte 0xb, 0xe, 0x1, 0x4, 0x7, 0xa, 0xd, 0x0 )
.byte 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61
.byte 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26
.byte 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d
+
+.LForward_ShiftRows:
+ .octa 0x0b06010c07020d08030e09040f0a0500
+
+.LReverse_ShiftRows:
+ .octa 0x0306090c0f0205080b0e0104070a0d00
+
+.Lror32by8:
+ .octa 0x0c0f0e0d080b0a090407060500030201
--
2.7.4
^ permalink raw reply related
* [PATCH 02/10] crypto: arm/aes-ce - remove cra_alignmask
From: Ard Biesheuvel @ 2017-01-17 15:22 UTC (permalink / raw)
To: linux-crypto, herbert; +Cc: linux-arm-kernel, Ard Biesheuvel
In-Reply-To: <1484666557-31458-1-git-send-email-ard.biesheuvel@linaro.org>
Remove the unnecessary alignmask: it is much more efficient to deal with
the misalignment in the core algorithm than relying on the crypto API to
copy the data to a suitably aligned buffer.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm/crypto/aes-ce-core.S | 84 ++++++++++----------
arch/arm/crypto/aes-ce-glue.c | 15 ++--
2 files changed, 47 insertions(+), 52 deletions(-)
diff --git a/arch/arm/crypto/aes-ce-core.S b/arch/arm/crypto/aes-ce-core.S
index 987aa632c9f0..ba8e6a32fdc9 100644
--- a/arch/arm/crypto/aes-ce-core.S
+++ b/arch/arm/crypto/aes-ce-core.S
@@ -169,19 +169,19 @@ ENTRY(ce_aes_ecb_encrypt)
.Lecbencloop3x:
subs r4, r4, #3
bmi .Lecbenc1x
- vld1.8 {q0-q1}, [r1, :64]!
- vld1.8 {q2}, [r1, :64]!
+ vld1.8 {q0-q1}, [r1]!
+ vld1.8 {q2}, [r1]!
bl aes_encrypt_3x
- vst1.8 {q0-q1}, [r0, :64]!
- vst1.8 {q2}, [r0, :64]!
+ vst1.8 {q0-q1}, [r0]!
+ vst1.8 {q2}, [r0]!
b .Lecbencloop3x
.Lecbenc1x:
adds r4, r4, #3
beq .Lecbencout
.Lecbencloop:
- vld1.8 {q0}, [r1, :64]!
+ vld1.8 {q0}, [r1]!
bl aes_encrypt
- vst1.8 {q0}, [r0, :64]!
+ vst1.8 {q0}, [r0]!
subs r4, r4, #1
bne .Lecbencloop
.Lecbencout:
@@ -195,19 +195,19 @@ ENTRY(ce_aes_ecb_decrypt)
.Lecbdecloop3x:
subs r4, r4, #3
bmi .Lecbdec1x
- vld1.8 {q0-q1}, [r1, :64]!
- vld1.8 {q2}, [r1, :64]!
+ vld1.8 {q0-q1}, [r1]!
+ vld1.8 {q2}, [r1]!
bl aes_decrypt_3x
- vst1.8 {q0-q1}, [r0, :64]!
- vst1.8 {q2}, [r0, :64]!
+ vst1.8 {q0-q1}, [r0]!
+ vst1.8 {q2}, [r0]!
b .Lecbdecloop3x
.Lecbdec1x:
adds r4, r4, #3
beq .Lecbdecout
.Lecbdecloop:
- vld1.8 {q0}, [r1, :64]!
+ vld1.8 {q0}, [r1]!
bl aes_decrypt
- vst1.8 {q0}, [r0, :64]!
+ vst1.8 {q0}, [r0]!
subs r4, r4, #1
bne .Lecbdecloop
.Lecbdecout:
@@ -226,10 +226,10 @@ ENTRY(ce_aes_cbc_encrypt)
vld1.8 {q0}, [r5]
prepare_key r2, r3
.Lcbcencloop:
- vld1.8 {q1}, [r1, :64]! @ get next pt block
+ vld1.8 {q1}, [r1]! @ get next pt block
veor q0, q0, q1 @ ..and xor with iv
bl aes_encrypt
- vst1.8 {q0}, [r0, :64]!
+ vst1.8 {q0}, [r0]!
subs r4, r4, #1
bne .Lcbcencloop
vst1.8 {q0}, [r5]
@@ -244,8 +244,8 @@ ENTRY(ce_aes_cbc_decrypt)
.Lcbcdecloop3x:
subs r4, r4, #3
bmi .Lcbcdec1x
- vld1.8 {q0-q1}, [r1, :64]!
- vld1.8 {q2}, [r1, :64]!
+ vld1.8 {q0-q1}, [r1]!
+ vld1.8 {q2}, [r1]!
vmov q3, q0
vmov q4, q1
vmov q5, q2
@@ -254,19 +254,19 @@ ENTRY(ce_aes_cbc_decrypt)
veor q1, q1, q3
veor q2, q2, q4
vmov q6, q5
- vst1.8 {q0-q1}, [r0, :64]!
- vst1.8 {q2}, [r0, :64]!
+ vst1.8 {q0-q1}, [r0]!
+ vst1.8 {q2}, [r0]!
b .Lcbcdecloop3x
.Lcbcdec1x:
adds r4, r4, #3
beq .Lcbcdecout
vmov q15, q14 @ preserve last round key
.Lcbcdecloop:
- vld1.8 {q0}, [r1, :64]! @ get next ct block
+ vld1.8 {q0}, [r1]! @ get next ct block
veor q14, q15, q6 @ combine prev ct with last key
vmov q6, q0
bl aes_decrypt
- vst1.8 {q0}, [r0, :64]!
+ vst1.8 {q0}, [r0]!
subs r4, r4, #1
bne .Lcbcdecloop
.Lcbcdecout:
@@ -300,15 +300,15 @@ ENTRY(ce_aes_ctr_encrypt)
rev ip, r6
add r6, r6, #1
vmov s11, ip
- vld1.8 {q3-q4}, [r1, :64]!
- vld1.8 {q5}, [r1, :64]!
+ vld1.8 {q3-q4}, [r1]!
+ vld1.8 {q5}, [r1]!
bl aes_encrypt_3x
veor q0, q0, q3
veor q1, q1, q4
veor q2, q2, q5
rev ip, r6
- vst1.8 {q0-q1}, [r0, :64]!
- vst1.8 {q2}, [r0, :64]!
+ vst1.8 {q0-q1}, [r0]!
+ vst1.8 {q2}, [r0]!
vmov s27, ip
b .Lctrloop3x
.Lctr1x:
@@ -318,10 +318,10 @@ ENTRY(ce_aes_ctr_encrypt)
vmov q0, q6
bl aes_encrypt
subs r4, r4, #1
- bmi .Lctrhalfblock @ blocks < 0 means 1/2 block
- vld1.8 {q3}, [r1, :64]!
+ bmi .Lctrtailblock @ blocks < 0 means tail block
+ vld1.8 {q3}, [r1]!
veor q3, q0, q3
- vst1.8 {q3}, [r0, :64]!
+ vst1.8 {q3}, [r0]!
adds r6, r6, #1 @ increment BE ctr
rev ip, r6
@@ -333,10 +333,8 @@ ENTRY(ce_aes_ctr_encrypt)
vst1.8 {q6}, [r5]
pop {r4-r6, pc}
-.Lctrhalfblock:
- vld1.8 {d1}, [r1, :64]
- veor d0, d0, d1
- vst1.8 {d0}, [r0, :64]
+.Lctrtailblock:
+ vst1.8 {q0}, [r0, :64] @ return just the key stream
pop {r4-r6, pc}
.Lctrcarry:
@@ -405,8 +403,8 @@ ENTRY(ce_aes_xts_encrypt)
.Lxtsenc3x:
subs r4, r4, #3
bmi .Lxtsenc1x
- vld1.8 {q0-q1}, [r1, :64]! @ get 3 pt blocks
- vld1.8 {q2}, [r1, :64]!
+ vld1.8 {q0-q1}, [r1]! @ get 3 pt blocks
+ vld1.8 {q2}, [r1]!
next_tweak q4, q3, q7, q6
veor q0, q0, q3
next_tweak q5, q4, q7, q6
@@ -416,8 +414,8 @@ ENTRY(ce_aes_xts_encrypt)
veor q0, q0, q3
veor q1, q1, q4
veor q2, q2, q5
- vst1.8 {q0-q1}, [r0, :64]! @ write 3 ct blocks
- vst1.8 {q2}, [r0, :64]!
+ vst1.8 {q0-q1}, [r0]! @ write 3 ct blocks
+ vst1.8 {q2}, [r0]!
vmov q3, q5
teq r4, #0
beq .Lxtsencout
@@ -426,11 +424,11 @@ ENTRY(ce_aes_xts_encrypt)
adds r4, r4, #3
beq .Lxtsencout
.Lxtsencloop:
- vld1.8 {q0}, [r1, :64]!
+ vld1.8 {q0}, [r1]!
veor q0, q0, q3
bl aes_encrypt
veor q0, q0, q3
- vst1.8 {q0}, [r0, :64]!
+ vst1.8 {q0}, [r0]!
subs r4, r4, #1
beq .Lxtsencout
next_tweak q3, q3, q7, q6
@@ -456,8 +454,8 @@ ENTRY(ce_aes_xts_decrypt)
.Lxtsdec3x:
subs r4, r4, #3
bmi .Lxtsdec1x
- vld1.8 {q0-q1}, [r1, :64]! @ get 3 ct blocks
- vld1.8 {q2}, [r1, :64]!
+ vld1.8 {q0-q1}, [r1]! @ get 3 ct blocks
+ vld1.8 {q2}, [r1]!
next_tweak q4, q3, q7, q6
veor q0, q0, q3
next_tweak q5, q4, q7, q6
@@ -467,8 +465,8 @@ ENTRY(ce_aes_xts_decrypt)
veor q0, q0, q3
veor q1, q1, q4
veor q2, q2, q5
- vst1.8 {q0-q1}, [r0, :64]! @ write 3 pt blocks
- vst1.8 {q2}, [r0, :64]!
+ vst1.8 {q0-q1}, [r0]! @ write 3 pt blocks
+ vst1.8 {q2}, [r0]!
vmov q3, q5
teq r4, #0
beq .Lxtsdecout
@@ -477,12 +475,12 @@ ENTRY(ce_aes_xts_decrypt)
adds r4, r4, #3
beq .Lxtsdecout
.Lxtsdecloop:
- vld1.8 {q0}, [r1, :64]!
+ vld1.8 {q0}, [r1]!
veor q0, q0, q3
add ip, r2, #32 @ 3rd round key
bl aes_decrypt
veor q0, q0, q3
- vst1.8 {q0}, [r0, :64]!
+ vst1.8 {q0}, [r0]!
subs r4, r4, #1
beq .Lxtsdecout
next_tweak q3, q3, q7, q6
diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c
index 8857531915bf..883b84d828c5 100644
--- a/arch/arm/crypto/aes-ce-glue.c
+++ b/arch/arm/crypto/aes-ce-glue.c
@@ -278,14 +278,15 @@ static int ctr_encrypt(struct skcipher_request *req)
u8 *tsrc = walk.src.virt.addr;
/*
- * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need
- * to tell aes_ctr_encrypt() to only read half a block.
+ * Tell aes_ctr_encrypt() to process a tail block.
*/
- blocks = (nbytes <= 8) ? -1 : 1;
+ blocks = -1;
- ce_aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc,
+ ce_aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc,
num_rounds(ctx), blocks, walk.iv);
- memcpy(tdst, tail, nbytes);
+ if (tdst != tsrc)
+ memcpy(tdst, tsrc, nbytes);
+ crypto_xor(tdst, tail, nbytes);
err = skcipher_walk_done(&walk, 0);
}
kernel_neon_end();
@@ -345,7 +346,6 @@ static struct skcipher_alg aes_algs[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_alignmask = 7,
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
@@ -361,7 +361,6 @@ static struct skcipher_alg aes_algs[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_alignmask = 7,
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
@@ -378,7 +377,6 @@ static struct skcipher_alg aes_algs[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_alignmask = 7,
.cra_module = THIS_MODULE,
},
.min_keysize = AES_MIN_KEY_SIZE,
@@ -396,7 +394,6 @@ static struct skcipher_alg aes_algs[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct crypto_aes_xts_ctx),
- .cra_alignmask = 7,
.cra_module = THIS_MODULE,
},
.min_keysize = 2 * AES_MIN_KEY_SIZE,
--
2.7.4
^ permalink raw reply related
* Re: [PATCH 3/8] random: trigger random_ready callback upon crng_init == 1
From: Theodore Ts'o @ 2017-01-18 4:12 UTC (permalink / raw)
To: Stephan Müller; +Cc: linux-kernel, linux-crypto
In-Reply-To: <3576149.HSfW5KObBF@positron.chronox.de>
On Tue, Dec 27, 2016 at 11:39:57PM +0100, Stephan Müller wrote:
> The random_ready callback mechanism is intended to replicate the
> getrandom system call behavior to in-kernel users. As the getrandom
> system call unblocks with crng_init == 1, trigger the random_ready
> wakeup call at the same time.
It was deliberate that random_ready would only get triggered with
crng_init==2.
In general I'm assuming kernel callers really want real randomness (as
opposed to using prandom), where as there's a lot of b.s. userspace
users of kernel randomness (for things that really don't require
cryptographic randomness, e.g., for salting Python dictionaries,
systemd/udev using /dev/urandom for non-cryptographic, non-security
applications etc.)
- Ted
^ permalink raw reply
* Re: [PATCH 4/8] random: remove unused branch in hot code path
From: Theodore Ts'o @ 2017-01-18 4:35 UTC (permalink / raw)
To: Stephan Müller; +Cc: linux-kernel, linux-crypto
In-Reply-To: <3380804.uZWydldqLU@positron.chronox.de>
On Tue, Dec 27, 2016 at 11:40:23PM +0100, Stephan Müller wrote:
> The variable ip is defined to be a __u64 which is always 8 bytes on any
> architecture. Thus, the check for sizeof(ip) > 4 will always be true.
>
> As the check happens in a hot code path, remove the branch.
The fact that it's a hot code path means the compiler will optimize it
out, so the fact that it's on the hot code path is irrelevant. The
main issue is that on platforms with a 32-bit IP's, ip >> 32 will
always be zero. It might be that we can just do this via
#if BITS_PER_LONG == 32
...
#else
...
#endif
I'm not sure that works for all platforms, though. More research is
needed...
- Ted
^ permalink raw reply
* Re: [RFC PATCH 0/6] Add bulk skcipher requests to crypto API and dm-crypt
From: Herbert Xu @ 2017-01-18 4:48 UTC (permalink / raw)
To: Ondrej Mosnáček
Cc: Binoy Jayan, Mike Snitzer, dm-devel, Mikulas Patocka,
linux-crypto, Milan Broz
In-Reply-To: <CAAUqJDsrFp2CczN5Gyy=rYLSDKi7=vQTQoyh=pE1kWue=c=fvw@mail.gmail.com>
On Tue, Jan 17, 2017 at 12:20:02PM +0100, Ondrej Mosnáček wrote:
> 2017-01-13 15:29 GMT+01:00 Herbert Xu <herbert@gondor.apana.org.au>:
> > What if the driver had hardware support for generating these IVs?
> > With your scheme this cannot be supported at all.
>
> That's true... I'm starting to think that this isn't really a good
> idea. I was mainly trying to keep the door open for the random IV
> support and also to keep the multi-key stuff (which was really only
> intended for loop-AES partition support) out of the crypto API, but
> both of these can be probably solved in a better way...
As you said that the multi-key stuff is legacy-only I too would like
to see a way to keep that complexity out of the common path.
> > With such a definition you could either generate the IVs in dm-crypt
> > or have them generated in the IV generator.
>
> That seems kind of hacky to me... but if that's what you prefer, then so be it.
I'm open to other proposals. The basic requirement is to be able to
process multiple blocks as one entity at the driver level, potentially
generating the IVs there too.
It's essentially the equivalent to full IPsec offload.
Thanks,
--
Email: Herbert Xu <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
--
dm-devel mailing list
dm-devel@redhat.com
https://www.redhat.com/mailman/listinfo/dm-devel
^ permalink raw reply
* [RFC PATCH v3] crypto: Add IV generation algorithms
From: Binoy Jayan @ 2017-01-18 9:40 UTC (permalink / raw)
To: Oded, Ofir
Cc: Herbert Xu, David S. Miller, linux-crypto, Mark Brown,
Arnd Bergmann, linux-kernel, Alasdair Kergon, Mike Snitzer,
dm-devel, Shaohua Li, linux-raid, Rajendra, Milan Broz, Gilad,
Binoy Jayan
In-Reply-To: <1484732425-10319-1-git-send-email-binoy.jayan@linaro.org>
Currently, the iv generation algorithms are implemented in dm-crypt.c.
The goal is to move these algorithms from the dm layer to the kernel
crypto layer by implementing them as template ciphers so they can be
implemented in hardware for performance. As part of this patchset, the
iv-generation code is moved from the dm layer to the crypto layer and
adapt the dm-layer to send a whole 'bio' (as defined in the block layer)
at a time. Each bio contains an in memory representation of physically
contiguous disk blocks. The dm layer sets up a chained scatterlist of
these blocks split into physically contiguous segments in memory so that
DMA can be performed. Also, the key management code is moved from dm layer
to the cryto layer since the key selection for encrypting neighboring
sectors depend on the keycount.
Synchronous crypto requests to encrypt/decrypt a sector are processed
sequentially. Asynchronous requests if processed in parallel, are freed
in the async callback. The dm layer allocates space for iv. The hardware
implementations can choose to make use of this space to generate their IVs
sequentially or allocate it on their own.
Interface to the crypto layer - include/crypto/geniv.h
Signed-off-by: Binoy Jayan <binoy.jayan@linaro.org>
---
drivers/md/dm-crypt.c | 1891 ++++++++++++++++++++++++++++++++++--------------
include/crypto/geniv.h | 47 ++
2 files changed, 1399 insertions(+), 539 deletions(-)
create mode 100644 include/crypto/geniv.h
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 7c6c572..7275b0f 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -32,170 +32,113 @@
#include <crypto/algapi.h>
#include <crypto/skcipher.h>
#include <keys/user-type.h>
-
#include <linux/device-mapper.h>
-
-#define DM_MSG_PREFIX "crypt"
-
-/*
- * context holding the current state of a multi-part conversion
- */
-struct convert_context {
- struct completion restart;
- struct bio *bio_in;
- struct bio *bio_out;
- struct bvec_iter iter_in;
- struct bvec_iter iter_out;
- sector_t cc_sector;
- atomic_t cc_pending;
- struct skcipher_request *req;
+#include <crypto/internal/skcipher.h>
+#include <linux/backing-dev.h>
+#include <linux/log2.h>
+#include <crypto/geniv.h>
+
+#define DM_MSG_PREFIX "crypt"
+#define MAX_SG_LIST (BIO_MAX_PAGES * 8)
+#define MIN_IOS 64
+#define LMK_SEED_SIZE 64 /* hash + 0 */
+#define TCW_WHITENING_SIZE 16
+
+struct geniv_ctx;
+struct geniv_req_ctx;
+
+/* Sub request for each of the skcipher_request's for a segment */
+struct geniv_subreq {
+ struct skcipher_request req CRYPTO_MINALIGN_ATTR;
+ struct scatterlist src;
+ struct scatterlist dst;
+ int n;
+ struct geniv_req_ctx *rctx;
};
-/*
- * per bio private data
- */
-struct dm_crypt_io {
- struct crypt_config *cc;
- struct bio *base_bio;
- struct work_struct work;
-
- struct convert_context ctx;
-
- atomic_t io_pending;
- int error;
- sector_t sector;
-
- struct rb_node rb_node;
-} CRYPTO_MINALIGN_ATTR;
-
-struct dm_crypt_request {
- struct convert_context *ctx;
- struct scatterlist sg_in;
- struct scatterlist sg_out;
+struct geniv_req_ctx {
+ struct geniv_subreq *subreq;
+ bool is_write;
sector_t iv_sector;
+ unsigned int nents;
+ u8 *iv;
+ struct completion restart;
+ atomic_t req_pending;
+ struct skcipher_request *req;
};
-struct crypt_config;
-
struct crypt_iv_operations {
- int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
- const char *opts);
- void (*dtr)(struct crypt_config *cc);
- int (*init)(struct crypt_config *cc);
- int (*wipe)(struct crypt_config *cc);
- int (*generator)(struct crypt_config *cc, u8 *iv,
- struct dm_crypt_request *dmreq);
- int (*post)(struct crypt_config *cc, u8 *iv,
- struct dm_crypt_request *dmreq);
+ int (*ctr)(struct geniv_ctx *ctx);
+ void (*dtr)(struct geniv_ctx *ctx);
+ int (*init)(struct geniv_ctx *ctx);
+ int (*wipe)(struct geniv_ctx *ctx);
+ int (*generator)(struct geniv_ctx *ctx,
+ struct geniv_req_ctx *rctx,
+ struct geniv_subreq *subreq);
+ int (*post)(struct geniv_ctx *ctx,
+ struct geniv_req_ctx *rctx,
+ struct geniv_subreq *subreq);
};
-struct iv_essiv_private {
+struct geniv_essiv_private {
struct crypto_ahash *hash_tfm;
u8 *salt;
};
-struct iv_benbi_private {
+struct geniv_benbi_private {
int shift;
};
-#define LMK_SEED_SIZE 64 /* hash + 0 */
-struct iv_lmk_private {
+struct geniv_lmk_private {
struct crypto_shash *hash_tfm;
u8 *seed;
};
-#define TCW_WHITENING_SIZE 16
-struct iv_tcw_private {
+struct geniv_tcw_private {
struct crypto_shash *crc32_tfm;
u8 *iv_seed;
u8 *whitening;
};
-/*
- * Crypt: maps a linear range of a block device
- * and encrypts / decrypts at the same time.
- */
-enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID,
- DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD };
-
-/*
- * The fields in here must be read only after initialization.
- */
-struct crypt_config {
- struct dm_dev *dev;
- sector_t start;
-
- /*
- * pool for per bio private data, crypto requests and
- * encryption requeusts/buffer pages
- */
- mempool_t *req_pool;
- mempool_t *page_pool;
- struct bio_set *bs;
- struct mutex bio_alloc_lock;
-
- struct workqueue_struct *io_queue;
- struct workqueue_struct *crypt_queue;
-
- struct task_struct *write_thread;
- wait_queue_head_t write_thread_wait;
- struct rb_root write_tree;
-
+struct geniv_ctx {
+ unsigned int tfms_count;
+ struct crypto_skcipher *child;
+ struct crypto_skcipher **tfms;
+ char *ivmode;
+ unsigned int iv_size;
+ char *ivopts;
char *cipher;
- char *cipher_string;
- char *key_string;
-
+ char *ciphermode;
const struct crypt_iv_operations *iv_gen_ops;
union {
- struct iv_essiv_private essiv;
- struct iv_benbi_private benbi;
- struct iv_lmk_private lmk;
- struct iv_tcw_private tcw;
+ struct geniv_essiv_private essiv;
+ struct geniv_benbi_private benbi;
+ struct geniv_lmk_private lmk;
+ struct geniv_tcw_private tcw;
} iv_gen_private;
- sector_t iv_offset;
- unsigned int iv_size;
-
- /* ESSIV: struct crypto_cipher *essiv_tfm */
void *iv_private;
- struct crypto_skcipher **tfms;
- unsigned tfms_count;
-
- /*
- * Layout of each crypto request:
- *
- * struct skcipher_request
- * context
- * padding
- * struct dm_crypt_request
- * padding
- * IV
- *
- * The padding is added so that dm_crypt_request and the IV are
- * correctly aligned.
- */
- unsigned int dmreq_start;
-
- unsigned int per_bio_data_size;
-
- unsigned long flags;
+ struct crypto_skcipher *tfm;
+ mempool_t *subreq_pool;
unsigned int key_size;
+ unsigned int key_extra_size;
unsigned int key_parts; /* independent parts in key buffer */
- unsigned int key_extra_size; /* additional keys length */
- u8 key[0];
+ enum setkey_op keyop;
+ char *msg;
+ u8 *key;
};
-#define MIN_IOS 64
-
-static void clone_init(struct dm_crypt_io *, struct bio *);
-static void kcryptd_queue_crypt(struct dm_crypt_io *io);
-static u8 *iv_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq);
+static struct crypto_skcipher *any_tfm(struct geniv_ctx *ctx)
+{
+ return ctx->tfms[0];
+}
-/*
- * Use this to access cipher attributes that are the same for each CPU.
- */
-static struct crypto_skcipher *any_tfm(struct crypt_config *cc)
+static inline
+struct geniv_req_ctx *geniv_req_ctx(struct skcipher_request *req)
{
- return cc->tfms[0];
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ unsigned long align = crypto_skcipher_alignmask(tfm);
+
+ return (void *) PTR_ALIGN((u8 *) skcipher_request_ctx(req), align + 1);
}
/*
@@ -245,44 +188,50 @@ static struct crypto_skcipher *any_tfm(struct crypt_config *cc)
* http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
*/
-static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv,
- struct dm_crypt_request *dmreq)
+static int crypt_iv_plain_gen(struct geniv_ctx *ctx,
+ struct geniv_req_ctx *rctx,
+ struct geniv_subreq *subreq)
{
- memset(iv, 0, cc->iv_size);
- *(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff);
+ u8 *iv = rctx->iv;
+
+ memset(iv, 0, ctx->iv_size);
+ *(__le32 *)iv = cpu_to_le32(rctx->iv_sector & 0xffffffff);
return 0;
}
-static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv,
- struct dm_crypt_request *dmreq)
+static int crypt_iv_plain64_gen(struct geniv_ctx *ctx,
+ struct geniv_req_ctx *rctx,
+ struct geniv_subreq *subreq)
{
- memset(iv, 0, cc->iv_size);
- *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
+ u8 *iv = rctx->iv;
+
+ memset(iv, 0, ctx->iv_size);
+ *(__le64 *)iv = cpu_to_le64(rctx->iv_sector);
return 0;
}
/* Initialise ESSIV - compute salt but no local memory allocations */
-static int crypt_iv_essiv_init(struct crypt_config *cc)
+static int crypt_iv_essiv_init(struct geniv_ctx *ctx)
{
- struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
- AHASH_REQUEST_ON_STACK(req, essiv->hash_tfm);
+ struct geniv_essiv_private *essiv = &ctx->iv_gen_private.essiv;
struct scatterlist sg;
struct crypto_cipher *essiv_tfm;
int err;
+ AHASH_REQUEST_ON_STACK(req, essiv->hash_tfm);
- sg_init_one(&sg, cc->key, cc->key_size);
+ sg_init_one(&sg, ctx->key, ctx->key_size);
ahash_request_set_tfm(req, essiv->hash_tfm);
ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL);
- ahash_request_set_crypt(req, &sg, essiv->salt, cc->key_size);
+ ahash_request_set_crypt(req, &sg, essiv->salt, ctx->key_size);
err = crypto_ahash_digest(req);
ahash_request_zero(req);
if (err)
return err;
- essiv_tfm = cc->iv_private;
+ essiv_tfm = ctx->iv_private;
err = crypto_cipher_setkey(essiv_tfm, essiv->salt,
crypto_ahash_digestsize(essiv->hash_tfm));
@@ -293,16 +242,16 @@ static int crypt_iv_essiv_init(struct crypt_config *cc)
}
/* Wipe salt and reset key derived from volume key */
-static int crypt_iv_essiv_wipe(struct crypt_config *cc)
+static int crypt_iv_essiv_wipe(struct geniv_ctx *ctx)
{
- struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
- unsigned salt_size = crypto_ahash_digestsize(essiv->hash_tfm);
+ struct geniv_essiv_private *essiv = &ctx->iv_gen_private.essiv;
+ unsigned int salt_size = crypto_ahash_digestsize(essiv->hash_tfm);
struct crypto_cipher *essiv_tfm;
int r, err = 0;
memset(essiv->salt, 0, salt_size);
- essiv_tfm = cc->iv_private;
+ essiv_tfm = ctx->iv_private;
r = crypto_cipher_setkey(essiv_tfm, essiv->salt, salt_size);
if (r)
err = r;
@@ -311,42 +260,40 @@ static int crypt_iv_essiv_wipe(struct crypt_config *cc)
}
/* Set up per cpu cipher state */
-static struct crypto_cipher *setup_essiv_cpu(struct crypt_config *cc,
- struct dm_target *ti,
- u8 *salt, unsigned saltsize)
+static struct crypto_cipher *setup_essiv_cpu(struct geniv_ctx *ctx,
+ u8 *salt, unsigned int saltsize)
{
struct crypto_cipher *essiv_tfm;
int err;
/* Setup the essiv_tfm with the given salt */
- essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC);
+ essiv_tfm = crypto_alloc_cipher(ctx->cipher, 0, CRYPTO_ALG_ASYNC);
+
if (IS_ERR(essiv_tfm)) {
- ti->error = "Error allocating crypto tfm for ESSIV";
+ DMERR("Error allocating crypto tfm for ESSIV\n");
return essiv_tfm;
}
if (crypto_cipher_blocksize(essiv_tfm) !=
- crypto_skcipher_ivsize(any_tfm(cc))) {
- ti->error = "Block size of ESSIV cipher does "
- "not match IV size of block cipher";
+ crypto_skcipher_ivsize(any_tfm(ctx))) {
+ DMERR("Block size of ESSIV cipher does not match IV size of block cipher\n");
crypto_free_cipher(essiv_tfm);
return ERR_PTR(-EINVAL);
}
err = crypto_cipher_setkey(essiv_tfm, salt, saltsize);
if (err) {
- ti->error = "Failed to set key for ESSIV cipher";
+ DMERR("Failed to set key for ESSIV cipher\n");
crypto_free_cipher(essiv_tfm);
return ERR_PTR(err);
}
-
return essiv_tfm;
}
-static void crypt_iv_essiv_dtr(struct crypt_config *cc)
+static void crypt_iv_essiv_dtr(struct geniv_ctx *ctx)
{
struct crypto_cipher *essiv_tfm;
- struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv;
+ struct geniv_essiv_private *essiv = &ctx->iv_gen_private.essiv;
crypto_free_ahash(essiv->hash_tfm);
essiv->hash_tfm = NULL;
@@ -354,52 +301,50 @@ static void crypt_iv_essiv_dtr(struct crypt_config *cc)
kzfree(essiv->salt);
essiv->salt = NULL;
- essiv_tfm = cc->iv_private;
+ essiv_tfm = ctx->iv_private;
if (essiv_tfm)
crypto_free_cipher(essiv_tfm);
- cc->iv_private = NULL;
+ ctx->iv_private = NULL;
}
-static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
- const char *opts)
+static int crypt_iv_essiv_ctr(struct geniv_ctx *ctx)
{
struct crypto_cipher *essiv_tfm = NULL;
struct crypto_ahash *hash_tfm = NULL;
u8 *salt = NULL;
int err;
- if (!opts) {
- ti->error = "Digest algorithm missing for ESSIV mode";
+ if (!ctx->ivopts) {
+ DMERR("Digest algorithm missing for ESSIV mode\n");
return -EINVAL;
}
/* Allocate hash algorithm */
- hash_tfm = crypto_alloc_ahash(opts, 0, CRYPTO_ALG_ASYNC);
+ hash_tfm = crypto_alloc_ahash(ctx->ivopts, 0, CRYPTO_ALG_ASYNC);
if (IS_ERR(hash_tfm)) {
- ti->error = "Error initializing ESSIV hash";
err = PTR_ERR(hash_tfm);
+ DMERR("Error initializing ESSIV hash. err=%d\n", err);
goto bad;
}
salt = kzalloc(crypto_ahash_digestsize(hash_tfm), GFP_KERNEL);
if (!salt) {
- ti->error = "Error kmallocing salt storage in ESSIV";
err = -ENOMEM;
goto bad;
}
- cc->iv_gen_private.essiv.salt = salt;
- cc->iv_gen_private.essiv.hash_tfm = hash_tfm;
+ ctx->iv_gen_private.essiv.salt = salt;
+ ctx->iv_gen_private.essiv.hash_tfm = hash_tfm;
- essiv_tfm = setup_essiv_cpu(cc, ti, salt,
+ essiv_tfm = setup_essiv_cpu(ctx, salt,
crypto_ahash_digestsize(hash_tfm));
if (IS_ERR(essiv_tfm)) {
- crypt_iv_essiv_dtr(cc);
+ crypt_iv_essiv_dtr(ctx);
return PTR_ERR(essiv_tfm);
}
- cc->iv_private = essiv_tfm;
+ ctx->iv_private = essiv_tfm;
return 0;
@@ -410,70 +355,73 @@ static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
return err;
}
-static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv,
- struct dm_crypt_request *dmreq)
+static int crypt_iv_essiv_gen(struct geniv_ctx *ctx,
+ struct geniv_req_ctx *rctx,
+ struct geniv_subreq *subreq)
{
- struct crypto_cipher *essiv_tfm = cc->iv_private;
+ u8 *iv = rctx->iv;
+ struct crypto_cipher *essiv_tfm = ctx->iv_private;
- memset(iv, 0, cc->iv_size);
- *(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
+ memset(iv, 0, ctx->iv_size);
+ *(__le64 *)iv = cpu_to_le64(rctx->iv_sector);
crypto_cipher_encrypt_one(essiv_tfm, iv, iv);
return 0;
}
-static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti,
- const char *opts)
+static int crypt_iv_benbi_ctr(struct geniv_ctx *ctx)
{
- unsigned bs = crypto_skcipher_blocksize(any_tfm(cc));
+ unsigned int bs = crypto_skcipher_blocksize(any_tfm(ctx));
int log = ilog2(bs);
/* we need to calculate how far we must shift the sector count
- * to get the cipher block count, we use this shift in _gen */
+ * to get the cipher block count, we use this shift in _gen
+ */
if (1 << log != bs) {
- ti->error = "cypher blocksize is not a power of 2";
+ DMERR("cypher blocksize is not a power of 2\n");
return -EINVAL;
}
if (log > 9) {
- ti->error = "cypher blocksize is > 512";
+ DMERR("cypher blocksize is > 512\n");
return -EINVAL;
}
- cc->iv_gen_private.benbi.shift = 9 - log;
+ ctx->iv_gen_private.benbi.shift = 9 - log;
return 0;
}
-static void crypt_iv_benbi_dtr(struct crypt_config *cc)
-{
-}
-
-static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv,
- struct dm_crypt_request *dmreq)
+static int crypt_iv_benbi_gen(struct geniv_ctx *ctx,
+ struct geniv_req_ctx *rctx,
+ struct geniv_subreq *subreq)
{
+ u8 *iv = rctx->iv;
__be64 val;
- memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */
+ memset(iv, 0, ctx->iv_size - sizeof(u64)); /* rest is cleared below */
- val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1);
- put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64)));
+ val = cpu_to_be64(((u64) rctx->iv_sector <<
+ ctx->iv_gen_private.benbi.shift) + 1);
+ put_unaligned(val, (__be64 *)(iv + ctx->iv_size - sizeof(u64)));
return 0;
}
-static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv,
- struct dm_crypt_request *dmreq)
+static int crypt_iv_null_gen(struct geniv_ctx *ctx,
+ struct geniv_req_ctx *rctx,
+ struct geniv_subreq *subreq)
{
- memset(iv, 0, cc->iv_size);
+ u8 *iv = rctx->iv;
+ memset(iv, 0, ctx->iv_size);
return 0;
}
-static void crypt_iv_lmk_dtr(struct crypt_config *cc)
+static void crypt_iv_lmk_dtr(struct geniv_ctx *ctx)
{
- struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
+ struct geniv_lmk_private *lmk = &ctx->iv_gen_private.lmk;
if (lmk->hash_tfm && !IS_ERR(lmk->hash_tfm))
crypto_free_shash(lmk->hash_tfm);
@@ -483,49 +431,49 @@ static void crypt_iv_lmk_dtr(struct crypt_config *cc)
lmk->seed = NULL;
}
-static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti,
- const char *opts)
+static int crypt_iv_lmk_ctr(struct geniv_ctx *ctx)
{
- struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
+ struct geniv_lmk_private *lmk = &ctx->iv_gen_private.lmk;
lmk->hash_tfm = crypto_alloc_shash("md5", 0, 0);
if (IS_ERR(lmk->hash_tfm)) {
- ti->error = "Error initializing LMK hash";
+ DMERR("Error initializing LMK hash; err=%ld\n",
+ PTR_ERR(lmk->hash_tfm));
return PTR_ERR(lmk->hash_tfm);
}
/* No seed in LMK version 2 */
- if (cc->key_parts == cc->tfms_count) {
+ if (ctx->key_parts == ctx->tfms_count) {
lmk->seed = NULL;
return 0;
}
lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL);
if (!lmk->seed) {
- crypt_iv_lmk_dtr(cc);
- ti->error = "Error kmallocing seed storage in LMK";
+ crypt_iv_lmk_dtr(ctx);
+ DMERR("Error kmallocing seed storage in LMK\n");
return -ENOMEM;
}
return 0;
}
-static int crypt_iv_lmk_init(struct crypt_config *cc)
+static int crypt_iv_lmk_init(struct geniv_ctx *ctx)
{
- struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
- int subkey_size = cc->key_size / cc->key_parts;
+ struct geniv_lmk_private *lmk = &ctx->iv_gen_private.lmk;
+ int subkey_size = ctx->key_size / ctx->key_parts;
/* LMK seed is on the position of LMK_KEYS + 1 key */
if (lmk->seed)
- memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size),
+ memcpy(lmk->seed, ctx->key + (ctx->tfms_count * subkey_size),
crypto_shash_digestsize(lmk->hash_tfm));
return 0;
}
-static int crypt_iv_lmk_wipe(struct crypt_config *cc)
+static int crypt_iv_lmk_wipe(struct geniv_ctx *ctx)
{
- struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
+ struct geniv_lmk_private *lmk = &ctx->iv_gen_private.lmk;
if (lmk->seed)
memset(lmk->seed, 0, LMK_SEED_SIZE);
@@ -533,15 +481,14 @@ static int crypt_iv_lmk_wipe(struct crypt_config *cc)
return 0;
}
-static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
- struct dm_crypt_request *dmreq,
- u8 *data)
+static int crypt_iv_lmk_one(struct geniv_ctx *ctx, u8 *iv,
+ struct geniv_req_ctx *rctx, u8 *data)
{
- struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
- SHASH_DESC_ON_STACK(desc, lmk->hash_tfm);
+ struct geniv_lmk_private *lmk = &ctx->iv_gen_private.lmk;
struct md5_state md5state;
__le32 buf[4];
int i, r;
+ SHASH_DESC_ON_STACK(desc, lmk->hash_tfm);
desc->tfm = lmk->hash_tfm;
desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
@@ -562,8 +509,9 @@ static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
return r;
/* Sector is cropped to 56 bits here */
- buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF);
- buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000);
+ buf[0] = cpu_to_le32(rctx->iv_sector & 0xFFFFFFFF);
+ buf[1] = cpu_to_le32((((u64)rctx->iv_sector >> 32) & 0x00FFFFFF)
+ | 0x80000000);
buf[2] = cpu_to_le32(4024);
buf[3] = 0;
r = crypto_shash_update(desc, (u8 *)buf, sizeof(buf));
@@ -577,50 +525,54 @@ static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
for (i = 0; i < MD5_HASH_WORDS; i++)
__cpu_to_le32s(&md5state.hash[i]);
- memcpy(iv, &md5state.hash, cc->iv_size);
+ memcpy(iv, &md5state.hash, ctx->iv_size);
return 0;
}
-static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv,
- struct dm_crypt_request *dmreq)
+static int crypt_iv_lmk_gen(struct geniv_ctx *ctx,
+ struct geniv_req_ctx *rctx,
+ struct geniv_subreq *subreq)
{
u8 *src;
+ u8 *iv = rctx->iv;
int r = 0;
- if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
- src = kmap_atomic(sg_page(&dmreq->sg_in));
- r = crypt_iv_lmk_one(cc, iv, dmreq, src + dmreq->sg_in.offset);
+ if (rctx->is_write) {
+ src = kmap_atomic(sg_page(&subreq->src));
+ r = crypt_iv_lmk_one(ctx, iv, rctx, src + subreq->src.offset);
kunmap_atomic(src);
} else
- memset(iv, 0, cc->iv_size);
+ memset(iv, 0, ctx->iv_size);
return r;
}
-static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv,
- struct dm_crypt_request *dmreq)
+static int crypt_iv_lmk_post(struct geniv_ctx *ctx,
+ struct geniv_req_ctx *rctx,
+ struct geniv_subreq *subreq)
{
u8 *dst;
+ u8 *iv = rctx->iv;
int r;
- if (bio_data_dir(dmreq->ctx->bio_in) == WRITE)
+ if (rctx->is_write)
return 0;
- dst = kmap_atomic(sg_page(&dmreq->sg_out));
- r = crypt_iv_lmk_one(cc, iv, dmreq, dst + dmreq->sg_out.offset);
+ dst = kmap_atomic(sg_page(&subreq->dst));
+ r = crypt_iv_lmk_one(ctx, iv, rctx, dst + subreq->dst.offset);
/* Tweak the first block of plaintext sector */
if (!r)
- crypto_xor(dst + dmreq->sg_out.offset, iv, cc->iv_size);
+ crypto_xor(dst + subreq->dst.offset, iv, ctx->iv_size);
kunmap_atomic(dst);
return r;
}
-static void crypt_iv_tcw_dtr(struct crypt_config *cc)
+static void crypt_iv_tcw_dtr(struct geniv_ctx *ctx)
{
- struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
+ struct geniv_tcw_private *tcw = &ctx->iv_gen_private.tcw;
kzfree(tcw->iv_seed);
tcw->iv_seed = NULL;
@@ -632,64 +584,65 @@ static void crypt_iv_tcw_dtr(struct crypt_config *cc)
tcw->crc32_tfm = NULL;
}
-static int crypt_iv_tcw_ctr(struct crypt_config *cc, struct dm_target *ti,
- const char *opts)
+static int crypt_iv_tcw_ctr(struct geniv_ctx *ctx)
{
- struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
+ struct geniv_tcw_private *tcw = &ctx->iv_gen_private.tcw;
- if (cc->key_size <= (cc->iv_size + TCW_WHITENING_SIZE)) {
- ti->error = "Wrong key size for TCW";
+ if (ctx->key_size <= (ctx->iv_size + TCW_WHITENING_SIZE)) {
+ DMERR("Wrong key size (%d) for TCW. Choose a value > %d bytes\n",
+ ctx->key_size,
+ ctx->iv_size + TCW_WHITENING_SIZE);
return -EINVAL;
}
tcw->crc32_tfm = crypto_alloc_shash("crc32", 0, 0);
if (IS_ERR(tcw->crc32_tfm)) {
- ti->error = "Error initializing CRC32 in TCW";
+ DMERR("Error initializing CRC32 in TCW; err=%ld\n",
+ PTR_ERR(tcw->crc32_tfm));
return PTR_ERR(tcw->crc32_tfm);
}
- tcw->iv_seed = kzalloc(cc->iv_size, GFP_KERNEL);
+ tcw->iv_seed = kzalloc(ctx->iv_size, GFP_KERNEL);
tcw->whitening = kzalloc(TCW_WHITENING_SIZE, GFP_KERNEL);
if (!tcw->iv_seed || !tcw->whitening) {
- crypt_iv_tcw_dtr(cc);
- ti->error = "Error allocating seed storage in TCW";
+ crypt_iv_tcw_dtr(ctx);
+ DMERR("Error allocating seed storage in TCW\n");
return -ENOMEM;
}
return 0;
}
-static int crypt_iv_tcw_init(struct crypt_config *cc)
+static int crypt_iv_tcw_init(struct geniv_ctx *ctx)
{
- struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
- int key_offset = cc->key_size - cc->iv_size - TCW_WHITENING_SIZE;
+ struct geniv_tcw_private *tcw = &ctx->iv_gen_private.tcw;
+ int key_offset = ctx->key_size - ctx->iv_size - TCW_WHITENING_SIZE;
- memcpy(tcw->iv_seed, &cc->key[key_offset], cc->iv_size);
- memcpy(tcw->whitening, &cc->key[key_offset + cc->iv_size],
+ memcpy(tcw->iv_seed, &ctx->key[key_offset], ctx->iv_size);
+ memcpy(tcw->whitening, &ctx->key[key_offset + ctx->iv_size],
TCW_WHITENING_SIZE);
return 0;
}
-static int crypt_iv_tcw_wipe(struct crypt_config *cc)
+static int crypt_iv_tcw_wipe(struct geniv_ctx *ctx)
{
- struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
+ struct geniv_tcw_private *tcw = &ctx->iv_gen_private.tcw;
- memset(tcw->iv_seed, 0, cc->iv_size);
+ memset(tcw->iv_seed, 0, ctx->iv_size);
memset(tcw->whitening, 0, TCW_WHITENING_SIZE);
return 0;
}
-static int crypt_iv_tcw_whitening(struct crypt_config *cc,
- struct dm_crypt_request *dmreq,
- u8 *data)
+static int crypt_iv_tcw_whitening(struct geniv_ctx *ctx,
+ struct geniv_req_ctx *rctx, u8 *data)
{
- struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
- __le64 sector = cpu_to_le64(dmreq->iv_sector);
+ struct geniv_tcw_private *tcw = &ctx->iv_gen_private.tcw;
+ __le64 sector = cpu_to_le64(rctx->iv_sector);
u8 buf[TCW_WHITENING_SIZE];
- SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm);
int i, r;
+ SHASH_DESC_ON_STACK(desc, tcw->crc32_tfm);
/* xor whitening with sector number */
memcpy(buf, tcw->whitening, TCW_WHITENING_SIZE);
@@ -713,99 +666,1006 @@ static int crypt_iv_tcw_whitening(struct crypt_config *cc,
crypto_xor(&buf[0], &buf[12], 4);
crypto_xor(&buf[4], &buf[8], 4);
- /* apply whitening (8 bytes) to whole sector */
- for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++)
- crypto_xor(data + i * 8, buf, 8);
-out:
- memzero_explicit(buf, sizeof(buf));
- return r;
-}
+ /* apply whitening (8 bytes) to whole sector */
+ for (i = 0; i < (SECTOR_SIZE / 8); i++)
+ crypto_xor(data + i * 8, buf, 8);
+out:
+ memzero_explicit(buf, sizeof(buf));
+ return r;
+}
+
+static int crypt_iv_tcw_gen(struct geniv_ctx *ctx,
+ struct geniv_req_ctx *rctx,
+ struct geniv_subreq *subreq)
+{
+ u8 *iv = rctx->iv;
+ struct geniv_tcw_private *tcw = &ctx->iv_gen_private.tcw;
+ __le64 sector = cpu_to_le64(rctx->iv_sector);
+ u8 *src;
+ int r = 0;
+
+ /* Remove whitening from ciphertext */
+ if (!rctx->is_write) {
+ src = kmap_atomic(sg_page(&subreq->src));
+ r = crypt_iv_tcw_whitening(ctx, rctx,
+ src + subreq->src.offset);
+ kunmap_atomic(src);
+ }
+
+ /* Calculate IV */
+ memcpy(iv, tcw->iv_seed, ctx->iv_size);
+ crypto_xor(iv, (u8 *)§or, 8);
+ if (ctx->iv_size > 8)
+ crypto_xor(&iv[8], (u8 *)§or, ctx->iv_size - 8);
+
+ return r;
+}
+
+static int crypt_iv_tcw_post(struct geniv_ctx *ctx,
+ struct geniv_req_ctx *rctx,
+ struct geniv_subreq *subreq)
+{
+ u8 *dst;
+ int r;
+
+ if (!rctx->is_write)
+ return 0;
+
+ /* Apply whitening on ciphertext */
+ dst = kmap_atomic(sg_page(&subreq->dst));
+ r = crypt_iv_tcw_whitening(ctx, rctx, dst + subreq->dst.offset);
+ kunmap_atomic(dst);
+
+ return r;
+}
+
+static const struct crypt_iv_operations crypt_iv_plain_ops = {
+ .generator = crypt_iv_plain_gen
+};
+
+static const struct crypt_iv_operations crypt_iv_plain64_ops = {
+ .generator = crypt_iv_plain64_gen
+};
+
+static const struct crypt_iv_operations crypt_iv_essiv_ops = {
+ .ctr = crypt_iv_essiv_ctr,
+ .dtr = crypt_iv_essiv_dtr,
+ .init = crypt_iv_essiv_init,
+ .wipe = crypt_iv_essiv_wipe,
+ .generator = crypt_iv_essiv_gen
+};
+
+static const struct crypt_iv_operations crypt_iv_benbi_ops = {
+ .ctr = crypt_iv_benbi_ctr,
+ .generator = crypt_iv_benbi_gen
+};
+
+static const struct crypt_iv_operations crypt_iv_null_ops = {
+ .generator = crypt_iv_null_gen
+};
+
+static const struct crypt_iv_operations crypt_iv_lmk_ops = {
+ .ctr = crypt_iv_lmk_ctr,
+ .dtr = crypt_iv_lmk_dtr,
+ .init = crypt_iv_lmk_init,
+ .wipe = crypt_iv_lmk_wipe,
+ .generator = crypt_iv_lmk_gen,
+ .post = crypt_iv_lmk_post
+};
+
+static const struct crypt_iv_operations crypt_iv_tcw_ops = {
+ .ctr = crypt_iv_tcw_ctr,
+ .dtr = crypt_iv_tcw_dtr,
+ .init = crypt_iv_tcw_init,
+ .wipe = crypt_iv_tcw_wipe,
+ .generator = crypt_iv_tcw_gen,
+ .post = crypt_iv_tcw_post
+};
+
+static int geniv_setkey_set(struct geniv_ctx *ctx)
+{
+ int ret = 0;
+
+ if (ctx->iv_gen_ops && ctx->iv_gen_ops->init)
+ ret = ctx->iv_gen_ops->init(ctx);
+ return ret;
+}
+
+static int geniv_setkey_wipe(struct geniv_ctx *ctx)
+{
+ int ret = 0;
+
+ if (ctx->iv_gen_ops && ctx->iv_gen_ops->wipe) {
+ ret = ctx->iv_gen_ops->wipe(ctx);
+ if (ret)
+ return ret;
+ }
+ return ret;
+}
+
+static int geniv_init_iv(struct geniv_ctx *ctx)
+{
+ int ret = -EINVAL;
+
+ DMDEBUG("IV Generation algorithm : %s\n", ctx->ivmode);
+
+ if (ctx->ivmode == NULL)
+ ctx->iv_gen_ops = NULL;
+ else if (strcmp(ctx->ivmode, "plain") == 0)
+ ctx->iv_gen_ops = &crypt_iv_plain_ops;
+ else if (strcmp(ctx->ivmode, "plain64") == 0)
+ ctx->iv_gen_ops = &crypt_iv_plain64_ops;
+ else if (strcmp(ctx->ivmode, "essiv") == 0)
+ ctx->iv_gen_ops = &crypt_iv_essiv_ops;
+ else if (strcmp(ctx->ivmode, "benbi") == 0)
+ ctx->iv_gen_ops = &crypt_iv_benbi_ops;
+ else if (strcmp(ctx->ivmode, "null") == 0)
+ ctx->iv_gen_ops = &crypt_iv_null_ops;
+ else if (strcmp(ctx->ivmode, "lmk") == 0)
+ ctx->iv_gen_ops = &crypt_iv_lmk_ops;
+ else if (strcmp(ctx->ivmode, "tcw") == 0) {
+ ctx->iv_gen_ops = &crypt_iv_tcw_ops;
+ ctx->key_parts += 2; /* IV + whitening */
+ ctx->key_extra_size = ctx->iv_size + TCW_WHITENING_SIZE;
+ } else {
+ ret = -EINVAL;
+ DMERR("Invalid IV mode %s\n", ctx->ivmode);
+ goto end;
+ }
+
+ /* Allocate IV */
+ if (ctx->iv_gen_ops && ctx->iv_gen_ops->ctr) {
+ ret = ctx->iv_gen_ops->ctr(ctx);
+ if (ret < 0) {
+ DMERR("Error creating IV for %s\n", ctx->ivmode);
+ goto end;
+ }
+ }
+
+ /* Initialize IV (set keys for ESSIV etc) */
+ if (ctx->iv_gen_ops && ctx->iv_gen_ops->init) {
+ ret = ctx->iv_gen_ops->init(ctx);
+ if (ret < 0)
+ DMERR("Error creating IV for %s\n", ctx->ivmode);
+ }
+ ret = 0;
+end:
+ return ret;
+}
+
+static void geniv_free_tfms(struct geniv_ctx *ctx)
+{
+ unsigned int i;
+
+ if (!ctx->tfms)
+ return;
+
+ for (i = 0; i < ctx->tfms_count; i++)
+ if (ctx->tfms[i] && !IS_ERR(ctx->tfms[i])) {
+ crypto_free_skcipher(ctx->tfms[i]);
+ ctx->tfms[i] = NULL;
+ }
+
+ kfree(ctx->tfms);
+ ctx->tfms = NULL;
+}
+
+/* Allocate memory for the underlying cipher algorithm. Ex: cbc(aes)
+ */
+
+static int geniv_alloc_tfms(struct crypto_skcipher *parent,
+ struct geniv_ctx *ctx)
+{
+ unsigned int i, reqsize, align;
+ int err = 0;
+
+ ctx->tfms = kcalloc(ctx->tfms_count, sizeof(struct crypto_skcipher *),
+ GFP_KERNEL);
+ if (!ctx->tfms) {
+ err = -ENOMEM;
+ goto end;
+ }
+
+ /* First instance is already allocated in geniv_init_tfm */
+ ctx->tfms[0] = ctx->child;
+ for (i = 1; i < ctx->tfms_count; i++) {
+ ctx->tfms[i] = crypto_alloc_skcipher(ctx->ciphermode, 0, 0);
+ if (IS_ERR(ctx->tfms[i])) {
+ err = PTR_ERR(ctx->tfms[i]);
+ geniv_free_tfms(ctx);
+ goto end;
+ }
+
+ /* Setup the current cipher's request structure */
+ align = crypto_skcipher_alignmask(parent);
+ align &= ~(crypto_tfm_ctx_alignment() - 1);
+ reqsize = align + sizeof(struct geniv_req_ctx) +
+ crypto_skcipher_reqsize(ctx->tfms[i]);
+ crypto_skcipher_set_reqsize(parent, reqsize);
+ }
+
+end:
+ return err;
+}
+
+/* Initialize the cipher's context with the key, ivmode and other parameters.
+ * Also allocate IV generation template ciphers and initialize them.
+ */
+
+static int geniv_setkey_init(struct crypto_skcipher *parent,
+ struct geniv_key_info *info)
+{
+ struct geniv_ctx *ctx = crypto_skcipher_ctx(parent);
+ int ret = -ENOMEM;
+
+ ctx->iv_size = crypto_skcipher_ivsize(parent);
+ ctx->tfms_count = info->tfms_count;
+ ctx->key = info->key;
+ ctx->key_size = info->key_size;
+ ctx->key_parts = info->key_parts;
+ ctx->ivopts = info->ivopts;
+
+ ret = geniv_alloc_tfms(parent, ctx);
+ if (ret)
+ goto end;
+
+ ret = geniv_init_iv(ctx);
+
+end:
+ return ret;
+}
+
+static int geniv_setkey_tfms(struct crypto_skcipher *parent,
+ struct geniv_ctx *ctx,
+ struct geniv_key_info *info)
+{
+ unsigned int subkey_size;
+ int ret = 0, i;
+
+ /* Ignore extra keys (which are used for IV etc) */
+ subkey_size = (ctx->key_size - ctx->key_extra_size)
+ >> ilog2(ctx->tfms_count);
+
+ for (i = 0; i < ctx->tfms_count; i++) {
+ struct crypto_skcipher *child = ctx->tfms[i];
+ char *subkey = ctx->key + (subkey_size) * i;
+
+ crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
+ crypto_skcipher_set_flags(child,
+ crypto_skcipher_get_flags(parent) &
+ CRYPTO_TFM_REQ_MASK);
+ ret = crypto_skcipher_setkey(child, subkey, subkey_size);
+ if (ret) {
+ DMERR("Error setting key for tfms[%d]\n", i);
+ break;
+ }
+ crypto_skcipher_set_flags(parent,
+ crypto_skcipher_get_flags(child) &
+ CRYPTO_TFM_RES_MASK);
+ }
+
+ return ret;
+}
+
+static int geniv_setkey(struct crypto_skcipher *parent,
+ const u8 *key, unsigned int keylen)
+{
+ int err = 0;
+ struct geniv_ctx *ctx = crypto_skcipher_ctx(parent);
+ struct geniv_key_info *info = (struct geniv_key_info *) key;
+
+ DMDEBUG("SETKEY Operation : %d\n", info->keyop);
+
+ switch (info->keyop) {
+ case SETKEY_OP_INIT:
+ err = geniv_setkey_init(parent, info);
+ break;
+ case SETKEY_OP_SET:
+ err = geniv_setkey_set(ctx);
+ break;
+ case SETKEY_OP_WIPE:
+ err = geniv_setkey_wipe(ctx);
+ break;
+ }
+
+ if (err)
+ goto end;
+
+ err = geniv_setkey_tfms(parent, ctx, info);
+
+end:
+ return err;
+}
+
+static void geniv_async_done(struct crypto_async_request *async_req, int error);
+
+static int geniv_alloc_subreq(struct skcipher_request *req,
+ struct geniv_ctx *ctx,
+ struct geniv_req_ctx *rctx)
+{
+ int key_index, r = 0;
+ struct skcipher_request *sreq;
+
+ if (!rctx->subreq) {
+ rctx->subreq = mempool_alloc(ctx->subreq_pool, GFP_NOIO);
+ if (!rctx->subreq)
+ r = -ENOMEM;
+ }
+
+ sreq = &rctx->subreq->req;
+ rctx->subreq->rctx = rctx;
+
+ key_index = rctx->iv_sector & (ctx->tfms_count - 1);
+
+ skcipher_request_set_tfm(sreq, ctx->tfms[key_index]);
+ skcipher_request_set_callback(sreq, req->base.flags,
+ geniv_async_done, rctx->subreq);
+ return r;
+}
+
+/* Asynchronous IO completion callback for each sector in a segment. When all
+ * pending i/o are completed the parent cipher's async function is called.
+ */
+
+static void geniv_async_done(struct crypto_async_request *async_req, int error)
+{
+ struct geniv_subreq *subreq =
+ (struct geniv_subreq *) async_req->data;
+ struct geniv_req_ctx *rctx = subreq->rctx;
+ struct skcipher_request *req = rctx->req;
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct geniv_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ /*
+ * A request from crypto driver backlog is going to be processed now,
+ * finish the completion and continue in crypt_convert().
+ * (Callback will be called for the second time for this request.)
+ */
+
+ if (error == -EINPROGRESS) {
+ complete(&rctx->restart);
+ return;
+ }
+
+ if (!error && ctx->iv_gen_ops && ctx->iv_gen_ops->post)
+ error = ctx->iv_gen_ops->post(ctx, rctx, subreq);
+
+ mempool_free(subreq, ctx->subreq_pool);
+
+ /* req_pending needs to be checked before req->base.complete is called
+ * as we need 'req_pending' to be equal to 1 to ensure all subrequests
+ * are processed.
+ */
+ if (!atomic_dec_and_test(&rctx->req_pending)) {
+ /* Call the parent cipher's completion function */
+ skcipher_request_complete(req, error);
+ }
+}
+
+static unsigned int geniv_get_sectors(struct scatterlist *sg1,
+ struct scatterlist *sg2,
+ unsigned int segments)
+{
+ unsigned int i, n1, n2, nents;
+
+ n1 = n2 = 0;
+ for (i = 0; i < segments ; i++)
+ n1 += sg1[i].length / SECTOR_SIZE;
+
+ for (i = 0; i < segments ; i++)
+ n2 += sg2[i].length / SECTOR_SIZE;
+
+ nents = n1 > n2 ? n1 : n2;
+ return nents;
+}
+
+/* Iterate scatterlist of segments to retrieve the 512-byte sectors so that
+ * unique IVs could be generated for each 512-byte sector. This split may not
+ * be necessary e.g. when these ciphers are modelled in hardware, where it can
+ * make use of the hardware's IV generation capabilities.
+ */
+
+static int geniv_iter_block(struct skcipher_request *req,
+ struct geniv_subreq *subreq,
+ struct geniv_req_ctx *rctx,
+ unsigned int *seg_no,
+ unsigned int *done)
+
+{
+ unsigned int srcoff, dstoff, len, rem;
+ struct scatterlist *src1, *dst1, *src2, *dst2;
+
+ if (unlikely(*seg_no >= rctx->nents))
+ return 0; /* done */
+
+ src1 = &req->src[*seg_no];
+ dst1 = &req->dst[*seg_no];
+ src2 = &subreq->src;
+ dst2 = &subreq->dst;
+
+ if (*done >= src1->length) {
+ (*seg_no)++;
+
+ if (*seg_no >= rctx->nents)
+ return 0; /* done */
+
+ src1 = &req->src[*seg_no];
+ dst1 = &req->dst[*seg_no];
+ *done = 0;
+ }
+
+ srcoff = src1->offset + *done;
+ dstoff = dst1->offset + *done;
+ rem = src1->length - *done;
+
+ len = rem > SECTOR_SIZE ? SECTOR_SIZE : rem;
+
+ DMDEBUG("segment:(%d/%u), srcoff:%d, dstoff:%d, done:%d, rem:%d\n",
+ *seg_no + 1, rctx->nents, srcoff, dstoff, *done, rem);
+
+ sg_set_page(src2, sg_page(src1), len, srcoff);
+ sg_set_page(dst2, sg_page(dst1), len, dstoff);
+
+ *done += len;
+
+ return len; /* bytes returned */
+}
+
+/* Common encryt/decrypt function for geniv template cipher. Before the crypto
+ * operation, it splits the memory segments (in the scatterlist) into 512 byte
+ * sectors. The initialization vector(IV) used is based on a unique sector
+ * number which is generated here.
+ */
+static inline int geniv_crypt(struct skcipher_request *req, bool encrypt)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct geniv_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct geniv_req_ctx *rctx = geniv_req_ctx(req);
+ struct geniv_req_info *rinfo = (struct geniv_req_info *) req->iv;
+ int i, bytes, cryptlen, ret = 0;
+ unsigned int sectors, segno = 0, done = 0;
+ char *str __maybe_unused = encrypt ? "encrypt" : "decrypt";
+
+ /* Instance of 'struct geniv_req_info' is stored in IV ptr */
+ rctx->is_write = rinfo->is_write;
+ rctx->iv_sector = rinfo->iv_sector;
+ rctx->nents = rinfo->nents;
+ rctx->iv = rinfo->iv;
+ rctx->req = req;
+ rctx->subreq = NULL;
+ cryptlen = req->cryptlen;
+
+ DMDEBUG("geniv:%s: starting sector=%d, #segments=%u\n", str,
+ (unsigned int) rctx->iv_sector, rctx->nents);
+
+ sectors = geniv_get_sectors(req->src, req->dst, rctx->nents);
+
+ init_completion(&rctx->restart);
+ atomic_set(&rctx->req_pending, 1);
+
+ for (i = 0; i < sectors; i++) {
+ struct geniv_subreq *subreq;
+
+ ret = geniv_alloc_subreq(req, ctx, rctx);
+ if (ret)
+ goto end;
+
+ subreq = rctx->subreq;
+ subreq->rctx = rctx;
+
+ atomic_inc(&rctx->req_pending);
+ bytes = geniv_iter_block(req, subreq, rctx, &segno, &done);
+
+ if (bytes == 0)
+ break;
+
+ cryptlen -= bytes;
+
+ if (ctx->iv_gen_ops)
+ ret = ctx->iv_gen_ops->generator(ctx, rctx, subreq);
+
+ if (ret < 0) {
+ DMERR("Error in generating IV ret: %d\n", ret);
+ goto end;
+ }
+
+ skcipher_request_set_crypt(&subreq->req, &subreq->src,
+ &subreq->dst, bytes, rctx->iv);
+
+ if (encrypt)
+ ret = crypto_skcipher_encrypt(&subreq->req);
+
+ else
+ ret = crypto_skcipher_decrypt(&subreq->req);
+
+ if (!ret && ctx->iv_gen_ops && ctx->iv_gen_ops->post)
+ ret = ctx->iv_gen_ops->post(ctx, rctx, subreq);
+
+ switch (ret) {
+ /*
+ * The request was queued by a crypto driver
+ * but the driver request queue is full, let's wait.
+ */
+ case -EBUSY:
+ wait_for_completion(&rctx->restart);
+ reinit_completion(&rctx->restart);
+ /* fall through */
+ /*
+ * The request is queued and processed asynchronously,
+ * completion function geniv_async_done() is called.
+ */
+ case -EINPROGRESS:
+ /* Marking this NULL lets the creation of a new sub-
+ * request when 'geniv_alloc_subreq' is called.
+ */
+ rctx->subreq = NULL;
+ rctx->iv_sector++;
+ cond_resched();
+ break;
+ /*
+ * The request was already processed (synchronously).
+ */
+ case 0:
+ atomic_dec(&rctx->req_pending);
+ rctx->iv_sector++;
+ cond_resched();
+ continue;
+
+ /* There was an error while processing the request. */
+ default:
+ atomic_dec(&rctx->req_pending);
+ return ret;
+ }
+
+ if (ret)
+ break;
+ }
+
+ if (rctx->subreq && atomic_read(&rctx->req_pending) == 1) {
+ DMDEBUG("geniv:%s: Freeing sub request\n", str);
+ mempool_free(rctx->subreq, ctx->subreq_pool);
+ }
+
+end:
+ return ret;
+}
+
+static int geniv_encrypt(struct skcipher_request *req)
+{
+ return geniv_crypt(req, true);
+}
+
+static int geniv_decrypt(struct skcipher_request *req)
+{
+ return geniv_crypt(req, false);
+}
+
+static int geniv_init_tfm(struct crypto_skcipher *tfm)
+{
+ struct geniv_ctx *ctx = crypto_skcipher_ctx(tfm);
+ const int psize = sizeof(struct geniv_subreq);
+ unsigned int reqsize, align;
+ char *algname, *chainmode;
+ int ret = 0;
+
+ algname = (char *) crypto_tfm_alg_name(crypto_skcipher_tfm(tfm));
+ ctx->ciphermode = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL);
+ if (!ctx->ciphermode) {
+ ret = -ENOMEM;
+ goto end;
+ }
+
+ /* Parse algorithm name 'ivmode(chainmode(cipher))' */
+ ctx->ivmode = strsep(&algname, "(");
+ chainmode = strsep(&algname, "(");
+ ctx->cipher = strsep(&algname, ")");
+
+ snprintf(ctx->ciphermode, CRYPTO_MAX_ALG_NAME, "%s(%s)",
+ chainmode, ctx->cipher);
+
+ DMDEBUG("ciphermode=%s, ivmode=%s\n", ctx->ciphermode, ctx->ivmode);
+
+ /*
+ * Usually the underlying cipher instances are spawned here, but since
+ * the value of tfms_count (which is equal to the key_count) is not
+ * known yet, create only one instance and delay the creation of the
+ * rest of the instances of the underlying cipher 'cbc(aes)' until
+ * the setkey operation is invoked.
+ * The first instance created i.e. ctx->child will later be assigned as
+ * the 1st element in the array ctx->tfms. Creation of atleast one
+ * instance of the cipher is necessary to be created here to uncover
+ * any errors earlier than during the setkey operation later where the
+ * remaining instances are created.
+ */
+ ctx->child = crypto_alloc_skcipher(ctx->ciphermode, 0, 0);
+ if (IS_ERR(ctx->child)) {
+ ret = PTR_ERR(ctx->child);
+ DMERR("Failed to create skcipher %s. err %d\n",
+ ctx->ciphermode, ret);
+ goto end;
+ }
+
+ /* Setup the current cipher's request structure */
+ align = crypto_skcipher_alignmask(tfm);
+ align &= ~(crypto_tfm_ctx_alignment() - 1);
+ reqsize = align + sizeof(struct geniv_req_ctx)
+ + crypto_skcipher_reqsize(ctx->child);
+ crypto_skcipher_set_reqsize(tfm, reqsize);
+
+ /* create memory pool for sub-request structure */
+ ctx->subreq_pool = mempool_create_kmalloc_pool(MIN_IOS, psize);
+ if (!ctx->subreq_pool) {
+ ret = -ENOMEM;
+ DMERR("Could not allocate crypt sub-request mempool\n");
+ }
+end:
+ return ret;
+}
+
+static void geniv_exit_tfm(struct crypto_skcipher *tfm)
+{
+ struct geniv_ctx *ctx = crypto_skcipher_ctx(tfm);
+
+ if (ctx->iv_gen_ops && ctx->iv_gen_ops->dtr)
+ ctx->iv_gen_ops->dtr(ctx);
+
+ mempool_destroy(ctx->subreq_pool);
+ geniv_free_tfms(ctx);
+ kfree(ctx->ciphermode);
+}
+
+static void geniv_free(struct skcipher_instance *inst)
+{
+ struct crypto_skcipher_spawn *spawn = skcipher_instance_ctx(inst);
+
+ crypto_drop_skcipher(spawn);
+ kfree(inst);
+}
+
+static int geniv_create(struct crypto_template *tmpl,
+ struct rtattr **tb, char *algname)
+{
+ struct crypto_attr_type *algt;
+ struct skcipher_instance *inst;
+ struct skcipher_alg *alg;
+ struct crypto_skcipher_spawn *spawn;
+ const char *cipher_name;
+ int err;
+
+ algt = crypto_get_attr_type(tb);
+
+ if (IS_ERR(algt))
+ return PTR_ERR(algt);
+
+ if ((algt->type ^ CRYPTO_ALG_TYPE_SKCIPHER) & algt->mask)
+ return -EINVAL;
+
+ cipher_name = crypto_attr_alg_name(tb[1]);
+
+ if (IS_ERR(cipher_name))
+ return PTR_ERR(cipher_name);
+
+ inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
+ if (!inst)
+ return -ENOMEM;
+
+ spawn = skcipher_instance_ctx(inst);
+
+ crypto_set_skcipher_spawn(spawn, skcipher_crypto_instance(inst));
+ err = crypto_grab_skcipher(spawn, cipher_name, 0,
+ crypto_requires_sync(algt->type,
+ algt->mask));
+
+ if (err)
+ goto err_free_inst;
+
+ alg = crypto_spawn_skcipher_alg(spawn);
+
+ err = -EINVAL;
+
+ /* Only support blocks of size which is of a power of 2 */
+ if (!is_power_of_2(alg->base.cra_blocksize))
+ goto err_drop_spawn;
+
+ /* algname: essiv, base.cra_name: cbc(aes) */
+ err = -ENAMETOOLONG;
+ if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
+ algname, alg->base.cra_name) >= CRYPTO_MAX_ALG_NAME)
+ goto err_drop_spawn;
+ if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
+ "%s(%s)", algname, alg->base.cra_driver_name) >=
+ CRYPTO_MAX_ALG_NAME)
+ goto err_drop_spawn;
+
+ inst->alg.base.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER;
+ inst->alg.base.cra_priority = alg->base.cra_priority;
+ inst->alg.base.cra_blocksize = alg->base.cra_blocksize;
+ inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
+ inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC;
+ inst->alg.ivsize = alg->base.cra_blocksize;
+ inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg);
+ inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg);
+ inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg);
+
+ inst->alg.setkey = geniv_setkey;
+ inst->alg.encrypt = geniv_encrypt;
+ inst->alg.decrypt = geniv_decrypt;
+
+ inst->alg.base.cra_ctxsize = sizeof(struct geniv_ctx);
+
+ inst->alg.init = geniv_init_tfm;
+ inst->alg.exit = geniv_exit_tfm;
+
+ inst->free = geniv_free;
+
+ err = skcipher_register_instance(tmpl, inst);
+ if (err)
+ goto err_drop_spawn;
+
+out:
+ return err;
+
+err_drop_spawn:
+ crypto_drop_skcipher(spawn);
+err_free_inst:
+ kfree(inst);
+ goto out;
+}
+
+static int crypto_plain_create(struct crypto_template *tmpl,
+ struct rtattr **tb)
+{
+ return geniv_create(tmpl, tb, "plain");
+}
+
+static int crypto_plain64_create(struct crypto_template *tmpl,
+ struct rtattr **tb)
+{
+ return geniv_create(tmpl, tb, "plain64");
+}
+
+static int crypto_essiv_create(struct crypto_template *tmpl,
+ struct rtattr **tb)
+{
+ return geniv_create(tmpl, tb, "essiv");
+}
+
+static int crypto_benbi_create(struct crypto_template *tmpl,
+ struct rtattr **tb)
+{
+ return geniv_create(tmpl, tb, "benbi");
+}
+
+static int crypto_null_create(struct crypto_template *tmpl,
+ struct rtattr **tb)
+{
+ return geniv_create(tmpl, tb, "null");
+}
+
+static int crypto_lmk_create(struct crypto_template *tmpl,
+ struct rtattr **tb)
+{
+ return geniv_create(tmpl, tb, "lmk");
+}
+
+static int crypto_tcw_create(struct crypto_template *tmpl,
+ struct rtattr **tb)
+{
+ return geniv_create(tmpl, tb, "tcw");
+}
+
+static struct crypto_template crypto_plain_tmpl = {
+ .name = "plain",
+ .create = crypto_plain_create,
+ .module = THIS_MODULE,
+};
+
+static struct crypto_template crypto_plain64_tmpl = {
+ .name = "plain64",
+ .create = crypto_plain64_create,
+ .module = THIS_MODULE,
+};
+
+static struct crypto_template crypto_essiv_tmpl = {
+ .name = "essiv",
+ .create = crypto_essiv_create,
+ .module = THIS_MODULE,
+};
+
+static struct crypto_template crypto_benbi_tmpl = {
+ .name = "benbi",
+ .create = crypto_benbi_create,
+ .module = THIS_MODULE,
+};
+
+static struct crypto_template crypto_null_tmpl = {
+ .name = "null",
+ .create = crypto_null_create,
+ .module = THIS_MODULE,
+};
+
+static struct crypto_template crypto_lmk_tmpl = {
+ .name = "lmk",
+ .create = crypto_lmk_create,
+ .module = THIS_MODULE,
+};
+
+static struct crypto_template crypto_tcw_tmpl = {
+ .name = "tcw",
+ .create = crypto_tcw_create,
+ .module = THIS_MODULE,
+};
+
+static int __init geniv_register_algs(void)
+{
+ int err;
+
+ err = crypto_register_template(&crypto_plain_tmpl);
+ if (err)
+ goto out;
+
+ err = crypto_register_template(&crypto_plain64_tmpl);
+ if (err)
+ goto out_undo_plain;
+
+ err = crypto_register_template(&crypto_essiv_tmpl);
+ if (err)
+ goto out_undo_plain64;
+
+ err = crypto_register_template(&crypto_benbi_tmpl);
+ if (err)
+ goto out_undo_essiv;
+
+ err = crypto_register_template(&crypto_null_tmpl);
+ if (err)
+ goto out_undo_benbi;
+
+ err = crypto_register_template(&crypto_lmk_tmpl);
+ if (err)
+ goto out_undo_null;
+
+ err = crypto_register_template(&crypto_tcw_tmpl);
+ if (!err)
+ goto out;
+
+ crypto_unregister_template(&crypto_lmk_tmpl);
+out_undo_null:
+ crypto_unregister_template(&crypto_null_tmpl);
+out_undo_benbi:
+ crypto_unregister_template(&crypto_benbi_tmpl);
+out_undo_essiv:
+ crypto_unregister_template(&crypto_essiv_tmpl);
+out_undo_plain64:
+ crypto_unregister_template(&crypto_plain64_tmpl);
+out_undo_plain:
+ crypto_unregister_template(&crypto_plain_tmpl);
+out:
+ return err;
+}
+
+static void __exit geniv_deregister_algs(void)
+{
+ crypto_unregister_template(&crypto_plain_tmpl);
+ crypto_unregister_template(&crypto_plain64_tmpl);
+ crypto_unregister_template(&crypto_essiv_tmpl);
+ crypto_unregister_template(&crypto_benbi_tmpl);
+ crypto_unregister_template(&crypto_null_tmpl);
+ crypto_unregister_template(&crypto_lmk_tmpl);
+ crypto_unregister_template(&crypto_tcw_tmpl);
+}
+
+/* End of geniv template cipher algorithms */
+
+/*
+ * context holding the current state of a multi-part conversion
+ */
+struct convert_context {
+ struct completion restart;
+ struct bio *bio_in;
+ struct bio *bio_out;
+ struct bvec_iter iter_in;
+ struct bvec_iter iter_out;
+ sector_t cc_sector;
+ atomic_t cc_pending;
+ struct skcipher_request *req;
+};
+
+/*
+ * per bio private data
+ */
+struct dm_crypt_io {
+ struct crypt_config *cc;
+ struct bio *base_bio;
+ struct work_struct work;
+
+ struct convert_context ctx;
-static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,
- struct dm_crypt_request *dmreq)
-{
- struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
- __le64 sector = cpu_to_le64(dmreq->iv_sector);
- u8 *src;
- int r = 0;
+ atomic_t io_pending;
+ int error;
+ sector_t sector;
- /* Remove whitening from ciphertext */
- if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
- src = kmap_atomic(sg_page(&dmreq->sg_in));
- r = crypt_iv_tcw_whitening(cc, dmreq, src + dmreq->sg_in.offset);
- kunmap_atomic(src);
- }
+ struct rb_node rb_node;
+} CRYPTO_MINALIGN_ATTR;
- /* Calculate IV */
- memcpy(iv, tcw->iv_seed, cc->iv_size);
- crypto_xor(iv, (u8 *)§or, 8);
- if (cc->iv_size > 8)
- crypto_xor(&iv[8], (u8 *)§or, cc->iv_size - 8);
+struct dm_crypt_request {
+ struct convert_context *ctx;
+ struct scatterlist *sg_in;
+ struct scatterlist *sg_out;
+ sector_t iv_sector;
+};
- return r;
-}
+struct crypt_config;
-static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv,
- struct dm_crypt_request *dmreq)
-{
- u8 *dst;
- int r;
+/*
+ * Crypt: maps a linear range of a block device
+ * and encrypts / decrypts at the same time.
+ */
+enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID,
+ DM_CRYPT_SAME_CPU, DM_CRYPT_NO_OFFLOAD };
- if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
- return 0;
+/*
+ * The fields in here must be read only after initialization.
+ */
+struct crypt_config {
+ struct dm_dev *dev;
+ sector_t start;
- /* Apply whitening on ciphertext */
- dst = kmap_atomic(sg_page(&dmreq->sg_out));
- r = crypt_iv_tcw_whitening(cc, dmreq, dst + dmreq->sg_out.offset);
- kunmap_atomic(dst);
+ /*
+ * pool for per bio private data, crypto requests and
+ * encryption requeusts/buffer pages
+ */
+ mempool_t *req_pool;
+ mempool_t *page_pool;
+ struct bio_set *bs;
+ struct mutex bio_alloc_lock;
- return r;
-}
+ struct workqueue_struct *io_queue;
+ struct workqueue_struct *crypt_queue;
-static const struct crypt_iv_operations crypt_iv_plain_ops = {
- .generator = crypt_iv_plain_gen
-};
+ struct task_struct *write_thread;
+ wait_queue_head_t write_thread_wait;
+ struct rb_root write_tree;
-static const struct crypt_iv_operations crypt_iv_plain64_ops = {
- .generator = crypt_iv_plain64_gen
-};
+ char *cipher;
+ char *cipher_string;
+ char *key_string;
-static const struct crypt_iv_operations crypt_iv_essiv_ops = {
- .ctr = crypt_iv_essiv_ctr,
- .dtr = crypt_iv_essiv_dtr,
- .init = crypt_iv_essiv_init,
- .wipe = crypt_iv_essiv_wipe,
- .generator = crypt_iv_essiv_gen
-};
+ sector_t iv_offset;
+ unsigned int iv_size;
-static const struct crypt_iv_operations crypt_iv_benbi_ops = {
- .ctr = crypt_iv_benbi_ctr,
- .dtr = crypt_iv_benbi_dtr,
- .generator = crypt_iv_benbi_gen
-};
+ /* ESSIV: struct crypto_cipher *essiv_tfm */
+ void *iv_private;
+ struct crypto_skcipher *tfm;
+ unsigned int tfms_count;
-static const struct crypt_iv_operations crypt_iv_null_ops = {
- .generator = crypt_iv_null_gen
-};
+ /*
+ * Layout of each crypto request:
+ *
+ * struct skcipher_request
+ * context
+ * padding
+ * struct dm_crypt_request
+ * padding
+ * IV
+ *
+ * The padding is added so that dm_crypt_request and the IV are
+ * correctly aligned.
+ */
+ unsigned int dmreq_start;
-static const struct crypt_iv_operations crypt_iv_lmk_ops = {
- .ctr = crypt_iv_lmk_ctr,
- .dtr = crypt_iv_lmk_dtr,
- .init = crypt_iv_lmk_init,
- .wipe = crypt_iv_lmk_wipe,
- .generator = crypt_iv_lmk_gen,
- .post = crypt_iv_lmk_post
-};
+ unsigned int per_bio_data_size;
-static const struct crypt_iv_operations crypt_iv_tcw_ops = {
- .ctr = crypt_iv_tcw_ctr,
- .dtr = crypt_iv_tcw_dtr,
- .init = crypt_iv_tcw_init,
- .wipe = crypt_iv_tcw_wipe,
- .generator = crypt_iv_tcw_gen,
- .post = crypt_iv_tcw_post
+ unsigned long flags;
+ unsigned int key_size;
+ unsigned int key_parts; /* independent parts in key buffer */
+ unsigned int key_extra_size; /* additional keys length */
+ u8 key[0];
};
+static void clone_init(struct dm_crypt_io *, struct bio *);
+static void kcryptd_queue_crypt(struct dm_crypt_io *io);
+static u8 *iv_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq);
+
static void crypt_convert_init(struct crypt_config *cc,
struct convert_context *ctx,
struct bio *bio_out, struct bio *bio_in,
@@ -837,53 +1697,7 @@ static u8 *iv_of_dmreq(struct crypt_config *cc,
struct dm_crypt_request *dmreq)
{
return (u8 *)ALIGN((unsigned long)(dmreq + 1),
- crypto_skcipher_alignmask(any_tfm(cc)) + 1);
-}
-
-static int crypt_convert_block(struct crypt_config *cc,
- struct convert_context *ctx,
- struct skcipher_request *req)
-{
- struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
- struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
- struct dm_crypt_request *dmreq;
- u8 *iv;
- int r;
-
- dmreq = dmreq_of_req(cc, req);
- iv = iv_of_dmreq(cc, dmreq);
-
- dmreq->iv_sector = ctx->cc_sector;
- dmreq->ctx = ctx;
- sg_init_table(&dmreq->sg_in, 1);
- sg_set_page(&dmreq->sg_in, bv_in.bv_page, 1 << SECTOR_SHIFT,
- bv_in.bv_offset);
-
- sg_init_table(&dmreq->sg_out, 1);
- sg_set_page(&dmreq->sg_out, bv_out.bv_page, 1 << SECTOR_SHIFT,
- bv_out.bv_offset);
-
- bio_advance_iter(ctx->bio_in, &ctx->iter_in, 1 << SECTOR_SHIFT);
- bio_advance_iter(ctx->bio_out, &ctx->iter_out, 1 << SECTOR_SHIFT);
-
- if (cc->iv_gen_ops) {
- r = cc->iv_gen_ops->generator(cc, iv, dmreq);
- if (r < 0)
- return r;
- }
-
- skcipher_request_set_crypt(req, &dmreq->sg_in, &dmreq->sg_out,
- 1 << SECTOR_SHIFT, iv);
-
- if (bio_data_dir(ctx->bio_in) == WRITE)
- r = crypto_skcipher_encrypt(req);
- else
- r = crypto_skcipher_decrypt(req);
-
- if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
- r = cc->iv_gen_ops->post(cc, iv, dmreq);
-
- return r;
+ crypto_skcipher_alignmask(cc->tfm) + 1);
}
static void kcryptd_async_done(struct crypto_async_request *async_req,
@@ -892,12 +1706,10 @@ static void kcryptd_async_done(struct crypto_async_request *async_req,
static void crypt_alloc_req(struct crypt_config *cc,
struct convert_context *ctx)
{
- unsigned key_index = ctx->cc_sector & (cc->tfms_count - 1);
-
if (!ctx->req)
ctx->req = mempool_alloc(cc->req_pool, GFP_NOIO);
- skcipher_request_set_tfm(ctx->req, cc->tfms[key_index]);
+ skcipher_request_set_tfm(ctx->req, cc->tfm);
/*
* Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
@@ -920,57 +1732,98 @@ static void crypt_free_req(struct crypt_config *cc,
/*
* Encrypt / decrypt data from one bio to another one (can be the same one)
*/
-static int crypt_convert(struct crypt_config *cc,
- struct convert_context *ctx)
+
+static int crypt_convert_bio(struct crypt_config *cc,
+ struct convert_context *ctx)
{
+ unsigned int cryptlen, n1, n2, nents, i = 0, bytes = 0;
+ struct skcipher_request *req;
+ struct dm_crypt_request *dmreq;
+ struct geniv_req_info rinfo;
+ struct bio_vec bv_in, bv_out;
int r;
+ u8 *iv;
atomic_set(&ctx->cc_pending, 1);
+ crypt_alloc_req(cc, ctx);
+
+ req = ctx->req;
+ dmreq = dmreq_of_req(cc, req);
+ iv = iv_of_dmreq(cc, dmreq);
- while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) {
+ n1 = bio_segments(ctx->bio_in);
+ n2 = bio_segments(ctx->bio_in);
+ nents = n1 > n2 ? n1 : n2;
+ nents = nents > MAX_SG_LIST ? MAX_SG_LIST : nents;
+ cryptlen = ctx->iter_in.bi_size;
- crypt_alloc_req(cc, ctx);
+ DMDEBUG("dm-crypt:%s: segments:[in=%u, out=%u] bi_size=%u\n",
+ bio_data_dir(ctx->bio_in) == WRITE ? "write" : "read",
+ n1, n2, cryptlen);
- atomic_inc(&ctx->cc_pending);
+ dmreq->sg_in = kcalloc(nents, sizeof(struct scatterlist), GFP_KERNEL);
+ dmreq->sg_out = kcalloc(nents, sizeof(struct scatterlist), GFP_KERNEL);
+ if (!dmreq->sg_in || !dmreq->sg_out) {
+ DMERR("dm-crypt: Failed to allocate scatterlist\n");
+ r = -ENOMEM;
+ goto end;
+ }
+ dmreq->ctx = ctx;
- r = crypt_convert_block(cc, ctx, ctx->req);
+ sg_init_table(dmreq->sg_in, nents);
+ sg_init_table(dmreq->sg_out, nents);
- switch (r) {
- /*
- * The request was queued by a crypto driver
- * but the driver request queue is full, let's wait.
- */
- case -EBUSY:
- wait_for_completion(&ctx->restart);
- reinit_completion(&ctx->restart);
- /* fall through */
- /*
- * The request is queued and processed asynchronously,
- * completion function kcryptd_async_done() will be called.
- */
- case -EINPROGRESS:
- ctx->req = NULL;
- ctx->cc_sector++;
- continue;
- /*
- * The request was already processed (synchronously).
- */
- case 0:
- atomic_dec(&ctx->cc_pending);
- ctx->cc_sector++;
- cond_resched();
- continue;
+ while (ctx->iter_in.bi_size && ctx->iter_out.bi_size && i < nents) {
+ bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
+ bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
- /* There was an error while processing the request. */
- default:
- atomic_dec(&ctx->cc_pending);
- return r;
- }
+ sg_set_page(&dmreq->sg_in[i], bv_in.bv_page, bv_in.bv_len,
+ bv_in.bv_offset);
+ sg_set_page(&dmreq->sg_out[i], bv_out.bv_page, bv_out.bv_len,
+ bv_out.bv_offset);
+
+ bio_advance_iter(ctx->bio_in, &ctx->iter_in, bv_in.bv_len);
+ bio_advance_iter(ctx->bio_out, &ctx->iter_out, bv_out.bv_len);
+
+ bytes += bv_in.bv_len;
+ i++;
}
- return 0;
+ DMDEBUG("dm-crypt: Processed %u of %u bytes\n", bytes, cryptlen);
+
+ rinfo.is_write = bio_data_dir(ctx->bio_in) == WRITE;
+ rinfo.iv_sector = ctx->cc_sector;
+ rinfo.nents = nents;
+ rinfo.iv = iv;
+
+ skcipher_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,
+ bytes, &rinfo);
+
+ if (bio_data_dir(ctx->bio_in) == WRITE)
+ r = crypto_skcipher_encrypt(req);
+ else
+ r = crypto_skcipher_decrypt(req);
+
+ switch (r) {
+ /* The request was queued so wait. */
+ case -EBUSY:
+ wait_for_completion(&ctx->restart);
+ reinit_completion(&ctx->restart);
+ /* fall through */
+ /*
+ * The request is queued and processed asynchronously,
+ * completion function kcryptd_async_done() is called.
+ */
+ case -EINPROGRESS:
+ ctx->req = NULL;
+ cond_resched();
+ break;
+ }
+end:
+ return r;
}
+
static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone);
/*
@@ -1070,11 +1923,17 @@ static void crypt_dec_pending(struct dm_crypt_io *io)
{
struct crypt_config *cc = io->cc;
struct bio *base_bio = io->base_bio;
+ struct dm_crypt_request *dmreq;
int error = io->error;
if (!atomic_dec_and_test(&io->io_pending))
return;
+ dmreq = dmreq_of_req(cc, io->ctx.req);
+ DMDEBUG("dm-crypt: Freeing scatterlists [sync]\n");
+ kfree(dmreq->sg_in);
+ kfree(dmreq->sg_out);
+
if (io->ctx.req)
crypt_free_req(cc, io->ctx.req, base_bio);
@@ -1313,7 +2172,7 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
sector += bio_sectors(clone);
crypt_inc_pending(io);
- r = crypt_convert(cc, &io->ctx);
+ r = crypt_convert_bio(cc, &io->ctx);
if (r)
io->error = -EIO;
crypt_finished = atomic_dec_and_test(&io->ctx.cc_pending);
@@ -1343,7 +2202,8 @@ static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)
crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio,
io->sector);
- r = crypt_convert(cc, &io->ctx);
+ r = crypt_convert_bio(cc, &io->ctx);
+
if (r < 0)
io->error = -EIO;
@@ -1371,12 +2231,13 @@ static void kcryptd_async_done(struct crypto_async_request *async_req,
return;
}
- if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post)
- error = cc->iv_gen_ops->post(cc, iv_of_dmreq(cc, dmreq), dmreq);
-
if (error < 0)
io->error = -EIO;
+ DMDEBUG("dm-crypt: Freeing scatterlists and request struct [async]\n");
+ kfree(dmreq->sg_in);
+ kfree(dmreq->sg_out);
+
crypt_free_req(cc, req_of_dmreq(cc, dmreq), io->base_bio);
if (!atomic_dec_and_test(&ctx->cc_pending))
@@ -1430,62 +2291,38 @@ static int crypt_decode_key(u8 *key, char *hex, unsigned int size)
return 0;
}
-static void crypt_free_tfms(struct crypt_config *cc)
+static void crypt_free_tfm(struct crypt_config *cc)
{
- unsigned i;
-
- if (!cc->tfms)
+ if (!cc->tfm)
return;
- for (i = 0; i < cc->tfms_count; i++)
- if (cc->tfms[i] && !IS_ERR(cc->tfms[i])) {
- crypto_free_skcipher(cc->tfms[i]);
- cc->tfms[i] = NULL;
- }
+ if (cc->tfm && !IS_ERR(cc->tfm))
+ crypto_free_skcipher(cc->tfm);
- kfree(cc->tfms);
- cc->tfms = NULL;
+ cc->tfm = NULL;
}
-static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)
+static int crypt_alloc_tfm(struct crypt_config *cc, char *ciphermode)
{
- unsigned i;
int err;
- cc->tfms = kzalloc(cc->tfms_count * sizeof(struct crypto_skcipher *),
- GFP_KERNEL);
- if (!cc->tfms)
- return -ENOMEM;
-
- for (i = 0; i < cc->tfms_count; i++) {
- cc->tfms[i] = crypto_alloc_skcipher(ciphermode, 0, 0);
- if (IS_ERR(cc->tfms[i])) {
- err = PTR_ERR(cc->tfms[i]);
- crypt_free_tfms(cc);
- return err;
- }
+ cc->tfm = crypto_alloc_skcipher(ciphermode, 0, 0);
+ if (IS_ERR(cc->tfm)) {
+ err = PTR_ERR(cc->tfm);
+ crypt_free_tfm(cc);
+ return err;
}
return 0;
}
-static int crypt_setkey(struct crypt_config *cc)
+static inline int crypt_setkey(struct crypt_config *cc, enum setkey_op keyop,
+ char *ivopts)
{
- unsigned subkey_size;
- int err = 0, i, r;
-
- /* Ignore extra keys (which are used for IV etc) */
- subkey_size = (cc->key_size - cc->key_extra_size) >> ilog2(cc->tfms_count);
-
- for (i = 0; i < cc->tfms_count; i++) {
- r = crypto_skcipher_setkey(cc->tfms[i],
- cc->key + (i * subkey_size),
- subkey_size);
- if (r)
- err = r;
- }
+ DECLARE_GENIV_KEY(kinfo, keyop, cc->tfms_count, cc->key, cc->key_size,
+ cc->key_parts, ivopts);
- return err;
+ return crypto_skcipher_setkey(cc->tfm, (u8 *) &kinfo, sizeof(kinfo));
}
#ifdef CONFIG_KEYS
@@ -1498,7 +2335,9 @@ static bool contains_whitespace(const char *str)
return false;
}
-static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
+static int crypt_set_keyring_key(struct crypt_config *cc,
+ const char *key_string,
+ enum setkey_op keyop, char *ivopts)
{
char *new_key_string, *key_desc;
int ret;
@@ -1559,7 +2398,7 @@ static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string
/* clear the flag since following operations may invalidate previously valid key */
clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
- ret = crypt_setkey(cc);
+ ret = crypt_setkey(cc, keyop, ivopts);
/* wipe the kernel key payload copy in each case */
memset(cc->key, 0, cc->key_size * sizeof(u8));
@@ -1599,7 +2438,9 @@ static int get_key_size(char **key_string)
#else
-static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
+static int crypt_set_keyring_key(struct crypt_config *cc,
+ const char *key_string,
+ enum setkey_op keyop, char *ivopts)
{
return -EINVAL;
}
@@ -1611,7 +2452,8 @@ static int get_key_size(char **key_string)
#endif
-static int crypt_set_key(struct crypt_config *cc, char *key)
+static int crypt_set_key(struct crypt_config *cc, enum setkey_op keyop,
+ char *key, char *ivopts)
{
int r = -EINVAL;
int key_string_len = strlen(key);
@@ -1622,7 +2464,7 @@ static int crypt_set_key(struct crypt_config *cc, char *key)
/* ':' means the key is in kernel keyring, short-circuit normal key processing */
if (key[0] == ':') {
- r = crypt_set_keyring_key(cc, key + 1);
+ r = crypt_set_keyring_key(cc, key + 1, keyop, ivopts);
goto out;
}
@@ -1636,7 +2478,7 @@ static int crypt_set_key(struct crypt_config *cc, char *key)
if (cc->key_size && crypt_decode_key(cc->key, key, cc->key_size) < 0)
goto out;
- r = crypt_setkey(cc);
+ r = crypt_setkey(cc, keyop, ivopts);
if (!r)
set_bit(DM_CRYPT_KEY_VALID, &cc->flags);
@@ -1647,6 +2489,17 @@ static int crypt_set_key(struct crypt_config *cc, char *key)
return r;
}
+static int crypt_init_key(struct dm_target *ti, char *key, char *ivopts)
+{
+ struct crypt_config *cc = ti->private;
+ int ret;
+
+ ret = crypt_set_key(cc, SETKEY_OP_INIT, key, ivopts);
+ if (ret < 0)
+ ti->error = "Error decoding and setting key";
+ return ret;
+}
+
static int crypt_wipe_key(struct crypt_config *cc)
{
clear_bit(DM_CRYPT_KEY_VALID, &cc->flags);
@@ -1654,7 +2507,7 @@ static int crypt_wipe_key(struct crypt_config *cc)
kzfree(cc->key_string);
cc->key_string = NULL;
- return crypt_setkey(cc);
+ return crypt_setkey(cc, SETKEY_OP_WIPE, NULL);
}
static void crypt_dtr(struct dm_target *ti)
@@ -1674,7 +2527,7 @@ static void crypt_dtr(struct dm_target *ti)
if (cc->crypt_queue)
destroy_workqueue(cc->crypt_queue);
- crypt_free_tfms(cc);
+ crypt_free_tfm(cc);
if (cc->bs)
bioset_free(cc->bs);
@@ -1682,9 +2535,6 @@ static void crypt_dtr(struct dm_target *ti)
mempool_destroy(cc->page_pool);
mempool_destroy(cc->req_pool);
- if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
- cc->iv_gen_ops->dtr(cc);
-
if (cc->dev)
dm_put_device(ti, cc->dev);
@@ -1762,22 +2612,30 @@ static int crypt_ctr_cipher(struct dm_target *ti,
if (!cipher_api)
goto bad_mem;
- ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME,
- "%s(%s)", chainmode, cipher);
+create_cipher:
+ /* For those ciphers which do not support IVs,
+ * use the 'null' template cipher
+ */
+
+ if (!ivmode)
+ ivmode = "null";
+
+ ret = snprintf(cipher_api, CRYPTO_MAX_ALG_NAME, "%s(%s(%s))",
+ ivmode, chainmode, cipher);
if (ret < 0) {
kfree(cipher_api);
goto bad_mem;
}
/* Allocate cipher */
- ret = crypt_alloc_tfms(cc, cipher_api);
+ ret = crypt_alloc_tfm(cc, cipher_api);
if (ret < 0) {
ti->error = "Error allocating crypto tfm";
goto bad;
}
/* Initialize IV */
- cc->iv_size = crypto_skcipher_ivsize(any_tfm(cc));
+ cc->iv_size = crypto_skcipher_ivsize(cc->tfm);
if (cc->iv_size)
/* at least a 64 bit sector number should fit in our buffer */
cc->iv_size = max(cc->iv_size,
@@ -1785,23 +2643,10 @@ static int crypt_ctr_cipher(struct dm_target *ti,
else if (ivmode) {
DMWARN("Selected cipher does not support IVs");
ivmode = NULL;
+ goto create_cipher;
}
- /* Choose ivmode, see comments at iv code. */
- if (ivmode == NULL)
- cc->iv_gen_ops = NULL;
- else if (strcmp(ivmode, "plain") == 0)
- cc->iv_gen_ops = &crypt_iv_plain_ops;
- else if (strcmp(ivmode, "plain64") == 0)
- cc->iv_gen_ops = &crypt_iv_plain64_ops;
- else if (strcmp(ivmode, "essiv") == 0)
- cc->iv_gen_ops = &crypt_iv_essiv_ops;
- else if (strcmp(ivmode, "benbi") == 0)
- cc->iv_gen_ops = &crypt_iv_benbi_ops;
- else if (strcmp(ivmode, "null") == 0)
- cc->iv_gen_ops = &crypt_iv_null_ops;
- else if (strcmp(ivmode, "lmk") == 0) {
- cc->iv_gen_ops = &crypt_iv_lmk_ops;
+ if (strcmp(ivmode, "lmk") == 0) {
/*
* Version 2 and 3 is recognised according
* to length of provided multi-key string.
@@ -1813,39 +2658,14 @@ static int crypt_ctr_cipher(struct dm_target *ti,
cc->key_extra_size = cc->key_size / cc->key_parts;
}
} else if (strcmp(ivmode, "tcw") == 0) {
- cc->iv_gen_ops = &crypt_iv_tcw_ops;
cc->key_parts += 2; /* IV + whitening */
cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE;
- } else {
- ret = -EINVAL;
- ti->error = "Invalid IV mode";
- goto bad;
}
/* Initialize and set key */
- ret = crypt_set_key(cc, key);
- if (ret < 0) {
- ti->error = "Error decoding and setting key";
+ ret = crypt_init_key(ti, key, ivopts);
+ if (ret < 0)
goto bad;
- }
-
- /* Allocate IV */
- if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) {
- ret = cc->iv_gen_ops->ctr(cc, ti, ivopts);
- if (ret < 0) {
- ti->error = "Error creating IV";
- goto bad;
- }
- }
-
- /* Initialize IV (set keys for ESSIV etc) */
- if (cc->iv_gen_ops && cc->iv_gen_ops->init) {
- ret = cc->iv_gen_ops->init(cc);
- if (ret < 0) {
- ti->error = "Error initialising IV";
- goto bad;
- }
- }
ret = 0;
bad:
@@ -1901,20 +2721,20 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
goto bad;
cc->dmreq_start = sizeof(struct skcipher_request);
- cc->dmreq_start += crypto_skcipher_reqsize(any_tfm(cc));
+ cc->dmreq_start += crypto_skcipher_reqsize(cc->tfm);
cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request));
- if (crypto_skcipher_alignmask(any_tfm(cc)) < CRYPTO_MINALIGN) {
+ if (crypto_skcipher_alignmask(cc->tfm) < CRYPTO_MINALIGN) {
/* Allocate the padding exactly */
iv_size_padding = -(cc->dmreq_start + sizeof(struct dm_crypt_request))
- & crypto_skcipher_alignmask(any_tfm(cc));
+ & crypto_skcipher_alignmask(cc->tfm);
} else {
/*
* If the cipher requires greater alignment than kmalloc
* alignment, we don't know the exact position of the
* initialization vector. We must assume worst case.
*/
- iv_size_padding = crypto_skcipher_alignmask(any_tfm(cc));
+ iv_size_padding = crypto_skcipher_alignmask(cc->tfm);
}
ret = -ENOMEM;
@@ -2072,8 +2892,9 @@ static int crypt_map(struct dm_target *ti, struct bio *bio)
if (bio_data_dir(io->base_bio) == READ) {
if (kcryptd_io_read(io, GFP_NOWAIT))
kcryptd_queue_read(io);
- } else
+ } else {
kcryptd_queue_crypt(io);
+ }
return DM_MAPIO_SUBMITTED;
}
@@ -2155,7 +2976,7 @@ static void crypt_resume(struct dm_target *ti)
static int crypt_message(struct dm_target *ti, unsigned argc, char **argv)
{
struct crypt_config *cc = ti->private;
- int key_size, ret = -EINVAL;
+ int key_size;
if (argc < 2)
goto error;
@@ -2173,19 +2994,9 @@ static int crypt_message(struct dm_target *ti, unsigned argc, char **argv)
return -EINVAL;
}
- ret = crypt_set_key(cc, argv[2]);
- if (ret)
- return ret;
- if (cc->iv_gen_ops && cc->iv_gen_ops->init)
- ret = cc->iv_gen_ops->init(cc);
- return ret;
+ return crypt_set_key(cc, SETKEY_OP_SET, argv[2], NULL);
}
if (argc == 2 && !strcasecmp(argv[1], "wipe")) {
- if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) {
- ret = cc->iv_gen_ops->wipe(cc);
- if (ret)
- return ret;
- }
return crypt_wipe_key(cc);
}
}
@@ -2216,7 +3027,7 @@ static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
static struct target_type crypt_target = {
.name = "crypt",
- .version = {1, 15, 0},
+ .version = {1, 16, 0},
.module = THIS_MODULE,
.ctr = crypt_ctr,
.dtr = crypt_dtr,
@@ -2234,6 +3045,7 @@ static int __init dm_crypt_init(void)
{
int r;
+ geniv_register_algs();
r = dm_register_target(&crypt_target);
if (r < 0)
DMERR("register failed %d", r);
@@ -2244,6 +3056,7 @@ static int __init dm_crypt_init(void)
static void __exit dm_crypt_exit(void)
{
dm_unregister_target(&crypt_target);
+ geniv_deregister_algs();
}
module_init(dm_crypt_init);
diff --git a/include/crypto/geniv.h b/include/crypto/geniv.h
new file mode 100644
index 0000000..b472507
--- /dev/null
+++ b/include/crypto/geniv.h
@@ -0,0 +1,47 @@
+/*
+ * geniv: common interface for IV generation algorithms
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ */
+#ifndef _CRYPTO_GENIV_
+#define _CRYPTO_GENIV_
+
+#define SECTOR_SIZE (1 << SECTOR_SHIFT)
+
+enum setkey_op {
+ SETKEY_OP_INIT,
+ SETKEY_OP_SET,
+ SETKEY_OP_WIPE,
+};
+
+struct geniv_key_info {
+ enum setkey_op keyop;
+ unsigned int tfms_count;
+ u8 *key;
+ unsigned int key_size;
+ unsigned int key_parts;
+ char *ivopts;
+};
+
+#define DECLARE_GENIV_KEY(c, op, n, k, sz, kp, opts) \
+ struct geniv_key_info c = { \
+ .keyop = op, \
+ .tfms_count = n, \
+ .key = k, \
+ .key_size = sz, \
+ .key_parts = kp, \
+ .ivopts = opts, \
+ }
+
+struct geniv_req_info {
+ bool is_write;
+ sector_t iv_sector;
+ unsigned int nents;
+ u8 *iv;
+};
+
+#endif
--
Binoy Jayan
^ permalink raw reply related
* [RFC PATCH v3] IV Generation algorithms for dm-crypt
From: Binoy Jayan @ 2017-01-18 9:40 UTC (permalink / raw)
To: Oded, Ofir
Cc: Herbert Xu, David S. Miller, linux-crypto, Mark Brown,
Arnd Bergmann, linux-kernel, Alasdair Kergon, Mike Snitzer,
dm-devel, Shaohua Li, linux-raid, Rajendra, Milan Broz, Gilad,
Binoy Jayan
===============================================================================
GENIV Template cipher
===============================================================================
Currently, the iv generation algorithms are implemented in dm-crypt.c. The goal
is to move these algorithms from the dm layer to the kernel crypto layer by
implementing them as template ciphers so they can be used in relation with
algorithms like aes, and with multiple modes like cbc, ecb etc. As part of this
patchset, the iv-generation code is moved from the dm layer to the crypto layer
and adapt the dm-layer to send a whole 'bio' (as defined in the block layer)
at a time. Each bio contains the in memory representation of physically
contiguous disk blocks. Since the bio itself may not be contiguous in main
memory, the dm layer sets up a chained scatterlist of these blocks split into
physically contiguous segments in memory so that DMA can be performed.
One challenge in doing so is that the IVs are generated based on a 512-byte
sector number. This infact limits the block sizes to 512 bytes. But this should
not be a problem if a hardware with iv generation support is used. The geniv
itself splits the segments into sectors so it could choose the IV based on
sector number. But it could be modelled in hardware effectively by not
splitting up the segments in the bio.
Another challenge faced is that dm-crypt has an option to use multiple keys.
The key selection is done based on the sector number. If the whole bio is
encrypted / decrypted with the same key, the encrypted volumes will not be
compatible with the original dm-crypt [without the changes]. So, the key
selection code is moved to crypto layer so the neighboring sectors are
encrypted with a different key.
The dm layer allocates space for iv. The hardware drivers can choose to make
use of this space to generate their IVs sequentially or allocate it on their
own. This can be moved to crypto layer too. Postponing this decision until
the requirement to integrate milan's changes are clear.
Interface to the crypto layer - include/crypto/geniv.h
Revisions:
v1: https://patchwork.kernel.org/patch/9439175
v2: https://patchwork.kernel.org/patch/9471923
v2 --> v3
----------
1. Moved iv algorithms in dm-crypt.c for control
2. Key management code moved from dm layer to cryto layer
so that cipher instance selection can be made depending on key_index
3. The revision v2 had scatterlist nodes created for every sector in the bio.
It is modified to create only once scatterlist node to reduce memory
foot print. Synchronous requests are processed sequentially. Asynchronous
requests are processed in parallel and is freed in the async callback.
4. Changed allocation for sub-requests using mempool
v1 --> v2
----------
1. dm-crypt changes to process larger block sizes (one segment in a bio)
2. Incorporated changes w.r.t. comments from Herbert.
Binoy Jayan (1):
crypto: Add IV generation algorithms
drivers/md/dm-crypt.c | 1891 ++++++++++++++++++++++++++++++++++--------------
include/crypto/geniv.h | 47 ++
2 files changed, 1399 insertions(+), 539 deletions(-)
create mode 100644 include/crypto/geniv.h
--
Binoy Jayan
^ permalink raw reply
* [PATCH] crypto: tcrypt - Add debug prints
From: Rabin Vincent @ 2017-01-18 13:54 UTC (permalink / raw)
To: herbert; +Cc: linux-crypto, Rabin Vincent
From: Rabin Vincent <rabinv@axis.com>
tcrypt is very tight-lipped when it succeeds, but a bit more feedback
would be useful when developing or debugging crypto drivers, especially
since even a successful run ends with the module failing to insert. Add
a couple of debug prints, which can be enabled with dynamic debug:
Before:
# insmod tcrypt.ko mode=10
insmod: can't insert 'tcrypt.ko': Resource temporarily unavailable
After:
# insmod tcrypt.ko mode=10 dyndbg
tcrypt: testing ecb(aes)
tcrypt: testing cbc(aes)
tcrypt: testing lrw(aes)
tcrypt: testing xts(aes)
tcrypt: testing ctr(aes)
tcrypt: testing rfc3686(ctr(aes))
tcrypt: all tests passed
insmod: can't insert 'tcrypt.ko': Resource temporarily unavailable
Signed-off-by: Rabin Vincent <rabinv@axis.com>
---
crypto/tcrypt.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index ae22f05..9a11f3c 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -22,6 +22,8 @@
*
*/
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
#include <crypto/aead.h>
#include <crypto/hash.h>
#include <crypto/skcipher.h>
@@ -1010,6 +1012,8 @@ static inline int tcrypt_test(const char *alg)
{
int ret;
+ pr_debug("testing %s\n", alg);
+
ret = alg_test(alg, alg, 0, 0);
/* non-fips algs return -EINVAL in fips mode */
if (fips_enabled && ret == -EINVAL)
@@ -2059,6 +2063,8 @@ static int __init tcrypt_mod_init(void)
if (err) {
printk(KERN_ERR "tcrypt: one or more tests failed!\n");
goto err_free_tv;
+ } else {
+ pr_debug("all tests passed\n");
}
/* We intentionaly return -EAGAIN to prevent keeping the module,
--
2.1.4
^ permalink raw reply related
* Re: [PATCH 00/13] crypto: copy AAD during encrypt for AEAD ciphers
From: Cyrille Pitchen @ 2017-01-18 14:57 UTC (permalink / raw)
To: Stephan Müller, herbert; +Cc: linux-crypto
In-Reply-To: <10526995.lyZ7Je1KMx@positron.chronox.de>
Hi Stephan,
this series of patches sounds like a good idea. I haven't tested it with
the Atmel AES hardware yet but I have many dummy questions:
Looking at some driver patches in the series, it seems you only add a call
to crypto_aead_copy_ad() but I don't see any removal of the previous driver
specific implementation to perform that copy.
I take the Atmel gcm(aes) driver case aside since looking at the current
code, it seems that I've forgotten to copy that assoc data from req->src
into req->dst scatter list. Then I assume that I was lucky so when I tested
with the test manager or IPSec connections, I always fell in the case where
req->src == req->dst. I thought the test manager also covered the case
req->src != req->dst but I don't remember very well and I haven't checked
recently, sorry!
Then I now look at the default drivers/gcm.c driver and, if I understand
this driver correctly, it doesn't copy the associated data from req->src
into req->dst when req->src != req->dst...
What does it mean? Did I misunderstand the gmc.c driver or is it a bug? Is
that copy the associated data needed after all?
Also looking at the drivers/authenc.c driver, this one does copy the
associated data.
So now I understand why I didn't make the copy for gcm(aes) but did it for
authenc(hmac(shaX),cbc(aes)) in atmel-aes.c: when I add new crypto
algorithms support in the Atmel drivers, I always take the crypto/*.c
drivers as reference.
So finally, what shall we do? copy or not copy? That is my question!
One last question: why do we copy those associated data only for encrypting
requests but not for decrypting requests?
The associated data might still be needed in the req->dst scatter list even
when it only refers to plain data so no other crypto operation is needed
after. However, let's take the example of an IPSec connection with ESP: the
first 8 bytes of the ESP header (4-byte SPI + 4-byte sequence number) are
used as associated data. They must be authenticated but they cannot be
ciphered as we need the plain SPI value to attach some IP packet to the
relevant IPSec session hence knowing the crypto algorithms to be used to
process the network packet.
However, once the received IPSec packet has been decrypted and
authenticated, the sequence number part the the ESP header might still be
needed in req->dst if for some reason the req->src is no longer available
when performing the anti-replay check.
Maybe the issue simply never occurs because req->src is always == req->dst
or maybe because the anti-replay check is always performed before the
crypto stuff. I dunno!
So why not copying the associated data also when processing decrypt
requests too?
Sorry for those newbie questions! I try to improve my understanding and
knowledge of the crypto subsystem and its interaction with the network
subsystem without digging too much in the source code :p
Best regards,
Cyrille
Le 10/01/2017 à 02:36, Stephan Müller a écrit :
> Hi,
>
> to all driver maintainers: the patches I added are compile tested, but
> I do not have the hardware to verify the code. May I ask the respective
> hardware maintainers to verify that the code is appropriate and works
> as intended? Thanks a lot.
>
> Herbert, this is my proprosal for our discussion around copying the
> AAD for algif_aead. Instead of adding the code to algif_aead and wait
> until it transpires to all cipher implementations, I thought it would
> be more helpful to fix all cipher implementations.
>
> To do so, the AAD copy function found in authenc is extracted to a global
> service function. Furthermore, the generic AEAD TFM initialization code
> now allocates the null cipher too. This allows us now to only invoke
> the AAD copy function in the various implementations without any additional
> allocation logic.
>
> The code for x86 and the generic code was tested with libkcapi.
>
> The code for the drivers is compile tested for drivers applicable to
> x86 only. All others are neither compile tested nor functionally tested.
>
> Stephan Mueller (13):
> crypto: service function to copy AAD from src to dst
> crypto: gcm_generic - copy AAD during encryption
> crypto: ccm_generic - copy AAD during encryption
> crypto: rfc4106-gcm-aesni - copy AAD during encryption
> crypto: ccm-aes-ce - copy AAD during encryption
> crypto: talitos - copy AAD during encryption
> crypto: picoxcell - copy AAD during encryption
> crypto: ixp4xx - copy AAD during encryption
> crypto: atmel - copy AAD during encryption
> crypto: caam - copy AAD during encryption
> crypto: chelsio - copy AAD during encryption
> crypto: nx - copy AAD during encryption
> crypto: qat - copy AAD during encryption
>
> arch/arm64/crypto/aes-ce-ccm-glue.c | 4 ++++
> arch/x86/crypto/aesni-intel_glue.c | 5 +++++
> crypto/Kconfig | 4 ++--
> crypto/aead.c | 36 ++++++++++++++++++++++++++++++--
> crypto/authenc.c | 36 ++++----------------------------
> crypto/ccm.c | 10 +++++++++
> crypto/gcm.c | 17 +++++++++++++++
> drivers/crypto/atmel-aes.c | 6 ++++++
> drivers/crypto/caam/caamalg.c | 8 +++++++
> drivers/crypto/chelsio/chcr_algo.c | 5 +++++
> drivers/crypto/ixp4xx_crypto.c | 6 ++++++
> drivers/crypto/nx/nx-aes-ccm.c | 4 ++++
> drivers/crypto/nx/nx-aes-gcm.c | 10 +++++++++
> drivers/crypto/picoxcell_crypto.c | 5 +++++
> drivers/crypto/qat/qat_common/qat_algs.c | 4 ++++
> drivers/crypto/talitos.c | 5 +++++
> include/crypto/aead.h | 2 ++
> include/crypto/internal/aead.h | 12 +++++++++++
> 18 files changed, 143 insertions(+), 36 deletions(-)
>
^ permalink raw reply
* Re: [RFC PATCH v3] crypto: Add IV generation algorithms
From: Gilad Ben-Yossef @ 2017-01-18 15:21 UTC (permalink / raw)
To: Binoy Jayan
Cc: Oded, Ofir, Herbert Xu, David S. Miller, linux-crypto, Mark Brown,
Arnd Bergmann, linux-kernel, Alasdair Kergon, Mike Snitzer,
dm-devel, Shaohua Li, linux-raid, Rajendra, Milan Broz
In-Reply-To: <1484732425-10319-2-git-send-email-binoy.jayan@linaro.org>
[-- Attachment #1: Type: text/plain, Size: 4693 bytes --]
Hi Binoy,
On Wed, Jan 18, 2017 at 11:40 AM, Binoy Jayan <binoy.jayan@linaro.org> wrote:
> Currently, the iv generation algorithms are implemented in dm-crypt.c.
> The goal is to move these algorithms from the dm layer to the kernel
> crypto layer by implementing them as template ciphers so they can be
> implemented in hardware for performance. As part of this patchset, the
> iv-generation code is moved from the dm layer to the crypto layer and
> adapt the dm-layer to send a whole 'bio' (as defined in the block layer)
> at a time. Each bio contains an in memory representation of physically
> contiguous disk blocks. The dm layer sets up a chained scatterlist of
> these blocks split into physically contiguous segments in memory so that
> DMA can be performed. Also, the key management code is moved from dm layer
> to the cryto layer since the key selection for encrypting neighboring
> sectors depend on the keycount.
>
> Synchronous crypto requests to encrypt/decrypt a sector are processed
> sequentially. Asynchronous requests if processed in parallel, are freed
> in the async callback. The dm layer allocates space for iv. The hardware
> implementations can choose to make use of this space to generate their IVs
> sequentially or allocate it on their own.
> Interface to the crypto layer - include/crypto/geniv.h
>
> Signed-off-by: Binoy Jayan <binoy.jayan@linaro.org>
> ---
I have some review comments and a bug report -
<snip>
> */
> -static int crypt_convert(struct crypt_config *cc,
> - struct convert_context *ctx)
> +
> +static int crypt_convert_bio(struct crypt_config *cc,
> + struct convert_context *ctx)
> {
> + unsigned int cryptlen, n1, n2, nents, i = 0, bytes = 0;
> + struct skcipher_request *req;
> + struct dm_crypt_request *dmreq;
> + struct geniv_req_info rinfo;
> + struct bio_vec bv_in, bv_out;
> int r;
> + u8 *iv;
>
> atomic_set(&ctx->cc_pending, 1);
> + crypt_alloc_req(cc, ctx);
> +
> + req = ctx->req;
> + dmreq = dmreq_of_req(cc, req);
> + iv = iv_of_dmreq(cc, dmreq);
>
> - while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) {
> + n1 = bio_segments(ctx->bio_in);
> + n2 = bio_segments(ctx->bio_in);
I'm pretty sure this needs to be
n2 = bio_segments(ctx->bio_out);
> + nents = n1 > n2 ? n1 : n2;
> + nents = nents > MAX_SG_LIST ? MAX_SG_LIST : nents;
> + cryptlen = ctx->iter_in.bi_size;
>
> - crypt_alloc_req(cc, ctx);
> + DMDEBUG("dm-crypt:%s: segments:[in=%u, out=%u] bi_size=%u\n",
> + bio_data_dir(ctx->bio_in) == WRITE ? "write" : "read",
> + n1, n2, cryptlen);
>
<Snip>
>
> - /* There was an error while processing the request. */
> - default:
> - atomic_dec(&ctx->cc_pending);
> - return r;
> - }
> + sg_set_page(&dmreq->sg_in[i], bv_in.bv_page, bv_in.bv_len,
> + bv_in.bv_offset);
> + sg_set_page(&dmreq->sg_out[i], bv_out.bv_page, bv_out.bv_len,
> + bv_out.bv_offset);
> +
> + bio_advance_iter(ctx->bio_in, &ctx->iter_in, bv_in.bv_len);
> + bio_advance_iter(ctx->bio_out, &ctx->iter_out, bv_out.bv_len);
> +
> + bytes += bv_in.bv_len;
> + i++;
> }
>
> - return 0;
> + DMDEBUG("dm-crypt: Processed %u of %u bytes\n", bytes, cryptlen);
> +
> + rinfo.is_write = bio_data_dir(ctx->bio_in) == WRITE;
Please consider wrapping the above boolean expression in parenthesis.
> + rinfo.iv_sector = ctx->cc_sector;
> + rinfo.nents = nents;
> + rinfo.iv = iv;
> +
> + skcipher_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,
Also, where do the scatterlist src2 and dst2 that you use
sg_set_page() get sg_init_table() called on?
I couldn't figure it out...
Last but not least, when performing the following sequence on Arm64
(on latest Qemu Virt platform) -
1. cryptsetup luksFormat fs3.img
2. cryptsetup open --type luks fs3.img croot
3. mke2fs /dev/mapper/croot
[ fs3.img is a 16MB file for loopback ]
The attached kernel panic happens. The same does not occur without the patch.
Let me know if you need any additional information to recreate it.
I've tried to debug it a little but did not came up with anything
useful aside from above review notes.
Thanks!
--
Gilad Ben-Yossef
Chief Coffee Drinker
"If you take a class in large-scale robotics, can you end up in a
situation where the homework eats your dog?"
-- Jean-Baptiste Queru
[-- Attachment #2: panic.txt --]
[-- Type: text/plain, Size: 8333 bytes --]
#
#
# mk
mkdir mkdosfs mke2fs mkfifo mknod mkpasswd mkswap mktemp
# mk
mkdir mkdosfs mke2fs mkfifo mknod mkpasswd mkswap mktemp
# mke2fs /dev/mapper/croot
Filesystem label=
OS type: Linux
Block size=1024 (log=0)
Fragment size=1024 (log=0)
3584 inodes, 14336 blocks
716 blocks (5%) reserved for the super user
First data block=1
Maximum filesystem blocks=262144
2 block groups
8192 blocks per group, 8192 fragments per group
1792 inodes per group
Superblock backups stored on blocks:
8193
[ 1288.475373] Unable to handle kernel paging request at virtual address 00001028
[ 1288.483568] pgd = ffff0000093ed000
[ 1288.483708] [00001028] *pgd=00000000beffe003, *pud=00000000beffd003, *pmd=0000000000000000
[ 1288.484071] Internal error: Oops: 96000006 [#1] PREEMPT SMP
[ 1288.484266] Modules linked in:
[ 1288.484526] CPU: 0 PID: 16 Comm: kworker/u2:1 Not tainted 4.10.0-rc4-00805-g9a4c309 #32
[ 1288.484712] Hardware name: linux,dummy-virt (DT)
[ 1288.485233] Workqueue: kcryptd kcryptd_crypt
[ 1288.485385] task: ffff80007c4abe80 task.stack: ffff80007c4dc000
[ 1288.485572] PC is at scatterwalk_copychunks+0x144/0x1e8
[ 1288.485750] LR is at scatterwalk_copychunks+0xdc/0x1e8
[ 1288.485904] pc : [<ffff00000834f46c>] lr : [<ffff00000834f404>] pstate: 20000145
[ 1288.486115] sp : ffff80007c4dfa50
[ 1288.486264] x29: ffff80007c4dfa50 x28: 0000000000000001
[ 1288.486539] x27: ffff80007c4abe80 x26: ffff80007bacab81
[ 1288.486709] x25: ffff80007c4dfb70 x24: 000000000000000f
[ 1288.486850] x23: 0000000000000001 x22: 0000000000000001
[ 1288.486991] x21: 000081ffffffffff x20: ffff80007c4abe80
[ 1288.487362] x19: ffff80007c4abe80 x18: 0000000000000001
[ 1288.487544] x17: 00000000004b3768 x16: ffff0000081ec678
[ 1288.487696] x15: ffffffffffffffff x14: ffff80007bab7c88
[ 1288.487846] x13: 0000000100000000 x12: 0000000000000000
[ 1288.487995] x11: 0000000000000010 x10: 0000000000000200
[ 1288.488178] x9 : 0000000000000000 x8 : 0000000000000000
[ 1288.488339] x7 : 0000000000000001 x6 : ffff800000040201
[ 1288.488540] x5 : ffff80007ba85b68 x4 : 0000000000001008
[ 1288.488690] x3 : 0000000000001008 x2 : 0000000000001008
[ 1288.488869] x1 : 0000000000000001 x0 : ffff80007ba838a0
[ 1288.489187]
[ 1288.489350] Process kworker/u2:1 (pid: 16, stack limit = 0xffff80007c4dc000)
[ 1288.489646] Stack: (0xffff80007c4dfa50 to 0xffff80007c4e0000)
[ 1288.489881] fa40: ffff80007c4dfac0 ffff0000083525d8
[ 1288.490134] fa60: ffff80007c4dfb38 0000000000000000 00000000000001f0 ffff80007ba859f0
[ 1288.490329] fa80: ffff80007babe900 0000000000000000 ffff80007b45adc8 ffff000008cec000
[ 1288.490548] faa0: ffff80007b45ad98 0000000000000200 ffff80007c4dfb38 0000000100000001
[ 1288.490764] fac0: ffff80007c4dfb00 ffff0000080b9774 000000000000000a ffff80007ba85ae8
[ 1288.490962] fae0: 0000000000000000 ffff80007ba859f0 ffff80007babe900 ffff0000080b9764
[ 1288.491155] fb00: ffff80007c4dfbd0 ffff000008391840 ffff80007ba83880 ffff80007babea80
[ 1288.491372] fb20: 0000000000000008 ffff80007b45ad00 ffff80007c4dfb60 00000000ffffffc8
[ 1288.491586] fb40: ffff80007bacab80 ffff80007c4dfba0 ffff80007bacab80 ffff80007ba83880
[ 1288.491798] fb60: ffff80007b45ae90 0000000000000010 ffff80007ba838a0 0000000000000001
[ 1288.491991] fb80: 0000000000000200 0000000000000000 0000000000001000 0000000000000000
[ 1288.492183] fba0: ffff80007bacab80 ffff80007b45ae80 ffff80007b45ae80 ffff800200000010
[ 1288.492394] fbc0: 0000001000000010 ffff800000000007 ffff80007c4dfbf0 ffff0000087969c4
[ 1288.492609] fbe0: ffff80007b45ad80 ffff80007ba83800 ffff80007c4dfc70 ffff000008796688
[ 1288.492836] fc00: 0000000000000000 ffff80007b45ad00 0000000000001000 fffffffffffffff8
[ 1288.493020] fc20: ffff80007b45ac78 ffff80007b45ac30 ffff80007b45ae60 0000000000001000
[ 1288.493213] fc40: 0000000000000001 0000000000000001 ffff000009373f75 ffff80007b45ada8
[ 1288.493482] fc60: 0000000000000001 0000020000000000 ffff80007c4dfd20 ffff000008797480
[ 1288.495928] fc80: ffff80007ba83f00 ffff80007b45ac30 ffff80007b45ac10 ffff80007c408000
[ 1288.496370] fca0: 0000000000000000 ffff80007b45ac10 ffff0000092a7edb ffff80007b45ac00
[ 1288.496762] fcc0: ffff80007c408078 ffff80007c4082a8 0000000000000001 ffff80007ba09800
[ 1288.497182] fce0: ffff00000924b000 ffff0000080e6e10 ffff80007c4dfd80 ffff000000001000
[ 1288.497590] fd00: ffff80007efdf800 0000000000000000 ffff000000000001 ffff80007b45ae80
[ 1288.498014] fd20: ffff80007c4dfdc0 ffff0000080d9d94 0000000000000020 ffff80007c4c5800
[ 1288.498412] fd40: ffff80007ba7dd00 ffff80007c408000 0000000000000000 ffff80007b45ac10
[ 1288.498787] fd60: ffff0000092a7edb ffff80007c408020 ffff80007c408078 ffff80007c4082a8
[ 1288.499147] fd80: ffff80007c4dfde0 ffff000008971800 ffff80007c4abe80 ffff80007c408000
[ 1288.499373] fda0: ffff80007c4c5830 ffff80007c408020 ffff000009287000 ffff80007b45ac30
[ 1288.499571] fdc0: ffff80007c4dfe00 ffff0000080d9f80 ffff80007c4c5800 ffff80007c408000
[ 1288.499766] fde0: ffff80007c4c5830 ffff80007c408020 ffff000009287000 ffff80007c4abe80
[ 1288.499963] fe00: ffff80007c4dfe60 ffff0000080dfe30 ffff80007c4c3b00 ffff80007c4c3680
[ 1288.500159] fe20: ffff00000938fb08 ffff80007c4abe80 ffff000008c50570 ffff80007c4c5800
[ 1288.500373] fe40: ffff0000080d9f38 ffff80007c4c3b38 ffff80007c487d20 0000000000000000
[ 1288.500569] fe60: 0000000000000000 ffff000008082ec0 ffff0000080dfd40 ffff80007c4c3680
[ 1288.500777] fe80: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 1288.500975] fea0: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 1288.501204] fec0: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 1288.501401] fee0: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 1288.501621] ff00: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 1288.501879] ff20: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 1288.502115] ff40: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 1288.502329] ff60: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 1288.502530] ff80: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 1288.502742] ffa0: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 1288.502996] ffc0: 0000000000000000 0000000000000005 0000000000000000 0000000000000000
[ 1288.503304] ffe0: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 1288.503761] Call trace:
[ 1288.504274] Exception stack(0xffff80007c4df880 to 0xffff80007c4df9b0)
[ 1288.504572] f880: ffff80007c4abe80 0001000000000000 ffff80007c4dfa50 ffff00000834f46c
[ 1288.504771] f8a0: ffff80007c4df8d0 ffff0000080f410c ffff80007ba09800 ffff80007efdf8e0
[ 1288.504968] f8c0: ffff80007c4df8d0 ffff0000080f4124 ffff80007c4df8e0 ffff0000083ca188
[ 1288.505239] f8e0: ffff80007c4df960 ffff0000083cce80 ffff000009395375 ffff0000089e5510
[ 1288.505439] f900: 0000000000000020 00000000000003e0 00000000fffffff8 ffff000009395328
[ 1288.505693] f920: ffff80007ba838a0 0000000000000001 0000000000001008 0000000000001008
[ 1288.505878] f940: 0000000000001008 ffff80007ba85b68 ffff800000040201 0000000000000001
[ 1288.506150] f960: 0000000000000000 0000000000000000 0000000000000200 0000000000000010
[ 1288.506455] f980: 0000000000000000 0000000100000000 ffff80007bab7c88 ffffffffffffffff
[ 1288.507201] f9a0: ffff0000081ec678 00000000004b3768
[ 1288.507466] [<ffff00000834f46c>] scatterwalk_copychunks+0x144/0x1e8
[ 1288.507627] [<ffff0000083525d8>] skcipher_walk_done+0x250/0x2b0
[ 1288.507781] [<ffff0000080b9774>] xts_decrypt+0x84/0xb0
[ 1288.507913] [<ffff000008391840>] simd_skcipher_decrypt+0x70/0xa8
[ 1288.508088] [<ffff0000087969c4>] geniv_decrypt+0x1b4/0x320
[ 1288.508226] [<ffff000008796688>] crypt_convert_bio+0x790/0x7e0
[ 1288.508368] [<ffff000008797480>] kcryptd_crypt+0x2f0/0x358
[ 1288.508504] [<ffff0000080d9d94>] process_one_work+0x1bc/0x360
[ 1288.508646] [<ffff0000080d9f80>] worker_thread+0x48/0x480
[ 1288.508778] [<ffff0000080dfe30>] kthread+0xf0/0x120
[ 1288.508905] [<ffff000008082ec0>] ret_from_fork+0x10/0x50
[ 1288.509154] Code: d34c7c42 f9400003 927ef463 8b021862 (f9401044)
[ 1288.509756] ---[ end trace 8de15ab91f16458a ]---
[ 1288.509951] note: kworker/u2:1[16] exited with preempt_count 1
^ permalink raw reply
* Re: [PATCH 7/8] random: remove noop function call to xfer_secondary_pool
From: Theodore Ts'o @ 2017-01-18 16:10 UTC (permalink / raw)
To: Stephan Müller; +Cc: linux-kernel, linux-crypto
In-Reply-To: <9955153.9pz8KSq437@positron.chronox.de>
On Tue, Dec 27, 2016 at 11:41:46PM +0100, Stephan Müller wrote:
> Since the introduction of the ChaCha20 DRNG, extract_entropy is only
> invoked with the input_pool. For this entropy pool, xfer_secondary_pool
> is a no-op and can therefore be safely removed.
>
> Signed-off-by: Stephan Mueller <smueller@chronox.de>
Instead of doing some minor deletions of single lines, what I want to
do is to look at a more comprehensive refactoring of the code. The
fact that we have extract_entropy() only being used for the input
pool, and extract_entropy_user() ony being used for the non-blocking
pool, is not obvious from the function name and the arguments that
these functions take.
Either the functions should be kept general (so someone else using
them in the future won't get confused about how they work), or they
should be made more speceific. But doing light modifications like
have the danger of causing confusion and bugs in the future.
- Ted
^ permalink raw reply
* [PATCH] crypto: tcrypt - Add mode to test specified algs
From: Rabin Vincent @ 2017-01-18 16:25 UTC (permalink / raw)
To: herbert; +Cc: linux-crypto, Rabin Vincent
From: Rabin Vincent <rabinv@axis.com>
tcrypt offers a bunch of mode= values to test various (groups of)
algorithms, but there is no way provided to test a subset of the
algorithms. This adds a new mode=2000 which interprets alg= as a
colon-separated list of algorithms to test with alg_test(). Colon is
used since the names may contain commas.
This is useful during driver development and also for regression testing
to avoid the errors that are otherwise generated when attempting to test
non-enabled algorithms.
# insmod tcrypt.ko dyndbg mode=2000 alg="cbc(aes):ecb(aes):hmac(sha256):sha256:xts(aes)"
[ 649.418569] tcrypt: testing cbc(aes)
[ 649.420809] tcrypt: testing ecb(aes)
[ 649.422627] tcrypt: testing hmac(sha256)
[ 649.424861] tcrypt: testing sha256
[ 649.426368] tcrypt: testing xts(aes)
[ 649.430014] tcrypt: all tests passed
Signed-off-by: Rabin Vincent <rabinv@axis.com>
---
crypto/tcrypt.c | 13 ++++++++++++-
1 file changed, 12 insertions(+), 1 deletion(-)
diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c
index 9a11f3c..fe5adf6 100644
--- a/crypto/tcrypt.c
+++ b/crypto/tcrypt.c
@@ -1021,7 +1021,7 @@ static inline int tcrypt_test(const char *alg)
return ret;
}
-static int do_test(const char *alg, u32 type, u32 mask, int m)
+static int do_test(char *alg, u32 type, u32 mask, int m)
{
int i;
int ret = 0;
@@ -2042,6 +2042,17 @@ static int do_test(const char *alg, u32 type, u32 mask, int m)
case 1000:
test_available();
break;
+
+ case 2000:
+ while (alg) {
+ char *tmp = strsep(&alg, ":");
+
+ if (!tmp || !*tmp)
+ break;
+
+ ret += tcrypt_test(tmp);
+ }
+ break;
}
return ret;
--
2.1.4
^ permalink raw reply related
* Re: [RFC PATCH 0/6] Add bulk skcipher requests to crypto API and dm-crypt
From: Binoy Jayan @ 2017-01-18 17:09 UTC (permalink / raw)
To: Ondrej Mosnáček, Arnd Bergmann, Mark Brown
Cc: Herbert Xu, linux-crypto, dm-devel, Mike Snitzer, Milan Broz,
Mikulas Patocka
In-Reply-To: <CAAUqJDtqN-ugcaz4VhuW0fzLbmKt4db8ggrRmaAq_zFvO+9Lvw@mail.gmail.com>
Hi Milan,
On 13 January 2017 at 17:31, Ondrej Mosnáček <omosnacek@gmail.com> wrote:
> 2017-01-13 11:41 GMT+01:00 Herbert Xu <herbert@gondor.apana.org.au>:
>> On Thu, Jan 12, 2017 at 01:59:52PM +0100, Ondrej Mosnacek wrote:
>>> the goal of this patchset is to allow those skcipher API users that need to
>>> process batches of small messages (especially dm-crypt) to do so efficiently.
>>
>> Please explain why this can't be done with the existing framework
>> using IV generators similar to the ones used for IPsec.
>
> As I already mentioned in another thread, there are basically two reasons:
>
> 1) Milan would like to add authenticated encryption support to
> dm-crypt (see [1]) and as part of this change, a new random IV mode
> would be introduced. This mode generates a random IV for each sector
> write, includes it in the authenticated data and stores it in the
> sector's metadata (in a separate part of the disk). In this case
> dm-crypt will need to have control over the IV generation (or at least
> be able to somehow retrieve it after the crypto operation... but
> passing RNG responsibility to drivers doesn't seem to be a good idea
> anyway).
>
> 2) With this API, drivers wouldn't have to provide implementations for
> specific IV generation modes, and just implement bulk requests for the
> common modes/algorithms (XTS, CBC, ...) while still getting
> performance benefit.
I just sent out v3 for the dm-crypt changes I was working on. I
came across your patches for authenticated encryption support.
Although I haven't looked at it entirely, I was wondering how it could
be put together including the points Ondrej was mentioning. Will look at
it more. Please keep me in cc when you send out the next revision if
that is possible.
Thanks,
Binoy
^ permalink raw reply
* Re: [PATCH 3/8] random: trigger random_ready callback upon crng_init == 1
From: Stephan Müller @ 2017-01-18 17:09 UTC (permalink / raw)
To: Theodore Ts'o; +Cc: linux-kernel, linux-crypto
In-Reply-To: <20170118041250.x7wszcasd4dplj3h@thunk.org>
Am Dienstag, 17. Januar 2017, 23:12:50 CET schrieb Theodore Ts'o:
Hi Theodore,
> On Tue, Dec 27, 2016 at 11:39:57PM +0100, Stephan Müller wrote:
> > The random_ready callback mechanism is intended to replicate the
> > getrandom system call behavior to in-kernel users. As the getrandom
> > system call unblocks with crng_init == 1, trigger the random_ready
> > wakeup call at the same time.
>
> It was deliberate that random_ready would only get triggered with
> crng_init==2.
>
> In general I'm assuming kernel callers really want real randomness (as
> opposed to using prandom), where as there's a lot of b.s. userspace
> users of kernel randomness (for things that really don't require
> cryptographic randomness, e.g., for salting Python dictionaries,
> systemd/udev using /dev/urandom for non-cryptographic, non-security
> applications etc.)
Users of getrandom want to ensure that they get random data from a DRNG that
is seeded, just like in-kernel users may want if they choose the callback-
approach.
I do not understand why there should be different treatment of in-kernel vs
user space callers in that respect.
(And yes, I do not want to open a discussion whether crng_init==1 can
considered as a sufficiently seeded DRNG as such discussion will lead
nowhere.)
Ciao
Stephan
^ permalink raw reply
* Re: [RFC PATCH v3] crypto: Add IV generation algorithms
From: Binoy Jayan @ 2017-01-19 4:42 UTC (permalink / raw)
To: Gilad Ben-Yossef
Cc: Oded, Ofir, Herbert Xu, David S. Miller, linux-crypto, Mark Brown,
Arnd Bergmann, Linux kernel mailing list, Alasdair Kergon,
Mike Snitzer, dm-devel, Shaohua Li, linux-raid, Rajendra,
Milan Broz
In-Reply-To: <CAOtvUMfhethgZ8TarKGDumL1W5fgjwy1Hi7f88mmCCD6vJ_hLg@mail.gmail.com>
Hi Gilad,
On 18 January 2017 at 20:51, Gilad Ben-Yossef <gilad@benyossef.com> wrote:
> I have some review comments and a bug report -
Thank you very much for testing this on ARM and for the comments.
> I'm pretty sure this needs to be
>
> n2 = bio_segments(ctx->bio_out);
Yes you are right, that was a typo :)
>> +
>> + rinfo.is_write = bio_data_dir(ctx->bio_in) == WRITE;
>
> Please consider wrapping the above boolean expression in parenthesis.
Well, I can do that to enhance the clarity.
>> + rinfo.iv_sector = ctx->cc_sector;
>> + rinfo.nents = nents;
>> + rinfo.iv = iv;
>> +
>> + skcipher_request_set_crypt(req, dmreq->sg_in, dmreq->sg_out,
>
> Also, where do the scatterlist src2 and dst2 that you use
> sg_set_page() get sg_init_table() called on?
> I couldn't figure it out...
Thank you pointing this out. I missed out to add sg_init_table(src2, 1)
and sg_init_table(dst2, 1), but sg_set_page is used in geniv_iter_block.
This is probably the reason for the panic on ARM platform. However it
ran fine under qemu-x86. May be I should setup an arm platform too
for testing.
Regards,
Binoy
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox