* [PATCH 2/6] crypto: arm/aes-neonbs - process 8 blocks in parallel if we can
From: Ard Biesheuvel @ 2017-01-02 18:21 UTC (permalink / raw)
To: linux-crypto; +Cc: herbert, linux-arm-kernel, Ard Biesheuvel
In-Reply-To: <1483381268-12987-1-git-send-email-ard.biesheuvel@linaro.org>
The bit-sliced NEON implementation of AES only performs optimally if
it can process 8 blocks of input in parallel. This is due to the nature
of bit slicing, where the n-th bit of each byte of AES state of each input
block is collected into NEON register 'n', for registers q0 - q7.
This implies that the amount of work for the transform is fixed,
regardless of whether we are handling just one block or 8 in parallel.
So let's try a bit harder to iterate over the input in suitably sized
chunks, by setting the newly introduced walksize attribute to 8x the value
of AES_BLOCK_SIZE, and tweaking the loops to only process multiples of the
walk size, unless we are handling the last chunk in the input stream.
Note that the skcipher walk API guarantees that a step in the walk never
returns less than 'walksize' bytes if there are at least that many bytes
of input still available. However, it does *not* guarantee that those steps
produce an exact multiple of the walk size.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm/crypto/aesbs-glue.c | 67 +++++++++++---------
1 file changed, 38 insertions(+), 29 deletions(-)
diff --git a/arch/arm/crypto/aesbs-glue.c b/arch/arm/crypto/aesbs-glue.c
index d8e06de72ef3..f3019333c2eb 100644
--- a/arch/arm/crypto/aesbs-glue.c
+++ b/arch/arm/crypto/aesbs-glue.c
@@ -121,39 +121,26 @@ static int aesbs_cbc_encrypt(struct skcipher_request *req)
return crypto_cbc_encrypt_walk(req, aesbs_encrypt_one);
}
-static inline void aesbs_decrypt_one(struct crypto_skcipher *tfm,
- const u8 *src, u8 *dst)
-{
- struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
-
- AES_decrypt(src, dst, &ctx->dec.rk);
-}
-
static int aesbs_cbc_decrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm);
struct skcipher_walk walk;
- unsigned int nbytes;
int err;
- for (err = skcipher_walk_virt(&walk, req, false);
- (nbytes = walk.nbytes); err = skcipher_walk_done(&walk, nbytes)) {
- u32 blocks = nbytes / AES_BLOCK_SIZE;
- u8 *dst = walk.dst.virt.addr;
- u8 *src = walk.src.virt.addr;
- u8 *iv = walk.iv;
-
- if (blocks >= 8) {
- kernel_neon_begin();
- bsaes_cbc_encrypt(src, dst, nbytes, &ctx->dec, iv);
- kernel_neon_end();
- nbytes %= AES_BLOCK_SIZE;
- continue;
- }
+ err = skcipher_walk_virt(&walk, req, false);
+
+ while (walk.nbytes) {
+ unsigned int nbytes = walk.nbytes;
+
+ if (nbytes < walk.total)
+ nbytes = round_down(nbytes, walk.stride);
- nbytes = crypto_cbc_decrypt_blocks(&walk, tfm,
- aesbs_decrypt_one);
+ kernel_neon_begin();
+ bsaes_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
+ nbytes, &ctx->dec, walk.iv);
+ kernel_neon_end();
+ err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
}
return err;
}
@@ -186,6 +173,12 @@ static int aesbs_ctr_encrypt(struct skcipher_request *req)
__be32 *ctr = (__be32 *)walk.iv;
u32 headroom = UINT_MAX - be32_to_cpu(ctr[3]);
+ if (walk.nbytes < walk.total) {
+ blocks = round_down(blocks,
+ walk.stride / AES_BLOCK_SIZE);
+ tail = walk.nbytes - blocks * AES_BLOCK_SIZE;
+ }
+
/* avoid 32 bit counter overflow in the NEON code */
if (unlikely(headroom < blocks)) {
blocks = headroom + 1;
@@ -198,6 +191,9 @@ static int aesbs_ctr_encrypt(struct skcipher_request *req)
kernel_neon_end();
inc_be128_ctr(ctr, blocks);
+ if (tail > 0 && tail < AES_BLOCK_SIZE)
+ break;
+
err = skcipher_walk_done(&walk, tail);
}
if (walk.nbytes) {
@@ -227,11 +223,16 @@ static int aesbs_xts_encrypt(struct skcipher_request *req)
AES_encrypt(walk.iv, walk.iv, &ctx->twkey);
while (walk.nbytes) {
+ unsigned int nbytes = walk.nbytes;
+
+ if (nbytes < walk.total)
+ nbytes = round_down(nbytes, walk.stride);
+
kernel_neon_begin();
bsaes_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
- walk.nbytes, &ctx->enc, walk.iv);
+ nbytes, &ctx->enc, walk.iv);
kernel_neon_end();
- err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
+ err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
}
return err;
}
@@ -249,11 +250,16 @@ static int aesbs_xts_decrypt(struct skcipher_request *req)
AES_encrypt(walk.iv, walk.iv, &ctx->twkey);
while (walk.nbytes) {
+ unsigned int nbytes = walk.nbytes;
+
+ if (nbytes < walk.total)
+ nbytes = round_down(nbytes, walk.stride);
+
kernel_neon_begin();
bsaes_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr,
- walk.nbytes, &ctx->dec, walk.iv);
+ nbytes, &ctx->dec, walk.iv);
kernel_neon_end();
- err = skcipher_walk_done(&walk, walk.nbytes % AES_BLOCK_SIZE);
+ err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
}
return err;
}
@@ -272,6 +278,7 @@ static struct skcipher_alg aesbs_algs[] = { {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
.setkey = aesbs_cbc_set_key,
.encrypt = aesbs_cbc_encrypt,
.decrypt = aesbs_cbc_decrypt,
@@ -290,6 +297,7 @@ static struct skcipher_alg aesbs_algs[] = { {
.max_keysize = AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
.chunksize = AES_BLOCK_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
.setkey = aesbs_ctr_set_key,
.encrypt = aesbs_ctr_encrypt,
.decrypt = aesbs_ctr_encrypt,
@@ -307,6 +315,7 @@ static struct skcipher_alg aesbs_algs[] = { {
.min_keysize = 2 * AES_MIN_KEY_SIZE,
.max_keysize = 2 * AES_MAX_KEY_SIZE,
.ivsize = AES_BLOCK_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
.setkey = aesbs_xts_set_key,
.encrypt = aesbs_xts_encrypt,
.decrypt = aesbs_xts_decrypt,
--
2.7.4
^ permalink raw reply related
* [PATCH 5/6] crypto: arm64/aes-blk - expose AES-CTR as synchronous cipher as well
From: Ard Biesheuvel @ 2017-01-02 18:21 UTC (permalink / raw)
To: linux-crypto; +Cc: herbert, linux-arm-kernel, Ard Biesheuvel
In-Reply-To: <1483381268-12987-1-git-send-email-ard.biesheuvel@linaro.org>
In addition to wrapping the AES-CTR cipher into the async SIMD wrapper,
which exposes it as an async skcipher that defers processing to process
context, expose our AES-CTR implementation directly as a synchronous cipher
as well, but with a lower priority.
This makes the AES-CTR transform usable in places where synchronous
transforms are required, such as the MAC802.11 encryption code, which
executes in sotfirq context, where SIMD processing is allowed on arm64.
Users of the async transform will keep the existing behavior.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/aes-glue.c | 25 ++++++++++++++++++--
1 file changed, 23 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index 4e3f8adb1793..5164aaf82c6a 100644
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -327,6 +327,23 @@ static struct skcipher_alg aes_algs[] = { {
.decrypt = ctr_encrypt,
}, {
.base = {
+ .cra_name = "ctr(aes)",
+ .cra_driver_name = "ctr-aes-" MODE,
+ .cra_priority = PRIO - 1,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct crypto_aes_ctx),
+ .cra_alignmask = 7,
+ .cra_module = THIS_MODULE,
+ },
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .chunksize = AES_BLOCK_SIZE,
+ .setkey = skcipher_aes_setkey,
+ .encrypt = ctr_encrypt,
+ .decrypt = ctr_encrypt,
+}, {
+ .base = {
.cra_name = "__xts(aes)",
.cra_driver_name = "__xts-aes-" MODE,
.cra_priority = PRIO,
@@ -350,8 +367,9 @@ static void aes_exit(void)
{
int i;
- for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++)
- simd_skcipher_free(aes_simd_algs[i]);
+ for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
+ if (aes_simd_algs[i])
+ simd_skcipher_free(aes_simd_algs[i]);
crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
}
@@ -370,6 +388,9 @@ static int __init aes_init(void)
return err;
for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
+ if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
+ continue;
+
algname = aes_algs[i].base.cra_name + 2;
drvname = aes_algs[i].base.cra_driver_name + 2;
basename = aes_algs[i].base.cra_driver_name;
--
2.7.4
^ permalink raw reply related
* [PATCH 3/6] crypto: arm/chacha20 - implement NEON version based on SSE3 code
From: Ard Biesheuvel @ 2017-01-02 18:21 UTC (permalink / raw)
To: linux-crypto; +Cc: linux-arm-kernel, herbert, Ard Biesheuvel
In-Reply-To: <1483381268-12987-1-git-send-email-ard.biesheuvel@linaro.org>
This is a straight port to ARM/NEON of the x86 SSE3 implementation
of the ChaCha20 stream cipher. It uses the new skcipher walksize
attribute to process the input in strides of 4x the block size.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm/crypto/Kconfig | 6 +
arch/arm/crypto/Makefile | 2 +
arch/arm/crypto/chacha20-neon-core.S | 524 ++++++++++++++++++++
arch/arm/crypto/chacha20-neon-glue.c | 128 +++++
4 files changed, 660 insertions(+)
diff --git a/arch/arm/crypto/Kconfig b/arch/arm/crypto/Kconfig
index 13f1b4c289d4..2f3339f015d3 100644
--- a/arch/arm/crypto/Kconfig
+++ b/arch/arm/crypto/Kconfig
@@ -130,4 +130,10 @@ config CRYPTO_CRC32_ARM_CE
depends on KERNEL_MODE_NEON && CRC32
select CRYPTO_HASH
+config CRYPTO_CHACHA20_NEON
+ tristate "NEON accelerated ChaCha20 symmetric cipher"
+ depends on KERNEL_MODE_NEON
+ select CRYPTO_BLKCIPHER
+ select CRYPTO_CHACHA20
+
endif
diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile
index b578a1820ab1..8d74e55eacd4 100644
--- a/arch/arm/crypto/Makefile
+++ b/arch/arm/crypto/Makefile
@@ -8,6 +8,7 @@ obj-$(CONFIG_CRYPTO_SHA1_ARM) += sha1-arm.o
obj-$(CONFIG_CRYPTO_SHA1_ARM_NEON) += sha1-arm-neon.o
obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o
obj-$(CONFIG_CRYPTO_SHA512_ARM) += sha512-arm.o
+obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha20-neon.o
ce-obj-$(CONFIG_CRYPTO_AES_ARM_CE) += aes-arm-ce.o
ce-obj-$(CONFIG_CRYPTO_SHA1_ARM_CE) += sha1-arm-ce.o
@@ -40,6 +41,7 @@ aes-arm-ce-y := aes-ce-core.o aes-ce-glue.o
ghash-arm-ce-y := ghash-ce-core.o ghash-ce-glue.o
crct10dif-arm-ce-y := crct10dif-ce-core.o crct10dif-ce-glue.o
crc32-arm-ce-y:= crc32-ce-core.o crc32-ce-glue.o
+chacha20-neon-y := chacha20-neon-core.o chacha20-neon-glue.o
quiet_cmd_perl = PERL $@
cmd_perl = $(PERL) $(<) > $(@)
diff --git a/arch/arm/crypto/chacha20-neon-core.S b/arch/arm/crypto/chacha20-neon-core.S
new file mode 100644
index 000000000000..ff1d337bdb4a
--- /dev/null
+++ b/arch/arm/crypto/chacha20-neon-core.S
@@ -0,0 +1,524 @@
+/*
+ * ChaCha20 256-bit cipher algorithm, RFC7539, ARM NEON functions
+ *
+ * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Based on:
+ * ChaCha20 256-bit cipher algorithm, RFC7539, x64 SSE3 functions
+ *
+ * Copyright (C) 2015 Martin Willi
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/linkage.h>
+
+ .text
+ .fpu neon
+ .align 5
+
+ENTRY(chacha20_block_xor_neon)
+ // r0: Input state matrix, s
+ // r1: 1 data block output, o
+ // r2: 1 data block input, i
+
+ //
+ // This function encrypts one ChaCha20 block by loading the state matrix
+ // in four NEON registers. It performs matrix operation on four words in
+ // parallel, but requireds shuffling to rearrange the words after each
+ // round.
+ //
+
+ // x0..3 = s0..3
+ add ip, r0, #0x20
+ vld1.32 {q0-q1}, [r0]
+ vld1.32 {q2-q3}, [ip]
+
+ vmov q8, q0
+ vmov q9, q1
+ vmov q10, q2
+ vmov q11, q3
+
+ mov r3, #10
+
+.Ldoubleround:
+ // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
+ vadd.i32 q0, q0, q1
+ veor q4, q3, q0
+ vshl.u32 q3, q4, #16
+ vsri.u32 q3, q4, #16
+
+ // x2 += x3, x1 = rotl32(x1 ^ x2, 12)
+ vadd.i32 q2, q2, q3
+ veor q4, q1, q2
+ vshl.u32 q1, q4, #12
+ vsri.u32 q1, q4, #20
+
+ // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
+ vadd.i32 q0, q0, q1
+ veor q4, q3, q0
+ vshl.u32 q3, q4, #8
+ vsri.u32 q3, q4, #24
+
+ // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
+ vadd.i32 q2, q2, q3
+ veor q4, q1, q2
+ vshl.u32 q1, q4, #7
+ vsri.u32 q1, q4, #25
+
+ // x1 = shuffle32(x1, MASK(0, 3, 2, 1))
+ vext.8 q1, q1, q1, #4
+ // x2 = shuffle32(x2, MASK(1, 0, 3, 2))
+ vext.8 q2, q2, q2, #8
+ // x3 = shuffle32(x3, MASK(2, 1, 0, 3))
+ vext.8 q3, q3, q3, #12
+
+ // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
+ vadd.i32 q0, q0, q1
+ veor q4, q3, q0
+ vshl.u32 q3, q4, #16
+ vsri.u32 q3, q4, #16
+
+ // x2 += x3, x1 = rotl32(x1 ^ x2, 12)
+ vadd.i32 q2, q2, q3
+ veor q4, q1, q2
+ vshl.u32 q1, q4, #12
+ vsri.u32 q1, q4, #20
+
+ // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
+ vadd.i32 q0, q0, q1
+ veor q4, q3, q0
+ vshl.u32 q3, q4, #8
+ vsri.u32 q3, q4, #24
+
+ // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
+ vadd.i32 q2, q2, q3
+ veor q4, q1, q2
+ vshl.u32 q1, q4, #7
+ vsri.u32 q1, q4, #25
+
+ // x1 = shuffle32(x1, MASK(2, 1, 0, 3))
+ vext.8 q1, q1, q1, #12
+ // x2 = shuffle32(x2, MASK(1, 0, 3, 2))
+ vext.8 q2, q2, q2, #8
+ // x3 = shuffle32(x3, MASK(0, 3, 2, 1))
+ vext.8 q3, q3, q3, #4
+
+ subs r3, r3, #1
+ bne .Ldoubleround
+
+ add ip, r2, #0x20
+ vld1.8 {q4-q5}, [r2]
+ vld1.8 {q6-q7}, [ip]
+
+ // o0 = i0 ^ (x0 + s0)
+ vadd.i32 q0, q0, q8
+ veor q0, q0, q4
+
+ // o1 = i1 ^ (x1 + s1)
+ vadd.i32 q1, q1, q9
+ veor q1, q1, q5
+
+ // o2 = i2 ^ (x2 + s2)
+ vadd.i32 q2, q2, q10
+ veor q2, q2, q6
+
+ // o3 = i3 ^ (x3 + s3)
+ vadd.i32 q3, q3, q11
+ veor q3, q3, q7
+
+ add ip, r1, #0x20
+ vst1.8 {q0-q1}, [r1]
+ vst1.8 {q2-q3}, [ip]
+
+ bx lr
+ENDPROC(chacha20_block_xor_neon)
+
+ .align 5
+ENTRY(chacha20_4block_xor_neon)
+ push {r4-r6, lr}
+ mov ip, sp // preserve the stack pointer
+ sub r3, sp, #0x20 // allocate a 32 byte buffer
+ bic r3, r3, #0x1f // aligned to 32 bytes
+ mov sp, r3
+
+ // r0: Input state matrix, s
+ // r1: 4 data blocks output, o
+ // r2: 4 data blocks input, i
+
+ //
+ // This function encrypts four consecutive ChaCha20 blocks by loading
+ // the state matrix in NEON registers four times. The algorithm performs
+ // each operation on the corresponding word of each state matrix, hence
+ // requires no word shuffling. For final XORing step we transpose the
+ // matrix by interleaving 32- and then 64-bit words, which allows us to
+ // do XOR in NEON registers.
+ //
+
+ // x0..15[0-3] = s0..3[0..3]
+ add r3, r0, #0x20
+ vld1.32 {q0-q1}, [r0]
+ vld1.32 {q2-q3}, [r3]
+
+ adr r3, CTRINC
+ vdup.32 q15, d7[1]
+ vdup.32 q14, d7[0]
+ vld1.32 {q11}, [r3, :128]
+ vdup.32 q13, d6[1]
+ vdup.32 q12, d6[0]
+ vadd.i32 q12, q12, q11 // x12 += counter values 0-3
+ vdup.32 q11, d5[1]
+ vdup.32 q10, d5[0]
+ vdup.32 q9, d4[1]
+ vdup.32 q8, d4[0]
+ vdup.32 q7, d3[1]
+ vdup.32 q6, d3[0]
+ vdup.32 q5, d2[1]
+ vdup.32 q4, d2[0]
+ vdup.32 q3, d1[1]
+ vdup.32 q2, d1[0]
+ vdup.32 q1, d0[1]
+ vdup.32 q0, d0[0]
+
+ mov r3, #10
+
+.Ldoubleround4:
+ // x0 += x4, x12 = rotl32(x12 ^ x0, 16)
+ // x1 += x5, x13 = rotl32(x13 ^ x1, 16)
+ // x2 += x6, x14 = rotl32(x14 ^ x2, 16)
+ // x3 += x7, x15 = rotl32(x15 ^ x3, 16)
+ vadd.i32 q0, q0, q4
+ vadd.i32 q1, q1, q5
+ vadd.i32 q2, q2, q6
+ vadd.i32 q3, q3, q7
+
+ veor q12, q12, q0
+ veor q13, q13, q1
+ veor q14, q14, q2
+ veor q15, q15, q3
+
+ vrev32.16 q12, q12
+ vrev32.16 q13, q13
+ vrev32.16 q14, q14
+ vrev32.16 q15, q15
+
+ // x8 += x12, x4 = rotl32(x4 ^ x8, 12)
+ // x9 += x13, x5 = rotl32(x5 ^ x9, 12)
+ // x10 += x14, x6 = rotl32(x6 ^ x10, 12)
+ // x11 += x15, x7 = rotl32(x7 ^ x11, 12)
+ vadd.i32 q8, q8, q12
+ vadd.i32 q9, q9, q13
+ vadd.i32 q10, q10, q14
+ vadd.i32 q11, q11, q15
+
+ vst1.32 {q8-q9}, [sp, :256]
+
+ veor q8, q4, q8
+ veor q9, q5, q9
+ vshl.u32 q4, q8, #12
+ vshl.u32 q5, q9, #12
+ vsri.u32 q4, q8, #20
+ vsri.u32 q5, q9, #20
+
+ veor q8, q6, q10
+ veor q9, q7, q11
+ vshl.u32 q6, q8, #12
+ vshl.u32 q7, q9, #12
+ vsri.u32 q6, q8, #20
+ vsri.u32 q7, q9, #20
+
+ // x0 += x4, x12 = rotl32(x12 ^ x0, 8)
+ // x1 += x5, x13 = rotl32(x13 ^ x1, 8)
+ // x2 += x6, x14 = rotl32(x14 ^ x2, 8)
+ // x3 += x7, x15 = rotl32(x15 ^ x3, 8)
+ vadd.i32 q0, q0, q4
+ vadd.i32 q1, q1, q5
+ vadd.i32 q2, q2, q6
+ vadd.i32 q3, q3, q7
+
+ veor q8, q12, q0
+ veor q9, q13, q1
+ vshl.u32 q12, q8, #8
+ vshl.u32 q13, q9, #8
+ vsri.u32 q12, q8, #24
+ vsri.u32 q13, q9, #24
+
+ veor q8, q14, q2
+ veor q9, q15, q3
+ vshl.u32 q14, q8, #8
+ vshl.u32 q15, q9, #8
+ vsri.u32 q14, q8, #24
+ vsri.u32 q15, q9, #24
+
+ vld1.32 {q8-q9}, [sp, :256]
+
+ // x8 += x12, x4 = rotl32(x4 ^ x8, 7)
+ // x9 += x13, x5 = rotl32(x5 ^ x9, 7)
+ // x10 += x14, x6 = rotl32(x6 ^ x10, 7)
+ // x11 += x15, x7 = rotl32(x7 ^ x11, 7)
+ vadd.i32 q8, q8, q12
+ vadd.i32 q9, q9, q13
+ vadd.i32 q10, q10, q14
+ vadd.i32 q11, q11, q15
+
+ vst1.32 {q8-q9}, [sp, :256]
+
+ veor q8, q4, q8
+ veor q9, q5, q9
+ vshl.u32 q4, q8, #7
+ vshl.u32 q5, q9, #7
+ vsri.u32 q4, q8, #25
+ vsri.u32 q5, q9, #25
+
+ veor q8, q6, q10
+ veor q9, q7, q11
+ vshl.u32 q6, q8, #7
+ vshl.u32 q7, q9, #7
+ vsri.u32 q6, q8, #25
+ vsri.u32 q7, q9, #25
+
+ vld1.32 {q8-q9}, [sp, :256]
+
+ // x0 += x5, x15 = rotl32(x15 ^ x0, 16)
+ // x1 += x6, x12 = rotl32(x12 ^ x1, 16)
+ // x2 += x7, x13 = rotl32(x13 ^ x2, 16)
+ // x3 += x4, x14 = rotl32(x14 ^ x3, 16)
+ vadd.i32 q0, q0, q5
+ vadd.i32 q1, q1, q6
+ vadd.i32 q2, q2, q7
+ vadd.i32 q3, q3, q4
+
+ veor q15, q15, q0
+ veor q12, q12, q1
+ veor q13, q13, q2
+ veor q14, q14, q3
+
+ vrev32.16 q15, q15
+ vrev32.16 q12, q12
+ vrev32.16 q13, q13
+ vrev32.16 q14, q14
+
+ // x10 += x15, x5 = rotl32(x5 ^ x10, 12)
+ // x11 += x12, x6 = rotl32(x6 ^ x11, 12)
+ // x8 += x13, x7 = rotl32(x7 ^ x8, 12)
+ // x9 += x14, x4 = rotl32(x4 ^ x9, 12)
+ vadd.i32 q10, q10, q15
+ vadd.i32 q11, q11, q12
+ vadd.i32 q8, q8, q13
+ vadd.i32 q9, q9, q14
+
+ vst1.32 {q8-q9}, [sp, :256]
+
+ veor q8, q7, q8
+ veor q9, q4, q9
+ vshl.u32 q7, q8, #12
+ vshl.u32 q4, q9, #12
+ vsri.u32 q7, q8, #20
+ vsri.u32 q4, q9, #20
+
+ veor q8, q5, q10
+ veor q9, q6, q11
+ vshl.u32 q5, q8, #12
+ vshl.u32 q6, q9, #12
+ vsri.u32 q5, q8, #20
+ vsri.u32 q6, q9, #20
+
+ // x0 += x5, x15 = rotl32(x15 ^ x0, 8)
+ // x1 += x6, x12 = rotl32(x12 ^ x1, 8)
+ // x2 += x7, x13 = rotl32(x13 ^ x2, 8)
+ // x3 += x4, x14 = rotl32(x14 ^ x3, 8)
+ vadd.i32 q0, q0, q5
+ vadd.i32 q1, q1, q6
+ vadd.i32 q2, q2, q7
+ vadd.i32 q3, q3, q4
+
+ veor q8, q15, q0
+ veor q9, q12, q1
+ vshl.u32 q15, q8, #8
+ vshl.u32 q12, q9, #8
+ vsri.u32 q15, q8, #24
+ vsri.u32 q12, q9, #24
+
+ veor q8, q13, q2
+ veor q9, q14, q3
+ vshl.u32 q13, q8, #8
+ vshl.u32 q14, q9, #8
+ vsri.u32 q13, q8, #24
+ vsri.u32 q14, q9, #24
+
+ vld1.32 {q8-q9}, [sp, :256]
+
+ // x10 += x15, x5 = rotl32(x5 ^ x10, 7)
+ // x11 += x12, x6 = rotl32(x6 ^ x11, 7)
+ // x8 += x13, x7 = rotl32(x7 ^ x8, 7)
+ // x9 += x14, x4 = rotl32(x4 ^ x9, 7)
+ vadd.i32 q10, q10, q15
+ vadd.i32 q11, q11, q12
+ vadd.i32 q8, q8, q13
+ vadd.i32 q9, q9, q14
+
+ vst1.32 {q8-q9}, [sp, :256]
+
+ veor q8, q7, q8
+ veor q9, q4, q9
+ vshl.u32 q7, q8, #7
+ vshl.u32 q4, q9, #7
+ vsri.u32 q7, q8, #25
+ vsri.u32 q4, q9, #25
+
+ veor q8, q5, q10
+ veor q9, q6, q11
+ vshl.u32 q5, q8, #7
+ vshl.u32 q6, q9, #7
+ vsri.u32 q5, q8, #25
+ vsri.u32 q6, q9, #25
+
+ subs r3, r3, #1
+ beq 0f
+
+ vld1.32 {q8-q9}, [sp, :256]
+ b .Ldoubleround4
+
+ // x0[0-3] += s0[0]
+ // x1[0-3] += s0[1]
+ // x2[0-3] += s0[2]
+ // x3[0-3] += s0[3]
+0: ldmia r0!, {r3-r6}
+ vdup.32 q8, r3
+ vdup.32 q9, r4
+ vadd.i32 q0, q0, q8
+ vadd.i32 q1, q1, q9
+ vdup.32 q8, r5
+ vdup.32 q9, r6
+ vadd.i32 q2, q2, q8
+ vadd.i32 q3, q3, q9
+
+ // x4[0-3] += s1[0]
+ // x5[0-3] += s1[1]
+ // x6[0-3] += s1[2]
+ // x7[0-3] += s1[3]
+ ldmia r0!, {r3-r6}
+ vdup.32 q8, r3
+ vdup.32 q9, r4
+ vadd.i32 q4, q4, q8
+ vadd.i32 q5, q5, q9
+ vdup.32 q8, r5
+ vdup.32 q9, r6
+ vadd.i32 q6, q6, q8
+ vadd.i32 q7, q7, q9
+
+ // interleave 32-bit words in state n, n+1
+ vzip.32 q0, q1
+ vzip.32 q2, q3
+ vzip.32 q4, q5
+ vzip.32 q6, q7
+
+ // interleave 64-bit words in state n, n+2
+ vswp d1, d4
+ vswp d3, d6
+ vswp d9, d12
+ vswp d11, d14
+
+ // xor with corresponding input, write to output
+ vld1.8 {q8-q9}, [r2]!
+ veor q8, q8, q0
+ veor q9, q9, q4
+ vst1.8 {q8-q9}, [r1]!
+
+ vld1.32 {q8-q9}, [sp, :256]
+
+ // x8[0-3] += s2[0]
+ // x9[0-3] += s2[1]
+ // x10[0-3] += s2[2]
+ // x11[0-3] += s2[3]
+ ldmia r0!, {r3-r6}
+ vdup.32 q0, r3
+ vdup.32 q4, r4
+ vadd.i32 q8, q8, q0
+ vadd.i32 q9, q9, q4
+ vdup.32 q0, r5
+ vdup.32 q4, r6
+ vadd.i32 q10, q10, q0
+ vadd.i32 q11, q11, q4
+
+ // x12[0-3] += s3[0]
+ // x13[0-3] += s3[1]
+ // x14[0-3] += s3[2]
+ // x15[0-3] += s3[3]
+ ldmia r0!, {r3-r6}
+ vdup.32 q0, r3
+ vdup.32 q4, r4
+ adr r3, CTRINC
+ vadd.i32 q12, q12, q0
+ vld1.32 {q0}, [r3, :128]
+ vadd.i32 q13, q13, q4
+ vadd.i32 q12, q12, q0 // x12 += counter values 0-3
+
+ vdup.32 q0, r5
+ vdup.32 q4, r6
+ vadd.i32 q14, q14, q0
+ vadd.i32 q15, q15, q4
+
+ // interleave 32-bit words in state n, n+1
+ vzip.32 q8, q9
+ vzip.32 q10, q11
+ vzip.32 q12, q13
+ vzip.32 q14, q15
+
+ // interleave 64-bit words in state n, n+2
+ vswp d17, d20
+ vswp d19, d22
+ vswp d25, d28
+ vswp d27, d30
+
+ vmov q4, q1
+
+ vld1.8 {q0-q1}, [r2]!
+ veor q0, q0, q8
+ veor q1, q1, q12
+ vst1.8 {q0-q1}, [r1]!
+
+ vld1.8 {q0-q1}, [r2]!
+ veor q0, q0, q2
+ veor q1, q1, q6
+ vst1.8 {q0-q1}, [r1]!
+
+ vld1.8 {q0-q1}, [r2]!
+ veor q0, q0, q10
+ veor q1, q1, q14
+ vst1.8 {q0-q1}, [r1]!
+
+ vld1.8 {q0-q1}, [r2]!
+ veor q0, q0, q4
+ veor q1, q1, q5
+ vst1.8 {q0-q1}, [r1]!
+
+ vld1.8 {q0-q1}, [r2]!
+ veor q0, q0, q9
+ veor q1, q1, q13
+ vst1.8 {q0-q1}, [r1]!
+
+ vld1.8 {q0-q1}, [r2]!
+ veor q0, q0, q3
+ veor q1, q1, q7
+ vst1.8 {q0-q1}, [r1]!
+
+ vld1.8 {q0-q1}, [r2]
+ veor q0, q0, q11
+ veor q1, q1, q15
+ vst1.8 {q0-q1}, [r1]
+
+ mov sp, ip
+ pop {r4-r6, pc}
+ENDPROC(chacha20_4block_xor_neon)
+
+ .align 4
+CTRINC: .word 0, 1, 2, 3
+
diff --git a/arch/arm/crypto/chacha20-neon-glue.c b/arch/arm/crypto/chacha20-neon-glue.c
new file mode 100644
index 000000000000..592f75ae4fa1
--- /dev/null
+++ b/arch/arm/crypto/chacha20-neon-glue.c
@@ -0,0 +1,128 @@
+/*
+ * ChaCha20 256-bit cipher algorithm, RFC7539, ARM NEON functions
+ *
+ * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Based on:
+ * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
+ *
+ * Copyright (C) 2015 Martin Willi
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <crypto/algapi.h>
+#include <crypto/chacha20.h>
+#include <crypto/internal/skcipher.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+#include <asm/simd.h>
+
+asmlinkage void chacha20_block_xor_neon(u32 *state, u8 *dst, const u8 *src);
+asmlinkage void chacha20_4block_xor_neon(u32 *state, u8 *dst, const u8 *src);
+
+static void chacha20_doneon(u32 *state, u8 *dst, const u8 *src,
+ unsigned int bytes)
+{
+ u8 buf[CHACHA20_BLOCK_SIZE];
+
+ while (bytes >= CHACHA20_BLOCK_SIZE * 4) {
+ chacha20_4block_xor_neon(state, dst, src);
+ bytes -= CHACHA20_BLOCK_SIZE * 4;
+ src += CHACHA20_BLOCK_SIZE * 4;
+ dst += CHACHA20_BLOCK_SIZE * 4;
+ state[12] += 4;
+ }
+ while (bytes >= CHACHA20_BLOCK_SIZE) {
+ chacha20_block_xor_neon(state, dst, src);
+ bytes -= CHACHA20_BLOCK_SIZE;
+ src += CHACHA20_BLOCK_SIZE;
+ dst += CHACHA20_BLOCK_SIZE;
+ state[12]++;
+ }
+ if (bytes) {
+ memcpy(buf, src, bytes);
+ chacha20_block_xor_neon(state, buf, buf);
+ memcpy(dst, buf, bytes);
+ }
+}
+
+static int chacha20_neon(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ u32 state[16];
+ int err;
+
+ if (req->cryptlen <= CHACHA20_BLOCK_SIZE || !may_use_simd())
+ return crypto_chacha20_crypt(req);
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ crypto_chacha20_init(state, ctx, walk.iv);
+
+ kernel_neon_begin();
+ while (walk.nbytes > 0) {
+ unsigned int nbytes = walk.nbytes;
+
+ if (nbytes < walk.total)
+ nbytes = round_down(nbytes, walk.stride);
+
+ chacha20_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
+ nbytes);
+ err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
+ }
+ kernel_neon_end();
+
+ return err;
+}
+
+static struct skcipher_alg alg = {
+ .base.cra_name = "chacha20",
+ .base.cra_driver_name = "chacha20-neon",
+ .base.cra_priority = 300,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct chacha20_ctx),
+ .base.cra_alignmask = 1,
+ .base.cra_module = THIS_MODULE,
+
+ .min_keysize = CHACHA20_KEY_SIZE,
+ .max_keysize = CHACHA20_KEY_SIZE,
+ .ivsize = CHACHA20_IV_SIZE,
+ .chunksize = CHACHA20_BLOCK_SIZE,
+ .walksize = 4 * CHACHA20_BLOCK_SIZE,
+ .setkey = crypto_chacha20_setkey,
+ .encrypt = chacha20_neon,
+ .decrypt = chacha20_neon,
+};
+
+static int __init chacha20_simd_mod_init(void)
+{
+ if (!(elf_hwcap & HWCAP_NEON))
+ return -ENODEV;
+
+ return crypto_register_skcipher(&alg);
+}
+
+static void __exit chacha20_simd_mod_fini(void)
+{
+ crypto_unregister_skcipher(&alg);
+}
+
+module_init(chacha20_simd_mod_init);
+module_exit(chacha20_simd_mod_fini);
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("chacha20");
--
2.7.4
^ permalink raw reply related
* [PATCH 4/6] crypto: arm64/chacha20 - implement NEON version based on SSE3 code
From: Ard Biesheuvel @ 2017-01-02 18:21 UTC (permalink / raw)
To: linux-crypto; +Cc: linux-arm-kernel, herbert, Ard Biesheuvel
In-Reply-To: <1483381268-12987-1-git-send-email-ard.biesheuvel@linaro.org>
This is a straight port to arm64/NEON of the x86 SSE3 implementation
of the ChaCha20 stream cipher. It uses the new skcipher walksize
attribute to process the input in strides of 4x the block size.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/Kconfig | 6 +
arch/arm64/crypto/Makefile | 3 +
arch/arm64/crypto/chacha20-neon-core.S | 450 ++++++++++++++++++++
arch/arm64/crypto/chacha20-neon-glue.c | 127 ++++++
4 files changed, 586 insertions(+)
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 450a85df041a..0bf0f531f539 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -72,4 +72,10 @@ config CRYPTO_CRC32_ARM64
depends on ARM64
select CRYPTO_HASH
+config CRYPTO_CHACHA20_NEON
+ tristate "NEON accelerated ChaCha20 symmetric cipher"
+ depends on KERNEL_MODE_NEON
+ select CRYPTO_BLKCIPHER
+ select CRYPTO_CHACHA20
+
endif
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index aa8888d7b744..9d2826c5fccf 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -41,6 +41,9 @@ sha256-arm64-y := sha256-glue.o sha256-core.o
obj-$(CONFIG_CRYPTO_SHA512_ARM64) += sha512-arm64.o
sha512-arm64-y := sha512-glue.o sha512-core.o
+obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha20-neon.o
+chacha20-neon-y := chacha20-neon-core.o chacha20-neon-glue.o
+
AFLAGS_aes-ce.o := -DINTERLEAVE=4
AFLAGS_aes-neon.o := -DINTERLEAVE=4
diff --git a/arch/arm64/crypto/chacha20-neon-core.S b/arch/arm64/crypto/chacha20-neon-core.S
new file mode 100644
index 000000000000..13c85e272c2a
--- /dev/null
+++ b/arch/arm64/crypto/chacha20-neon-core.S
@@ -0,0 +1,450 @@
+/*
+ * ChaCha20 256-bit cipher algorithm, RFC7539, arm64 NEON functions
+ *
+ * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Based on:
+ * ChaCha20 256-bit cipher algorithm, RFC7539, x64 SSSE3 functions
+ *
+ * Copyright (C) 2015 Martin Willi
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/linkage.h>
+
+ .text
+ .align 6
+
+ENTRY(chacha20_block_xor_neon)
+ // x0: Input state matrix, s
+ // x1: 1 data block output, o
+ // x2: 1 data block input, i
+
+ //
+ // This function encrypts one ChaCha20 block by loading the state matrix
+ // in four NEON registers. It performs matrix operation on four words in
+ // parallel, but requires shuffling to rearrange the words after each
+ // round.
+ //
+
+ // x0..3 = s0..3
+ adr x3, ROT8
+ ld1 {v0.4s-v3.4s}, [x0]
+ ld1 {v8.4s-v11.4s}, [x0]
+ ld1 {v12.4s}, [x3]
+
+ mov x3, #10
+
+.Ldoubleround:
+ // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
+ add v0.4s, v0.4s, v1.4s
+ eor v3.16b, v3.16b, v0.16b
+ rev32 v3.8h, v3.8h
+
+ // x2 += x3, x1 = rotl32(x1 ^ x2, 12)
+ add v2.4s, v2.4s, v3.4s
+ eor v4.16b, v1.16b, v2.16b
+ shl v1.4s, v4.4s, #12
+ sri v1.4s, v4.4s, #20
+
+ // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
+ add v0.4s, v0.4s, v1.4s
+ eor v3.16b, v3.16b, v0.16b
+ tbl v3.16b, {v3.16b}, v12.16b
+
+ // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
+ add v2.4s, v2.4s, v3.4s
+ eor v4.16b, v1.16b, v2.16b
+ shl v1.4s, v4.4s, #7
+ sri v1.4s, v4.4s, #25
+
+ // x1 = shuffle32(x1, MASK(0, 3, 2, 1))
+ ext v1.16b, v1.16b, v1.16b, #4
+ // x2 = shuffle32(x2, MASK(1, 0, 3, 2))
+ ext v2.16b, v2.16b, v2.16b, #8
+ // x3 = shuffle32(x3, MASK(2, 1, 0, 3))
+ ext v3.16b, v3.16b, v3.16b, #12
+
+ // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
+ add v0.4s, v0.4s, v1.4s
+ eor v3.16b, v3.16b, v0.16b
+ rev32 v3.8h, v3.8h
+
+ // x2 += x3, x1 = rotl32(x1 ^ x2, 12)
+ add v2.4s, v2.4s, v3.4s
+ eor v4.16b, v1.16b, v2.16b
+ shl v1.4s, v4.4s, #12
+ sri v1.4s, v4.4s, #20
+
+ // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
+ add v0.4s, v0.4s, v1.4s
+ eor v3.16b, v3.16b, v0.16b
+ tbl v3.16b, {v3.16b}, v12.16b
+
+ // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
+ add v2.4s, v2.4s, v3.4s
+ eor v4.16b, v1.16b, v2.16b
+ shl v1.4s, v4.4s, #7
+ sri v1.4s, v4.4s, #25
+
+ // x1 = shuffle32(x1, MASK(2, 1, 0, 3))
+ ext v1.16b, v1.16b, v1.16b, #12
+ // x2 = shuffle32(x2, MASK(1, 0, 3, 2))
+ ext v2.16b, v2.16b, v2.16b, #8
+ // x3 = shuffle32(x3, MASK(0, 3, 2, 1))
+ ext v3.16b, v3.16b, v3.16b, #4
+
+ subs x3, x3, #1
+ b.ne .Ldoubleround
+
+ ld1 {v4.16b-v7.16b}, [x2]
+
+ // o0 = i0 ^ (x0 + s0)
+ add v0.4s, v0.4s, v8.4s
+ eor v0.16b, v0.16b, v4.16b
+
+ // o1 = i1 ^ (x1 + s1)
+ add v1.4s, v1.4s, v9.4s
+ eor v1.16b, v1.16b, v5.16b
+
+ // o2 = i2 ^ (x2 + s2)
+ add v2.4s, v2.4s, v10.4s
+ eor v2.16b, v2.16b, v6.16b
+
+ // o3 = i3 ^ (x3 + s3)
+ add v3.4s, v3.4s, v11.4s
+ eor v3.16b, v3.16b, v7.16b
+
+ st1 {v0.16b-v3.16b}, [x1]
+
+ ret
+ENDPROC(chacha20_block_xor_neon)
+
+ .align 6
+ENTRY(chacha20_4block_xor_neon)
+ // x0: Input state matrix, s
+ // x1: 4 data blocks output, o
+ // x2: 4 data blocks input, i
+
+ //
+ // This function encrypts four consecutive ChaCha20 blocks by loading
+ // the state matrix in NEON registers four times. The algorithm performs
+ // each operation on the corresponding word of each state matrix, hence
+ // requires no word shuffling. For final XORing step we transpose the
+ // matrix by interleaving 32- and then 64-bit words, which allows us to
+ // do XOR in NEON registers.
+ //
+ adr x3, CTRINC // ... and ROT8
+ ld1 {v30.4s-v31.4s}, [x3]
+
+ // x0..15[0-3] = s0..3[0..3]
+ mov x4, x0
+ ld4r { v0.4s- v3.4s}, [x4], #16
+ ld4r { v4.4s- v7.4s}, [x4], #16
+ ld4r { v8.4s-v11.4s}, [x4], #16
+ ld4r {v12.4s-v15.4s}, [x4]
+
+ // x12 += counter values 0-3
+ add v12.4s, v12.4s, v30.4s
+
+ mov x3, #10
+
+.Ldoubleround4:
+ // x0 += x4, x12 = rotl32(x12 ^ x0, 16)
+ // x1 += x5, x13 = rotl32(x13 ^ x1, 16)
+ // x2 += x6, x14 = rotl32(x14 ^ x2, 16)
+ // x3 += x7, x15 = rotl32(x15 ^ x3, 16)
+ add v0.4s, v0.4s, v4.4s
+ add v1.4s, v1.4s, v5.4s
+ add v2.4s, v2.4s, v6.4s
+ add v3.4s, v3.4s, v7.4s
+
+ eor v12.16b, v12.16b, v0.16b
+ eor v13.16b, v13.16b, v1.16b
+ eor v14.16b, v14.16b, v2.16b
+ eor v15.16b, v15.16b, v3.16b
+
+ rev32 v12.8h, v12.8h
+ rev32 v13.8h, v13.8h
+ rev32 v14.8h, v14.8h
+ rev32 v15.8h, v15.8h
+
+ // x8 += x12, x4 = rotl32(x4 ^ x8, 12)
+ // x9 += x13, x5 = rotl32(x5 ^ x9, 12)
+ // x10 += x14, x6 = rotl32(x6 ^ x10, 12)
+ // x11 += x15, x7 = rotl32(x7 ^ x11, 12)
+ add v8.4s, v8.4s, v12.4s
+ add v9.4s, v9.4s, v13.4s
+ add v10.4s, v10.4s, v14.4s
+ add v11.4s, v11.4s, v15.4s
+
+ eor v16.16b, v4.16b, v8.16b
+ eor v17.16b, v5.16b, v9.16b
+ eor v18.16b, v6.16b, v10.16b
+ eor v19.16b, v7.16b, v11.16b
+
+ shl v4.4s, v16.4s, #12
+ shl v5.4s, v17.4s, #12
+ shl v6.4s, v18.4s, #12
+ shl v7.4s, v19.4s, #12
+
+ sri v4.4s, v16.4s, #20
+ sri v5.4s, v17.4s, #20
+ sri v6.4s, v18.4s, #20
+ sri v7.4s, v19.4s, #20
+
+ // x0 += x4, x12 = rotl32(x12 ^ x0, 8)
+ // x1 += x5, x13 = rotl32(x13 ^ x1, 8)
+ // x2 += x6, x14 = rotl32(x14 ^ x2, 8)
+ // x3 += x7, x15 = rotl32(x15 ^ x3, 8)
+ add v0.4s, v0.4s, v4.4s
+ add v1.4s, v1.4s, v5.4s
+ add v2.4s, v2.4s, v6.4s
+ add v3.4s, v3.4s, v7.4s
+
+ eor v12.16b, v12.16b, v0.16b
+ eor v13.16b, v13.16b, v1.16b
+ eor v14.16b, v14.16b, v2.16b
+ eor v15.16b, v15.16b, v3.16b
+
+ tbl v12.16b, {v12.16b}, v31.16b
+ tbl v13.16b, {v13.16b}, v31.16b
+ tbl v14.16b, {v14.16b}, v31.16b
+ tbl v15.16b, {v15.16b}, v31.16b
+
+ // x8 += x12, x4 = rotl32(x4 ^ x8, 7)
+ // x9 += x13, x5 = rotl32(x5 ^ x9, 7)
+ // x10 += x14, x6 = rotl32(x6 ^ x10, 7)
+ // x11 += x15, x7 = rotl32(x7 ^ x11, 7)
+ add v8.4s, v8.4s, v12.4s
+ add v9.4s, v9.4s, v13.4s
+ add v10.4s, v10.4s, v14.4s
+ add v11.4s, v11.4s, v15.4s
+
+ eor v16.16b, v4.16b, v8.16b
+ eor v17.16b, v5.16b, v9.16b
+ eor v18.16b, v6.16b, v10.16b
+ eor v19.16b, v7.16b, v11.16b
+
+ shl v4.4s, v16.4s, #7
+ shl v5.4s, v17.4s, #7
+ shl v6.4s, v18.4s, #7
+ shl v7.4s, v19.4s, #7
+
+ sri v4.4s, v16.4s, #25
+ sri v5.4s, v17.4s, #25
+ sri v6.4s, v18.4s, #25
+ sri v7.4s, v19.4s, #25
+
+ // x0 += x5, x15 = rotl32(x15 ^ x0, 16)
+ // x1 += x6, x12 = rotl32(x12 ^ x1, 16)
+ // x2 += x7, x13 = rotl32(x13 ^ x2, 16)
+ // x3 += x4, x14 = rotl32(x14 ^ x3, 16)
+ add v0.4s, v0.4s, v5.4s
+ add v1.4s, v1.4s, v6.4s
+ add v2.4s, v2.4s, v7.4s
+ add v3.4s, v3.4s, v4.4s
+
+ eor v15.16b, v15.16b, v0.16b
+ eor v12.16b, v12.16b, v1.16b
+ eor v13.16b, v13.16b, v2.16b
+ eor v14.16b, v14.16b, v3.16b
+
+ rev32 v15.8h, v15.8h
+ rev32 v12.8h, v12.8h
+ rev32 v13.8h, v13.8h
+ rev32 v14.8h, v14.8h
+
+ // x10 += x15, x5 = rotl32(x5 ^ x10, 12)
+ // x11 += x12, x6 = rotl32(x6 ^ x11, 12)
+ // x8 += x13, x7 = rotl32(x7 ^ x8, 12)
+ // x9 += x14, x4 = rotl32(x4 ^ x9, 12)
+ add v10.4s, v10.4s, v15.4s
+ add v11.4s, v11.4s, v12.4s
+ add v8.4s, v8.4s, v13.4s
+ add v9.4s, v9.4s, v14.4s
+
+ eor v16.16b, v5.16b, v10.16b
+ eor v17.16b, v6.16b, v11.16b
+ eor v18.16b, v7.16b, v8.16b
+ eor v19.16b, v4.16b, v9.16b
+
+ shl v5.4s, v16.4s, #12
+ shl v6.4s, v17.4s, #12
+ shl v7.4s, v18.4s, #12
+ shl v4.4s, v19.4s, #12
+
+ sri v5.4s, v16.4s, #20
+ sri v6.4s, v17.4s, #20
+ sri v7.4s, v18.4s, #20
+ sri v4.4s, v19.4s, #20
+
+ // x0 += x5, x15 = rotl32(x15 ^ x0, 8)
+ // x1 += x6, x12 = rotl32(x12 ^ x1, 8)
+ // x2 += x7, x13 = rotl32(x13 ^ x2, 8)
+ // x3 += x4, x14 = rotl32(x14 ^ x3, 8)
+ add v0.4s, v0.4s, v5.4s
+ add v1.4s, v1.4s, v6.4s
+ add v2.4s, v2.4s, v7.4s
+ add v3.4s, v3.4s, v4.4s
+
+ eor v15.16b, v15.16b, v0.16b
+ eor v12.16b, v12.16b, v1.16b
+ eor v13.16b, v13.16b, v2.16b
+ eor v14.16b, v14.16b, v3.16b
+
+ tbl v15.16b, {v15.16b}, v31.16b
+ tbl v12.16b, {v12.16b}, v31.16b
+ tbl v13.16b, {v13.16b}, v31.16b
+ tbl v14.16b, {v14.16b}, v31.16b
+
+ // x10 += x15, x5 = rotl32(x5 ^ x10, 7)
+ // x11 += x12, x6 = rotl32(x6 ^ x11, 7)
+ // x8 += x13, x7 = rotl32(x7 ^ x8, 7)
+ // x9 += x14, x4 = rotl32(x4 ^ x9, 7)
+ add v10.4s, v10.4s, v15.4s
+ add v11.4s, v11.4s, v12.4s
+ add v8.4s, v8.4s, v13.4s
+ add v9.4s, v9.4s, v14.4s
+
+ eor v16.16b, v5.16b, v10.16b
+ eor v17.16b, v6.16b, v11.16b
+ eor v18.16b, v7.16b, v8.16b
+ eor v19.16b, v4.16b, v9.16b
+
+ shl v5.4s, v16.4s, #7
+ shl v6.4s, v17.4s, #7
+ shl v7.4s, v18.4s, #7
+ shl v4.4s, v19.4s, #7
+
+ sri v5.4s, v16.4s, #25
+ sri v6.4s, v17.4s, #25
+ sri v7.4s, v18.4s, #25
+ sri v4.4s, v19.4s, #25
+
+ subs x3, x3, #1
+ b.ne .Ldoubleround4
+
+ ld4r {v16.4s-v19.4s}, [x0], #16
+ ld4r {v20.4s-v23.4s}, [x0], #16
+
+ // x12 += counter values 0-3
+ add v12.4s, v12.4s, v30.4s
+
+ // x0[0-3] += s0[0]
+ // x1[0-3] += s0[1]
+ // x2[0-3] += s0[2]
+ // x3[0-3] += s0[3]
+ add v0.4s, v0.4s, v16.4s
+ add v1.4s, v1.4s, v17.4s
+ add v2.4s, v2.4s, v18.4s
+ add v3.4s, v3.4s, v19.4s
+
+ ld4r {v24.4s-v27.4s}, [x0], #16
+ ld4r {v28.4s-v31.4s}, [x0]
+
+ // x4[0-3] += s1[0]
+ // x5[0-3] += s1[1]
+ // x6[0-3] += s1[2]
+ // x7[0-3] += s1[3]
+ add v4.4s, v4.4s, v20.4s
+ add v5.4s, v5.4s, v21.4s
+ add v6.4s, v6.4s, v22.4s
+ add v7.4s, v7.4s, v23.4s
+
+ // x8[0-3] += s2[0]
+ // x9[0-3] += s2[1]
+ // x10[0-3] += s2[2]
+ // x11[0-3] += s2[3]
+ add v8.4s, v8.4s, v24.4s
+ add v9.4s, v9.4s, v25.4s
+ add v10.4s, v10.4s, v26.4s
+ add v11.4s, v11.4s, v27.4s
+
+ // x12[0-3] += s3[0]
+ // x13[0-3] += s3[1]
+ // x14[0-3] += s3[2]
+ // x15[0-3] += s3[3]
+ add v12.4s, v12.4s, v28.4s
+ add v13.4s, v13.4s, v29.4s
+ add v14.4s, v14.4s, v30.4s
+ add v15.4s, v15.4s, v31.4s
+
+ // interleave 32-bit words in state n, n+1
+ zip1 v16.4s, v0.4s, v1.4s
+ zip2 v17.4s, v0.4s, v1.4s
+ zip1 v18.4s, v2.4s, v3.4s
+ zip2 v19.4s, v2.4s, v3.4s
+ zip1 v20.4s, v4.4s, v5.4s
+ zip2 v21.4s, v4.4s, v5.4s
+ zip1 v22.4s, v6.4s, v7.4s
+ zip2 v23.4s, v6.4s, v7.4s
+ zip1 v24.4s, v8.4s, v9.4s
+ zip2 v25.4s, v8.4s, v9.4s
+ zip1 v26.4s, v10.4s, v11.4s
+ zip2 v27.4s, v10.4s, v11.4s
+ zip1 v28.4s, v12.4s, v13.4s
+ zip2 v29.4s, v12.4s, v13.4s
+ zip1 v30.4s, v14.4s, v15.4s
+ zip2 v31.4s, v14.4s, v15.4s
+
+ // interleave 64-bit words in state n, n+2
+ zip1 v0.2d, v16.2d, v18.2d
+ zip2 v4.2d, v16.2d, v18.2d
+ zip1 v8.2d, v17.2d, v19.2d
+ zip2 v12.2d, v17.2d, v19.2d
+ ld1 {v16.16b-v19.16b}, [x2], #64
+
+ zip1 v1.2d, v20.2d, v22.2d
+ zip2 v5.2d, v20.2d, v22.2d
+ zip1 v9.2d, v21.2d, v23.2d
+ zip2 v13.2d, v21.2d, v23.2d
+ ld1 {v20.16b-v23.16b}, [x2], #64
+
+ zip1 v2.2d, v24.2d, v26.2d
+ zip2 v6.2d, v24.2d, v26.2d
+ zip1 v10.2d, v25.2d, v27.2d
+ zip2 v14.2d, v25.2d, v27.2d
+ ld1 {v24.16b-v27.16b}, [x2], #64
+
+ zip1 v3.2d, v28.2d, v30.2d
+ zip2 v7.2d, v28.2d, v30.2d
+ zip1 v11.2d, v29.2d, v31.2d
+ zip2 v15.2d, v29.2d, v31.2d
+ ld1 {v28.16b-v31.16b}, [x2]
+
+ // xor with corresponding input, write to output
+ eor v16.16b, v16.16b, v0.16b
+ eor v17.16b, v17.16b, v1.16b
+ eor v18.16b, v18.16b, v2.16b
+ eor v19.16b, v19.16b, v3.16b
+ eor v20.16b, v20.16b, v4.16b
+ eor v21.16b, v21.16b, v5.16b
+ st1 {v16.16b-v19.16b}, [x1], #64
+ eor v22.16b, v22.16b, v6.16b
+ eor v23.16b, v23.16b, v7.16b
+ eor v24.16b, v24.16b, v8.16b
+ eor v25.16b, v25.16b, v9.16b
+ st1 {v20.16b-v23.16b}, [x1], #64
+ eor v26.16b, v26.16b, v10.16b
+ eor v27.16b, v27.16b, v11.16b
+ eor v28.16b, v28.16b, v12.16b
+ st1 {v24.16b-v27.16b}, [x1], #64
+ eor v29.16b, v29.16b, v13.16b
+ eor v30.16b, v30.16b, v14.16b
+ eor v31.16b, v31.16b, v15.16b
+ st1 {v28.16b-v31.16b}, [x1]
+
+ ret
+ENDPROC(chacha20_4block_xor_neon)
+
+CTRINC: .word 0, 1, 2, 3
+ROT8: .word 0x02010003, 0x06050407, 0x0a09080b, 0x0e0d0c0f
diff --git a/arch/arm64/crypto/chacha20-neon-glue.c b/arch/arm64/crypto/chacha20-neon-glue.c
new file mode 100644
index 000000000000..a7f2337d46cf
--- /dev/null
+++ b/arch/arm64/crypto/chacha20-neon-glue.c
@@ -0,0 +1,127 @@
+/*
+ * ChaCha20 256-bit cipher algorithm, RFC7539, arm64 NEON functions
+ *
+ * Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Based on:
+ * ChaCha20 256-bit cipher algorithm, RFC7539, SIMD glue code
+ *
+ * Copyright (C) 2015 Martin Willi
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <crypto/algapi.h>
+#include <crypto/chacha20.h>
+#include <crypto/internal/skcipher.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+#include <asm/hwcap.h>
+#include <asm/neon.h>
+
+asmlinkage void chacha20_block_xor_neon(u32 *state, u8 *dst, const u8 *src);
+asmlinkage void chacha20_4block_xor_neon(u32 *state, u8 *dst, const u8 *src);
+
+static void chacha20_doneon(u32 *state, u8 *dst, const u8 *src,
+ unsigned int bytes)
+{
+ u8 buf[CHACHA20_BLOCK_SIZE];
+
+ while (bytes >= CHACHA20_BLOCK_SIZE * 4) {
+ chacha20_4block_xor_neon(state, dst, src);
+ bytes -= CHACHA20_BLOCK_SIZE * 4;
+ src += CHACHA20_BLOCK_SIZE * 4;
+ dst += CHACHA20_BLOCK_SIZE * 4;
+ state[12] += 4;
+ }
+ while (bytes >= CHACHA20_BLOCK_SIZE) {
+ chacha20_block_xor_neon(state, dst, src);
+ bytes -= CHACHA20_BLOCK_SIZE;
+ src += CHACHA20_BLOCK_SIZE;
+ dst += CHACHA20_BLOCK_SIZE;
+ state[12]++;
+ }
+ if (bytes) {
+ memcpy(buf, src, bytes);
+ chacha20_block_xor_neon(state, buf, buf);
+ memcpy(dst, buf, bytes);
+ }
+}
+
+static int chacha20_neon(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct chacha20_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ u32 state[16];
+ int err;
+
+ if (req->cryptlen <= CHACHA20_BLOCK_SIZE)
+ return crypto_chacha20_crypt(req);
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ crypto_chacha20_init(state, ctx, walk.iv);
+
+ kernel_neon_begin();
+ while (walk.nbytes > 0) {
+ unsigned int nbytes = walk.nbytes;
+
+ if (nbytes < walk.total)
+ nbytes = round_down(nbytes, walk.stride);
+
+ chacha20_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
+ nbytes);
+ err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
+ }
+ kernel_neon_end();
+
+ return err;
+}
+
+static struct skcipher_alg alg = {
+ .base.cra_name = "chacha20",
+ .base.cra_driver_name = "chacha20-neon",
+ .base.cra_priority = 300,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct chacha20_ctx),
+ .base.cra_alignmask = 1,
+ .base.cra_module = THIS_MODULE,
+
+ .min_keysize = CHACHA20_KEY_SIZE,
+ .max_keysize = CHACHA20_KEY_SIZE,
+ .ivsize = CHACHA20_IV_SIZE,
+ .chunksize = CHACHA20_BLOCK_SIZE,
+ .walksize = 4 * CHACHA20_BLOCK_SIZE,
+ .setkey = crypto_chacha20_setkey,
+ .encrypt = chacha20_neon,
+ .decrypt = chacha20_neon,
+};
+
+static int __init chacha20_simd_mod_init(void)
+{
+ if (!(elf_hwcap & HWCAP_ASIMD))
+ return -ENODEV;
+
+ return crypto_register_skcipher(&alg);
+}
+
+static void __exit chacha20_simd_mod_fini(void)
+{
+ crypto_unregister_skcipher(&alg);
+}
+
+module_init(chacha20_simd_mod_init);
+module_exit(chacha20_simd_mod_fini);
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("chacha20");
--
2.7.4
^ permalink raw reply related
* [PATCH 6/6] crypto: arm64/aes - reimplement bit-sliced ARM/NEON implementation for arm64
From: Ard Biesheuvel @ 2017-01-02 18:21 UTC (permalink / raw)
To: linux-crypto; +Cc: linux-arm-kernel, herbert, Ard Biesheuvel
In-Reply-To: <1483381268-12987-1-git-send-email-ard.biesheuvel@linaro.org>
This is a reimplementation of the NEON version of the bit-sliced AES
algorithm. This code is heavily based on Andy Polyakov's OpenSSL version
for ARM, which is also available in the kernel. This is an alternative for
the existing NEON implementation for arm64 authored by me, which suffers
from poor performance due to its reliance on the pathologically slow four
register variant of the tbl/tbx NEON instruction.
This version is about ~30% (*) faster than the generic C code, but only in
cases where the input can be 8x interleaved (this is a fundamental property
of bit slicing). For this reason, only the chaining modes ECB, XTS and CTR
are implemented. (The significance of ECB is that it could potentially be
used by other chaining modes)
* Measured on Cortex-A57. Note that this is still an order of magnitude
slower than the implementations that use the dedicated AES instructions
introduced in ARMv8, but those are part of an optional extension, and so
it is good to have a fallback.
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
arch/arm64/crypto/Kconfig | 7 +
arch/arm64/crypto/Makefile | 3 +
arch/arm64/crypto/aes-neonbs-core.S | 879 ++++++++++++++++++++
arch/arm64/crypto/aes-neonbs-glue.c | 344 ++++++++
4 files changed, 1233 insertions(+)
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 0bf0f531f539..7c4249ad4935 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -78,4 +78,11 @@ config CRYPTO_CHACHA20_NEON
select CRYPTO_BLKCIPHER
select CRYPTO_CHACHA20
+config CRYPTO_AES_NEON_BS
+ tristate "AES in ECB/CTR/XTS modes using bit-sliced NEON algorithm"
+ depends on KERNEL_MODE_NEON
+ select CRYPTO_BLKCIPHER
+ select CRYPTO_AES
+ select CRYPTO_SIMD
+
endif
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index 9d2826c5fccf..df3c0584b05c 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -44,6 +44,9 @@ sha512-arm64-y := sha512-glue.o sha512-core.o
obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha20-neon.o
chacha20-neon-y := chacha20-neon-core.o chacha20-neon-glue.o
+obj-$(CONFIG_CRYPTO_AES_NEON_BS) += aes-neon-bs.o
+aes-neon-bs-y := aes-neonbs-core.o aes-neonbs-glue.o
+
AFLAGS_aes-ce.o := -DINTERLEAVE=4
AFLAGS_aes-neon.o := -DINTERLEAVE=4
diff --git a/arch/arm64/crypto/aes-neonbs-core.S b/arch/arm64/crypto/aes-neonbs-core.S
new file mode 100644
index 000000000000..f5e1f76e8ee8
--- /dev/null
+++ b/arch/arm64/crypto/aes-neonbs-core.S
@@ -0,0 +1,879 @@
+/*
+ * Bit sliced AES using NEON instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/*
+ * The algorithm implemented here is described in detail by the paper
+ * 'Faster and Timing-Attack Resistant AES-GCM' by Emilia Kaesper and
+ * Peter Schwabe (https://eprint.iacr.org/2009/129.pdf)
+ *
+ * This implementation is based primarily on the OpenSSL implementation
+ * for 32-bit ARM written by Andy Polyakov <appro@openssl.org>
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+ .text
+
+ rounds .req x11
+ bskey .req x12
+
+ .macro in_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
+ eor \b2, \b2, \b1
+ eor \b5, \b5, \b6
+ eor \b3, \b3, \b0
+ eor \b6, \b6, \b2
+ eor \b5, \b5, \b0
+ eor \b6, \b6, \b3
+ eor \b3, \b3, \b7
+ eor \b7, \b7, \b5
+ eor \b3, \b3, \b4
+ eor \b4, \b4, \b5
+ eor \b2, \b2, \b7
+ eor \b3, \b3, \b1
+ eor \b1, \b1, \b5
+ .endm
+
+ .macro out_bs_ch, b0, b1, b2, b3, b4, b5, b6, b7
+ eor \b0, \b0, \b6
+ eor \b1, \b1, \b4
+ eor \b4, \b4, \b6
+ eor \b2, \b2, \b0
+ eor \b6, \b6, \b1
+ eor \b1, \b1, \b5
+ eor \b5, \b5, \b3
+ eor \b3, \b3, \b7
+ eor \b7, \b7, \b5
+ eor \b2, \b2, \b5
+ eor \b4, \b4, \b7
+ .endm
+
+ .macro inv_in_bs_ch, b6, b1, b2, b4, b7, b0, b3, b5
+ eor \b1, \b1, \b7
+ eor \b4, \b4, \b7
+ eor \b7, \b7, \b5
+ eor \b1, \b1, \b3
+ eor \b2, \b2, \b5
+ eor \b3, \b3, \b7
+ eor \b6, \b6, \b1
+ eor \b2, \b2, \b0
+ eor \b5, \b5, \b3
+ eor \b4, \b4, \b6
+ eor \b0, \b0, \b6
+ eor \b1, \b1, \b4
+ .endm
+
+ .macro inv_out_bs_ch, b6, b5, b0, b3, b7, b1, b4, b2
+ eor \b1, \b1, \b5
+ eor \b2, \b2, \b7
+ eor \b3, \b3, \b1
+ eor \b4, \b4, \b5
+ eor \b7, \b7, \b5
+ eor \b3, \b3, \b4
+ eor \b5, \b5, \b0
+ eor \b3, \b3, \b7
+ eor \b6, \b6, \b2
+ eor \b2, \b2, \b1
+ eor \b6, \b6, \b3
+ eor \b3, \b3, \b0
+ eor \b5, \b5, \b6
+ .endm
+
+ .macro mul_gf4, x0, x1, y0, y1, t0, t1
+ eor \t0, \y0, \y1
+ and \t0, \t0, \x0
+ eor \x0, \x0, \x1
+ and \t1, \x1, \y0
+ and \x0, \x0, \y1
+ eor \x1, \t1, \t0
+ eor \x0, \x0, \t1
+ .endm
+
+ .macro mul_gf4_n_gf4, x0, x1, y0, y1, t0, x2, x3, y2, y3, t1
+ eor \t0, \y0, \y1
+ eor \t1, \y2, \y3
+ and \t0, \t0, \x0
+ and \t1, \t1, \x2
+ eor \x0, \x0, \x1
+ eor \x2, \x2, \x3
+ and \x1, \x1, \y0
+ and \x3, \x3, \y2
+ and \x0, \x0, \y1
+ and \x2, \x2, \y3
+ eor \x1, \x1, \x0
+ eor \x2, \x2, \x3
+ eor \x0, \x0, \t0
+ eor \x3, \x3, \t1
+ .endm
+
+ .macro mul_gf16_2, x0, x1, x2, x3, x4, x5, x6, x7, \
+ y0, y1, y2, y3, t0, t1, t2, t3
+ eor \t0, \x0, \x2
+ eor \t1, \x1, \x3
+ mul_gf4 \x0, \x1, \y0, \y1, \t2, \t3
+ eor \y0, \y0, \y2
+ eor \y1, \y1, \y3
+ mul_gf4_n_gf4 \t0, \t1, \y0, \y1, \t3, \x2, \x3, \y2, \y3, \t2
+ eor \x0, \x0, \t0
+ eor \x2, \x2, \t0
+ eor \x1, \x1, \t1
+ eor \x3, \x3, \t1
+ eor \t0, \x4, \x6
+ eor \t1, \x5, \x7
+ mul_gf4_n_gf4 \t0, \t1, \y0, \y1, \t3, \x6, \x7, \y2, \y3, \t2
+ eor \y0, \y0, \y2
+ eor \y1, \y1, \y3
+ mul_gf4 \x4, \x5, \y0, \y1, \t2, \t3
+ eor \x4, \x4, \t0
+ eor \x6, \x6, \t0
+ eor \x5, \x5, \t1
+ eor \x7, \x7, \t1
+ .endm
+
+ .macro inv_gf256, x0, x1, x2, x3, x4, x5, x6, x7, \
+ t0, t1, t2, t3, s0, s1, s2, s3
+ eor \t3, \x4, \x6
+ eor \t0, \x5, \x7
+ eor \t1, \x1, \x3
+ eor \s1, \x7, \x6
+ eor \s0, \x0, \x2
+ eor \s3, \t3, \t0
+ orr \t2, \t0, \t1
+ and \s2, \t3, \s0
+ orr \t3, \t3, \s0
+ eor \s0, \s0, \t1
+ and \t0, \t0, \t1
+ eor \t1, \x3, \x2
+ and \s3, \s3, \s0
+ and \s1, \s1, \t1
+ eor \t1, \x4, \x5
+ eor \s0, \x1, \x0
+ eor \t3, \t3, \s1
+ eor \t2, \t2, \s1
+ and \s1, \t1, \s0
+ orr \t1, \t1, \s0
+ eor \t3, \t3, \s3
+ eor \t0, \t0, \s1
+ eor \t2, \t2, \s2
+ eor \t1, \t1, \s3
+ eor \t0, \t0, \s2
+ and \s0, \x7, \x3
+ eor \t1, \t1, \s2
+ and \s1, \x6, \x2
+ and \s2, \x5, \x1
+ orr \s3, \x4, \x0
+ eor \t3, \t3, \s0
+ eor \t1, \t1, \s2
+ eor \s0, \t0, \s3
+ eor \t2, \t2, \s1
+ and \s2, \t3, \t1
+ eor \s1, \t2, \s2
+ eor \s3, \s0, \s2
+ bsl \s1, \t1, \s0
+ not \t0, \s0
+ bsl \s0, \s1, \s3
+ bsl \t0, \s1, \s3
+ bsl \s3, \t3, \t2
+ eor \t3, \t3, \t2
+ and \s2, \s0, \s3
+ eor \t1, \t1, \t0
+ eor \s2, \s2, \t3
+ mul_gf16_2 \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
+ \s3, \s2, \s1, \t1, \s0, \t0, \t2, \t3
+ .endm
+
+ .macro sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
+ t0, t1, t2, t3, s0, s1, s2, s3
+ in_bs_ch \b0\().16b, \b1\().16b, \b2\().16b, \b3\().16b, \
+ \b4\().16b, \b5\().16b, \b6\().16b, \b7\().16b
+ inv_gf256 \b6\().16b, \b5\().16b, \b0\().16b, \b3\().16b, \
+ \b7\().16b, \b1\().16b, \b4\().16b, \b2\().16b, \
+ \t0\().16b, \t1\().16b, \t2\().16b, \t3\().16b, \
+ \s0\().16b, \s1\().16b, \s2\().16b, \s3\().16b
+ out_bs_ch \b7\().16b, \b1\().16b, \b4\().16b, \b2\().16b, \
+ \b6\().16b, \b5\().16b, \b0\().16b, \b3\().16b
+ .endm
+
+ .macro inv_sbox, b0, b1, b2, b3, b4, b5, b6, b7, \
+ t0, t1, t2, t3, s0, s1, s2, s3
+ inv_in_bs_ch \b0\().16b, \b1\().16b, \b2\().16b, \b3\().16b, \
+ \b4\().16b, \b5\().16b, \b6\().16b, \b7\().16b
+ inv_gf256 \b5\().16b, \b1\().16b, \b2\().16b, \b6\().16b, \
+ \b3\().16b, \b7\().16b, \b0\().16b, \b4\().16b, \
+ \t0\().16b, \t1\().16b, \t2\().16b, \t3\().16b, \
+ \s0\().16b, \s1\().16b, \s2\().16b, \s3\().16b
+ inv_out_bs_ch \b3\().16b, \b7\().16b, \b0\().16b, \b4\().16b, \
+ \b5\().16b, \b1\().16b, \b2\().16b, \b6\().16b
+ .endm
+
+ .macro enc_next_rk
+ ldp q16, q17, [bskey], #128
+ ldp q18, q19, [bskey, #-96]
+ ldp q20, q21, [bskey, #-64]
+ ldp q22, q23, [bskey, #-32]
+ .endm
+
+ .macro dec_next_rk
+ ldp q16, q17, [bskey, #-128]!
+ ldp q18, q19, [bskey, #32]
+ ldp q20, q21, [bskey, #64]
+ ldp q22, q23, [bskey, #96]
+ .endm
+
+ .macro add_round_key, x0, x1, x2, x3, x4, x5, x6, x7
+ eor \x0\().16b, \x0\().16b, v16.16b
+ eor \x1\().16b, \x1\().16b, v17.16b
+ eor \x2\().16b, \x2\().16b, v18.16b
+ eor \x3\().16b, \x3\().16b, v19.16b
+ eor \x4\().16b, \x4\().16b, v20.16b
+ eor \x5\().16b, \x5\().16b, v21.16b
+ eor \x6\().16b, \x6\().16b, v22.16b
+ eor \x7\().16b, \x7\().16b, v23.16b
+ .endm
+
+ .macro shift_rows, x0, x1, x2, x3, x4, x5, x6, x7, mask
+ tbl \x0\().16b, {\x0\().16b}, \mask\().16b
+ tbl \x1\().16b, {\x1\().16b}, \mask\().16b
+ tbl \x2\().16b, {\x2\().16b}, \mask\().16b
+ tbl \x3\().16b, {\x3\().16b}, \mask\().16b
+ tbl \x4\().16b, {\x4\().16b}, \mask\().16b
+ tbl \x5\().16b, {\x5\().16b}, \mask\().16b
+ tbl \x6\().16b, {\x6\().16b}, \mask\().16b
+ tbl \x7\().16b, {\x7\().16b}, \mask\().16b
+ .endm
+
+ .macro mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
+ t0, t1, t2, t3, t4, t5, t6, t7, inv
+ ext \t0\().16b, \x0\().16b, \x0\().16b, #12
+ ext \t1\().16b, \x1\().16b, \x1\().16b, #12
+ eor \x0\().16b, \x0\().16b, \t0\().16b
+ ext \t2\().16b, \x2\().16b, \x2\().16b, #12
+ eor \x1\().16b, \x1\().16b, \t1\().16b
+ ext \t3\().16b, \x3\().16b, \x3\().16b, #12
+ eor \x2\().16b, \x2\().16b, \t2\().16b
+ ext \t4\().16b, \x4\().16b, \x4\().16b, #12
+ eor \x3\().16b, \x3\().16b, \t3\().16b
+ ext \t5\().16b, \x5\().16b, \x5\().16b, #12
+ eor \x4\().16b, \x4\().16b, \t4\().16b
+ ext \t6\().16b, \x6\().16b, \x6\().16b, #12
+ eor \x5\().16b, \x5\().16b, \t5\().16b
+ ext \t7\().16b, \x7\().16b, \x7\().16b, #12
+ eor \x6\().16b, \x6\().16b, \t6\().16b
+ eor \t1\().16b, \t1\().16b, \x0\().16b
+ eor \x7\().16b, \x7\().16b, \t7\().16b
+ ext \x0\().16b, \x0\().16b, \x0\().16b, #8
+ eor \t2\().16b, \t2\().16b, \x1\().16b
+ eor \t0\().16b, \t0\().16b, \x7\().16b
+ eor \t1\().16b, \t1\().16b, \x7\().16b
+ ext \x1\().16b, \x1\().16b, \x1\().16b, #8
+ eor \t5\().16b, \t5\().16b, \x4\().16b
+ eor \x0\().16b, \x0\().16b, \t0\().16b
+ eor \t6\().16b, \t6\().16b, \x5\().16b
+ eor \x1\().16b, \x1\().16b, \t1\().16b
+ ext \t0\().16b, \x4\().16b, \x4\().16b, #8
+ eor \t4\().16b, \t4\().16b, \x3\().16b
+ ext \t1\().16b, \x5\().16b, \x5\().16b, #8
+ eor \t7\().16b, \t7\().16b, \x6\().16b
+ ext \x4\().16b, \x3\().16b, \x3\().16b, #8
+ eor \t3\().16b, \t3\().16b, \x2\().16b
+ ext \x5\().16b, \x7\().16b, \x7\().16b, #8
+ eor \t4\().16b, \t4\().16b, \x7\().16b
+ ext \x3\().16b, \x6\().16b, \x6\().16b, #8
+ eor \t3\().16b, \t3\().16b, \x7\().16b
+ ext \x6\().16b, \x2\().16b, \x2\().16b, #8
+ eor \x7\().16b, \t1\().16b, \t5\().16b
+ .ifb \inv
+ eor \x2\().16b, \t0\().16b, \t4\().16b
+ eor \x4\().16b, \x4\().16b, \t3\().16b
+ eor \x5\().16b, \x5\().16b, \t7\().16b
+ eor \x3\().16b, \x3\().16b, \t6\().16b
+ eor \x6\().16b, \x6\().16b, \t2\().16b
+ .else
+ eor \t3\().16b, \t3\().16b, \x4\().16b
+ eor \x5\().16b, \x5\().16b, \t7\().16b
+ eor \x2\().16b, \x3\().16b, \t6\().16b
+ eor \x3\().16b, \t0\().16b, \t4\().16b
+ eor \x4\().16b, \x6\().16b, \t2\().16b
+ mov \x6\().16b, \t3\().16b
+ .endif
+ .endm
+
+ .macro inv_mix_cols, x0, x1, x2, x3, x4, x5, x6, x7, \
+ t0, t1, t2, t3, t4, t5, t6, t7
+ ext \t0\().16b, \x0\().16b, \x0\().16b, #8
+ ext \t6\().16b, \x6\().16b, \x6\().16b, #8
+ ext \t7\().16b, \x7\().16b, \x7\().16b, #8
+ eor \t0\().16b, \t0\().16b, \x0\().16b
+ ext \t1\().16b, \x1\().16b, \x1\().16b, #8
+ eor \t6\().16b, \t6\().16b, \x6\().16b
+ ext \t2\().16b, \x2\().16b, \x2\().16b, #8
+ eor \t7\().16b, \t7\().16b, \x7\().16b
+ ext \t3\().16b, \x3\().16b, \x3\().16b, #8
+ eor \t1\().16b, \t1\().16b, \x1\().16b
+ ext \t4\().16b, \x4\().16b, \x4\().16b, #8
+ eor \t2\().16b, \t2\().16b, \x2\().16b
+ ext \t5\().16b, \x5\().16b, \x5\().16b, #8
+ eor \t3\().16b, \t3\().16b, \x3\().16b
+ eor \t4\().16b, \t4\().16b, \x4\().16b
+ eor \t5\().16b, \t5\().16b, \x5\().16b
+ eor \x0\().16b, \x0\().16b, \t6\().16b
+ eor \x1\().16b, \x1\().16b, \t6\().16b
+ eor \x2\().16b, \x2\().16b, \t0\().16b
+ eor \x4\().16b, \x4\().16b, \t2\().16b
+ eor \x3\().16b, \x3\().16b, \t1\().16b
+ eor \x1\().16b, \x1\().16b, \t7\().16b
+ eor \x2\().16b, \x2\().16b, \t7\().16b
+ eor \x4\().16b, \x4\().16b, \t6\().16b
+ eor \x5\().16b, \x5\().16b, \t3\().16b
+ eor \x3\().16b, \x3\().16b, \t6\().16b
+ eor \x6\().16b, \x6\().16b, \t4\().16b
+ eor \x4\().16b, \x4\().16b, \t7\().16b
+ eor \x5\().16b, \x5\().16b, \t7\().16b
+ eor \x7\().16b, \x7\().16b, \t5\().16b
+ mix_cols \x0, \x1, \x2, \x3, \x4, \x5, \x6, \x7, \
+ \t0, \t1, \t2, \t3, \t4, \t5, \t6, \t7, 1
+ .endm
+
+ .macro swapmove_2x, a0, b0, a1, b1, n, mask, t0, t1
+ ushr \t0\().2d, \b0\().2d, #\n
+ ushr \t1\().2d, \b1\().2d, #\n
+ eor \t0\().16b, \t0\().16b, \a0\().16b
+ eor \t1\().16b, \t1\().16b, \a1\().16b
+ and \t0\().16b, \t0\().16b, \mask\().16b
+ and \t1\().16b, \t1\().16b, \mask\().16b
+ eor \a0\().16b, \a0\().16b, \t0\().16b
+ shl \t0\().2d, \t0\().2d, #\n
+ eor \a1\().16b, \a1\().16b, \t1\().16b
+ shl \t1\().2d, \t1\().2d, #\n
+ eor \b0\().16b, \b0\().16b, \t0\().16b
+ eor \b1\().16b, \b1\().16b, \t1\().16b
+ .endm
+
+ .macro bitslice, x7, x6, x5, x4, x3, x2, x1, x0, t0, t1, t2, t3
+ movi \t0\().16b, #0x55
+ movi \t1\().16b, #0x33
+ swapmove_2x \x0, \x1, \x2, \x3, 1, \t0, \t2, \t3
+ swapmove_2x \x4, \x5, \x6, \x7, 1, \t0, \t2, \t3
+ movi \t0\().16b, #0x0f
+ swapmove_2x \x0, \x2, \x1, \x3, 2, \t1, \t2, \t3
+ swapmove_2x \x4, \x6, \x5, \x7, 2, \t1, \t2, \t3
+ swapmove_2x \x0, \x4, \x1, \x5, 4, \t0, \t2, \t3
+ swapmove_2x \x2, \x6, \x3, \x7, 4, \t0, \t2, \t3
+ .endm
+
+
+ .align 6
+M0: .octa 0x0004080c0105090d02060a0e03070b0f
+
+M0SR: .octa 0x0004080c05090d010a0e02060f03070b
+SR: .octa 0x0f0e0d0c0a09080b0504070600030201
+SRM0: .octa 0x01060b0c0207080d0304090e00050a0f
+
+M0ISR: .octa 0x0004080c0d0105090a0e0206070b0f03
+ISR: .octa 0x0f0e0d0c080b0a090504070602010003
+ISRM0: .octa 0x0306090c00070a0d01040b0e0205080f
+
+ /*
+ * void aesbs_convert_key(u8 out[], u32 const rk[], int rounds)
+ */
+ENTRY(aesbs_convert_key)
+ ld1 {v7.4s}, [x1], #16 // load round 0 key
+ ld1 {v17.4s}, [x1], #16 // load round 1 key
+
+ movi v8.16b, #0x01 // bit masks
+ movi v9.16b, #0x02
+ movi v10.16b, #0x04
+ movi v11.16b, #0x08
+ movi v12.16b, #0x10
+ movi v13.16b, #0x20
+ movi v14.16b, #0x40
+ movi v15.16b, #0x80
+ ldr q16, M0
+
+ sub x2, x2, #1
+ str q7, [x0], #16 // save round 0 key
+
+.Lkey_loop:
+ tbl v7.16b ,{v17.16b}, v16.16b
+ ld1 {v17.4s}, [x1], #16 // load next round key
+
+ cmtst v0.16b, v7.16b, v8.16b
+ cmtst v1.16b, v7.16b, v9.16b
+ cmtst v2.16b, v7.16b, v10.16b
+ cmtst v3.16b, v7.16b, v11.16b
+ cmtst v4.16b, v7.16b, v12.16b
+ cmtst v5.16b, v7.16b, v13.16b
+ cmtst v6.16b, v7.16b, v14.16b
+ cmtst v7.16b, v7.16b, v15.16b
+ not v0.16b, v0.16b
+ not v1.16b, v1.16b
+ not v5.16b, v5.16b
+ not v6.16b, v6.16b
+
+ subs x2, x2, #1
+ stp q0, q1, [x0], #128
+ stp q2, q3, [x0, #-96]
+ stp q4, q5, [x0, #-64]
+ stp q6, q7, [x0, #-32]
+ b.ne .Lkey_loop
+
+ movi v7.16b, #0x63 // compose .L63
+ eor v17.16b, v17.16b, v7.16b
+ str q17, [x0]
+ ret
+ENDPROC(aesbs_convert_key)
+
+ .align 4
+aesbs_encrypt8:
+ ldr q9, [bskey], #16 // round 0 key
+ ldr q8, M0SR
+ ldr q24, SR
+
+ eor v10.16b, v0.16b, v9.16b // xor with round0 key
+ eor v11.16b, v1.16b, v9.16b
+ tbl v0.16b, {v10.16b}, v8.16b
+ eor v12.16b, v2.16b, v9.16b
+ tbl v1.16b, {v11.16b}, v8.16b
+ eor v13.16b, v3.16b, v9.16b
+ tbl v2.16b, {v12.16b}, v8.16b
+ eor v14.16b, v4.16b, v9.16b
+ tbl v3.16b, {v13.16b}, v8.16b
+ eor v15.16b, v5.16b, v9.16b
+ tbl v4.16b, {v14.16b}, v8.16b
+ eor v10.16b, v6.16b, v9.16b
+ tbl v5.16b, {v15.16b}, v8.16b
+ eor v11.16b, v7.16b, v9.16b
+ tbl v6.16b, {v10.16b}, v8.16b
+ tbl v7.16b, {v11.16b}, v8.16b
+
+ bitslice v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11
+
+ sub rounds, rounds, #1
+ b .Lenc_sbox
+
+.Lenc_loop:
+ shift_rows v0, v1, v2, v3, v4, v5, v6, v7, v24
+.Lenc_sbox:
+ sbox v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, \
+ v13, v14, v15
+ subs rounds, rounds, #1
+ b.cc .Lenc_done
+
+ enc_next_rk
+
+ mix_cols v0, v1, v4, v6, v3, v7, v2, v5, v8, v9, v10, v11, v12, \
+ v13, v14, v15
+
+ add_round_key v0, v1, v2, v3, v4, v5, v6, v7
+
+ b.ne .Lenc_loop
+ ldr q24, SRM0
+ b .Lenc_loop
+
+.Lenc_done:
+ ldr q12, [bskey] // last round key
+
+ bitslice v0, v1, v4, v6, v3, v7, v2, v5, v8, v9, v10, v11
+
+ eor v0.16b, v0.16b, v12.16b
+ eor v1.16b, v1.16b, v12.16b
+ eor v4.16b, v4.16b, v12.16b
+ eor v6.16b, v6.16b, v12.16b
+ eor v3.16b, v3.16b, v12.16b
+ eor v7.16b, v7.16b, v12.16b
+ eor v2.16b, v2.16b, v12.16b
+ eor v5.16b, v5.16b, v12.16b
+ ret
+ENDPROC(aesbs_encrypt8)
+
+ .align 4
+aesbs_decrypt8:
+ lsl x9, rounds, #7
+ add bskey, bskey, x9
+
+ ldr q9, [bskey, #-112]! // round 0 key
+ ldr q8, M0ISR
+ ldr q24, ISR
+
+ eor v10.16b, v0.16b, v9.16b // xor with round0 key
+ eor v11.16b, v1.16b, v9.16b
+ tbl v0.16b, {v10.16b}, v8.16b
+ eor v12.16b, v2.16b, v9.16b
+ tbl v1.16b, {v11.16b}, v8.16b
+ eor v13.16b, v3.16b, v9.16b
+ tbl v2.16b, {v12.16b}, v8.16b
+ eor v14.16b, v4.16b, v9.16b
+ tbl v3.16b, {v13.16b}, v8.16b
+ eor v15.16b, v5.16b, v9.16b
+ tbl v4.16b, {v14.16b}, v8.16b
+ eor v10.16b, v6.16b, v9.16b
+ tbl v5.16b, {v15.16b}, v8.16b
+ eor v11.16b, v7.16b, v9.16b
+ tbl v6.16b, {v10.16b}, v8.16b
+ tbl v7.16b, {v11.16b}, v8.16b
+
+ bitslice v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11
+
+ sub rounds, rounds, #1
+ b .Ldec_sbox
+
+.Ldec_loop:
+ shift_rows v0, v1, v2, v3, v4, v5, v6, v7, v24
+.Ldec_sbox:
+ inv_sbox v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, \
+ v13, v14, v15
+ subs rounds, rounds, #1
+ b.cc .Ldec_done
+
+ dec_next_rk
+
+ add_round_key v0, v1, v6, v4, v2, v7, v3, v5
+
+ inv_mix_cols v0, v1, v6, v4, v2, v7, v3, v5, v8, v9, v10, v11, v12, \
+ v13, v14, v15
+
+ b.ne .Ldec_loop
+ ldr q24, ISRM0
+ b .Ldec_loop
+.Ldec_done:
+ ldr q12, [bskey, #-16] // last round key
+
+ bitslice v0, v1, v6, v4, v2, v7, v3, v5, v8, v9, v10, v11
+
+ eor v0.16b, v0.16b, v12.16b
+ eor v1.16b, v1.16b, v12.16b
+ eor v6.16b, v6.16b, v12.16b
+ eor v4.16b, v4.16b, v12.16b
+ eor v2.16b, v2.16b, v12.16b
+ eor v7.16b, v7.16b, v12.16b
+ eor v3.16b, v3.16b, v12.16b
+ eor v5.16b, v5.16b, v12.16b
+ ret
+ENDPROC(aesbs_decrypt8)
+
+ /*
+ * aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+ * int blocks)
+ * aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+ * int blocks)
+ */
+ .macro __ecb_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
+ stp x29, x30, [sp, #-16]!
+ mov x29, sp
+
+99: mov x5, #1
+ lsl x5, x5, x4
+ subs w4, w4, #8
+ csel x4, x4, xzr, pl
+ csel x5, x5, xzr, mi
+
+ ld1 {v0.16b}, [x1], #16
+ tbnz x5, #1, 0f
+ ld1 {v1.16b}, [x1], #16
+ tbnz x5, #2, 0f
+ ld1 {v2.16b}, [x1], #16
+ tbnz x5, #3, 0f
+ ld1 {v3.16b}, [x1], #16
+ tbnz x5, #4, 0f
+ ld1 {v4.16b}, [x1], #16
+ tbnz x5, #5, 0f
+ ld1 {v5.16b}, [x1], #16
+ tbnz x5, #6, 0f
+ ld1 {v6.16b}, [x1], #16
+ tbnz x5, #7, 0f
+ ld1 {v7.16b}, [x1], #16
+
+0: mov bskey, x2
+ mov rounds, x3
+ bl \do8
+
+ st1 {\o0\().16b}, [x0], #16
+ tbnz x5, #1, 1f
+ st1 {\o1\().16b}, [x0], #16
+ tbnz x5, #2, 1f
+ st1 {\o2\().16b}, [x0], #16
+ tbnz x5, #3, 1f
+ st1 {\o3\().16b}, [x0], #16
+ tbnz x5, #4, 1f
+ st1 {\o4\().16b}, [x0], #16
+ tbnz x5, #5, 1f
+ st1 {\o5\().16b}, [x0], #16
+ tbnz x5, #6, 1f
+ st1 {\o6\().16b}, [x0], #16
+ tbnz x5, #7, 1f
+ st1 {\o7\().16b}, [x0], #16
+
+ cbnz x4, 99b
+
+1: ldp x29, x30, [sp], #16
+ ret
+ .endm
+
+ .align 4
+ENTRY(aesbs_ecb_encrypt)
+ __ecb_crypt aesbs_encrypt8, v0, v1, v4, v6, v3, v7, v2, v5
+ENDPROC(aesbs_ecb_encrypt)
+
+ .align 4
+ENTRY(aesbs_ecb_decrypt)
+ __ecb_crypt aesbs_decrypt8, v0, v1, v6, v4, v2, v7, v3, v5
+ENDPROC(aesbs_ecb_decrypt)
+
+ .macro next_tweak, out, in, const, tmp
+ sshr \tmp\().2d, \in\().2d, #63
+ and \tmp\().16b, \tmp\().16b, \const\().16b
+ add \out\().2d, \in\().2d, \in\().2d
+ ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8
+ eor \out\().16b, \out\().16b, \tmp\().16b
+ .endm
+
+ .align 4
+.Lxts_mul_x:
+CPU_LE( .quad 1, 0x87 )
+CPU_BE( .quad 0x87, 1 )
+
+ /*
+ * aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+ * int blocks, u8 iv[])
+ * aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
+ * int blocks, u8 iv[])
+ */
+__xts_crypt8:
+ mov x6, #1
+ lsl x6, x6, x4
+ subs w4, w4, #8
+ csel x4, x4, xzr, pl
+ csel x6, x6, xzr, mi
+
+ ld1 {v0.16b}, [x1], #16
+ next_tweak v26, v25, v30, v31
+ eor v0.16b, v0.16b, v25.16b
+ tbnz x6, #1, 0f
+
+ ld1 {v1.16b}, [x1], #16
+ next_tweak v27, v26, v30, v31
+ eor v1.16b, v1.16b, v26.16b
+ tbnz x6, #2, 0f
+
+ ld1 {v2.16b}, [x1], #16
+ next_tweak v28, v27, v30, v31
+ eor v2.16b, v2.16b, v27.16b
+ tbnz x6, #3, 0f
+
+ ld1 {v3.16b}, [x1], #16
+ next_tweak v29, v28, v30, v31
+ eor v3.16b, v3.16b, v28.16b
+ tbnz x6, #4, 0f
+
+ ld1 {v4.16b}, [x1], #16
+ str q29, [sp, #16]
+ eor v4.16b, v4.16b, v29.16b
+ next_tweak v29, v29, v30, v31
+ tbnz x6, #5, 0f
+
+ ld1 {v5.16b}, [x1], #16
+ str q29, [sp, #32]
+ eor v5.16b, v5.16b, v29.16b
+ next_tweak v29, v29, v30, v31
+ tbnz x6, #6, 0f
+
+ ld1 {v6.16b}, [x1], #16
+ str q29, [sp, #48]
+ eor v6.16b, v6.16b, v29.16b
+ next_tweak v29, v29, v30, v31
+ tbnz x6, #7, 0f
+
+ ld1 {v7.16b}, [x1], #16
+ str q29, [sp, #64]
+ eor v7.16b, v7.16b, v29.16b
+ next_tweak v29, v29, v30, v31
+
+0: mov bskey, x2
+ mov rounds, x3
+ br x7
+ENDPROC(__xts_crypt8)
+
+ .macro __xts_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
+ stp x29, x30, [sp, #-80]!
+ mov x29, sp
+
+ ldr q30, .Lxts_mul_x
+ ld1 {v25.16b}, [x5]
+
+99: adr x7, \do8
+ bl __xts_crypt8
+
+ ldp q16, q17, [sp, #16]
+ ldp q18, q19, [sp, #48]
+
+ eor \o0\().16b, \o0\().16b, v25.16b
+ eor \o1\().16b, \o1\().16b, v26.16b
+ eor \o2\().16b, \o2\().16b, v27.16b
+ eor \o3\().16b, \o3\().16b, v28.16b
+
+ st1 {\o0\().16b}, [x0], #16
+ mov v25.16b, v26.16b
+ tbnz x6, #1, 1f
+ st1 {\o1\().16b}, [x0], #16
+ mov v25.16b, v27.16b
+ tbnz x6, #2, 1f
+ st1 {\o2\().16b}, [x0], #16
+ mov v25.16b, v28.16b
+ tbnz x6, #3, 1f
+ st1 {\o3\().16b}, [x0], #16
+ mov v25.16b, v29.16b
+ tbnz x6, #4, 1f
+
+ eor \o4\().16b, \o4\().16b, v16.16b
+ eor \o5\().16b, \o5\().16b, v17.16b
+ eor \o6\().16b, \o6\().16b, v18.16b
+ eor \o7\().16b, \o7\().16b, v19.16b
+
+ st1 {\o4\().16b}, [x0], #16
+ tbnz x6, #5, 1f
+ st1 {\o5\().16b}, [x0], #16
+ tbnz x6, #6, 1f
+ st1 {\o6\().16b}, [x0], #16
+ tbnz x6, #7, 1f
+ st1 {\o7\().16b}, [x0], #16
+
+ cbnz x4, 99b
+
+1: st1 {v25.16b}, [x5]
+ ldp x29, x30, [sp], #80
+ ret
+ .endm
+
+ENTRY(aesbs_xts_encrypt)
+ __xts_crypt aesbs_encrypt8, v0, v1, v4, v6, v3, v7, v2, v5
+ENDPROC(aesbs_xts_encrypt)
+
+ENTRY(aesbs_xts_decrypt)
+ __xts_crypt aesbs_decrypt8, v0, v1, v6, v4, v2, v7, v3, v5
+ENDPROC(aesbs_xts_decrypt)
+
+ .macro next_ctr, v
+ mov \v\().d[1], x8
+ adds x8, x8, #1
+ mov \v\().d[0], x7
+ adc x7, x7, xzr
+ rev64 \v\().16b, \v\().16b
+ .endm
+
+ /*
+ * aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
+ * int rounds, int blocks, u8 iv[], bool final)
+ */
+ENTRY(aesbs_ctr_encrypt)
+ stp x29, x30, [sp, #-16]!
+ mov x29, sp
+
+ add x4, x4, x6 // do one extra block if final
+
+ ldp x7, x8, [x5]
+ ld1 {v0.16b}, [x5]
+CPU_LE( rev x7, x7 )
+CPU_LE( rev x8, x8 )
+ adds x8, x8, #1
+ adc x7, x7, xzr
+
+99: mov x9, #1
+ lsl x9, x9, x4
+ subs w4, w4, #8
+ csel x4, x4, xzr, pl
+ csel x9, x9, xzr, le
+
+ next_ctr v1
+ next_ctr v2
+ next_ctr v3
+ next_ctr v4
+ next_ctr v5
+ next_ctr v6
+ next_ctr v7
+
+0: mov bskey, x2
+ mov rounds, x3
+ bl aesbs_encrypt8
+
+ lsr x9, x9, x6 // disregard the extra block
+ tbnz x9, #0, 0f
+
+ ld1 {v8.16b}, [x1], #16
+ eor v0.16b, v0.16b, v8.16b
+ st1 {v0.16b}, [x0], #16
+ tbnz x9, #1, 1f
+
+ ld1 {v9.16b}, [x1], #16
+ eor v1.16b, v1.16b, v9.16b
+ st1 {v1.16b}, [x0], #16
+ tbnz x9, #2, 2f
+
+ ld1 {v10.16b}, [x1], #16
+ eor v4.16b, v4.16b, v10.16b
+ st1 {v4.16b}, [x0], #16
+ tbnz x9, #3, 3f
+
+ ld1 {v11.16b}, [x1], #16
+ eor v6.16b, v6.16b, v11.16b
+ st1 {v6.16b}, [x0], #16
+ tbnz x9, #4, 4f
+
+ ld1 {v12.16b}, [x1], #16
+ eor v3.16b, v3.16b, v12.16b
+ st1 {v3.16b}, [x0], #16
+ tbnz x9, #5, 5f
+
+ ld1 {v13.16b}, [x1], #16
+ eor v7.16b, v7.16b, v13.16b
+ st1 {v7.16b}, [x0], #16
+ tbnz x9, #6, 6f
+
+ ld1 {v14.16b}, [x1], #16
+ eor v2.16b, v2.16b, v14.16b
+ st1 {v2.16b}, [x0], #16
+ tbnz x9, #7, 7f
+
+ ld1 {v15.16b}, [x1], #16
+ eor v5.16b, v5.16b, v15.16b
+ st1 {v5.16b}, [x0], #16
+
+ next_ctr v0
+ cbnz x4, 99b
+
+0: st1 {v0.16b}, [x5]
+8: ldp x29, x30, [sp], #16
+ ret
+
+ /*
+ * If we are handling the tail of the input (x6 == 1), return the
+ * final keystream block back to the caller via the IV buffer.
+ */
+1: cbz x6, 8b
+ st1 {v1.16b}, [x5]
+ b 8b
+2: cbz x6, 8b
+ st1 {v4.16b}, [x5]
+ b 8b
+3: cbz x6, 8b
+ st1 {v6.16b}, [x5]
+ b 8b
+4: cbz x6, 8b
+ st1 {v3.16b}, [x5]
+ b 8b
+5: cbz x6, 8b
+ st1 {v7.16b}, [x5]
+ b 8b
+6: cbz x6, 8b
+ st1 {v2.16b}, [x5]
+ b 8b
+7: cbz x6, 8b
+ st1 {v5.16b}, [x5]
+ b 8b
+ENDPROC(aesbs_ctr_encrypt)
diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c
new file mode 100644
index 000000000000..45c1862f86a7
--- /dev/null
+++ b/arch/arm64/crypto/aes-neonbs-glue.c
@@ -0,0 +1,344 @@
+/*
+ * Bit sliced AES using NEON instructions
+ *
+ * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <asm/neon.h>
+#include <crypto/aes.h>
+#include <crypto/internal/simd.h>
+#include <crypto/internal/skcipher.h>
+#include <crypto/xts.h>
+#include <linux/module.h>
+
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+
+MODULE_ALIAS_CRYPTO("ecb(aes)");
+MODULE_ALIAS_CRYPTO("ctr(aes)");
+MODULE_ALIAS_CRYPTO("xts(aes)");
+
+asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds);
+
+asmlinkage void aesbs_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks);
+asmlinkage void aesbs_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks);
+
+asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks, u8 iv[], bool final);
+
+asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks, u8 iv[]);
+asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks, u8 iv[]);
+
+struct aesbs_key {
+ u8 key[13 * (8 * AES_BLOCK_SIZE) + 32];
+} __aligned(AES_BLOCK_SIZE);
+
+struct aesbs_ctx {
+ struct aesbs_key bskey;
+ int rounds;
+};
+
+struct aesbs_xts_ctx {
+ struct crypto_aes_ctx tweak; /* keep at the beginning */
+ struct aesbs_key bskey;
+ int rounds;
+};
+
+static int aesbs_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct crypto_aes_ctx rk;
+ int err;
+
+ err = crypto_aes_expand_key(&rk, in_key, key_len);
+ if (err)
+ return err;
+
+ ctx->rounds = 6 + key_len / 4;
+
+ kernel_neon_begin();
+ aesbs_convert_key(ctx->bskey.key, rk.key_enc, ctx->rounds);
+ kernel_neon_end();
+
+ return 0;
+}
+
+static int __ecb_crypt(struct skcipher_request *req,
+ void (*fn)(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks))
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ kernel_neon_begin();
+ while (walk.nbytes >= AES_BLOCK_SIZE) {
+ unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
+
+ if (walk.nbytes < walk.total)
+ blocks = round_down(blocks,
+ walk.stride / AES_BLOCK_SIZE);
+
+ fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->bskey.key,
+ ctx->rounds, blocks);
+ err = skcipher_walk_done(&walk,
+ walk.nbytes - blocks * AES_BLOCK_SIZE);
+ }
+ kernel_neon_end();
+
+ return err;
+}
+
+static int ecb_encrypt(struct skcipher_request *req)
+{
+ return __ecb_crypt(req, aesbs_ecb_encrypt);
+}
+
+static int ecb_decrypt(struct skcipher_request *req)
+{
+ return __ecb_crypt(req, aesbs_ecb_decrypt);
+}
+
+static int ctr_encrypt(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct aesbs_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ kernel_neon_begin();
+ while (walk.nbytes > 0) {
+ unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
+ bool final = (walk.total % AES_BLOCK_SIZE) != 0;
+
+ if (walk.nbytes < walk.total) {
+ blocks = round_down(blocks,
+ walk.stride / AES_BLOCK_SIZE);
+ final = false;
+ }
+
+ aesbs_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
+ ctx->bskey.key, ctx->rounds, blocks, walk.iv,
+ final);
+
+ if (final) {
+ u8 *dst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
+ u8 *src = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
+
+ if (dst != src)
+ memcpy(dst, src, walk.total % AES_BLOCK_SIZE);
+ crypto_xor(dst, walk.iv, walk.total % AES_BLOCK_SIZE);
+
+ err = skcipher_walk_done(&walk, 0);
+ break;
+ }
+ err = skcipher_walk_done(&walk,
+ walk.nbytes - blocks * AES_BLOCK_SIZE);
+ }
+ kernel_neon_end();
+
+ return err;
+}
+
+static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct crypto_aes_ctx rk;
+ int err;
+
+ err = xts_verify_key(tfm, in_key, key_len);
+ if (err)
+ return err;
+
+ key_len /= 2;
+ err = crypto_aes_expand_key(&ctx->tweak, in_key + key_len, key_len);
+ if (err)
+ return err;
+
+ err = crypto_aes_expand_key(&rk, in_key, key_len);
+ if (err)
+ return err;
+
+ ctx->rounds = 6 + key_len / 4;
+
+ kernel_neon_begin();
+ aesbs_convert_key(ctx->bskey.key, rk.key_enc, ctx->rounds);
+ kernel_neon_end();
+
+ return 0;
+}
+
+static int __xts_crypt(struct skcipher_request *req,
+ void (*fn)(u8 out[], u8 const in[], u8 const rk[],
+ int rounds, int blocks, u8 iv[]))
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
+ struct skcipher_walk walk;
+ int err;
+
+ err = skcipher_walk_virt(&walk, req, true);
+
+ crypto_aes_encrypt(crypto_skcipher_tfm(tfm), walk.iv, walk.iv);
+
+ kernel_neon_begin();
+ while (walk.nbytes >= AES_BLOCK_SIZE) {
+ unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
+
+ if (walk.nbytes < walk.total)
+ blocks = round_down(blocks,
+ walk.stride / AES_BLOCK_SIZE);
+
+ fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->bskey.key,
+ ctx->rounds, blocks, walk.iv);
+ err = skcipher_walk_done(&walk,
+ walk.nbytes - blocks * AES_BLOCK_SIZE);
+ }
+ kernel_neon_end();
+
+ return err;
+}
+
+static int xts_encrypt(struct skcipher_request *req)
+{
+ return __xts_crypt(req, aesbs_xts_encrypt);
+}
+
+static int xts_decrypt(struct skcipher_request *req)
+{
+ return __xts_crypt(req, aesbs_xts_decrypt);
+}
+
+static struct skcipher_alg aes_algs[] = { {
+ .base.cra_name = "__ecb(aes)",
+ .base.cra_driver_name = "__ecb-aes-neonbs",
+ .base.cra_priority = 200,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct aesbs_ctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_flags = CRYPTO_ALG_INTERNAL,
+
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
+ .setkey = aesbs_setkey,
+ .encrypt = ecb_encrypt,
+ .decrypt = ecb_decrypt,
+}, {
+ .base.cra_name = "__ctr(aes)",
+ .base.cra_driver_name = "__ctr-aes-neonbs",
+ .base.cra_priority = 200,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct aesbs_ctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_flags = CRYPTO_ALG_INTERNAL,
+
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .chunksize = AES_BLOCK_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aesbs_setkey,
+ .encrypt = ctr_encrypt,
+ .decrypt = ctr_encrypt,
+}, {
+ .base.cra_name = "ctr(aes)",
+ .base.cra_driver_name = "ctr-aes-neonbs",
+ .base.cra_priority = 200 - 1,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct aesbs_ctx),
+ .base.cra_module = THIS_MODULE,
+
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .chunksize = AES_BLOCK_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aesbs_setkey,
+ .encrypt = ctr_encrypt,
+ .decrypt = ctr_encrypt,
+}, {
+ .base.cra_name = "__xts(aes)",
+ .base.cra_driver_name = "__xts-aes-neonbs",
+ .base.cra_priority = 200,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct aesbs_xts_ctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_flags = CRYPTO_ALG_INTERNAL,
+
+ .min_keysize = 2 * AES_MIN_KEY_SIZE,
+ .max_keysize = 2 * AES_MAX_KEY_SIZE,
+ .walksize = 8 * AES_BLOCK_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aesbs_xts_setkey,
+ .encrypt = xts_encrypt,
+ .decrypt = xts_decrypt,
+} };
+
+static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
+
+static void aes_exit(void)
+{
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
+ if (aes_simd_algs[i])
+ simd_skcipher_free(aes_simd_algs[i]);
+
+ crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
+}
+
+static int __init aes_init(void)
+{
+ struct simd_skcipher_alg *simd;
+ const char *basename;
+ const char *algname;
+ const char *drvname;
+ int err;
+ int i;
+
+ if (!(elf_hwcap & HWCAP_ASIMD))
+ return -ENODEV;
+
+ err = crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
+ if (err)
+ return err;
+
+ for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
+ if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
+ continue;
+
+ algname = aes_algs[i].base.cra_name + 2;
+ drvname = aes_algs[i].base.cra_driver_name + 2;
+ basename = aes_algs[i].base.cra_driver_name;
+ simd = simd_skcipher_create_compat(algname, drvname, basename);
+ err = PTR_ERR(simd);
+ if (IS_ERR(simd))
+ goto unregister_simds;
+
+ aes_simd_algs[i] = simd;
+ }
+ return 0;
+
+unregister_simds:
+ aes_exit();
+ return err;
+}
+
+module_init(aes_init);
+module_exit(aes_exit);
--
2.7.4
^ permalink raw reply related
* Re: [PATCH 1/5] ARM: wire up HWCAP2 feature bits to the CPU modalias
From: Ard Biesheuvel @ 2017-01-02 21:06 UTC (permalink / raw)
To: Russell King - ARM Linux
Cc: Steve Capper, linux-crypto@vger.kernel.org,
linux-arm-kernel@lists.infradead.org, Herbert Xu
In-Reply-To: <20161031161319.GF1041@n2100.armlinux.org.uk>
On 31 October 2016 at 16:13, Russell King - ARM Linux
<linux@armlinux.org.uk> wrote:
> On Sat, Oct 29, 2016 at 11:08:36AM +0100, Ard Biesheuvel wrote:
>> On 18 October 2016 at 11:52, Ard Biesheuvel <ard.biesheuvel@linaro.org> wrote:
>> > Wire up the generic support for exposing CPU feature bits via the
>> > modalias in /sys/device/system/cpu. This allows udev to automatically
>> > load modules for things like crypto algorithms that are implemented
>> > using optional instructions.
>> >
>> > Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
>> > ---
>> > arch/arm/Kconfig | 1 +
>> > arch/arm/include/asm/cpufeature.h | 32 ++++++++++++++++++++
>> > 2 files changed, 33 insertions(+)
>> >
>>
>> Russell,
>>
>> do you have any concerns regarding this patch? If not, I will drop it
>> into the patch system.
>
> It's still something I need to look at... I've been offline last week,
> and sort-of offline the previous week, so I'm catching up.
>
Hi Russell,
Any thoughts yet?
Thanks,
Ard.
^ permalink raw reply
* Re: [PATCH 1/5] ARM: wire up HWCAP2 feature bits to the CPU modalias
From: Russell King - ARM Linux @ 2017-01-02 23:40 UTC (permalink / raw)
To: Ard Biesheuvel
Cc: linux-arm-kernel@lists.infradead.org,
linux-crypto@vger.kernel.org, Herbert Xu, Steve Capper
In-Reply-To: <CAKv+Gu-hLD0xnLyywceJvPiB8ZkENNwWZTM--OHFvzw6MahYmw@mail.gmail.com>
On Mon, Jan 02, 2017 at 09:06:04PM +0000, Ard Biesheuvel wrote:
> On 31 October 2016 at 16:13, Russell King - ARM Linux
> <linux@armlinux.org.uk> wrote:
> > On Sat, Oct 29, 2016 at 11:08:36AM +0100, Ard Biesheuvel wrote:
> >> On 18 October 2016 at 11:52, Ard Biesheuvel <ard.biesheuvel@linaro.org> wrote:
> >> > Wire up the generic support for exposing CPU feature bits via the
> >> > modalias in /sys/device/system/cpu. This allows udev to automatically
> >> > load modules for things like crypto algorithms that are implemented
> >> > using optional instructions.
> >> >
> >> > Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
> >> > ---
> >> > arch/arm/Kconfig | 1 +
> >> > arch/arm/include/asm/cpufeature.h | 32 ++++++++++++++++++++
> >> > 2 files changed, 33 insertions(+)
> >> >
> >>
> >> Russell,
> >>
> >> do you have any concerns regarding this patch? If not, I will drop it
> >> into the patch system.
> >
> > It's still something I need to look at... I've been offline last week,
> > and sort-of offline the previous week, so I'm catching up.
> >
>
> Hi Russell,
>
> Any thoughts yet?
None, and the patch is well buried now that it'll take me a while to
find... back in mid-October? Yea, I'll have to drop everything and
go digging through my mailboxes to find it... and I'm just catching
up (again) after a week and a bit's time offline - yep, it's wonderful
timing. Sorry, no time to look at it right now, you're not the only
one wanting my attention at the moment.
Please try again in about a week's time - don't leave it a few months,
and please include the patch.
--
RMK's Patch system: http://www.armlinux.org.uk/developer/patches/
FTTC broadband for 0.8mile line: currently at 9.6Mbps down 400kbps up
according to speedtest.net.
^ permalink raw reply
* (unknown),
From: системы администратор @ 2017-01-03 6:48 UTC (permalink / raw)
внимания;
Ваши сообщения превысил лимит памяти, который составляет 5 Гб, определенных администратором, который в настоящее время работает на 10.9GB, Вы не сможете отправить или получить новую почту, пока вы повторно не проверить ваш почтовый ящик почты. Чтобы восстановить работоспособность Вашего почтового ящика, отправьте следующую информацию ниже:
имя:
Имя пользователя:
пароль:
Подтверждение пароля:
Адрес электронной почты:
телефон:
Если вы не в состоянии перепроверить сообщения, ваш почтовый ящик будет отключен!
Приносим извинения за неудобства.
Проверочный код: EN: Ru...776774990..2017
Почты технической поддержки ©2017
спасибо
системы администратор
^ permalink raw reply
* (unknown),
From: системы администратор @ 2017-01-03 6:48 UTC (permalink / raw)
внимания;
Ваши сообщения превысил лимит памяти, который составляет 5 Гб, определенных администратором, который в настоящее время работает на 10.9GB, Вы не сможете отправить или получить новую почту, пока вы повторно не проверить ваш почтовый ящик почты. Чтобы восстановить работоспособность Вашего почтового ящика, отправьте следующую информацию ниже:
имя:
Имя пользователя:
пароль:
Подтверждение пароля:
Адрес электронной почты:
телефон:
Если вы не в состоянии перепроверить сообщения, ваш почтовый ящик будет отключен!
Приносим извинения за неудобства.
Проверочный код: EN: Ru...776774990..2017
Почты технической поддержки ©2017
спасибо
системы администратор
^ permalink raw reply
* (unknown),
From: системы администратор @ 2017-01-03 6:57 UTC (permalink / raw)
внимания;
Ваши сообщения превысил лимит памяти, который составляет 5 Гб, определенных администратором, который в настоящее время работает на 10.9GB, Вы не сможете отправить или получить новую почту, пока вы повторно не проверить ваш почтовый ящик почты. Чтобы восстановить работоспособность Вашего почтового ящика, отправьте следующую информацию ниже:
имя:
Имя пользователя:
пароль:
Подтверждение пароля:
Адрес электронной почты:
телефон:
Если вы не в состоянии перепроверить сообщения, ваш почтовый ящик будет отключен!
Приносим извинения за неудобства.
Проверочный код: EN: Ru...776774990..2017
Почты технической поддержки ©2017
спасибо
системы администратор
^ permalink raw reply
* [PATCH] crypto: mediatek: don't return garbage err on successful return
From: Colin King @ 2017-01-03 13:21 UTC (permalink / raw)
To: Herbert Xu, David S . Miller, Matthias Brugger, Ryder Lee,
linux-crypto-u79uwXL29TY76Z2rM5mHXA,
linux-arm-kernel-IAPFreCvJWM7uuMidbF8XUB+6BGkLq7r,
linux-mediatek-IAPFreCvJWM7uuMidbF8XUB+6BGkLq7r
Cc: linux-kernel-u79uwXL29TY76Z2rM5mHXA
From: Colin Ian King <colin.king-Z7WLFzj8eWMS+FvcfC7Uqw@public.gmane.org>
In the case where keylen <= bs mtk_sha_setkey returns an uninitialized
return value in err. Fix this by returning 0 instead of err.
Issue detected by static analysis with cppcheck.
Signed-off-by: Colin Ian King <colin.king-Z7WLFzj8eWMS+FvcfC7Uqw@public.gmane.org>
---
drivers/crypto/mediatek/mtk-sha.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c
index 8951363..8e1b440 100644
--- a/drivers/crypto/mediatek/mtk-sha.c
+++ b/drivers/crypto/mediatek/mtk-sha.c
@@ -878,7 +878,7 @@ static int mtk_sha_setkey(struct crypto_ahash *tfm,
bctx->opad[i] ^= 0x5c;
}
- return err;
+ return 0;
}
static int mtk_sha_export(struct ahash_request *req, void *out)
--
2.10.2
^ permalink raw reply related
* Re: [RFC PATCH v2] crypto: Add IV generation algorithms
From: Gilad Ben-Yossef @ 2017-01-03 14:23 UTC (permalink / raw)
To: Binoy Jayan
Cc: Oded, Ofir, Herbert Xu, David S. Miller, linux-crypto, Mark Brown,
Arnd Bergmann, linux-kernel, Alasdair Kergon, Mike Snitzer,
dm-devel, Shaohua Li, linux-raid, Rajendra, gilad.benyossef
In-Reply-To: <1481618949-20086-2-git-send-email-binoy.jayan@linaro.org>
Hi Binoy,
On Tue, Dec 13, 2016 at 02:19:09PM +0530, Binoy Jayan wrote:
> Currently, the iv generation algorithms are implemented in dm-crypt.c.
> The goal is to move these algorithms from the dm layer to the kernel
> crypto layer by implementing them as template ciphers so they can be
> implemented in hardware for performance. As part of this patchset, the
> iv-generation code is moved from the dm layer to the crypto layer and
> adapt the dm-layer to send a whole 'bio' (as defined in the block layer)
> at a time. Each bio contains the in memory representation of physically
> contiguous disk blocks. The dm layer sets up a chained scatterlist of
> these blocks split into physically contiguous segments in memory so that
> DMA can be performed. The iv generation algorithms implemented in geniv.c
> include plain, plain64, essiv, benbi, null, lmk and tcw.
>
Good idea. I wanted to test the patch but alas it does not apply cleanly.
You seem to have a blank line at the end of files and other small
transgressions that makes checkpatch grumpy.
<snip>
Also...
>
> Not-signed-off-by: Binoy Jayan <binoy.jayan@linaro.org>
What is Not-signed-off-by ? :-)
Thanks,
Gilad Ben-Yossef
^ permalink raw reply
* Re: Geode LX AES/RNG driver triggers warning
From: One Thousand Gnomes @ 2017-01-03 14:52 UTC (permalink / raw)
To: David Gstir
Cc: linux-geode, linux-crypto, linux-kernel, teheo, prasannatsmkumar,
Richard Weinberger
In-Reply-To: <A0185338-47A4-42BE-B2D4-F1EB082593E3@sigma-star.at>
On Sat, 31 Dec 2016 00:58:54 +0100
David Gstir <david@sigma-star.at> wrote:
> Hi!
>
> I recently tested kernel v4.9 on my AMD Geode platform and noticed that its AES hardware driver triggers this warning on initialization:
...
> I narrowed it down to commit 6e9b5e76882c ("hwrng: geode - Migrate to managed API") which seems to introduce this. It looks to me like some issue between devres, the Geode hwrng and AES drivers which both use the same PCI device.
It does
> I'm no expert here, but I curious if this will cause any issues when using the hardware crypto drivers and also what's the best way to get rid of this?
Probably to create an mfd device that turns the PCI device into two MFD
devices and bind AES and hwrng one to each MFD device. Take a look in
drivers/mfd. That would also fix the uglies in mod_init for the rng
driver.
Alan
^ permalink raw reply
* Re: [PATCH 0/6] crypto: ARM/arm64 - AES and ChaCha20 updates for v4.11
From: Ard Biesheuvel @ 2017-01-03 20:01 UTC (permalink / raw)
To: linux-crypto@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org, Herbert Xu, Ard Biesheuvel
In-Reply-To: <1483381268-12987-1-git-send-email-ard.biesheuvel@linaro.org>
On 2 January 2017 at 18:21, Ard Biesheuvel <ard.biesheuvel@linaro.org> wrote:
> This series adds SIMD implementations for arm64 and ARM of ChaCha20 (*),
> and a port of the ARM bit-sliced AES algorithm to arm64, and
>
> Patch #1 is a prerequisite for the AES-XTS implementation in #6, which needs
> a secondary AES transform to generate the initial tweak.
>
Herbert,
I actually have a scalar AES implementation for arm64 which I could
use instead, making this patch unnecessary.
I could respin the entire series, or you could simply disregard #1 and
#6 for now, whichever you prefer.
Thanks,
Ard.
> Patch #2 optimizes the bit-sliced AES glue code for ARM to iterate over the
> input in the most efficient manner possible.
>
> Patch #3 adds a NEON implementation of ChaCha20 for ARM.
>
> Patch #4 adds a NEON implementation of ChaCha20 for arm64.
>
> Patch #5 modifies the existing NEON and ARMv8 Crypto Extensions implementations
> of AES-CTR to be available as a synchronous skcipher as well. This is intended
> for the mac80211 code, which uses synchronous encapsulations of ctr(aes)
> [ccm, gcm] in softirq context, which supports SIMD algorithms on arm64.
>
> Patch #6 adds a port of the ARM bit-sliced AES code to arm64, in ECB, CTR
> and XTS modes.
>
> Ard Biesheuvel (6):
> crypto: generic/aes - export encrypt and decrypt entry points
> crypto: arm/aes-neonbs - process 8 blocks in parallel if we can
> crypto: arm/chacha20 - implement NEON version based on SSE3 code
> crypto: arm64/chacha20 - implement NEON version based on SSE3 code
> crypto: arm64/aes-blk - expose AES-CTR as synchronous cipher as well
> crypto: arm64/aes - reimplement bit-sliced ARM/NEON implementation for
> arm64
>
> arch/arm/crypto/Kconfig | 6 +
> arch/arm/crypto/Makefile | 2 +
> arch/arm/crypto/aesbs-glue.c | 67 +-
> arch/arm/crypto/chacha20-neon-core.S | 524 ++++++++++++
> arch/arm/crypto/chacha20-neon-glue.c | 128 +++
> arch/arm64/crypto/Kconfig | 13 +
> arch/arm64/crypto/Makefile | 6 +
> arch/arm64/crypto/aes-glue.c | 25 +-
> arch/arm64/crypto/aes-neonbs-core.S | 879 ++++++++++++++++++++
> arch/arm64/crypto/aes-neonbs-glue.c | 344 ++++++++
> arch/arm64/crypto/chacha20-neon-core.S | 450 ++++++++++
> arch/arm64/crypto/chacha20-neon-glue.c | 127 +++
> crypto/aes_generic.c | 10 +-
> include/crypto/aes.h | 3 +
> 14 files changed, 2549 insertions(+), 35 deletions(-)
> create mode 100644 arch/arm/crypto/chacha20-neon-core.S
> create mode 100644 arch/arm/crypto/chacha20-neon-glue.c
> create mode 100644 arch/arm64/crypto/aes-neonbs-core.S
> create mode 100644 arch/arm64/crypto/aes-neonbs-glue.c
> create mode 100644 arch/arm64/crypto/chacha20-neon-core.S
> create mode 100644 arch/arm64/crypto/chacha20-neon-glue.c
>
> --
> 2.7.4
>
^ permalink raw reply
* Re: [RFC PATCH v2] crypto: Add IV generation algorithms
From: Binoy Jayan @ 2017-01-04 5:20 UTC (permalink / raw)
To: Gilad Ben-Yossef
Cc: Oded, Ofir, Herbert Xu, David S. Miller, linux-crypto, Mark Brown,
Arnd Bergmann, Linux kernel mailing list, Alasdair Kergon,
Mike Snitzer, dm-devel, Shaohua Li, linux-raid, Rajendra,
gilad.benyossef
In-Reply-To: <CAOtvUMc_X1DUuR8NTvOLjW7h-p=X14tFHFqe_dB8t08-nYBPcw@mail.gmail.com>
Hi Gilad,
On 3 January 2017 at 19:53, Gilad Ben-Yossef <gilad@benyossef.com> wrote:
> Good idea. I wanted to test the patch but alas it does not apply cleanly.
> You seem to have a blank line at the end of files and other small
> transgressions that makes checkpatch grumpy.
I think that is because there were some key structure changes in dm-crypt
after I sent out v2. I have resolved them while working on v3. Please wait for
the next version of the patchset. I'll send it probably by next week.
I wanted to incorporate a few changes suggested by Herbert before sending them.
> What is Not-signed-off-by ? :-)
It was just an RFC patch, not ready for merging.
Thanks,
Binoy
^ permalink raw reply
* Re: [PATCH] crypto: Replaced gcc specific attributes with macros from compiler.h
From: Gideon D'souza @ 2017-01-04 15:15 UTC (permalink / raw)
To: linux-kernel@vger.kernel.org, davem, nhorman, Joe Perches,
Andrew Morton, linux-crypto, Geert Uytterhoeven
Any update on this patch, should I base it on another tree, this was
based off of linus's tree right when he released 4.10-rc2
Should I send it close to the next merge window?
On Sat, Dec 31, 2016 at 9:26 PM, <gidisrael@gmail.com> wrote:
> From: Gideon Israel Dsouza <gidisrael@gmail.com>
>
> Continuing from this commit: 52f5684c8e1e
> ("kernel: use macros from compiler.h instead of __attribute__((...))")
>
> I submitted 4 total patches. They are part of task I've taken up to
> increase compiler portability in the kernel. I've cleaned up the
> subsystems under /kernel /mm /block and /security, this patch targets
> /crypto.
>
> There is <linux/compiler.h> which provides macros for various gcc specific
> constructs. Eg: __weak for __attribute__((weak)). I've cleaned all
> instances of gcc specific attributes with the right macros for the crypto
> subsystem.
>
> I had to make one additional change into compiler-gcc.h for the case when
> one wants to use this: __attribute__((aligned) and not specify an alignment
> factor. From the gcc docs, this will result in the largest alignment for
> that data type on the target machine so I've named the macro
> __aligned_largest. Please advise if another name is more appropriate.
>
> Signed-off-by: Gideon Israel Dsouza <gidisrael@gmail.com>
> ---
> crypto/ablkcipher.c | 5 +++--
> crypto/acompress.c | 3 ++-
> crypto/aead.c | 3 ++-
> crypto/ahash.c | 3 ++-
> crypto/akcipher.c | 3 ++-
> crypto/blkcipher.c | 7 ++++---
> crypto/cts.c | 5 +++--
> crypto/kpp.c | 3 ++-
> crypto/pcbc.c | 3 ++-
> crypto/rng.c | 3 ++-
> crypto/scompress.c | 3 ++-
> crypto/shash.c | 9 +++++----
> crypto/skcipher.c | 3 ++-
> include/linux/compiler-gcc.h | 1 +
> 14 files changed, 34 insertions(+), 20 deletions(-)
>
> diff --git a/crypto/ablkcipher.c b/crypto/ablkcipher.c
> index d676fc5..d880a48 100644
> --- a/crypto/ablkcipher.c
> +++ b/crypto/ablkcipher.c
> @@ -19,6 +19,7 @@
> #include <linux/slab.h>
> #include <linux/seq_file.h>
> #include <linux/cryptouser.h>
> +#include <linux/compiler.h>
> #include <net/netlink.h>
>
> #include <crypto/scatterwalk.h>
> @@ -394,7 +395,7 @@ static int crypto_ablkcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
> #endif
>
> static void crypto_ablkcipher_show(struct seq_file *m, struct crypto_alg *alg)
> - __attribute__ ((unused));
> + __maybe_unused;
> static void crypto_ablkcipher_show(struct seq_file *m, struct crypto_alg *alg)
> {
> struct ablkcipher_alg *ablkcipher = &alg->cra_ablkcipher;
> @@ -468,7 +469,7 @@ static int crypto_givcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
> #endif
>
> static void crypto_givcipher_show(struct seq_file *m, struct crypto_alg *alg)
> - __attribute__ ((unused));
> + __maybe_unused;
> static void crypto_givcipher_show(struct seq_file *m, struct crypto_alg *alg)
> {
> struct ablkcipher_alg *ablkcipher = &alg->cra_ablkcipher;
> diff --git a/crypto/acompress.c b/crypto/acompress.c
> index 887783d..47d1162 100644
> --- a/crypto/acompress.c
> +++ b/crypto/acompress.c
> @@ -20,6 +20,7 @@
> #include <linux/crypto.h>
> #include <crypto/algapi.h>
> #include <linux/cryptouser.h>
> +#include <linux/compiler.h>
> #include <net/netlink.h>
> #include <crypto/internal/acompress.h>
> #include <crypto/internal/scompress.h>
> @@ -50,7 +51,7 @@ static int crypto_acomp_report(struct sk_buff *skb, struct crypto_alg *alg)
> #endif
>
> static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg)
> - __attribute__ ((unused));
> + __maybe_unused;
>
> static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg)
> {
> diff --git a/crypto/aead.c b/crypto/aead.c
> index 3f5c5ff..f794b30 100644
> --- a/crypto/aead.c
> +++ b/crypto/aead.c
> @@ -24,6 +24,7 @@
> #include <linux/slab.h>
> #include <linux/seq_file.h>
> #include <linux/cryptouser.h>
> +#include <linux/compiler.h>
> #include <net/netlink.h>
>
> #include "internal.h"
> @@ -132,7 +133,7 @@ static int crypto_aead_report(struct sk_buff *skb, struct crypto_alg *alg)
> #endif
>
> static void crypto_aead_show(struct seq_file *m, struct crypto_alg *alg)
> - __attribute__ ((unused));
> + __maybe_unused;
> static void crypto_aead_show(struct seq_file *m, struct crypto_alg *alg)
> {
> struct aead_alg *aead = container_of(alg, struct aead_alg, base);
> diff --git a/crypto/ahash.c b/crypto/ahash.c
> index 2ce8bcb..e58c497 100644
> --- a/crypto/ahash.c
> +++ b/crypto/ahash.c
> @@ -23,6 +23,7 @@
> #include <linux/slab.h>
> #include <linux/seq_file.h>
> #include <linux/cryptouser.h>
> +#include <linux/compiler.h>
> #include <net/netlink.h>
>
> #include "internal.h"
> @@ -493,7 +494,7 @@ static int crypto_ahash_report(struct sk_buff *skb, struct crypto_alg *alg)
> #endif
>
> static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg)
> - __attribute__ ((unused));
> + __maybe_unused;
> static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg)
> {
> seq_printf(m, "type : ahash\n");
> diff --git a/crypto/akcipher.c b/crypto/akcipher.c
> index def301e..cfbdb06 100644
> --- a/crypto/akcipher.c
> +++ b/crypto/akcipher.c
> @@ -17,6 +17,7 @@
> #include <linux/slab.h>
> #include <linux/string.h>
> #include <linux/crypto.h>
> +#include <linux/compiler.h>
> #include <crypto/algapi.h>
> #include <linux/cryptouser.h>
> #include <net/netlink.h>
> @@ -47,7 +48,7 @@ static int crypto_akcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
> #endif
>
> static void crypto_akcipher_show(struct seq_file *m, struct crypto_alg *alg)
> - __attribute__ ((unused));
> + __maybe_unused;
>
> static void crypto_akcipher_show(struct seq_file *m, struct crypto_alg *alg)
> {
> diff --git a/crypto/blkcipher.c b/crypto/blkcipher.c
> index a832426..6c43a0a 100644
> --- a/crypto/blkcipher.c
> +++ b/crypto/blkcipher.c
> @@ -1,6 +1,6 @@
> /*
> * Block chaining cipher operations.
> - *
> + *
> * Generic encrypt/decrypt wrapper for ciphers, handles operations across
> * multiple page boundaries by using temporary blocks. In user context,
> * the kernel is given a chance to schedule us once per page.
> @@ -9,7 +9,7 @@
> *
> * This program is free software; you can redistribute it and/or modify it
> * under the terms of the GNU General Public License as published by the Free
> - * Software Foundation; either version 2 of the License, or (at your option)
> + * Software Foundation; either version 2 of the License, or (at your option)
> * any later version.
> *
> */
> @@ -25,6 +25,7 @@
> #include <linux/slab.h>
> #include <linux/string.h>
> #include <linux/cryptouser.h>
> +#include <linux/compiler.h>
> #include <net/netlink.h>
>
> #include "internal.h"
> @@ -534,7 +535,7 @@ static int crypto_blkcipher_report(struct sk_buff *skb, struct crypto_alg *alg)
> #endif
>
> static void crypto_blkcipher_show(struct seq_file *m, struct crypto_alg *alg)
> - __attribute__ ((unused));
> + __maybe_unused;
> static void crypto_blkcipher_show(struct seq_file *m, struct crypto_alg *alg)
> {
> seq_printf(m, "type : blkcipher\n");
> diff --git a/crypto/cts.c b/crypto/cts.c
> index 00254d7..a1335d6 100644
> --- a/crypto/cts.c
> +++ b/crypto/cts.c
> @@ -49,6 +49,7 @@
> #include <linux/scatterlist.h>
> #include <crypto/scatterwalk.h>
> #include <linux/slab.h>
> +#include <linux/compiler.h>
>
> struct crypto_cts_ctx {
> struct crypto_skcipher *child;
> @@ -103,7 +104,7 @@ static int cts_cbc_encrypt(struct skcipher_request *req)
> struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
> struct skcipher_request *subreq = &rctx->subreq;
> int bsize = crypto_skcipher_blocksize(tfm);
> - u8 d[bsize * 2] __attribute__ ((aligned(__alignof__(u32))));
> + u8 d[bsize * 2] __aligned(__alignof__(u32));
> struct scatterlist *sg;
> unsigned int offset;
> int lastn;
> @@ -183,7 +184,7 @@ static int cts_cbc_decrypt(struct skcipher_request *req)
> struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
> struct skcipher_request *subreq = &rctx->subreq;
> int bsize = crypto_skcipher_blocksize(tfm);
> - u8 d[bsize * 2] __attribute__ ((aligned(__alignof__(u32))));
> + u8 d[bsize * 2] __aligned(__alignof__(u32));
> struct scatterlist *sg;
> unsigned int offset;
> u8 *space;
> diff --git a/crypto/kpp.c b/crypto/kpp.c
> index d36ce05..a90edc2 100644
> --- a/crypto/kpp.c
> +++ b/crypto/kpp.c
> @@ -19,6 +19,7 @@
> #include <linux/crypto.h>
> #include <crypto/algapi.h>
> #include <linux/cryptouser.h>
> +#include <linux/compiler.h>
> #include <net/netlink.h>
> #include <crypto/kpp.h>
> #include <crypto/internal/kpp.h>
> @@ -47,7 +48,7 @@ static int crypto_kpp_report(struct sk_buff *skb, struct crypto_alg *alg)
> #endif
>
> static void crypto_kpp_show(struct seq_file *m, struct crypto_alg *alg)
> - __attribute__ ((unused));
> + __maybe_unused;
>
> static void crypto_kpp_show(struct seq_file *m, struct crypto_alg *alg)
> {
> diff --git a/crypto/pcbc.c b/crypto/pcbc.c
> index e4538e0..11d2486 100644
> --- a/crypto/pcbc.c
> +++ b/crypto/pcbc.c
> @@ -20,6 +20,7 @@
> #include <linux/kernel.h>
> #include <linux/module.h>
> #include <linux/slab.h>
> +#include <linux/compiler.h>
>
> struct crypto_pcbc_ctx {
> struct crypto_cipher *child;
> @@ -146,7 +147,7 @@ static int crypto_pcbc_decrypt_inplace(struct skcipher_request *req,
> unsigned int nbytes = walk->nbytes;
> u8 *src = walk->src.virt.addr;
> u8 *iv = walk->iv;
> - u8 tmpbuf[bsize] __attribute__ ((aligned(__alignof__(u32))));
> + u8 tmpbuf[bsize] __aligned(__alignof__(u32));
>
> do {
> memcpy(tmpbuf, src, bsize);
> diff --git a/crypto/rng.c b/crypto/rng.c
> index b81cffb..f46dac5 100644
> --- a/crypto/rng.c
> +++ b/crypto/rng.c
> @@ -23,6 +23,7 @@
> #include <linux/slab.h>
> #include <linux/string.h>
> #include <linux/cryptouser.h>
> +#include <linux/compiler.h>
> #include <net/netlink.h>
>
> #include "internal.h"
> @@ -95,7 +96,7 @@ static int crypto_rng_report(struct sk_buff *skb, struct crypto_alg *alg)
> #endif
>
> static void crypto_rng_show(struct seq_file *m, struct crypto_alg *alg)
> - __attribute__ ((unused));
> + __maybe_unused;
> static void crypto_rng_show(struct seq_file *m, struct crypto_alg *alg)
> {
> seq_printf(m, "type : rng\n");
> diff --git a/crypto/scompress.c b/crypto/scompress.c
> index 35e396d..6b048b3 100644
> --- a/crypto/scompress.c
> +++ b/crypto/scompress.c
> @@ -18,6 +18,7 @@
> #include <linux/slab.h>
> #include <linux/string.h>
> #include <linux/crypto.h>
> +#include <linux/compiler.h>
> #include <linux/vmalloc.h>
> #include <crypto/algapi.h>
> #include <linux/cryptouser.h>
> @@ -57,7 +58,7 @@ static int crypto_scomp_report(struct sk_buff *skb, struct crypto_alg *alg)
> #endif
>
> static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg)
> - __attribute__ ((unused));
> + __maybe_unused;
>
> static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg)
> {
> diff --git a/crypto/shash.c b/crypto/shash.c
> index a051541..5e31c8d 100644
> --- a/crypto/shash.c
> +++ b/crypto/shash.c
> @@ -19,6 +19,7 @@
> #include <linux/seq_file.h>
> #include <linux/cryptouser.h>
> #include <net/netlink.h>
> +#include <linux/compiler.h>
>
> #include "internal.h"
>
> @@ -67,7 +68,7 @@ EXPORT_SYMBOL_GPL(crypto_shash_setkey);
> static inline unsigned int shash_align_buffer_size(unsigned len,
> unsigned long mask)
> {
> - typedef u8 __attribute__ ((aligned)) u8_aligned;
> + typedef u8 __aligned_largest u8_aligned;
> return len + (mask & ~(__alignof__(u8_aligned) - 1));
> }
>
> @@ -80,7 +81,7 @@ static int shash_update_unaligned(struct shash_desc *desc, const u8 *data,
> unsigned int unaligned_len = alignmask + 1 -
> ((unsigned long)data & alignmask);
> u8 ubuf[shash_align_buffer_size(unaligned_len, alignmask)]
> - __attribute__ ((aligned));
> + __aligned_largest;
> u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1);
> int err;
>
> @@ -116,7 +117,7 @@ static int shash_final_unaligned(struct shash_desc *desc, u8 *out)
> struct shash_alg *shash = crypto_shash_alg(tfm);
> unsigned int ds = crypto_shash_digestsize(tfm);
> u8 ubuf[shash_align_buffer_size(ds, alignmask)]
> - __attribute__ ((aligned));
> + __aligned_largest;
> u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1);
> int err;
>
> @@ -403,7 +404,7 @@ static int crypto_shash_report(struct sk_buff *skb, struct crypto_alg *alg)
> #endif
>
> static void crypto_shash_show(struct seq_file *m, struct crypto_alg *alg)
> - __attribute__ ((unused));
> + __maybe_unused;
> static void crypto_shash_show(struct seq_file *m, struct crypto_alg *alg)
> {
> struct shash_alg *salg = __crypto_shash_alg(alg);
> diff --git a/crypto/skcipher.c b/crypto/skcipher.c
> index 0e1e6c3..1a0bd92 100644
> --- a/crypto/skcipher.c
> +++ b/crypto/skcipher.c
> @@ -19,6 +19,7 @@
> #include <crypto/scatterwalk.h>
> #include <linux/bug.h>
> #include <linux/cryptouser.h>
> +#include <linux/compiler.h>
> #include <linux/list.h>
> #include <linux/module.h>
> #include <linux/rtnetlink.h>
> @@ -807,7 +808,7 @@ static void crypto_skcipher_free_instance(struct crypto_instance *inst)
> }
>
> static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg)
> - __attribute__ ((unused));
> + __maybe_unused;
> static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg)
> {
> struct skcipher_alg *skcipher = container_of(alg, struct skcipher_alg,
> diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h
> index 0444b13..fddd1a5 100644
> --- a/include/linux/compiler-gcc.h
> +++ b/include/linux/compiler-gcc.h
> @@ -116,6 +116,7 @@
> */
> #define __pure __attribute__((pure))
> #define __aligned(x) __attribute__((aligned(x)))
> +#define __aligned_largest __attribute__((aligned))
> #define __printf(a, b) __attribute__((format(printf, a, b)))
> #define __scanf(a, b) __attribute__((format(scanf, a, b)))
> #define __attribute_const__ __attribute__((__const__))
> --
> 2.7.4
>
^ permalink raw reply
* [PATCH] crypto: arm64/aes - add scalar implementation
From: Ard Biesheuvel @ 2017-01-04 16:19 UTC (permalink / raw)
To: linux-crypto, linux-arm-kernel; +Cc: herbert, Ard Biesheuvel
This adds a scalar implementation of AES, based on the precomputed tables
that are exposed by the generic AES code. Since rotates are cheap on arm64,
this implementation only uses the 4 core tables (of 1 KB each), and avoids
the prerotated ones, reducing the D-cache footprint by 75%.
On Cortex-A57, this code manages 13.0 cycles per byte, which is ~34% faster
than the generic C code. (Note that this is still >13x slower than the code
that uses the optional ARMv8 Crypto Extensions, which manages <1 cycles per
byte.)
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
---
Raw performance data after the patch, which was generated on a 2 GHz
Cortex-A57 (AMD Seattle B1).
arch/arm64/crypto/Kconfig | 4 +
arch/arm64/crypto/Makefile | 3 +
arch/arm64/crypto/aes-cipher-core.S | 126 ++++++++++++++++++++
arch/arm64/crypto/aes-cipher-glue.c | 69 +++++++++++
4 files changed, 202 insertions(+)
diff --git a/arch/arm64/crypto/Kconfig b/arch/arm64/crypto/Kconfig
index 0bf0f531f539..0826f8e599a6 100644
--- a/arch/arm64/crypto/Kconfig
+++ b/arch/arm64/crypto/Kconfig
@@ -41,6 +41,10 @@ config CRYPTO_CRC32_ARM64_CE
depends on KERNEL_MODE_NEON && CRC32
select CRYPTO_HASH
+config CRYPTO_AES_ARM64
+ tristate "AES core cipher using scalar instructions"
+ select CRYPTO_AES
+
config CRYPTO_AES_ARM64_CE
tristate "AES core cipher using ARMv8 Crypto Extensions"
depends on ARM64 && KERNEL_MODE_NEON
diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile
index 9d2826c5fccf..a893507629eb 100644
--- a/arch/arm64/crypto/Makefile
+++ b/arch/arm64/crypto/Makefile
@@ -44,6 +44,9 @@ sha512-arm64-y := sha512-glue.o sha512-core.o
obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha20-neon.o
chacha20-neon-y := chacha20-neon-core.o chacha20-neon-glue.o
+obj-$(CONFIG_CRYPTO_AES_ARM64) += aes-arm64.o
+aes-arm64-y := aes-cipher-core.o aes-cipher-glue.o
+
AFLAGS_aes-ce.o := -DINTERLEAVE=4
AFLAGS_aes-neon.o := -DINTERLEAVE=4
diff --git a/arch/arm64/crypto/aes-cipher-core.S b/arch/arm64/crypto/aes-cipher-core.S
new file mode 100644
index 000000000000..22d1bc46feba
--- /dev/null
+++ b/arch/arm64/crypto/aes-cipher-core.S
@@ -0,0 +1,126 @@
+/*
+ * Scalar AES core transform
+ *
+ * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+ .text
+ .align 5
+
+ rk .req x0
+ out .req x1
+ in .req x2
+ rounds .req x3
+ tt .req x4
+ lt .req x2
+
+ .macro __hround, out0, out1, in0, in1, in2, in3, t0, t1, enc
+ ldp \out0, \out1, [rk], #8
+
+ ubfx w13, \in0, #0, #8
+ ubfx w14, \in1, #8, #8
+ ldr w13, [tt, w13, uxtw #2]
+ ldr w14, [tt, w14, uxtw #2]
+
+ ubfx w15, \in2, #16, #8
+ ubfx w16, \in3, #24, #8
+ ldr w15, [tt, w15, uxtw #2]
+ ldr w16, [tt, w16, uxtw #2]
+
+ .if \enc
+ ubfx w17, \in1, #0, #8
+ ubfx w18, \in2, #8, #8
+ .else
+ ubfx w17, \in3, #0, #8
+ ubfx w18, \in0, #8, #8
+ .endif
+ ldr w17, [tt, w17, uxtw #2]
+ ldr w18, [tt, w18, uxtw #2]
+
+ .if \enc
+ ubfx \t0, \in3, #16, #8
+ ubfx \t1, \in0, #24, #8
+ .else
+ ubfx \t0, \in1, #16, #8
+ ubfx \t1, \in2, #24, #8
+ .endif
+ ldr \t0, [tt, \t0, uxtw #2]
+ ldr \t1, [tt, \t1, uxtw #2]
+
+ eor \out0, \out0, w13
+ eor \out1, \out1, w17
+ eor \out0, \out0, w14, ror #24
+ eor \out1, \out1, w18, ror #24
+ eor \out0, \out0, w15, ror #16
+ eor \out1, \out1, \t0, ror #16
+ eor \out0, \out0, w16, ror #8
+ eor \out1, \out1, \t1, ror #8
+ .endm
+
+ .macro fround, out0, out1, out2, out3, in0, in1, in2, in3
+ __hround \out0, \out1, \in0, \in1, \in2, \in3, \out2, \out3, 1
+ __hround \out2, \out3, \in2, \in3, \in0, \in1, \in1, \in2, 1
+ .endm
+
+ .macro iround, out0, out1, out2, out3, in0, in1, in2, in3
+ __hround \out0, \out1, \in0, \in3, \in2, \in1, \out2, \out3, 0
+ __hround \out2, \out3, \in2, \in1, \in0, \in3, \in1, \in0, 0
+ .endm
+
+ .macro do_crypt, round, ttab, ltab
+ ldp w5, w6, [in]
+ ldp w7, w8, [in, #8]
+ ldp w9, w10, [rk], #16
+ ldp w11, w12, [rk, #-8]
+
+CPU_BE( rev w5, w5 )
+CPU_BE( rev w6, w6 )
+CPU_BE( rev w7, w7 )
+CPU_BE( rev w8, w8 )
+
+ eor w5, w5, w9
+ eor w6, w6, w10
+ eor w7, w7, w11
+ eor w8, w8, w12
+
+ ldr tt, =\ttab
+ ldr lt, =\ltab
+
+ tbnz rounds, #1, 1f
+
+0: \round w9, w10, w11, w12, w5, w6, w7, w8
+ \round w5, w6, w7, w8, w9, w10, w11, w12
+
+1: subs rounds, rounds, #4
+ \round w9, w10, w11, w12, w5, w6, w7, w8
+ csel tt, tt, lt, hi
+ \round w5, w6, w7, w8, w9, w10, w11, w12
+ b.hi 0b
+
+CPU_BE( rev w5, w5 )
+CPU_BE( rev w6, w6 )
+CPU_BE( rev w7, w7 )
+CPU_BE( rev w8, w8 )
+
+ stp w5, w6, [out]
+ stp w7, w8, [out, #8]
+ ret
+
+ .align 4
+ .ltorg
+ .endm
+
+ENTRY(__aes_arm64_encrypt)
+ do_crypt fround, crypto_ft_tab, crypto_fl_tab
+ENDPROC(__aes_arm64_encrypt)
+
+ENTRY(__aes_arm64_decrypt)
+ do_crypt iround, crypto_it_tab, crypto_il_tab
+ENDPROC(__aes_arm64_decrypt)
diff --git a/arch/arm64/crypto/aes-cipher-glue.c b/arch/arm64/crypto/aes-cipher-glue.c
new file mode 100644
index 000000000000..7288e7cbebff
--- /dev/null
+++ b/arch/arm64/crypto/aes-cipher-glue.c
@@ -0,0 +1,69 @@
+/*
+ * Scalar AES core transform
+ *
+ * Copyright (C) 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <crypto/aes.h>
+#include <linux/crypto.h>
+#include <linux/module.h>
+
+asmlinkage void __aes_arm64_encrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
+EXPORT_SYMBOL(__aes_arm64_encrypt);
+
+asmlinkage void __aes_arm64_decrypt(u32 *rk, u8 *out, const u8 *in, int rounds);
+EXPORT_SYMBOL(__aes_arm64_decrypt);
+
+static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ int rounds = 6 + ctx->key_length / 4;
+
+ __aes_arm64_encrypt(ctx->key_enc, out, in, rounds);
+}
+
+static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct crypto_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ int rounds = 6 + ctx->key_length / 4;
+
+ __aes_arm64_decrypt(ctx->key_dec, out, in, rounds);
+}
+
+static struct crypto_alg aes_alg = {
+ .cra_name = "aes",
+ .cra_driver_name = "aes-arm64",
+ .cra_priority = 200,
+ .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct crypto_aes_ctx),
+ .cra_module = THIS_MODULE,
+
+ .cra_cipher.cia_min_keysize = AES_MIN_KEY_SIZE,
+ .cra_cipher.cia_max_keysize = AES_MAX_KEY_SIZE,
+ .cra_cipher.cia_setkey = crypto_aes_set_key,
+ .cra_cipher.cia_encrypt = aes_encrypt,
+ .cra_cipher.cia_decrypt = aes_decrypt
+};
+
+static int __init aes_init(void)
+{
+ return crypto_register_alg(&aes_alg);
+}
+
+static void __exit aes_fini(void)
+{
+ crypto_unregister_alg(&aes_alg);
+}
+
+module_init(aes_init);
+module_exit(aes_fini);
+
+MODULE_DESCRIPTION("Scalar AES cipher for arm64");
+MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
+MODULE_LICENSE("GPL v2");
+MODULE_ALIAS_CRYPTO("aes");
--
2.7.4
testing speed of async ecb(aes) (ecb(aes-generic)) encryption
test 0 (128 bit key, 16 byte blocks): 4594689 operations in 1 seconds (73515024 bytes)
test 1 (128 bit key, 64 byte blocks): 1585137 operations in 1 seconds (101448768 bytes)
test 2 (128 bit key, 256 byte blocks): 435173 operations in 1 seconds (111404288 bytes)
test 3 (128 bit key, 1024 byte blocks): 111505 operations in 1 seconds (114181120 bytes)
test 4 (128 bit key, 8192 byte blocks): 14093 operations in 1 seconds (115449856 bytes)
test 5 (192 bit key, 16 byte blocks): 4078345 operations in 1 seconds (65253520 bytes)
test 6 (192 bit key, 64 byte blocks): 1349425 operations in 1 seconds (86363200 bytes)
test 7 (192 bit key, 256 byte blocks): 365631 operations in 1 seconds (93601536 bytes)
test 8 (192 bit key, 1024 byte blocks): 93362 operations in 1 seconds (95602688 bytes)
test 9 (192 bit key, 8192 byte blocks): 11729 operations in 1 seconds (96083968 bytes)
test 10 (256 bit key, 16 byte blocks): 3692945 operations in 1 seconds (59087120 bytes)
test 11 (256 bit key, 64 byte blocks): 1182522 operations in 1 seconds (75681408 bytes)
test 12 (256 bit key, 256 byte blocks): 317285 operations in 1 seconds (81224960 bytes)
test 13 (256 bit key, 1024 byte blocks): 80459 operations in 1 seconds (82390016 bytes)
test 14 (256 bit key, 8192 byte blocks): 10138 operations in 1 seconds (83050496 bytes)
testing speed of async ecb(aes) (ecb(aes-arm64)) encryption
test 0 (128 bit key, 16 byte blocks): 5455304 operations in 1 seconds (87284864 bytes)
test 1 (128 bit key, 64 byte blocks): 2000321 operations in 1 seconds (128020544 bytes)
test 2 (128 bit key, 256 byte blocks): 574174 operations in 1 seconds (146988544 bytes)
test 3 (128 bit key, 1024 byte blocks): 148497 operations in 1 seconds (152060928 bytes)
test 4 (128 bit key, 8192 byte blocks): 18836 operations in 1 seconds (154304512 bytes)
test 5 (192 bit key, 16 byte blocks): 4962478 operations in 1 seconds (79399648 bytes)
test 6 (192 bit key, 64 byte blocks): 1740157 operations in 1 seconds (111370048 bytes)
test 7 (192 bit key, 256 byte blocks): 490443 operations in 1 seconds (125553408 bytes)
test 8 (192 bit key, 1024 byte blocks): 126165 operations in 1 seconds (129192960 bytes)
test 9 (192 bit key, 8192 byte blocks): 15897 operations in 1 seconds (130228224 bytes)
test 10 (256 bit key, 16 byte blocks): 4527784 operations in 1 seconds (72444544 bytes)
test 11 (256 bit key, 64 byte blocks): 1527235 operations in 1 seconds (97743040 bytes)
test 12 (256 bit key, 256 byte blocks): 425302 operations in 1 seconds (108877312 bytes)
test 13 (256 bit key, 1024 byte blocks): 109013 operations in 1 seconds (111629312 bytes)
test 14 (256 bit key, 8192 byte blocks): 13778 operations in 1 seconds (112869376 bytes)
^ permalink raw reply related
* Re: [PATCH 1/5] ARM: wire up HWCAP2 feature bits to the CPU modalias
From: Ard Biesheuvel @ 2017-01-04 16:42 UTC (permalink / raw)
To: Russell King - ARM Linux
Cc: linux-arm-kernel@lists.infradead.org,
linux-crypto@vger.kernel.org, Herbert Xu, Steve Capper
In-Reply-To: <20170102234013.GK14217@n2100.armlinux.org.uk>
On 2 January 2017 at 23:40, Russell King - ARM Linux
<linux@armlinux.org.uk> wrote:
> On Mon, Jan 02, 2017 at 09:06:04PM +0000, Ard Biesheuvel wrote:
>> On 31 October 2016 at 16:13, Russell King - ARM Linux
>> <linux@armlinux.org.uk> wrote:
>> > On Sat, Oct 29, 2016 at 11:08:36AM +0100, Ard Biesheuvel wrote:
>> >> On 18 October 2016 at 11:52, Ard Biesheuvel <ard.biesheuvel@linaro.org> wrote:
>> >> > Wire up the generic support for exposing CPU feature bits via the
>> >> > modalias in /sys/device/system/cpu. This allows udev to automatically
>> >> > load modules for things like crypto algorithms that are implemented
>> >> > using optional instructions.
>> >> >
>> >> > Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
>> >> > ---
>> >> > arch/arm/Kconfig | 1 +
>> >> > arch/arm/include/asm/cpufeature.h | 32 ++++++++++++++++++++
>> >> > 2 files changed, 33 insertions(+)
>> >> >
>> >>
>> >> Russell,
>> >>
>> >> do you have any concerns regarding this patch? If not, I will drop it
>> >> into the patch system.
>> >
>> > It's still something I need to look at... I've been offline last week,
>> > and sort-of offline the previous week, so I'm catching up.
>> >
>>
>> Hi Russell,
>>
>> Any thoughts yet?
>
> None, and the patch is well buried now that it'll take me a while to
> find... back in mid-October? Yea, I'll have to drop everything and
> go digging through my mailboxes to find it... and I'm just catching
> up (again) after a week and a bit's time offline - yep, it's wonderful
> timing. Sorry, no time to look at it right now, you're not the only
> one wanting my attention at the moment.
>
No worries. It is not exactly urgent, but it is a useful enhancement
nonetheless.
> Please try again in about a week's time - don't leave it a few months,
> and please include the patch.
>
OK
^ permalink raw reply
* Re: console noise after commit c1e9b3b0eea
From: Shannon Nelson @ 2017-01-05 0:44 UTC (permalink / raw)
To: Anatoly Pugachev; +Cc: SOWMINI.VARADHAN, sparclinux, linux-crypto
In-Reply-To: <a0a9e8ae-38a8-9bb8-d3b7-2f4b50809096@oracle.com>
Resurrecting an old thread, pulled out of
http://www.spinics.net/lists/linux-crypto/msg19192.html
On Wed, Apr 20, 2016 at 9:18 AM, Anatoly Pugachev <matorola@xxxxxxxxx>
wrote:
> On Wed, Apr 20, 2016 at 1:33 AM, Sowmini Varadhan
> <sowmini.varadhan@xxxxxxxxxx> wrote:
>>
>> Hi Anatoly,
>>
>> after commit c1e9b3b0eea1 ("hwrng: n2 - Attach on T5/M5, T7/M7 SPARC
> CPUs")
>> I get a *lot* of console noise on my T5-2, of the form:
>>
>> n2rng f028f21c: Selftest failed on unit 0
>> n2rng f028f21c: Test buffer slot 0 [0x0000000000000000]
>> n2rng f028f21c: Test buffer slot 1 [0xe63f56d6a22eb116]
>> n2rng f028f21c: Test buffer slot 2 [0xe63f56d6a22eb116]
>> n2rng f028f21c: Test buffer slot 3 [0xe63f56d6a22eb116]
>> n2rng f028f21c: Test buffer slot 4 [0xe63f56d6a22eb116]
>> n2rng f028f21c: Test buffer slot 5 [0xe63f56d6a22eb116]
>> n2rng f028f21c: Test buffer slot 6 [0xe63f56d6a22eb116]
>> n2rng f028f21c: Test buffer slot 7 [0xe63f56d6a22eb116]
>>
>> Why/when is your commit needed on my T5-2?
>>
>> I'm not sure how this was tested, but if you need to revise it and test
>> on sparc, please let me know- I think it needs more work on sparc.
>
> Sowmini,
>
> the patch/commit is actually quite trivial, it just adds device_id
> matches for newer T5/M7 CPUs to n2rng_match structure. Without this
> patch, n2rng does not work on this newer CPUs. Works well on my T5-2
> LDOM (tested with rng-tools and gpg --gen-key). I don't have M7
^^^^
Anatoly, I think your LDOM is why you don't see the problem. Yes, your
patch works just fine when running in a client LDOM, but we see a
problem when running this on sparc "bare metal". Did you test this on
the bare metal so that the self-test would run?
It seems there's an issue with the self-test in the newer hardware and
the driver will never stop trying to retest the hardware. I'm
contemplating a patch to limit the self-test attempts, at least until we
can figure out what is the root of the issue.
sln
> machine to test it with.
>
> Why the n2rng selftest fails on your machine - I've no idea... Just to
> silence it, you can blacklist this module, since it does not work your
> hardware anyway.
>
> Can you please send me "prtconf -pv" output from your machine, as well
> information on how do you run linux as LDOM container or baremetal
> T5-2 ?
>
> Mine T5-2 is one of the last firmware releases (run from solaris 11.3
> control domain):
>
> root@deimos:/home/sysadmin# prtdiag -v
> ============================ FW Version ============================
> Sun System Firmware 9.5.3 2015/11/25 09:50
>
> sysadmin@deimos:~$ ldm -V
>
> Logical Domains Manager (v 3.3.0.0.17)
> Hypervisor control protocol v 1.12
> Using Hypervisor MD v 1.4
>
> System PROM:
> Hostconfig v. 1.6.3 @(#)Hostconfig 1.6.3 2015/11/25
> 08:57
> Hypervisor v. 1.15.3 @(#)Hypervisor 1.15.3 2015/11/11
> 17:15
> OpenBoot v. 4.38.3 @(#)OpenBoot 4.38.3 2015/11/11
> 10:38
>
>
> Can you please check what firmware release if yours T5-2 server and
> probably update? I'm not sure it would help, but anyway.
>
>
>
^ permalink raw reply
* Re: [RFC PATCH v2] crypto: Add IV generation algorithms
From: Binoy Jayan @ 2017-01-05 6:06 UTC (permalink / raw)
To: Herbert Xu
Cc: Milan Broz, Oded, Ofir, David S. Miller, linux-crypto, Mark Brown,
Arnd Bergmann, Linux kernel mailing list, Alasdair Kergon,
Mike Snitzer, dm-devel, Shaohua Li, linux-raid, Rajendra
In-Reply-To: <20170102065325.GA19553@gondor.apana.org.au>
Hi Herbert,
On 2 January 2017 at 12:23, Herbert Xu <herbert@gondor.apana.org.au> wrote:
> On Mon, Jan 02, 2017 at 12:16:45PM +0530, Binoy Jayan wrote:
>
> Right. The actual number of underlying tfms that do the work
> won't change compared to the status quo. We're just structuring
> it such that if the overall scheme is supported by the hardware
> then we can feed more than one sector at a time to it.
I was thinking of continuing to have the iv generation algorithms as template
ciphers instead of regular 'skcipher' as it is easier to inherit the parameters
from the underlying cipher (e.g. aes) like cra_blocksize, cra_alignmask,
ivsize, chunksize etc.
Usually, the underlying cipher for the template ciphers are instantiated
in the following function:
skcipher_instance:skcipher_alg:init()
Since the number of such cipher instances depend on the key count, which is
not known at the time of creation of the cipher (it's passed to as an argument
to the setkey api), the creation of those have to be delayed until the setkey
operation of the template cipher. But as Mark pointed out, the users of this
cipher may get confused if the creation of the underlying cipher fails while
trying to do a 'setkey' on the template cipher. I was wondering if I can create
a single instance of the cipher and assign it to tfms[0] and allocate the
remaining instances when the setkey operation is called later with the encoded
key_count so that errors during cipher creation are uncovered earlier.
Thanks,
Binoy
^ permalink raw reply
* Re: [PATCH v3 1/3] drivers: crypto: Add Support for Octeon-tx CPT Engine
From: George Cherian @ 2017-01-06 6:19 UTC (permalink / raw)
To: Corentin Labbe, george.cherian
Cc: herbert, davem, linux-kernel, linux-crypto, david.daney
In-Reply-To: <20161221132347.GA21051@Red>
Hi Corentin,
Thank you very much for the review.
I was on vacation and now am back, I will fix your comments and send a
new version.
On 12/21/2016 06:53 PM, Corentin Labbe wrote:
> Hello
>
> I have some comment inline
>
> On Wed, Dec 21, 2016 at 11:56:11AM +0000, george.cherian@cavium.com wrote:
>> From: George Cherian <george.cherian@cavium.com>
>>
>> Enable the Physical Function diver for the Cavium Crypto Engine (CPT)
>
> typo driver
okay
>
>> found in Octeon-tx series of SoC's. CPT is the Cryptographic Acceleration
>> Unit. CPT includes microcoded GigaCypher symmetric engines (SEs) and
>> asymmetric engines (AEs).
>>
>> Signed-off-by: George Cherian <george.cherian@cavium.com>
>> Reviewed-by: David Daney <david.daney@cavium.com>
>> ---
>> drivers/crypto/cavium/cpt/Kconfig | 16 +
>> drivers/crypto/cavium/cpt/Makefile | 2 +
>> drivers/crypto/cavium/cpt/cpt_common.h | 158 +++++++
>> drivers/crypto/cavium/cpt/cpt_hw_types.h | 658 +++++++++++++++++++++++++++++
>> drivers/crypto/cavium/cpt/cptpf.h | 69 +++
>> drivers/crypto/cavium/cpt/cptpf_main.c | 703 +++++++++++++++++++++++++++++++
>> drivers/crypto/cavium/cpt/cptpf_mbox.c | 163 +++++++
>> 7 files changed, 1769 insertions(+)
>> create mode 100644 drivers/crypto/cavium/cpt/Kconfig
>> create mode 100644 drivers/crypto/cavium/cpt/Makefile
>> create mode 100644 drivers/crypto/cavium/cpt/cpt_common.h
>> create mode 100644 drivers/crypto/cavium/cpt/cpt_hw_types.h
>> create mode 100644 drivers/crypto/cavium/cpt/cptpf.h
>> create mode 100644 drivers/crypto/cavium/cpt/cptpf_main.c
>> create mode 100644 drivers/crypto/cavium/cpt/cptpf_mbox.c
>>
>> diff --git a/drivers/crypto/cavium/cpt/Kconfig b/drivers/crypto/cavium/cpt/Kconfig
>> new file mode 100644
>> index 0000000..247f1cb
>> --- /dev/null
>> +++ b/drivers/crypto/cavium/cpt/Kconfig
>> @@ -0,0 +1,16 @@
>> +#
>> +# Cavium crypto device configuration
>> +#
>> +
>> +config CRYPTO_DEV_CPT
>> + tristate
>> +
>> +config CAVIUM_CPT
>> + tristate "Cavium Cryptographic Accelerator driver"
>> + depends on ARCH_THUNDER
>> + select CRYPTO_DEV_CPT
>
> Could you add some COMPILE_TEST ?
You meant depends on ARCH_THUNDER || COMPILE_TEST?
>
> [...]
>> +struct microcode {
>> + u8 is_mc_valid;
>> + u8 is_ae;
>> + u8 group;
>> + u8 num_cores;
>> + u32 code_size;
>> + u64 core_mask;
>> + u8 version[32];
>
> I see this "32" in some other place, perhaps you could use a define
okay
>
> [...]
>> +#include <linux/version.h>
>> +#include <linux/module.h>
>> +#include <linux/moduleparam.h>
>> +#include <linux/printk.h>
>> +#include <linux/device.h>
>> +#include <linux/interrupt.h>
>> +#include <linux/firmware.h>
>> +#include <linux/pci.h>
>
> Header need to be sorted
will do
>
> [...]
>> +static void cpt_disable_cores(struct cpt_device *cpt, u64 coremask,
>> + u8 type, u8 grp)
>> +{
>> + u64 pf_exe_ctl;
>> + u32 timeout = 0xFFFFFFFF;
>> + u64 grpmask = 0;
>> + struct device *dev = &cpt->pdev->dev;
>> +
>> + if (type == AE_TYPES)
>> + coremask = (coremask << cpt->max_se_cores);
>> +
>> + /* Disengage the cores from groups */
>> + grpmask = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp));
>> + cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp),
>> + (grpmask & ~coremask));
>> + udelay(CSR_DELAY);
>> + grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0));
>> + while (grp & coremask) {
>> + dev_err(dev, "Cores still busy %llx", coremask);
>> + grp = cpt_read_csr64(cpt->reg_base,
>> + CPTX_PF_EXEC_BUSY(0));
>> + if (timeout--)
>> + break;
>
> The timeout seems enormous and you will flooding syslog with dev_err()
will reduce.
>
>> + }
>> +
>> + /* Disable the cores */
>> + pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0));
>> + cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0),
>> + (pf_exe_ctl & ~coremask));
>> + udelay(CSR_DELAY);
>> +}
>> +
>> +/*
>> + * Enable cores specified by coremask
>> + */
>> +static void cpt_enable_cores(struct cpt_device *cpt, u64 coremask,
>> + u8 type)
>> +{
>> + u64 pf_exe_ctl;
>> +
>> + if (type == AE_TYPES)
>> + coremask = (coremask << cpt->max_se_cores);
>> +
>> + pf_exe_ctl = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0));
>> + cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0),
>> + (pf_exe_ctl | coremask));
>> + udelay(CSR_DELAY);
>> +}
>> +
>> +static void cpt_configure_group(struct cpt_device *cpt, u8 grp,
>> + u64 coremask, u8 type)
>> +{
>> + u64 pf_gx_en = 0;
>> +
>> + if (type == AE_TYPES)
>> + coremask = (coremask << cpt->max_se_cores);
>> +
>> + pf_gx_en = cpt_read_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp));
>> + cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp),
>> + (pf_gx_en | coremask));
>> + udelay(CSR_DELAY);
>> +}
>> +
>> +static void cpt_disable_mbox_interrupts(struct cpt_device *cpt)
>> +{
>> + /* Clear mbox(0) interupts for all vfs */
>> + cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1CX(0, 0), ~0ull);
>> +}
>> +
>> +static void cpt_disable_ecc_interrupts(struct cpt_device *cpt)
>> +{
>> + /* Clear ecc(0) interupts for all vfs */
>> + cpt_write_csr64(cpt->reg_base, CPTX_PF_ECC0_ENA_W1C(0), ~0ull);
>> +}
>> +
>> +static void cpt_disable_exec_interrupts(struct cpt_device *cpt)
>> +{
>> + /* Clear exec interupts for all vfs */
>> + cpt_write_csr64(cpt->reg_base, CPTX_PF_EXEC_ENA_W1C(0), ~0ull);
>> +}
>> +
>> +static void cpt_disable_all_interrupts(struct cpt_device *cpt)
>> +{
>> + cpt_disable_mbox_interrupts(cpt);
>> + cpt_disable_ecc_interrupts(cpt);
>> + cpt_disable_exec_interrupts(cpt);
>> +}
>> +
>> +static void cpt_enable_mbox_interrupts(struct cpt_device *cpt)
>> +{
>> + /* Set mbox(0) interupts for all vfs */
>> + cpt_write_csr64(cpt->reg_base, CPTX_PF_MBOX_ENA_W1SX(0, 0), ~0ull);
>> +}
>> +
>> +static int cpt_load_microcode(struct cpt_device *cpt, struct microcode *mcode)
>> +{
>> + int ret = 0, core = 0, shift = 0;
>> + u32 total_cores = 0;
>> + struct device *dev = &cpt->pdev->dev;
>> +
>> + if (!mcode || !mcode->code) {
>> + dev_err(dev, "Either the mcode is null or data is NULL\n");
>> + return 1;
>
> This is not a standard error code
>
Yes will return standard error codes.
>> + }
>> +
>> + if (mcode->code_size == 0) {
>> + dev_err(dev, "microcode size is 0\n");
>> + return 1;
>
> the same
>
>> + }
>> +
>> + /* Assumes 0-9 are SE cores for UCODE_BASE registers and
>> + * AE core bases follow
>> + */
>> + if (mcode->is_ae) {
>> + core = CPT_MAX_SE_CORES; /* start couting from 10 */
>> + total_cores = CPT_MAX_TOTAL_CORES; /* upto 15 */
>> + } else {
>> + core = 0; /* start couting from 0 */
>> + total_cores = CPT_MAX_SE_CORES; /* upto 9 */
>> + }
>> +
>> + /* Point to microcode for each core of the group */
>> + for (; core < total_cores ; core++, shift++) {
>> + if (mcode->core_mask & (1 << shift)) {
>> + cpt_write_csr64(cpt->reg_base,
>> + CPTX_PF_ENGX_UCODE_BASE(0, core),
>> + (u64)mcode->phys_base);
>> + }
>> + }
>> + return ret;
>> +}
>> +
>> +static int do_cpt_init(struct cpt_device *cpt, struct microcode *mcode)
>> +{
>> + int ret = 0;
>> + struct device *dev = &cpt->pdev->dev;
>> +
>> + /* Make device not ready */
>> + cpt->flags &= ~CPT_FLAG_DEVICE_READY;
>> + /* Disable All PF interrupts */
>> + cpt_disable_all_interrupts(cpt);
>> + /* Calculate mcode group and coremasks */
>> + if (mcode->is_ae) {
>> + if (mcode->num_cores > cpt->max_ae_cores) {
>> + dev_err(dev, "Requested for more cores than available AE cores\n");
>> + ret = -1;
>
> This is not a standard error code
>
>> + goto cpt_init_fail;
>> + }
>> +
>> + if (cpt->next_group >= CPT_MAX_CORE_GROUPS) {
>> + dev_err(dev, "Can't load, all eight microcode groups in use");
>> + return -ENFILE;
>> + }
>> +
>> + mcode->group = cpt->next_group;
>> + /* Convert requested cores to mask */
>> + mcode->core_mask = GENMASK(mcode->num_cores, 0);
>> + cpt_disable_cores(cpt, mcode->core_mask, AE_TYPES,
>> + mcode->group);
>> + /* Load microcode for AE engines */
>> + if (cpt_load_microcode(cpt, mcode)) {
>> + dev_err(dev, "Microcode load Failed for %s\n",
>> + mcode->version);
>> + ret = -1;
>
> again and you loose the error code given by cpt_load_microcode
okay
>
>> + goto cpt_init_fail;
>> + }
>> + cpt->next_group++;
>> + /* Configure group mask for the mcode */
>> + cpt_configure_group(cpt, mcode->group, mcode->core_mask,
>> + AE_TYPES);
>> + /* Enable AE cores for the group mask */
>> + cpt_enable_cores(cpt, mcode->core_mask, AE_TYPES);
>> + } else {
>> + if (mcode->num_cores > cpt->max_se_cores) {
>> + dev_err(dev, "Requested for more cores than available SE cores\n");
>> + ret = -1;
>
> Again
>
>> + goto cpt_init_fail;
>> + }
>> + if (cpt->next_group >= CPT_MAX_CORE_GROUPS) {
>> + dev_err(dev, "Can't load, all eight microcode groups in use");
>> + return -ENFILE;
>> + }
>> +
>> + mcode->group = cpt->next_group;
>> + /* Covert requested cores to mask */
>> + mcode->core_mask = GENMASK(mcode->num_cores, 0);
>> + cpt_disable_cores(cpt, mcode->core_mask, SE_TYPES,
>> + mcode->group);
>> + /* Load microcode for SE engines */
>> + if (cpt_load_microcode(cpt, mcode)) {
>> + dev_err(dev, "Microcode load Failed for %s\n",
>> + mcode->version);
>> + ret = -1;
>
> Again
>
>> + goto cpt_init_fail;
>> + }
>> + cpt->next_group++;
>> + /* Configure group mask for the mcode */
>> + cpt_configure_group(cpt, mcode->group, mcode->core_mask,
>> + SE_TYPES);
>> + /* Enable SE cores for the group mask */
>> + cpt_enable_cores(cpt, mcode->core_mask, SE_TYPES);
>> + }
>> +
>> + /* Enabled PF mailbox interrupts */
>> + cpt_enable_mbox_interrupts(cpt);
>> + cpt->flags |= CPT_FLAG_DEVICE_READY;
>> +
>> + return ret;
>> +
>> +cpt_init_fail:
>> + /* Enabled PF mailbox interrupts */
>> + cpt_enable_mbox_interrupts(cpt);
>> +
>> + return ret;
>> +}
>> +
>> +struct ucode_header {
>> + u8 version[32];
>> + u32 code_length;
>> + u32 data_length;
>> + u64 sram_address;
>> +};
>> +
>> +static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae)
>> +{
>> + const struct firmware *fw_entry;
>> + struct device *dev = &cpt->pdev->dev;
>> + struct ucode_header *ucode;
>> + struct microcode *mcode;
>> + int j, ret = 0;
>> +
>> + ret = request_firmware(&fw_entry, fw, dev);
>> + if (ret)
>> + return ret;
>
> I think you could also check for a minimal firmware size
Yes will add the check.
>
> [...]
>> +static void cpt_disable_all_cores(struct cpt_device *cpt)
>> +{
>> + u32 grp, timeout = 0xFFFFFFFF;
>> + struct device *dev = &cpt->pdev->dev;
>> +
>> + /* Disengage the cores from groups */
>> + for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) {
>> + cpt_write_csr64(cpt->reg_base, CPTX_PF_GX_EN(0, grp), 0);
>> + udelay(CSR_DELAY);
>> + }
>> +
>> + grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0));
>> + while (grp) {
>> + dev_err(dev, "Cores still busy");
>> + grp = cpt_read_csr64(cpt->reg_base,
>> + CPTX_PF_EXEC_BUSY(0));
>> + if (timeout--)
>> + break;
>> + }
>
> Same problem than cpt_disable_cores
Will adjust the timeout.
>
>> + /* Disable the cores */
>> + cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), 0);
>> +}
>> +
>> +/**
>> + * Ensure all cores are disenganed from all groups by
>
> typo engaged
>
>> + * calling cpt_disable_all_cores() before calling this
>> + * function.
>> + */
>> +static void cpt_unload_microcode(struct cpt_device *cpt)
>> +{
>> + u32 grp = 0, core;
>> +
>> + /* Free microcode bases and reset group masks */
>> + for (grp = 0; grp < CPT_MAX_CORE_GROUPS; grp++) {
>> + struct microcode *mcode = &cpt->mcode[grp];
>> +
>> + if (cpt->mcode[grp].code)
>> + dma_free_coherent(&cpt->pdev->dev, mcode->code_size,
>> + mcode->code, mcode->phys_base);
>> + mcode->code = NULL;
>> + //mcode->base = NULL;
>
> This is not a standard comment
Will get this removed.
>
>> + }
>> + /* Clear UCODE_BASE registers for all engines */
>> + for (core = 0; core < CPT_MAX_TOTAL_CORES; core++)
>> + cpt_write_csr64(cpt->reg_base,
>> + CPTX_PF_ENGX_UCODE_BASE(0, core), 0ull);
>> +}
>> +
>> +static int cpt_device_init(struct cpt_device *cpt)
>> +{
>> + u64 bist;
>> + struct device *dev = &cpt->pdev->dev;
>> +
>> + /* Reset the PF when probed first */
>> + cpt_reset(cpt);
>> + mdelay((100));
>
> double parenthesis
okay
>
>> +
>> + /*Check BIST status*/
>> + bist = (u64)cpt_check_bist_status(cpt);
>> + if (bist) {
>> + dev_err(dev, "RAM BIST failed with code 0x%llx", bist);
>> + return -ENODEV;
>> + }
>> +
>> + bist = cpt_check_exe_bist_status(cpt);
>> + if (bist) {
>> + dev_err(dev, "Engine BIST failed with code 0x%llx", bist);
>> + return -ENODEV;
>> + }
>> +
>> + /*Get CLK frequency*/
>> + /*Get max enabled cores */
>> + cpt_find_max_enabled_cores(cpt);
>> + /*Disable all cores*/
>> + cpt_disable_all_cores(cpt);
>> + /*Reset device parameters*/
>> + cpt->next_mc_idx = 0;
>> + cpt->next_group = 0;
>> + /* PF is ready */
>> + cpt->flags |= CPT_FLAG_DEVICE_READY;
>> +
>> + return 0;
>> +}
>> +
>> +static int cpt_register_interrupts(struct cpt_device *cpt)
>> +{
>> + int ret;
>> + struct device *dev = &cpt->pdev->dev;
>> +
>> + /* Enable MSI-X */
>> + ret = cpt_enable_msix(cpt);
>> + if (ret)
>> + return ret;
>> +
>> + /* Register mailbox interrupt handlers */
>> + ret = request_irq(cpt->msix_entries[CPT_PF_INT_VEC_E_MBOXX(0)].vector,
>> + cpt_mbx0_intr_handler, 0, "CPT Mbox0", cpt);
>> + if (ret)
>> + goto fail;
>> +
>> + cpt->irq_allocated[CPT_PF_INT_VEC_E_MBOXX(0)] = true;
>> +
>> + /* Enable mailbox interrupt */
>> + cpt_enable_mbox_interrupts(cpt);
>> + return 0;
>> +
>> +fail:
>> + dev_err(dev, "Request irq failed\n");
>> + cpt_free_all_interrupts(cpt);
>> + return ret;
>> +}
>> +
>> +static void cpt_unregister_interrupts(struct cpt_device *cpt)
>> +{
>> + cpt_free_all_interrupts(cpt);
>> + cpt_disable_msix(cpt);
>> +}
>> +
>> +static int cpt_sriov_init(struct cpt_device *cpt, int num_vfs)
>> +{
>> + int pos = 0;
>> + int err;
>> + u16 total_vf_cnt;
>> + struct pci_dev *pdev = cpt->pdev;
>> +
>> + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
>> + if (!pos) {
>> + dev_err(&pdev->dev, "SRIOV capability is not found in PCIe config space\n");
>> + return -ENODEV;
>> + }
>> +
>> + cpt->num_vf_en = num_vfs; /* User requested VFs */
>> + pci_read_config_word(pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf_cnt);
>> + if (total_vf_cnt < cpt->num_vf_en)
>> + cpt->num_vf_en = total_vf_cnt;
>> +
>> + if (!total_vf_cnt)
>> + return 0;
>> +
>> + /*Enabled the available VFs */
>> + err = pci_enable_sriov(pdev, cpt->num_vf_en);
>> + if (err) {
>> + dev_err(&pdev->dev, "SRIOV enable failed, num VF is %d\n",
>> + cpt->num_vf_en);
>> + cpt->num_vf_en = 0;
>> + return err;
>> + }
>> +
>> + /* TODO: Optionally enable static VQ priorities feature */
>> +
>> + dev_info(&pdev->dev, "SRIOV enabled, number of VF available %d\n",
>> + cpt->num_vf_en);
>> +
>> + cpt->flags |= CPT_FLAG_SRIOV_ENABLED;
>> +
>> + return 0;
>> +}
>> +
>> +static int cpt_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
>> +{
>> + struct device *dev = &pdev->dev;
>> + struct cpt_device *cpt;
>> + int err;
>> +
>> + if (num_vfs > 16) {
>> + pr_warn("Invalid vf count %d, Resetting it to 4(default)\n",
>> + num_vfs);
>
> Why not using dev_warn ?
>
>> + num_vfs = 4;
>> + }
>> +
>> + cpt = devm_kzalloc(dev, sizeof(struct cpt_device), GFP_KERNEL);
>
> Use sizeof(*cpt) like checkpatch will said.
>
> [...]
>> +static void cpt_shutdown(struct pci_dev *pdev)
>> +{
>> + struct cpt_device *cpt = pci_get_drvdata(pdev);
>> +
>> + if (!cpt)
>> + return;
>> +
>> + dev_info(&pdev->dev, "Shutdown device %x:%x.\n",
>> + (u32)pdev->vendor, (u32)pdev->device);
>> +
>> + cpt_unregister_interrupts(cpt);
>> + pci_release_regions(pdev);
>> + pci_disable_device(pdev);
>> + pci_set_drvdata(pdev, NULL);
>> + kzfree(cpt);
>
> since cpt is allocated with devm_, this kzfree is unnecessary
Noted!!
>
> Thanks
> Regards
> Corentin Labbe
>
^ permalink raw reply
* Re: [PATCH v3 2/3] drivers: crypto: Add the Virtual Function driver for CPT
From: George Cherian @ 2017-01-06 7:03 UTC (permalink / raw)
To: Corentin Labbe, george.cherian
Cc: herbert, davem, linux-kernel, linux-crypto, david.daney
In-Reply-To: <20161221140155.GB21051@Red>
Hi Corentin,
On 12/21/2016 07:31 PM, Corentin Labbe wrote:
> Hello
>
> I have some comment inline
>
> On Wed, Dec 21, 2016 at 11:56:12AM +0000, george.cherian@cavium.com wrote:
>> From: George Cherian <george.cherian@cavium.com>
>>
>> Enable the CPT VF driver. CPT is the cryptographic Accelaration Unit
>
> typo acceleration
will fix
>
> [...]
>> +static inline void update_input_data(struct cpt_request_info *req_info,
>> + struct scatterlist *inp_sg,
>> + u32 nbytes, u32 *argcnt)
>> +{
>> + req_info->req.dlen += nbytes;
>> +
>> + while (nbytes) {
>> + u32 len = min(nbytes, inp_sg->length);
>> + u8 *ptr = page_address(sg_page(inp_sg)) + inp_sg->offset;
>
> You could use sg_virt instead.
Thanks for pointing it out, Yes will replace with sg_virt.
>
> But do you have tested your accelerator with user space data (via cryptodev/AF_ALG) ?
No I have tested only using in kernel applications, Not used
cryptodev/AF_ALG
> In my memory, you better use kmap() instead of this direct memory address.
>
> [...]
>> +static inline u32 cvm_enc_dec(struct ablkcipher_request *req, u32 enc,
>> + u32 cipher_type)
>> +{
>> + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
>> + struct cvm_enc_ctx *ctx = crypto_ablkcipher_ctx(tfm);
>> + u32 key_type = AES_128_BIT;
>> + struct cvm_req_ctx *rctx = ablkcipher_request_ctx(req);
>> + u32 enc_iv_len = crypto_ablkcipher_ivsize(tfm);
>> + struct fc_context *fctx = &rctx->fctx;
>> + struct cpt_request_info *req_info = &rctx->cpt_req;
>> + void *cdev = NULL;
>> + u32 status = -1;
>
> Doable but dangerous
> Furthermore, cptvf_do_request return int so why use u32 ?
will fix it.
>
> [...]
>> +void cvm_enc_dec_exit(struct crypto_tfm *tfm)
>> +{
>> + return;
>> +}
>
> So you could remove all reference to this function
>
okay
> [...]
>> +static inline int cav_register_algs(void)
>> +{
>> + int err = 0;
>> +
>> + err = crypto_register_algs(algs, ARRAY_SIZE(algs));
>> + if (err) {
>> + pr_err("Error in aes module init %d\n", err);
>> + return -1;
>
> This is not a standard error code
>
okay
> [...]
>> diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.h b/drivers/crypto/cavium/cpt/cptvf_algs.h
>> new file mode 100644
>> index 0000000..fcb287b
>> --- /dev/null
>> +++ b/drivers/crypto/cavium/cpt/cptvf_algs.h
> [...]
>> +
>> +u32 cptvf_do_request(void *cptvf, struct cpt_request_info *req);
>
> latter this function is set "return int"
>
> [...]
>> +static int cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
>> +{
>> + struct device *dev = &pdev->dev;
>> + struct cpt_vf *cptvf;
>> + int err;
>> +
>> + cptvf = devm_kzalloc(dev, sizeof(struct cpt_vf), GFP_KERNEL);
>
> use sizeof(*cptvf) and checkpatch
>
okay
> [...]
>> +static int setup_sgio_components(struct cpt_vf *cptvf, struct buf_ptr *list,
>> + int buf_count, u8 *buffer)
>> +{
>> + int ret = 0, i, j;
>> + int components;
>> + struct sglist_component *sg_ptr = NULL;
>> + struct pci_dev *pdev = cptvf->pdev;
>> +
>> + if (unlikely(!list)) {
>> + pr_err("Input List pointer is NULL\n");
>> + ret = -EFAULT;
>> + return ret;
>
> You could directly return -EFAULT and use dev_err()
>
okay
>> + }
>> +
>> + for (i = 0; i < buf_count; i++) {
>> + if (likely(list[i].vptr)) {
>> + list[i].dma_addr = dma_map_single(&pdev->dev,
>> + list[i].vptr,
>> + list[i].size,
>> + DMA_BIDIRECTIONAL);
>> + if (unlikely(dma_mapping_error(&pdev->dev,
>> + list[i].dma_addr))) {
>> + pr_err("DMA map kernel buffer failed for component: %d\n",
>> + i);
>
> Use dev_err
>
> [...]
>> + u16 g_sz_bytes = 0, s_sz_bytes = 0;
>> + int ret = 0;
>> + struct pci_dev *pdev = cptvf->pdev;
>> +
>> + if (req->incnt > MAX_SG_IN_CNT || req->outcnt > MAX_SG_OUT_CNT) {
>> + pr_err("Requestes SG components are higher than supported\n");
>
> typo request and use dev_err
>
> In all files you have some pr_x that could be better use as dev_x
okay
>
>> + ret = -EINVAL;
>> + goto scatter_gather_clean;
>> + }
>> +
>> + /* Setup gather (input) components */
>> + g_sz_bytes = ((req->incnt + 3) / 4) * sizeof(struct sglist_component);
>> + info->gather_components = kzalloc((g_sz_bytes), GFP_KERNEL);
>
> unnecessary parenthesis
>
>> + if (!info->gather_components) {
>> + ret = -ENOMEM;
>> + goto scatter_gather_clean;
>> + }
>> +
>> + ret = setup_sgio_components(cptvf, req->in,
>> + req->incnt,
>> + info->gather_components);
>> + if (ret) {
>> + pr_err("Failed to setup gather list\n");
>> + ret = -EFAULT;
>> + goto scatter_gather_clean;
>> + }
>> +
>> + /* Setup scatter (output) components */
>> + s_sz_bytes = ((req->outcnt + 3) / 4) * sizeof(struct sglist_component);
>> + info->scatter_components = kzalloc((s_sz_bytes), GFP_KERNEL);
>
> again
>
>> + if (!info->scatter_components) {
>> + ret = -ENOMEM;
>> + goto scatter_gather_clean;
>> + }
>> +
>> + ret = setup_sgio_components(cptvf, req->out,
>> + req->outcnt,
>> + info->scatter_components);
>> + if (ret) {
>> + pr_err("Failed to setup gather list\n");
>> + ret = -EFAULT;
>> + goto scatter_gather_clean;
>
> double space
okay
>
>> + }
>> +
>> + /* Create and initialize DPTR */
>> + info->dlen = g_sz_bytes + s_sz_bytes + SG_LIST_HDR_SIZE;
>> + info->in_buffer = kzalloc((info->dlen), GFP_KERNEL);
>
> double parenthesis
> I will stop here, you have lots of that in all your alloc
>
okay
> [...]
>> +
>> + ret = send_cpt_command(cptvf, &cptinst, queue);
>> + spin_unlock_bh(&pqueue->lock);
>> + if (unlikely(ret)) {
>> + spin_unlock_bh(&pqueue->lock);
>
> Double unlock
>
Yes will fix it.
> [...]
>> diff --git a/drivers/crypto/cavium/cpt/request_manager.h b/drivers/crypto/cavium/cpt/request_manager.h
>> new file mode 100644
>> index 0000000..df6c306
>> --- /dev/null
>> +++ b/drivers/crypto/cavium/cpt/request_manager.h
>> @@ -0,0 +1,147 @@
>> +/*
>> + * Copyright (C) 2016 Cavium, Inc.
>> + *
>> + * This program is free software; you can redistribute it and/or modify it
>> + * under the terms of version 2 of the GNU General Public License
>> + * as published by the Free Software Foundation.
>> + */
>> +
>> +#ifndef __REQUEST_MANGER_H
>> +#define __REQUEST_MANGER_H
>
> typo manager
>
okay
> Thanks
> Regards
> Corentin Labbe
>
^ permalink raw reply
* [PATCH v1 0/8] crypto:chcr- Bug fixes
From: Harsh Jain @ 2017-01-06 8:31 UTC (permalink / raw)
To: hariprasad, netdev, herbert, linux-crypto; +Cc: Harsh Jain
The patch series is based on Herbert's cryptodev-2.6 tree.
It include bug fixes.
Atul Gupta (4):
crypto:chcr-Change flow IDs
crypto:chcr- Fix panic on dma_unmap_sg
crypto:chcr- Check device is allocated before use
crypto:chcr- Fix wrong typecasting
Harsh Jain (4):
crypto:chcr- Fix key length for RFC4106
crypto:chcr- Use cipher instead of Block Cipher in gcm setkey
crypto:chcr: Change cra_flags for cipher algos
crypto:chcr- Change algo priority
drivers/crypto/chelsio/chcr_algo.c | 97 ++++++++++++++-------------
drivers/crypto/chelsio/chcr_algo.h | 9 +--
drivers/crypto/chelsio/chcr_core.c | 27 ++++----
drivers/crypto/chelsio/chcr_core.h | 1 +
drivers/crypto/chelsio/chcr_crypto.h | 5 +-
drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h | 8 +++
6 files changed, 80 insertions(+), 67 deletions(-)
--
1.8.2.3
^ permalink raw reply
* [PATCH v1 2/8] crypto:chcr- Fix panic on dma_unmap_sg
From: Harsh Jain @ 2017-01-06 8:31 UTC (permalink / raw)
To: hariprasad, netdev, herbert, linux-crypto; +Cc: Harsh Jain, Atul Gupta
In-Reply-To: <cover.1483599449.git.harsh@chelsio.com>
Save DMA mapped sg list addresses to request context buffer.
Signed-off-by: Atul Gupta <atul.gupta@chelsio.com>
---
drivers/crypto/chelsio/chcr_algo.c | 49 +++++++++++++++++++-----------------
drivers/crypto/chelsio/chcr_crypto.h | 3 +++
2 files changed, 29 insertions(+), 23 deletions(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index 1d7dfcf..deec7c0 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -158,7 +158,7 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
case CRYPTO_ALG_TYPE_AEAD:
ctx_req.req.aead_req = (struct aead_request *)req;
ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req);
- dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.req.aead_req->dst,
+ dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst,
ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE);
if (ctx_req.ctx.reqctx->skb) {
kfree_skb(ctx_req.ctx.reqctx->skb);
@@ -1364,8 +1364,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
struct chcr_wr *chcr_req;
struct cpl_rx_phys_dsgl *phys_cpl;
struct phys_sge_parm sg_param;
- struct scatterlist *src, *dst;
- struct scatterlist src_sg[2], dst_sg[2];
+ struct scatterlist *src;
unsigned int frags = 0, transhdr_len;
unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0;
unsigned int kctx_len = 0;
@@ -1385,19 +1384,21 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0)
goto err;
- src = scatterwalk_ffwd(src_sg, req->src, req->assoclen);
- dst = src;
+ src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
+ reqctx->dst = src;
+
if (req->src != req->dst) {
err = chcr_copy_assoc(req, aeadctx);
if (err)
return ERR_PTR(err);
- dst = scatterwalk_ffwd(dst_sg, req->dst, req->assoclen);
+ reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
+ req->assoclen);
}
if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) {
null = 1;
assoclen = 0;
}
- reqctx->dst_nents = sg_nents_for_len(dst, req->cryptlen +
+ reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
(op_type ? -authsize : authsize));
if (reqctx->dst_nents <= 0) {
pr_err("AUTHENC:Invalid Destination sg entries\n");
@@ -1462,7 +1463,7 @@ static struct sk_buff *create_authenc_wr(struct aead_request *req,
sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
sg_param.qid = qid;
sg_param.align = 0;
- if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, dst,
+ if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst,
&sg_param))
goto dstmap_fail;
@@ -1713,8 +1714,7 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
struct chcr_wr *chcr_req;
struct cpl_rx_phys_dsgl *phys_cpl;
struct phys_sge_parm sg_param;
- struct scatterlist *src, *dst;
- struct scatterlist src_sg[2], dst_sg[2];
+ struct scatterlist *src;
unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE;
unsigned int dst_size = 0, kctx_len;
unsigned int sub_type;
@@ -1730,17 +1730,19 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0)
goto err;
sub_type = get_aead_subtype(tfm);
- src = scatterwalk_ffwd(src_sg, req->src, req->assoclen);
- dst = src;
+ src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
+ reqctx->dst = src;
+
if (req->src != req->dst) {
err = chcr_copy_assoc(req, aeadctx);
if (err) {
pr_err("AAD copy to destination buffer fails\n");
return ERR_PTR(err);
}
- dst = scatterwalk_ffwd(dst_sg, req->dst, req->assoclen);
+ reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
+ req->assoclen);
}
- reqctx->dst_nents = sg_nents_for_len(dst, req->cryptlen +
+ reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
(op_type ? -authsize : authsize));
if (reqctx->dst_nents <= 0) {
pr_err("CCM:Invalid Destination sg entries\n");
@@ -1779,7 +1781,7 @@ static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
sg_param.qid = qid;
sg_param.align = 0;
- if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, dst,
+ if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst,
&sg_param))
goto dstmap_fail;
@@ -1811,8 +1813,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
struct chcr_wr *chcr_req;
struct cpl_rx_phys_dsgl *phys_cpl;
struct phys_sge_parm sg_param;
- struct scatterlist *src, *dst;
- struct scatterlist src_sg[2], dst_sg[2];
+ struct scatterlist *src;
unsigned int frags = 0, transhdr_len;
unsigned int ivsize = AES_BLOCK_SIZE;
unsigned int dst_size = 0, kctx_len;
@@ -1834,13 +1835,14 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
if (sg_nents_for_len(req->src, req->assoclen + req->cryptlen) < 0)
goto err;
- src = scatterwalk_ffwd(src_sg, req->src, req->assoclen);
- dst = src;
+ src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
+ reqctx->dst = src;
if (req->src != req->dst) {
err = chcr_copy_assoc(req, aeadctx);
if (err)
return ERR_PTR(err);
- dst = scatterwalk_ffwd(dst_sg, req->dst, req->assoclen);
+ reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, req->dst,
+ req->assoclen);
}
if (!req->cryptlen)
@@ -1850,7 +1852,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
crypt_len = AES_BLOCK_SIZE;
else
crypt_len = req->cryptlen;
- reqctx->dst_nents = sg_nents_for_len(dst, req->cryptlen +
+ reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
(op_type ? -authsize : authsize));
if (reqctx->dst_nents <= 0) {
pr_err("GCM:Invalid Destination sg entries\n");
@@ -1925,7 +1927,7 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
sg_param.qid = qid;
sg_param.align = 0;
- if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, dst,
+ if (map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl, reqctx->dst,
&sg_param))
goto dstmap_fail;
@@ -1939,7 +1941,8 @@ static struct sk_buff *create_gcm_wr(struct aead_request *req,
write_sg_to_skb(skb, &frags, src, req->cryptlen);
} else {
aes_gcm_empty_pld_pad(req->dst, authsize - 1);
- write_sg_to_skb(skb, &frags, dst, crypt_len);
+ write_sg_to_skb(skb, &frags, reqctx->dst, crypt_len);
+
}
create_wreq(ctx, chcr_req, req, skb, kctx_len, size, 1,
diff --git a/drivers/crypto/chelsio/chcr_crypto.h b/drivers/crypto/chelsio/chcr_crypto.h
index d5af7d6..7ec0a8f 100644
--- a/drivers/crypto/chelsio/chcr_crypto.h
+++ b/drivers/crypto/chelsio/chcr_crypto.h
@@ -158,6 +158,9 @@ struct ablk_ctx {
};
struct chcr_aead_reqctx {
struct sk_buff *skb;
+ struct scatterlist *dst;
+ struct scatterlist srcffwd[2];
+ struct scatterlist dstffwd[2];
short int dst_nents;
u16 verify;
u8 iv[CHCR_MAX_CRYPTO_IV_LEN];
--
1.8.2.3
^ permalink raw reply related
* [PATCH v1 3/8] crypto:chcr- Fix key length for RFC4106
From: Harsh Jain @ 2017-01-06 8:31 UTC (permalink / raw)
To: hariprasad, netdev, herbert, linux-crypto; +Cc: Harsh Jain
In-Reply-To: <cover.1483599449.git.harsh@chelsio.com>
Check keylen before copying salt to avoid wrap around of Integer.
Signed-off-by: Harsh Jain <harsh@chelsio.com>
---
drivers/crypto/chelsio/chcr_algo.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c
index deec7c0..6c2dea3 100644
--- a/drivers/crypto/chelsio/chcr_algo.c
+++ b/drivers/crypto/chelsio/chcr_algo.c
@@ -2194,8 +2194,8 @@ static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
unsigned int ck_size;
int ret = 0, key_ctx_size = 0;
- if (get_aead_subtype(aead) ==
- CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
+ if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
+ keylen > 3) {
keylen -= 4; /* nonce/salt is present in the last 4 bytes */
memcpy(aeadctx->salt, key + keylen, 4);
}
--
1.8.2.3
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox